Ascend/htknow
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial project import
Some checks failed
CI / Frontend (Vite) (push) Successful in 24s
Details
CI / Docker Release Tar (push) Has been skipped
Details
CI / Backend (Rust) (push) Failing after 10m15s
Details

Browse Source
This commit is contained in:
Defeng 2026-07-24 10:28:31 +08:00
commit fe8f0a875a
93 changed files with 48547 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.cargo/config.toml Normal file
View File

@ -0,0 +1,11 @@
[build]
rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup"]
[target.x86_64-apple-darwin]
rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"]
[target.aarch64-apple-darwin]
rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"]
[target.aarch64-unknown-linux-gnu]
linker = "aarch64-linux-gnu-gcc"

43
.dockerignore Normal file
View File

@ -0,0 +1,43 @@
# Rust build artifacts (except the release binary we need)
target/*/incremental
target/*/build
target/*/deps
target/*/examples
target/*/.fingerprint
*.pdb
# Git
.git
.gitignore
# IDE
.vscode
.idea
*.swp
*.swo
*~
# Local data (will be mounted as volumes in production)
data/
# Logs
*.log
# OS files
.DS_Store
Thumbs.db
# Documentation
README.md
docs/
# CI/CD
.gitea
.github
.gitlab-ci.yml
# Development files
clear.sh
# Frontend is embedded into the Rust binary before docker build
frontend/

247
.gitea/scripts/setup-rust.sh Normal file
View File

@ -0,0 +1,247 @@
#!/usr/bin/env bash
set -euo pipefail
REQUIRED_RUST_VERSION="${REQUIRED_RUST_VERSION:-1.88.0}"
DEFAULT_RUSTUP_INIT_URL="https://rsproxy.cn/rustup-init.sh"
DEFAULT_RUSTUP_DIST_SERVER="https://rsproxy.cn"
DEFAULT_RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup"
RUSTUP_INIT_URL="${RUSTUP_INIT_URL:-${DEFAULT_RUSTUP_INIT_URL}}"
export RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-${DEFAULT_RUSTUP_DIST_SERVER}}"
export RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-${DEFAULT_RUSTUP_UPDATE_ROOT}}"
APT_UPDATED=0
PACMAN_SYNCED=0
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install system packages."
exit 1
fi
}
ensure_pkg() {
if command -v pacman >/dev/null 2>&1; then
if [ "${PACMAN_SYNCED}" -ne 1 ]; then
as_root pacman -Sy --noconfirm
PACMAN_SYNCED=1
fi
as_root pacman --noconfirm --needed "$@"
elif command -v apt-get >/dev/null 2>&1; then
if [ "${APT_UPDATED}" -ne 1 ]; then
as_root apt-get update
APT_UPDATED=1
fi
as_root apt-get install -y --no-install-recommends "$@"
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache "$@"
elif command -v dnf >/dev/null 2>&1; then
as_root dnf install -y "$@"
else
echo "Unsupported package manager. Please preinstall required tools: $*"
exit 1
fi
}
has_pkg_config() {
command -v pkg-config >/dev/null 2>&1 || command -v pkgconf >/dev/null 2>&1
}
ensure_cc_toolchain() {
if command -v cc >/dev/null 2>&1 && command -v make >/dev/null 2>&1 && has_pkg_config; then
return
fi
if command -v pacman >/dev/null 2>&1; then
ensure_pkg gcc make pkgconf
elif command -v apt-get >/dev/null 2>&1; then
ensure_pkg build-essential pkg-config
elif command -v apk >/dev/null 2>&1; then
ensure_pkg build-base pkgconf
elif command -v dnf >/dev/null 2>&1; then
ensure_pkg gcc gcc-c++ make pkgconf-pkg-config
else
echo "Unsupported package manager. Please preinstall C toolchain (cc/make/pkg-config)."
exit 1
fi
}
resolve_protoc_include() {
local protoc_bin candidate proto_file
protoc_bin="$(command -v protoc || true)"
for candidate in \
"${PROTOC_INCLUDE:-}" \
/usr/include \
/usr/local/include \
/usr/include/x86_64-linux-gnu \
/usr/include/aarch64-linux-gnu \
/usr/lib/protobuf/include \
"$(dirname "${protoc_bin}")/../include"; do
[ -n "${candidate}" ] || continue
if [ -f "${candidate}/google/protobuf/empty.proto" ]; then
(
cd "${candidate}"
pwd
)
return 0
fi
done
if command -v find >/dev/null 2>&1; then
proto_file="$(find /usr /usr/local -type f -path '*/google/protobuf/empty.proto' 2>/dev/null | head -n 1 || true)"
if [ -n "${proto_file}" ]; then
(
cd "$(dirname "$(dirname "$(dirname "${proto_file}")")")"
pwd
)
return 0
fi
fi
return 1
}
ensure_protoc() {
if command -v protoc >/dev/null 2>&1 && resolve_protoc_include >/dev/null 2>&1; then
return
fi
if command -v pacman >/dev/null 2>&1; then
ensure_pkg protobuf
elif command -v apt-get >/dev/null 2>&1; then
ensure_pkg protobuf-compiler libprotobuf-dev
elif command -v apk >/dev/null 2>&1; then
ensure_pkg protobuf protobuf-dev
elif command -v dnf >/dev/null 2>&1; then
ensure_pkg protobuf-compiler protobuf-devel
else
echo "Unsupported package manager. Please preinstall protoc."
exit 1
fi
if ! command -v protoc >/dev/null 2>&1; then
echo "protoc not found after installation."
exit 1
fi
if ! resolve_protoc_include >/dev/null 2>&1; then
echo "protoc found but standard protobuf include files are missing."
exit 1
fi
}
version_ge() {
[ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -n 1)" = "$2" ]
}
has_usable_system_rust() {
local current_rust_version
if ! command -v cargo >/dev/null 2>&1 || ! command -v rustc >/dev/null 2>&1; then
return 1
fi
current_rust_version="$(rustc --version | awk '{print $2}')"
if [ -z "${current_rust_version}" ] || ! version_ge "${current_rust_version}" "${REQUIRED_RUST_VERSION}"; then
return 1
fi
cargo fmt --version >/dev/null 2>&1
}
ensure_downloader() {
if command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; then
return
fi
ensure_pkg curl
}
install_rustup_with() {
local init_url
init_url="$1"
if command -v curl >/dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -fsSL "${init_url}" \
| sh -s -- -y --profile minimal --default-toolchain none
else
wget -qO- "${init_url}" \
| sh -s -- -y --profile minimal --default-toolchain none
fi
}
setup_rustup_toolchain() {
ensure_downloader
if ! command -v rustup >/dev/null 2>&1; then
if ! install_rustup_with "${RUSTUP_INIT_URL}"; then
if [ "${RUSTUP_INIT_URL}" = "${DEFAULT_RUSTUP_INIT_URL}" ] \
&& [ "${RUSTUP_DIST_SERVER}" = "${DEFAULT_RUSTUP_DIST_SERVER}" ] \
&& [ "${RUSTUP_UPDATE_ROOT}" = "${DEFAULT_RUSTUP_UPDATE_ROOT}" ]; then
echo "Primary rustup mirror is unavailable, falling back to official endpoints."
export RUSTUP_DIST_SERVER="https://static.rust-lang.org"
export RUSTUP_UPDATE_ROOT="https://static.rust-lang.org/rustup"
install_rustup_with "https://sh.rustup.rs"
else
echo "Failed to install rustup from ${RUSTUP_INIT_URL}."
exit 1
fi
fi
fi
if [ -f "${HOME}/.cargo/env" ]; then
. "${HOME}/.cargo/env"
fi
export PATH="${HOME}/.cargo/bin:${PATH}"
rustup set auto-self-update disable || true
rustup set profile minimal
if ! rustup toolchain list | awk '{print $1}' | grep -Eq "^${REQUIRED_RUST_VERSION}(-|$)"; then
rustup toolchain install "${REQUIRED_RUST_VERSION}" --profile minimal --component rustfmt --no-self-update
fi
rustup default "${REQUIRED_RUST_VERSION}"
rustup component add rustfmt --toolchain "${REQUIRED_RUST_VERSION}" || true
}
ensure_cc_toolchain
ensure_protoc
if ! has_usable_system_rust; then
setup_rustup_toolchain
fi
if ! has_usable_system_rust; then
echo "Rust toolchain ${REQUIRED_RUST_VERSION} with rustfmt is not available after setup."
exit 1
fi
PROTOC_BIN="$(command -v protoc || true)"
if [ -z "${PROTOC_BIN}" ]; then
echo "protoc not found after installation."
exit 1
fi
PROTOC_INC="$(resolve_protoc_include || true)"
if [ -z "${PROTOC_INC}" ]; then
echo "Could not locate google/protobuf include dir for protoc."
exit 1
fi
if [ -n "${GITHUB_ENV:-}" ]; then
echo "PROTOC=${PROTOC_BIN}" >> "${GITHUB_ENV}"
echo "PROTOC_INCLUDE=${PROTOC_INC}" >> "${GITHUB_ENV}"
fi
export PROTOC="${PROTOC_BIN}"
export PROTOC_INCLUDE="${PROTOC_INC}"
if [ -d "${HOME}/.cargo/bin" ] && [ -n "${GITHUB_PATH:-}" ]; then
echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}"
fi
cargo --version
rustc --version
protoc --version

638
.gitea/workflows/ci.yml Normal file
View File

@ -0,0 +1,638 @@
name: CI
on:
push:
branches:
- master
- main
tags:
- "*"
pull_request:
workflow_dispatch:
env:
REQUIRED_RUST_VERSION: 1.91.0
jobs:
backend:
name: Backend (Rust)
if: github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')
runs-on: amd
steps:
- name: Checkout (shell)
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
SERVER_URL="${CI_REPO_BASE_URL:-${GITHUB_SERVER_URL%/}}"
PRIMARY_URL="${SERVER_URL}/${GITHUB_REPOSITORY}.git"
URLS="${PRIMARY_URL}"
case "${PRIMARY_URL}" in
https://*) URLS="${URLS} http://${PRIMARY_URL#https://}" ;;
http://*) URLS="${URLS} https://${PRIMARY_URL#http://}" ;;
esac
[ -d .git ] || git init .
BASIC_AUTH=""
if [ -n "${GITHUB_TOKEN:-}" ]; then
BASIC_AUTH="$(printf '%s' "${GITHUB_ACTOR}:${GITHUB_TOKEN}" | base64 | tr -d '\n')"
fi
git_fetch() {
if [ -n "${BASIC_AUTH}" ]; then
git -c "http.extraHeader=Authorization: Basic ${BASIC_AUTH}" fetch --depth 1 origin "${GITHUB_SHA}"
else
git fetch --depth 1 origin "${GITHUB_SHA}"
fi
}
FETCH_OK=0
for URL in ${URLS}; do
if git remote get-url origin >/dev/null 2>&1; then
git remote set-url origin "${URL}"
else
git remote add origin "${URL}"
fi
if git_fetch; then
FETCH_OK=1
break
fi
done
if [ "${FETCH_OK}" -ne 1 ]; then
echo "Failed to fetch ${GITHUB_SHA} from ${PRIMARY_URL} (tried http/https)."
exit 1
fi
git checkout --force FETCH_HEAD
- name: Setup Rust
run: bash .gitea/scripts/setup-rust.sh
- name: Check format
run: |
set -euo pipefail
NIGHTLY_ONLY_KEYS='^(indent_style|wrap_comments|normalize_comments|format_macro_matchers|format_macro_bodies|struct_lit_single_line|fn_single_line|imports_indent|imports_layout|imports_granularity|group_imports|type_punctuation_density|space_before_colon|space_after_colon|spaces_around_ranges|binop_separator|overflow_delimited_expr|match_arm_blocks|force_multiline_blocks|brace_style|control_brace_style|trailing_semicolon|trailing_comma)\s*='
if [ -f .rustfmt.toml ] && grep -Eq "${NIGHTLY_ONLY_KEYS}" .rustfmt.toml; then
echo "Skip rustfmt check: .rustfmt.toml includes nightly-only options."
exit 0
fi
cargo fmt --all -- --check
- name: Setup Node (shell)
run: |
set -euo pipefail
if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then
node --version
npm --version
exit 0
fi
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install nodejs/npm."
exit 1
fi
}
if command -v pacman >/dev/null 2>&1; then
as_root pacman -Sy --noconfirm --needed nodejs npm
elif command -v apt-get >/dev/null 2>&1; then
as_root apt-get update
as_root apt-get install -y nodejs npm
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache nodejs npm
elif command -v dnf >/dev/null 2>&1; then
as_root dnf install -y nodejs npm
else
echo "Unsupported package manager. Preinstall nodejs/npm on runner."
exit 1
fi
node --version
npm --version
- name: Build frontend assets for RustEmbed
working-directory: frontend
run: |
npm config set registry https://registry.npmjs.org
npm config set replace-registry-host always
npm ci --cache .npm-cache --prefer-online
npm run build
- name: Run tests
run: cargo test --workspace --all-features
frontend:
name: Frontend (Vite)
if: github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')
runs-on: amd
steps:
- name: Checkout (shell)
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
SERVER_URL="${CI_REPO_BASE_URL:-${GITHUB_SERVER_URL%/}}"
PRIMARY_URL="${SERVER_URL}/${GITHUB_REPOSITORY}.git"
URLS="${PRIMARY_URL}"
case "${PRIMARY_URL}" in
https://*) URLS="${URLS} http://${PRIMARY_URL#https://}" ;;
http://*) URLS="${URLS} https://${PRIMARY_URL#http://}" ;;
esac
[ -d .git ] || git init .
BASIC_AUTH=""
if [ -n "${GITHUB_TOKEN:-}" ]; then
BASIC_AUTH="$(printf '%s' "${GITHUB_ACTOR}:${GITHUB_TOKEN}" | base64 | tr -d '\n')"
fi
git_fetch() {
if [ -n "${BASIC_AUTH}" ]; then
git -c "http.extraHeader=Authorization: Basic ${BASIC_AUTH}" fetch --depth 1 origin "${GITHUB_SHA}"
else
git fetch --depth 1 origin "${GITHUB_SHA}"
fi
}
FETCH_OK=0
for URL in ${URLS}; do
if git remote get-url origin >/dev/null 2>&1; then
git remote set-url origin "${URL}"
else
git remote add origin "${URL}"
fi
if git_fetch; then
FETCH_OK=1
break
fi
done
if [ "${FETCH_OK}" -ne 1 ]; then
echo "Failed to fetch ${GITHUB_SHA} from ${PRIMARY_URL} (tried http/https)."
exit 1
fi
git checkout --force FETCH_HEAD
- name: Setup Node (shell)
run: |
set -euo pipefail
if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then
node --version
npm --version
exit 0
fi
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install nodejs/npm."
exit 1
fi
}
if command -v pacman >/dev/null 2>&1; then
as_root pacman -Sy --noconfirm --needed nodejs npm
elif command -v apt-get >/dev/null 2>&1; then
as_root apt-get update
as_root apt-get install -y nodejs npm
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache nodejs npm
elif command -v dnf >/dev/null 2>&1; then
as_root dnf install -y nodejs npm
else
echo "Unsupported package manager. Preinstall nodejs/npm on runner."
exit 1
fi
node --version
npm --version
- name: Install dependencies
working-directory: frontend
run: |
npm config set registry https://registry.npmjs.org
npm config set replace-registry-host always
npm ci --cache .npm-cache --prefer-online
- name: Build
working-directory: frontend
run: npm run build
docker_release:
name: Docker Release Tar
if: startsWith(github.ref, 'refs/tags/')
runs-on: amd
permissions:
contents: write
releases: write
steps:
- name: Checkout (shell)
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
SERVER_URL="${CI_REPO_BASE_URL:-${GITHUB_SERVER_URL%/}}"
PRIMARY_URL="${SERVER_URL}/${GITHUB_REPOSITORY}.git"
URLS="${PRIMARY_URL}"
case "${PRIMARY_URL}" in
https://*) URLS="${URLS} http://${PRIMARY_URL#https://}" ;;
http://*) URLS="${URLS} https://${PRIMARY_URL#http://}" ;;
esac
[ -d .git ] || git init .
BASIC_AUTH=""
if [ -n "${GITHUB_TOKEN:-}" ]; then
BASIC_AUTH="$(printf '%s' "${GITHUB_ACTOR}:${GITHUB_TOKEN}" | base64 | tr -d '\n')"
fi
git_fetch() {
if [ -n "${BASIC_AUTH}" ]; then
git -c "http.extraHeader=Authorization: Basic ${BASIC_AUTH}" fetch --depth 1 origin "${GITHUB_SHA}"
else
git fetch --depth 1 origin "${GITHUB_SHA}"
fi
}
FETCH_OK=0
for URL in ${URLS}; do
if git remote get-url origin >/dev/null 2>&1; then
git remote set-url origin "${URL}"
else
git remote add origin "${URL}"
fi
if git_fetch; then
FETCH_OK=1
break
fi
done
if [ "${FETCH_OK}" -ne 1 ]; then
echo "Failed to fetch ${GITHUB_SHA} from ${PRIMARY_URL} (tried http/https)."
exit 1
fi
git checkout --force FETCH_HEAD
- name: Setup Rust
run: bash .gitea/scripts/setup-rust.sh
- name: Setup Node (shell)
run: |
set -euo pipefail
if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then
node --version
npm --version
exit 0
fi
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install nodejs/npm."
exit 1
fi
}
if command -v pacman >/dev/null 2>&1; then
as_root pacman -Sy --noconfirm --needed nodejs npm
elif command -v apt-get >/dev/null 2>&1; then
as_root apt-get update
as_root apt-get install -y nodejs npm
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache nodejs npm
elif command -v dnf >/dev/null 2>&1; then
as_root dnf install -y nodejs npm
else
echo "Unsupported package manager. Preinstall nodejs/npm on runner."
exit 1
fi
node --version
npm --version
- name: Setup Docker (shell)
run: |
set -euo pipefail
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install/start docker."
exit 1
fi
}
if ! command -v docker >/dev/null 2>&1; then
if command -v pacman >/dev/null 2>&1; then
as_root pacman -Sy --noconfirm --needed docker
elif command -v apt-get >/dev/null 2>&1; then
as_root apt-get update
as_root apt-get install -y docker.io
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache docker
elif command -v dnf >/dev/null 2>&1; then
if ! as_root dnf install -y docker; then
as_root dnf install -y moby-engine
fi
else
echo "Unsupported package manager. Please preinstall docker."
exit 1
fi
fi
if ! docker info >/dev/null 2>&1; then
if command -v dockerd >/dev/null 2>&1; then
nohup dockerd >/tmp/dockerd.log 2>&1 &
for _ in $(seq 1 60); do
if docker info >/dev/null 2>&1; then
break
fi
sleep 1
done
fi
fi
if ! docker info >/dev/null 2>&1; then
echo "Docker daemon is not available. Runner must support docker (daemon or mounted /var/run/docker.sock)."
[ -f /tmp/dockerd.log ] && tail -n 80 /tmp/dockerd.log || true
exit 1
fi
docker version
- name: Build Docker image by script
run: |
set -euo pipefail
TAG_NAME="${GITHUB_REF_NAME}"
SAFE_TAG="$(echo "${TAG_NAME}" | tr '/' '-')"
npm config set registry https://registry.npmjs.org
npm config set replace-registry-host always
IMAGE_TAG="${SAFE_TAG}" sh build-docker.sh
docker save "htknow:${SAFE_TAG}" -o "htknow-${SAFE_TAG}.tar"
gzip -f "htknow-${SAFE_TAG}.tar"
echo "TAG_NAME=${TAG_NAME}" >> "${GITHUB_ENV}"
echo "SAFE_TAG=${SAFE_TAG}" >> "${GITHUB_ENV}"
echo "ASSET_FILE=htknow-${SAFE_TAG}.tar.gz" >> "${GITHUB_ENV}"
- name: Create release (if needed) and upload tar
env:
RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }}
run: |
set -euo pipefail
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install system packages."
exit 1
fi
}
ensure_pkg() {
if command -v pacman >/dev/null 2>&1; then
as_root pacman -Sy --noconfirm --needed "$@"
elif command -v apt-get >/dev/null 2>&1; then
as_root apt-get update
as_root apt-get install -y "$@"
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache "$@"
elif command -v dnf >/dev/null 2>&1; then
as_root dnf install -y "$@"
else
echo "Unsupported package manager. Please preinstall required tools: $*"
exit 1
fi
}
command -v curl >/dev/null 2>&1 || ensure_pkg curl
command -v jq >/dev/null 2>&1 || ensure_pkg jq
if [ -z "${RELEASE_TOKEN:-}" ]; then
echo "secrets.RELEASE_TOKEN is required to create releases and upload assets."
exit 1
fi
REPO_OWNER="${GITHUB_REPOSITORY%%/*}"
REPO_NAME="${GITHUB_REPOSITORY##*/}"
API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
AUTH_HEADER="Authorization: token ${RELEASE_TOKEN}"
RELEASE_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/releases"
LOOKUP_CODE="$(curl -sS -o release.json -w '%{http_code}' \
-H "${AUTH_HEADER}" \
"${RELEASE_URL}/tags/${TAG_NAME}")"
if [ "${LOOKUP_CODE}" = "200" ]; then
echo "Release for tag ${TAG_NAME} already exists."
elif [ "${LOOKUP_CODE}" = "404" ]; then
cat > create-release.json <<EOF
{"tag_name":"${TAG_NAME}","name":"${TAG_NAME}","body":"Automated Docker image tar.gz for ${TAG_NAME}.","draft":false,"prerelease":false}
EOF
curl -sS -f -X POST \
-H "${AUTH_HEADER}" \
-H "Content-Type: application/json" \
--data @create-release.json \
"${RELEASE_URL}" \
-o release.json
echo "Created release for ${TAG_NAME}."
else
echo "Unexpected response (${LOOKUP_CODE}) while querying release:"
cat release.json
exit 1
fi
RELEASE_ID="$(jq -r '.id' release.json)"
if [ -z "${RELEASE_ID}" ] || [ "${RELEASE_ID}" = "null" ]; then
echo "Failed to resolve release id."
cat release.json
exit 1
fi
EXISTING_ASSET_ID="$(curl -sS -f \
-H "${AUTH_HEADER}" \
"${RELEASE_URL}/${RELEASE_ID}/assets" \
| jq -r --arg name "${ASSET_FILE}" '.[] | select(.name == $name) | .id')"
if [ -n "${EXISTING_ASSET_ID}" ] && [ "${EXISTING_ASSET_ID}" != "null" ]; then
curl -sS -f -X DELETE \
-H "${AUTH_HEADER}" \
"${RELEASE_URL}/${RELEASE_ID}/assets/${EXISTING_ASSET_ID}" >/dev/null
echo "Deleted existing asset ${ASSET_FILE}."
fi
curl -sS -f -X POST \
-H "${AUTH_HEADER}" \
-H "Content-Type: application/octet-stream" \
--data-binary @"${ASSET_FILE}" \
"${RELEASE_URL}/${RELEASE_ID}/assets?name=${ASSET_FILE}" >/dev/null
echo "Uploaded ${ASSET_FILE} to release ${TAG_NAME}."
- name: Deploy to target server
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
DEPLOY_PASSWORD: ${{ secrets.DEPLOY_PASSWORD }}
DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_COMMAND: ${{ secrets.DEPLOY_COMMAND }}
run: |
set -euo pipefail
if [ -z "${DEPLOY_HOST:-}" ] || [ -z "${DEPLOY_USER:-}" ]; then
echo "Deploy secrets not fully configured (DEPLOY_HOST/DEPLOY_USER). Skip deployment."
exit 0
fi
if [ -z "${DEPLOY_PASSWORD:-}" ] && [ -z "${DEPLOY_SSH_KEY:-}" ]; then
echo "Neither DEPLOY_PASSWORD nor DEPLOY_SSH_KEY is configured. Skip deployment."
exit 0
fi
as_root() {
if [ "$(id -u)" -eq 0 ]; then
"$@"
elif command -v sudo >/dev/null 2>&1; then
sudo "$@"
else
echo "Need root privileges to install openssh client."
exit 1
fi
}
ensure_pkg() {
if command -v pacman >/dev/null 2>&1; then
as_root pacman -Sy --noconfirm --needed "$@"
elif command -v apt-get >/dev/null 2>&1; then
as_root apt-get update
as_root apt-get install -y "$@"
elif command -v apk >/dev/null 2>&1; then
as_root apk add --no-cache "$@"
elif command -v dnf >/dev/null 2>&1; then
as_root dnf install -y "$@"
else
echo "Unsupported package manager. Please preinstall openssh client."
exit 1
fi
}
ensure_ssh_client() {
if command -v ssh >/dev/null 2>&1 && command -v scp >/dev/null 2>&1 && command -v ssh-keyscan >/dev/null 2>&1; then
return
fi
if command -v pacman >/dev/null 2>&1; then
ensure_pkg openssh
elif command -v apt-get >/dev/null 2>&1; then
ensure_pkg openssh-client
elif command -v apk >/dev/null 2>&1; then
ensure_pkg openssh-client
elif command -v dnf >/dev/null 2>&1; then
ensure_pkg openssh-clients
else
echo "Unsupported package manager. Please preinstall ssh/scp."
exit 1
fi
}
ensure_ssh_client
PORT="${DEPLOY_PORT:-22}"
REMOTE_PATH_RAW="${DEPLOY_PATH:-/tmp/htknow-release}"
case "${REMOTE_PATH_RAW}" in
"~") REMOTE_PATH_REMOTE="\$HOME" ;;
"~/"*) REMOTE_PATH_REMOTE="\$HOME/${REMOTE_PATH_RAW#~/}" ;;
*) REMOTE_PATH_REMOTE="${REMOTE_PATH_RAW}" ;;
esac
KNOWN_HOSTS="${HOME}/.ssh/known_hosts"
mkdir -p ~/.ssh
chmod 700 ~/.ssh
touch "${KNOWN_HOSTS}"
chmod 600 "${KNOWN_HOSTS}"
ssh-keyscan -T 10 -p "${PORT}" -H "${DEPLOY_HOST}" >> "${KNOWN_HOSTS}" 2>/dev/null || true
run_ssh() {
ssh -p "${PORT}" \
-o BatchMode=yes \
-o ConnectTimeout=10 \
-o ServerAliveInterval=10 \
-o ServerAliveCountMax=3 \
-o StrictHostKeyChecking=accept-new \
-o UserKnownHostsFile="${KNOWN_HOSTS}" \
-i "${HOME}/.ssh/id_deploy" \
"$@"
}
run_scp() {
scp -P "${PORT}" \
-o BatchMode=yes \
-o ConnectTimeout=10 \
-o ServerAliveInterval=10 \
-o ServerAliveCountMax=3 \
-o StrictHostKeyChecking=accept-new \
-o UserKnownHostsFile="${KNOWN_HOSTS}" \
-i "${HOME}/.ssh/id_deploy" \
"$@"
}
if [ -n "${DEPLOY_PASSWORD:-}" ]; then
command -v sshpass >/dev/null 2>&1 || ensure_pkg sshpass
export SSHPASS="${DEPLOY_PASSWORD}"
run_ssh() {
sshpass -e ssh -p "${PORT}" \
-o PreferredAuthentications=password \
-o PubkeyAuthentication=no \
-o ConnectTimeout=10 \
-o ServerAliveInterval=10 \
-o ServerAliveCountMax=3 \
-o StrictHostKeyChecking=accept-new \
-o UserKnownHostsFile="${KNOWN_HOSTS}" \
"$@"
}
run_scp() {
sshpass -e scp -P "${PORT}" \
-o PreferredAuthentications=password \
-o PubkeyAuthentication=no \
-o ConnectTimeout=10 \
-o ServerAliveInterval=10 \
-o ServerAliveCountMax=3 \
-o StrictHostKeyChecking=accept-new \
-o UserKnownHostsFile="${KNOWN_HOSTS}" \
"$@"
}
else
printf '%s\n' "${DEPLOY_SSH_KEY}" | sed 's/\r$//' > ~/.ssh/id_deploy
chmod 600 ~/.ssh/id_deploy
fi
if ! run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "echo 'ssh connectivity ok'"; then
echo "SSH connectivity check failed. Verify DEPLOY_HOST/DEPLOY_PORT/DEPLOY_USER and DEPLOY_PASSWORD or DEPLOY_SSH_KEY."
exit 1
fi
if [ ! -f "${ASSET_FILE}" ]; then
echo "Local asset file not found: ${ASSET_FILE}"
ls -lah
exit 1
fi
run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc 'set -e; REMOTE_PATH=\"${REMOTE_PATH_REMOTE}\"; REMOTE_PATH=\$(eval echo \"\${REMOTE_PATH}\"); mkdir -p \"\${REMOTE_PATH}\"'"
run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc 'set -e; REMOTE_PATH=\"${REMOTE_PATH_REMOTE}\"; REMOTE_PATH=\$(eval echo \"\${REMOTE_PATH}\"); cat > \"\${REMOTE_PATH}/${ASSET_FILE}\"'" < "${ASSET_FILE}"
run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc 'set -e; REMOTE_PATH=\"${REMOTE_PATH_REMOTE}\"; REMOTE_PATH=\$(eval echo \"\${REMOTE_PATH}\"); ls -lh \"\${REMOTE_PATH}/${ASSET_FILE}\"'"
if [ -n "${DEPLOY_COMMAND:-}" ]; then
DEPLOY_CMD_B64="$(printf '%s' "${DEPLOY_COMMAND}" | base64 | tr -d '\n')"
run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" \
"ASSET_FILE='${ASSET_FILE}' SAFE_TAG='${SAFE_TAG}' DEPLOY_PATH='${REMOTE_PATH_REMOTE}' DEPLOY_CMD_B64='${DEPLOY_CMD_B64}' sh -lc 'DEPLOY_PATH=\$(eval echo \"\$DEPLOY_PATH\"); printf %s \"\$DEPLOY_CMD_B64\" | base64 -d | sh'"
else
run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc '
set -e
DEPLOY_PATH=\"${REMOTE_PATH_REMOTE}\"
DEPLOY_PATH=\$(eval echo \"\$DEPLOY_PATH\")
EXTRACTED_TAR_FILE=\"${ASSET_FILE%.gz}\"
gzip -dc \"\$DEPLOY_PATH/${ASSET_FILE}\" > \"\$DEPLOY_PATH/\$EXTRACTED_TAR_FILE\"
docker load -i \"\$DEPLOY_PATH/\$EXTRACTED_TAR_FILE\"
rm -f \"\$DEPLOY_PATH/\$EXTRACTED_TAR_FILE\"
docker rm -f htknow >/dev/null 2>&1 || true
docker run -d --name htknow --restart unless-stopped -p 8080:8080 -v /opt/htknow/data:/app/data \"htknow:${SAFE_TAG}\"
'"
fi
echo "Deployed htknow:${SAFE_TAG} to ${DEPLOY_USER}@${DEPLOY_HOST}:${REMOTE_PATH_RAW}"

6
.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
/target
app.sqlite*
/data
*.log
.DS_Store
*.pdf

60
.rustfmt.toml Normal file
View File

@ -0,0 +1,60 @@
# .rustfmt.toml
# 基本设置
edition = "2024"
max_width = 120 # 每行最大字符数（建议 80~120）
hard_tabs = false # 使用空格而不是制表符
tab_spaces = 4 # 一个缩进等于 4 个空格
newline_style = "Unix" # 换行符使用 \n（Linux/macOS/WSL 风格）
use_small_heuristics = "Max" # 尽量将短的 use 合并成一行
# 空格与括号
space_before_colon = false # 类型声明中冒号前不加空格: `foo: i32`
space_after_colon = true # 冒号后加空格
spaces_around_ranges = false # 不在 range 周围加空格: `..` 而不是 ` .. `
# 控制结构格式
indent_style = "Block" # 使用块式缩进（而非 Visual）
control_brace_style = "AlwaysSameLine" # if/else、fn 的 `{` 放下一行
brace_style = "PreferSameLine" # 结构体、enum 的 `{` 放同一行（除非有 where）
# 函数相关
fn_params_layout = "Compressed" # 函数参数紧凑布局（少换行）
fn_single_line = false # 不强制简单函数写成单行
# 结构体与元组
struct_lit_single_line = true # 简短结构体字面量放在一行: `Point { x: 1, y: 2 }`
# use 语句的分组方式
group_imports = "StdExternalCrate" # 可选: "StdExternalCrate", "Preserve", "One"
# 导入排序
reorder_imports = true # 自动对 use 导入语句排序
reorder_modules = true # 是否对模块内的 use 按照路径排序
imports_indent = "Block" # use 导入缩进为块风格
imports_layout = "Horizontal" # 垂直排列导入（每行一个）
# 导入同一模块的类型，应该置于同一个块内
imports_granularity = "Crate"
# 模式匹配
match_arm_blocks = true # match 分支使用块时换行
match_arm_leading_pipes = "Never" # 不在 match 分支前加 `|`
force_multiline_blocks = false # 不强制复杂块换行
# 表达式
trailing_comma = "Vertical" # 垂直列表末尾加逗号
trailing_semicolon = true # 语句结尾加分号
wrap_comments = false # 不自动换行注释（保留手动格式）
# 类型与泛型
type_punctuation_density = "Compressed" # 类型标点紧凑: `Vec<i32>` 而非 `Vec< i32 >`
binop_separator = "Front" # 二元操作符换行时，操作符放下一行开头
# 宏
format_macro_matchers = true # 格式化 macro_rules! 中的匹配模式
format_macro_bodies = true # 格式化宏体内容
# 高级（按需启用）
normalize_comments = false # 不自动规范化注释大小写或空格
overflow_delimited_expr = true # 允许括号表达式溢出时不换行（实验性）

159
CLAUDE.md Normal file
View File

@ -0,0 +1,159 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
HTKnow is a Chinese knowledge base management system. The backend is Rust (axum + SQLite + Tantivy + LanceDB) and the frontend is a Vue 3 SPA built with Vite and Tailwind CSS 4. The built frontend is embedded into the Rust binary via `rust-embed`.
## Common Commands
### Backend
```bash
# Run locally (requires external services; see below)
cargo run
# Release build
cargo build --release
# Run all tests
cargo test
# Run a specific integration test
cargo test knowledge_base_flow
# Run only integration tests
cargo test --test api_integration
# Format code (uses .rustfmt.toml config)
cargo fmt
# Build with etcd config center support
cargo build --features etcd
# Build with jemalloc heap profiling (debug only)
cargo build --features profiling
```
Rust edition is 2024 and the project uses the nightly toolchain. Formatting is configured in `.rustfmt.toml` (120 char width, 4-space indentation, `StdExternalCrate` import grouping).
### Frontend
```bash
cd frontend
npm ci
npm run dev # Dev server proxies /api to localhost:3000
npm run build # Outputs to frontend/dist/ (embedded in binary)
```
### Docker
```bash
# Run with docker-compose (port 8080 -> 3000)
docker compose up -d
# Build Docker image (builds frontend + Rust binary)
./build-docker.sh
# Debug build with profiling tools
./build-docker.sh debug
```
## Architecture
### Entry Point and Routing (`src/main.rs`, `src/lib.rs`, `src/api/mod.rs`)
The axum server is started in `main.rs`. Routes are defined in `src/api/mod.rs` and organized into nested routers:
- `/api/v1/knowledge/knowledge_base/` — KB CRUD, tree, reparse
- `/api/v1/knowledge/files/` — upload, list, download, slices, highlighted PDF
- `/api/v1/knowledge/search/` — full-text, vector, image, graph-enhanced, advanced stream, lexicon/synonym management
- `/api/v1/knowledge/graph/` — entity search, entity detail, graph stats
- `/api/v1/knowledge/system/` — heap profiling, LanceDB compact, index force merge/rebuild
All `/api/v1/knowledge/*` routes require the `auth` middleware (`src/lib.rs`), which extracts `x-user-id` and `x-role` from headers into an `AuthUser` extension. `x-user-name` is optional and base64-decoded if valid UTF-8.
Swagger UI is at `/docs` and the OpenAPI JSON is at `/api-docs/openapi.json`.
### Database (`src/db.rs`, `src/init.sql`)
SQLite with WAL mode enabled. The schema is auto-initialized on startup from `src/init.sql`. Key tables include `knowledge_base`, `file`, `file_slice`, `entity`, `relation`, `mention`, `lexicon`, and `synonym`. The pool is created via `sqlx` and passed as axum state to handlers.
### Search (`src/search/`)
`SearchEngine` (`src/search/mod.rs`) orchestrates three search backends:
- **Tantivy** (`src/search/tantivy_engine.rs`) — full-text search with a custom Chinese tokenizer based on `jieba-rs`. Maintains two indexes: a default index and a "full" index. Supports lexicon-based synonym expansion.
- **LanceDB** (`src/search/lancedb.rs`) — vector search. Stores text and image embeddings in separate tables.
- **Embedding client** (`src/search/embedding.rs`) — calls external embedding services for text and image vectors.
The search module also handles reranking via an external service and provides an advanced streaming search endpoint (`src/search/advanced.rs`).
### Background File Processing (`src/processor.rs`)
`FileProcessor` runs as a background task polling the database for pending files. It:
1. Converts Office docs to PDF via an external service.
2. Parses PDFs via MinerU (or a custom parse service if configured).
3. Transcribes audio files via an external service.
4. Splits content into slices (smart or fixed overlap).
5. Stores slices in SQLite, indexes them in Tantivy, and generates embeddings for LanceDB.
6. Optionally builds a knowledge graph via LLM extraction (if `HTKNOW_BUILD_KNOWLEDGE_GRAPH=true`).
File statuses: `0=pending`, `1=completed`, `2=processing`, `3=skipped`, `-1=failed`.
### Knowledge Graph (`src/graph/`)
- `graph_manager.rs` — manages entities, relations, and mentions in SQLite.
- `llm_extractor.rs` — uses an LLM API to extract entities and relations from file content.
Graph data is stored relationally in SQLite and queried through the graph API endpoints.
### Configuration (`src/config.rs`)
All configuration is environment-variable based with sensible defaults. No config files. Key prefixes:
- `HTKNOW_SERVER_*` — server settings
- `HTKNOW_MINERU_URL`, `HTKNOW_OFFICE_CONVERT_URL`, `HTKNOW_EMBEDDING_URL`, etc. — external services
- `DATABASE_URL` — SQLite connection string
- `HTKNOW_DATA_DIR`, `HTKNOW_LANCEDB_PATH`, `HTKNOW_TANTIVY_INDEX_PATH` — storage paths
- `LLM_API_URL`, `LLM_API_KEY`, `LLM_MODEL` — optional LLM for knowledge graph
See `README.md` for the full environment variable table.
### Frontend (`frontend/`)
Vue 3 SPA using Vite and Tailwind CSS 4. The `vite.config.js` proxies `/api` to `http://localhost:3000` in dev mode. Built assets in `frontend/dist/` are served by the Rust binary via `rust-embed` (`src/frontend.rs`).
## External Service Dependencies
For full functionality, these external services must be running:
- **MinerU** — PDF parsing (`HTKNOW_MINERU_URL`)
- **Office converter** — Office to PDF (`HTKNOW_OFFICE_CONVERT_URL`)
- **Embedding service** — text embeddings (`HTKNOW_EMBEDDING_URL`)
- **Image embedding service** — image embeddings (`HTKNOW_IMAGE_EMBEDDING_URL`)
- **Rerank service** — result reranking (`HTKNOW_RERANK_URL`)
- **Audio transcription** — audio to text (`HTKNOW_AUDIO_TRANSCRIPTION_URL`)
Default values in `docker-compose.yml` and `src/config.rs` point to internal network IPs.
## Testing
Integration tests live in `tests/api_integration.rs`. They set up an isolated test environment with a temp directory and a fresh SQLite database, then exercise the full axum router.
The `APP` router is initialized once via `tokio::sync::OnceCell` and reused across tests. Environment variables are set via `unsafe { std::env::set_var(...) }` during test setup.
## Memory Profiling (Debug Builds)
Debug builds use jemalloc as the global allocator. With the `profiling` feature, heap profiling is enabled and accessible via:
- `GET /api/v1/knowledge/system/heap` — returns a heap profile
- `GET /api/v1/knowledge/system/heap/pdf` — returns a PDF visualization
## Notes
- LanceDB auto-compact runs on a cron schedule (`HTKNOW_LANCEDB_COMPACT_CRON`, default `0 0 3 * * *`). Compaction holds a lock to prevent concurrent writes.
- If Tantivy indexes become corrupted (e.g., from an unclean shutdown), the app will panic on startup. Recovery: back up and remove the `tantivy_index` and `tantivy_full_index` directories, then trigger a rebuild via the system endpoints.
- The `cargo build --features etcd` flag enables etcd as a config center, but the etcd integration is optional and not enabled by default.

7367
Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

96
Cargo.toml Normal file
View File

@ -0,0 +1,96 @@
[package]
name = "htknow"
version = "0.1.0"
edition = "2024"
[dependencies]
anyhow = "1.0.100"
axum = { version = "0.8.8", features = ["multipart"] }
chrono = "0.4"
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
tokio-util = { version = "0.7", features = ["io"] }
tokio-cron-scheduler = "0.13"
sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls", "sqlite", "chrono", "derive"] }
lazy_static = "1.4"
log = "0.4"
log4rs = "1.3.0"
lancedb = "0.27.2"
arrow-array = "57"
arrow-schema = "57"
futures = "0.3"
bytes = "1"
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "http2", "stream"] }
once_cell = "1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
# Swagger/OpenAPI 文档
utoipa = { version = "5.3", features = ["axum_extras", "chrono"] }
utoipa-swagger-ui = { version = "9.0.2", features = ["axum", "vendored"] }
tantivy = "0.24.2"
jieba-rs = "0.6"
sha2 = "0.10"
hex = "0.4"
rust-embed = "8"
mime_guess = "2"
uuid = "1.19.0"
base64 = "0.22.1"
# 知识图谱相关依赖
petgraph = "0.6" # 图数据结构
bincode = "1.3" # 序列化图结构
aho-corasick = "1.0" # 高效字符串匹配（词典）
regex = "1" # 正则表达式
# PDF 操作
lopdf = { version = "0.36", default-features = false }
tempfile = "3"
# Excel 解析
calamine = "0.26"
# 压缩文件解压
zip = { version = "2", default-features = false, features = ["aes-crypto", "bzip2", "deflate", "deflate64", "lzma", "time", "zstd"] }
tar = "0.4"
flate2 = "1"
bzip2 = "0.4"
xz2 = "0.1"
# 错误处理
thiserror = "1"
# 编码处理（ZIP 中文文件名 GBK/GB18030 解码）
encoding_rs = "0.8"
# etcd 配置中心（可选）
etcd-client = { version = "0.14", optional = true }
serde_with = "3.16.1"
sysinfo = "0.29"
# jemalloc allocator
# Note: profiling is only needed for debug/analysis, disabled in release for better performance
tikv-jemallocator = { version = "0.6" }
tikv-jemalloc-ctl = { version = "0.6", optional = true, features = ["use_std"] }
[features]
default = []
etcd = ["etcd-client"]
# Enable jemalloc profiling for debug/analysis builds only
profiling = ["tikv-jemalloc-ctl"]
[dev-dependencies]
http-body-util = "0.1"
tower = "0.5"
[profile.release]
opt-level = "s"
lto = "thin"
codegen-units = 1
panic = "abort"
strip = "symbols"

184
DEBUG_MEMORY.md Normal file
View File

@ -0,0 +1,184 @@
# Debug 镜像内存分析指南
## 构建 Debug 镜像
Debug 镜像包含 jemalloc 内存分析工具:
- `libjemalloc-dev` - jemalloc 开发库
- `perl` - jeprof 脚本依赖
- `graphviz` - 生成可视化图表
- `binutils` - 符号解析工具
### 构建命令
```bash
# 构建 debug 镜像
./build-docker.sh debug
# 构建 release 镜像 (不包含分析工具)
./build-docker.sh
```
## 导出 Heap Dump
### 方法 1: 直接获取 PDF 报告 (最简单 🚀)
```bash
# 一键生成并下载 PDF 报告
curl -H "x-user-id: 1" \
-H "x-user-name: admin" \
-H "x-role: admin" \
-o heap_analysis.pdf \
http://localhost:8080/api/v1/knowledge/system/heap/pdf
# 然后直接打开查看
open heap_analysis.pdf # macOS
xdg-open heap_analysis.pdf # Linux
```
### 方法 2: 通过 API 导出原始 heap 文件
```bash
# 导出当前内存快照
curl -H "x-user-id: 1" \
-H "x-user-name: admin" \
-H "x-role: admin" \
-o htknow.heap.$(date +%s).heap \
http://localhost:8080/api/v1/knowledge/system/heap
# 查看堆分析状态
curl -H "x-user-id: 1" \
-H "x-user-name: admin" \
-H "x-role: admin" \
http://localhost:8080/api/v1/knowledge/system/heap/status | jq
# 查看内存占用
curl -H "x-user-id: 1" \
-H "x-user-name: admin" \
-H "x-role: admin" \
http://localhost:8080/api/v1/knowledge/system/memory | jq
```
### 方法 3: 进入容器手动导出
```bash
# 进入容器
docker exec -it htknow bash
# 安装 jeprof (如果需要在容器内分析)
apt update && apt install -y google-perftools
# 使用 jeprof 生成报告
jeprof --show_bytes --text /app/htknow /path/to/heap.file
```
## 分析 Heap Dump
### 手动分析
```bash
# 安装 gperftools (macOS)
brew install gperftools
# 安装 gperftools (Linux)
apt-get install google-perftools
# 生成 PDF 报告
jeprof --show_bytes --pdf target/debug/htknow htknow.heap.*.heap > report.pdf
# 生成文本报告
jeprof --show_bytes --text target/debug/htknow htknow.heap.*.heap > report.txt
# 查看前 10 个内存分配点
jeprof --show_bytes --text target/debug/htknow htknow.heap.*.heap | head -20
```
## 采样率配置
当前配置在 `src/main.rs:6`:
```rust
pub static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:10\0";
```
| lg_prof_sample | 采样间隔 | 适用场景 | 性能开销 |
|----------------|----------|----------|----------|
| 0 | 每次分配 | 详细分析 | 非常高 |
| **10** | 1 KB | **推荐** | 中等 |
| 15 | 32 KB | 轻量监控 | 较低 |
| 19 | 512 KB | 只看大块 | 很低 |
## 内存优化
如果发现内存占用过高,可以调整以下配置 (在 `docker-compose.yml`):
### 1. 降低 Tantivy 缓存
```yaml
- HTKNOW_TANTIVY_MEMORY_MB=25 # 默认 50
```
### 2. 减少数据库连接
```yaml
- HTKNOW_DB_MAX_CONNECTIONS=5 # 默认 10
```
### 3. 调整搜索结果数量
```yaml
- HTKNOW_SEARCH_LIMIT=5 # 默认 10
```
## 容器内分析工具
进入容器后可用的工具:
```bash
# 查看进程内存
htop
# 查看系统内存
free -h
# 查看磁盘占用
df -h
# 查看目录大小
du -sh /app/data/*
# 实时监控容器
docker stats htknow
```
## 常见问题
### Q: 为什么 heap dump 文件很小?
A: 采样率太高 (lg_prof_sample:19),降低到 10 可获得更详细的数据。
### Q: 容器内存 251MB 但 heap 只有 1.7MB?
A: 可能原因:
1. 采样率过高,未记录小内存分配
2. 向量数据库 (LanceDB) 占用大量内存
3. 全文索引 (Tantivy) 缓存
4. Tokio 运行时和其他系统内存
### Q: Debug 镜像可以用于生产吗?
A: 不推荐。Debug 镜像:
- 包含额外的分析工具 (~100MB)
- 使用 debug 二进制 (更大,更慢)
- 启用了详细日志 (RUST_LOG=debug)
- 启用了内存分析 (性能开销)
生产环境请使用 release 镜像:
```bash
./build-docker.sh
```
## 参考链接
- [jemalloc Documentation](http://jemalloc.net/)
- [jeprof Manual](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling)
- [Rust jemalloc](https://github.com/tikv/jemallocator)

77
Dockerfile Normal file
View File

@ -0,0 +1,77 @@
# Runtime image for htknow
# Binary should be compiled outside the container using: cargo build --release
FROM docker.1ms.run/debian:bookworm-20260406-slim
ARG DEBIAN_FRONTEND=noninteractive
ARG APT_MIRROR=http://mirrors.aliyun.com
COPY docker/apt-fast /usr/local/sbin/apt-fast
# Switch Debian APT to Aliyun mirror and bootstrap vendored apt-fast.
RUN set -eux; \
for sources in /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list; do \
if [ -f "${sources}" ]; then \
sed -i \
-e "s|http://deb.debian.org/debian|${APT_MIRROR}/debian|g" \
-e "s|https://deb.debian.org/debian|${APT_MIRROR}/debian|g" \
-e "s|http://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
-e "s|https://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
-e "s|http://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
-e "s|https://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
"${sources}"; \
fi; \
done; \
apt-get update; \
apt-get install -y --no-install-recommends ca-certificates aria2; \
chmod +x /usr/local/sbin/apt-fast; \
printf '%s\n' \
'_APTMGR=apt-get' \
'DOWNLOADBEFORE=true' \
'_MAXNUM=8' \
'_MAXCONPERSRV=8' \
'_SPLITCON=8' \
"MIRRORS=(" \
" '${APT_MIRROR}/debian,http://mirrors.tuna.tsinghua.edu.cn/debian,http://mirrors.ustc.edu.cn/debian'" \
" '${APT_MIRROR}/debian-security,http://mirrors.tuna.tsinghua.edu.cn/debian-security,http://mirrors.ustc.edu.cn/debian-security'" \
')' \
> /etc/apt-fast.conf; \
rm -rf /var/lib/apt/lists/*
# Install runtime dependencies
RUN apt-fast update && apt-fast install -y \
# Timezone data
tzdata \
# sqlite3
sqlite3 \
# Image normalization for formats MinerU does not accept directly (ICO / AVIF / SVG)
imagemagick \
libheif1 \
librsvg2-bin \
&& rm -rf /var/lib/apt/lists/*
# Configure timezone
ENV TZ=Asia/Shanghai
RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
# Create application directory
WORKDIR /app
# Create data directories
RUN mkdir -p /app/data/images /app/data/temp /app/data/db
# Copy the compiled binary from host
# Make sure to build with: cargo build --release
COPY target/release/htknow /app/htknow
# Set permissions
RUN chmod +x /app/htknow
# Expose the application port (adjust based on your app's configuration)
EXPOSE 8080
# Set environment variables
ENV RUST_LOG=info
# Run the application
CMD ["/app/htknow"]

88
Dockerfile.debug Normal file
View File

@ -0,0 +1,88 @@
# Debug image for htknow with jemalloc profiling tools
# Binary should be compiled outside the container using: cargo build --profile dev
FROM docker.hjkl01.cn/debian:bookworm-slim
ARG DEBIAN_FRONTEND=noninteractive
ARG APT_MIRROR=http://mirrors.aliyun.com
COPY docker/apt-fast /usr/local/sbin/apt-fast
# Switch Debian APT to Aliyun mirror and bootstrap vendored apt-fast.
RUN set -eux; \
for sources in /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list; do \
if [ -f "${sources}" ]; then \
sed -i \
-e "s|http://deb.debian.org/debian|${APT_MIRROR}/debian|g" \
-e "s|https://deb.debian.org/debian|${APT_MIRROR}/debian|g" \
-e "s|http://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
-e "s|https://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
-e "s|http://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
-e "s|https://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \
"${sources}"; \
fi; \
done; \
apt-get update; \
apt-get install -y --no-install-recommends ca-certificates aria2; \
chmod +x /usr/local/sbin/apt-fast; \
printf '%s\n' \
'_APTMGR=apt-get' \
'DOWNLOADBEFORE=true' \
'_MAXNUM=8' \
'_MAXCONPERSRV=8' \
'_SPLITCON=8' \
"MIRRORS=(" \
" '${APT_MIRROR}/debian,http://mirrors.tuna.tsinghua.edu.cn/debian,http://mirrors.ustc.edu.cn/debian'" \
" '${APT_MIRROR}/debian-security,http://mirrors.tuna.tsinghua.edu.cn/debian-security,http://mirrors.ustc.edu.cn/debian-security'" \
')' \
> /etc/apt-fast.conf; \
rm -rf /var/lib/apt/lists/*
# Install runtime dependencies + debug tools
RUN apt-fast update && apt-fast install -y \
# Timezone data
tzdata \
# sqlite3
sqlite3 \
# Image normalization for formats MinerU does not accept directly (ICO / AVIF / SVG)
imagemagick \
libheif1 \
librsvg2-bin \
# Jemalloc profiling tools
libjemalloc-dev \
google-perftools \
perl \
graphviz \
binutils \
# Optional: useful debug utilities
curl \
htop \
# Clean up
&& rm -rf /var/lib/apt/lists/*
# Configure timezone
ENV TZ=Asia/Shanghai
RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
# Create application directory
WORKDIR /app
# Create data directories
RUN mkdir -p /app/data/images /app/data/temp /app/data/db
# Copy the compiled binary from host (debug build)
# Make sure to build with: cargo build --profile dev
COPY target/debug/htknow /app/htknow
# Set permissions
RUN chmod +x /app/htknow
# Expose the application port (adjust based on your app's configuration)
EXPOSE 8080
# Set environment variables for debug
ENV RUST_LOG=debug
ENV RUST_BACKTRACE=1
# Run the application
CMD ["/app/htknow"]

178
README.md Normal file
View File

@ -0,0 +1,178 @@
# HTKnow
HTKnow 知识库管理系统，提供文档上传、检索与知识图谱能力，内置前端界面与 OpenAPI 文档。
## 功能概览
- 知识库管理、文件上传与解析
- 全文/向量/图谱增强搜索
- 知识图谱查询与可视化
- 内置前端界面与 Swagger API 文档
## 快速开始
### 本地运行
1. 准备外部服务：MinerU、Embedding、图片 Embedding、Rerank（见“配置”）。
2. 启动服务：
```shell
cargo run
# 或
cargo build --release
./target/release/htknow
```
### Docker Compose
```shell
docker compose up -d
```
`docker-compose.yml` 默认将 `8080 -> 3000`。
### 构建 Docker 镜像
```shell
./build-docker.sh
```
脚本会先编译二进制，再构建镜像并给出运行示例。
### 访问入口
- 前端界面: `http://localhost:3000/`
- API 文档: `http://localhost:3000/docs`
- OpenAPI JSON: `http://localhost:3000/api-docs/openapi.json`
> 使用 docker-compose 时，请将端口替换为 `8080`。
## API 认证
`/api/v1/knowledge/*` 需要请求头：
- `x-user-id`（必填）
- `x-role`（必填）
- `x-user-name`（可选）
示例：
```shell
curl -H 'x-user-id: 1' -H 'x-role: admin' -H 'x-user-name: testuser' \
http://localhost:3000/api/v1/knowledge/knowledge_base/
```
## 启动 mineru
```shell
docker run -d --name mineru-api --restart unless-stopped --ipc host -p 10001:10001 -e MINERU_MODEL_SOURCE=local --ulimit memlock=-1 --ulimit stack=67108864 --gpus all alexsuntop/mineru:latest mineru-api --host 0.0.0.0 --port 10001
```
## 配置
支持通过环境变量覆盖配置，未设置时使用默认值。
### 服务器
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `HTKNOW_SERVER_HOST` | `0.0.0.0` | 监听地址 |
| `HTKNOW_SERVER_PORT` | `3000` | 监听端口 |
| `HTKNOW_SERVER_UPLOAD_LIMIT_MB` | `500` | 上传大小限制（MB） |
| `HTKNOW_SERVER_PROCESS_INTERVAL_SECS` | `10` | 文件处理间隔（秒） |
| `HTKNOW_SERVER_PROCESS_CONCURRENCY` | `1` | 后台文件处理并发数 |
| `HTKNOW_PARSE_ENABLED` | `true` | 是否启动后台文件解析（false 时仅即时解析生效） |
| `HTKNOW_REUSE_DUPLICATE_FILES` | `true` | 是否复用重复文件的已解析结果 |
| `HTKNOW_BUILD_KNOWLEDGE_GRAPH` | `false` | 文件解析完成后是否构建知识图谱（依赖 LLM 配置） |
| `HTKNOW_LANCEDB_COMPACT_CRON` | `0 0 3 * * *` | LanceDB 自动压缩 cron 表达式（本地时区，off/disabled/0 禁用） |
### 外部服务
默认值为示例地址，请按实际部署环境调整。
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `HTKNOW_MINERU_URL` | `http://192.168.0.46:10001/file_parse` | MinerU PDF 解析 |
| `HTKNOW_REQUEST_TIMEOUT_SECS` | `600` | 外部接口请求超时（秒），适用于文件解析相关接口 |
| `HTKNOW_MINERU_MAX_PAGES` | `50` | MinerU 单次解析 PDF 最大页数（0 表示不限制） |
| `HTKNOW_OFFICE_CONVERT_URL` | `http://192.168.0.46:8003/convert` | Office 文档转 PDF 服务，使用 multipart `file` 字段并自动追加 `target_format=pdf` |
| `HTKNOW_CUSTOM_PARSE_URL` | 空 | 自定义解析服务地址（配置后仅 Word/PPT/PDF 解析走该服务，需返回已切片数据） |
| `HTKNOW_CUSTOM_PARSE_REUSE_URL` | 空 | 自定义解析复用服务地址（仅输入 pdf_contents，不包含图片） |
| `HTKNOW_AUDIO_TRANSCRIPTION_URL` | `http://192.168.0.46:59805/api/v1/audio/transcriptions` | 音频转写服务 |
| `HTKNOW_AUDIO_TRANSCRIPTION_KEY` | 空 | 音频转写服务 API Key |
| `HTKNOW_EMBEDDING_URL` | `http://222.190.139.186:59700/v1/embeddings` | 文本向量服务 |
| `HTKNOW_IMAGE_EMBEDDING_URL` | `http://192.168.0.46:59802/v1/embeddings/file` | 图片向量服务 |
| `HTKNOW_RERANK_URL` | `http://222.190.139.186:59600/v1/rerank` | Rerank 服务 |
### AI 模型
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `HTKNOW_EMBEDDING_MODEL` | `bge-m3` | Embedding 模型 |
| `HTKNOW_EMBEDDING_DIM` | `1024` | Embedding 维度 |
| `HTKNOW_IMAGE_EMBEDDING_DIM` | `2048` | 图片 Embedding 维度 |
| `HTKNOW_EMBEDDING_BATCH_SIZE` | `8` | Embedding 批量请求批次大小 |
| `HTKNOW_RERANK_MODEL` | `bge-rerank` | Rerank 模型 |
| `HTKNOW_RERANK_THRESHOLD` | `0.1` | Rerank 阈值 |
### 数据库
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `DATABASE_URL` | `sqlite://data/app.sqlite` | 数据库连接 |
| `HTKNOW_DB_MAX_CONNECTIONS` | `16` | 最大连接数 |
| `HTKNOW_DB_MIN_CONNECTIONS` | `2` | 最小空闲连接数 |
| `HTKNOW_DB_BUSY_TIMEOUT_MS` | `5000` | busy_timeout（毫秒） |
| `HTKNOW_DB_INIT_DEFAULT_KBS` | `true` | 是否初始化默认知识库 |
### 存储路径
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `HTKNOW_DATA_DIR` | `data` | 数据目录 |
| `HTKNOW_LANCEDB_PATH` | `data/lancedb_data` | LanceDB 路径 |
| `HTKNOW_TEMP_PATH` | `data/temp` | 临时目录 |
| `HTKNOW_IMAGES_PATH` | `data/images` | 图片目录 |
| `HTKNOW_PDF_PATH` | `data/pdfs` | PDF 目录 |
| `HTKNOW_FILES_PATH` | `data/files` | 文件目录 |
| `HTKNOW_ARCHIVES_PATH` | `data/archives` | 压缩文件解压目录 |
| `HTKNOW_CONTENTS_PATH` | `data/contents` | 文件解析后完整文本目录 |
### 搜索
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `HTKNOW_SEARCH_LIMIT` | `10` | 搜索结果限制 |
| `HTKNOW_TANTIVY_INDEX_PATH` | `data/tantivy_index` | Tantivy 索引路径 |
| `HTKNOW_TANTIVY_FULL_INDEX_PATH` | `data/tantivy_full_index` | Tantivy 全文索引 |
| `HTKNOW_TANTIVY_MEMORY_MB` | `50` | Tantivy 内存（MB） |
| `HTKNOW_SEARCH_TANTIVY_REBUILD_BATCH_SIZE` | `100` | Tantivy 索引重建批次大小 |
| `HTKNOW_SEARCH_LANCEDB_REBUILD_BATCH_SIZE` | `100` | LanceDB 从 SQLite 重建批次大小 |
| `HTKNOW_SEARCH_EMBEDDING_TIMEOUT_SECS` | `30` | embedding / 图片 embedding 请求超时（秒） |
| `HTKNOW_SEARCH_RERANK_TIMEOUT_SECS` | `20` | rerank 请求超时（秒） |
| `HTKNOW_SEARCH_SYNONYM_ENABLED` | `true` | 是否启用同义词查询扩展 |
| `HTKNOW_SEARCH_SYNONYM_BOOST` | `0.7` | 同义词权重因子（与行权重相乘） |
| `HTKNOW_SEARCH_MAX_SYNONYMS_PER_TERM` | `5` | 每个词最多扩展同义词数 |
| `HTKNOW_SEARCH_MAX_TOTAL_SYNONYMS` | `30` | 单次查询最多扩展同义词总数 |
| `HTKNOW_HIGHLIGHT_PAGE_MIN_POSITIONS` | `20` | 高亮页码选择阈值：首选页位置数少于该值时优先使用第二页 |
### 切片
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `HTKNOW_SMART_SLICE_MAX_CHARS` | `8000` | 智能切片最大字数 |
| `HTKNOW_FIXED_SLICE_OVERLAP_CHARS` | `100` | 固定切片重叠字数 |
### LLM（可选）
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `LLM_API_URL` | 空 | LLM API 地址 |
| `LLM_API_KEY` | 空 | LLM API Key |
| `LLM_MODEL` | `gpt-3.5-turbo` | LLM 模型 |
## 启用 etcd 配置
`cargo build --features etcd`
## 日志级别（RUST_LOG）
默认日志级别为 `info`，可通过环境变量覆盖：
```shell
RUST_LOG=debug ./htknow
RUST_LOG=warn,htknow::search=debug ./htknow
```
## 问题处理
1. 全文索引 tantivy 异常
可能是异常停止导致的，报错
thread 'main' (1) panicked at src/search/mod.rs:903:29:failed to create tantivy index reader: Failed to open file for read: 'FileDoestiotExist("/app/data/tantivy index/eafseaef4f2340...
run with 'RuST_BAcKTRAcE=l environment variable to display a backtrace
**方案一**：在 `meta.json` 中去掉报错的索引，注意 `meta.json` 中记录的名称有 `-`
**方案二**：
先去掉原本的索引使应用正常启动
```shell
mv tantivy_index tantivy_index_bak0423
mv tantivy_full_index tantivy_full_index_bak0423
docker start htknow
```
然后在词典可以重建全文检索的索引

62
build-docker.sh Normal file
View File

@ -0,0 +1,62 @@
#!/bin/bash
set -e
# Parse command line arguments
BUILD_MODE="release"
PROFILING_FLAG=""
CUSTOM_IMAGE_TAG="${IMAGE_TAG:-}"
if [ "${1:-}" = "debug" ] || [ "${1:-}" = "--debug" ]; then
BUILD_MODE="debug"
PROFILING_FLAG="--features profiling"
fi
# Build frontend
cd frontend
npm install
npm run build
cd ..
if [ "$BUILD_MODE" = "debug" ]; then
echo "Building htknow for debug with profiling..."
cargo build ${PROFILING_FLAG}
if [ -n "$CUSTOM_IMAGE_TAG" ]; then
IMAGE_TAG="${CUSTOM_IMAGE_TAG}-debug"
else
IMAGE_TAG="$(date +%Y%m%d%H%M)-debug"
fi
DOCKERFILE="Dockerfile.debug"
echo "Using Dockerfile.debug with jemalloc profiling tools"
else
echo "Building htknow for release (profiling disabled)..."
cargo build --release
if [ -n "$CUSTOM_IMAGE_TAG" ]; then
IMAGE_TAG="${CUSTOM_IMAGE_TAG}"
else
IMAGE_TAG="$(date +%Y%m%d%H%M)"
fi
DOCKERFILE="Dockerfile"
fi
# Build Docker image
echo "Building Docker image with ${DOCKERFILE}..."
docker build -f "${DOCKERFILE}" -t "htknow:${IMAGE_TAG}" -t htknow:latest .
echo "Build complete!"
echo ""
echo "Image tag: htknow:${IMAGE_TAG}"
echo "Latest tag: htknow:latest"
echo "BUILT_IMAGE=htknow:${IMAGE_TAG}"
echo "BUILT_LATEST_IMAGE=htknow:latest"
echo "Build mode: ${BUILD_MODE}"
echo ""
echo "To run the container:"
echo " docker-compose up -d"
echo ""
echo "Or manually:"
echo " docker run -d -p 8080:8080 -v \$(pwd)/data:/app/data --name htknow htknow:${IMAGE_TAG}"
echo ""
echo "Usage:"
echo " ./build-docker.sh # Build in release mode (default)"
echo " ./build-docker.sh debug # Build in debug mode"
echo " IMAGE_TAG=v1.2.3 ./build-docker.sh # Use custom image tag"

24
deploy_ci.md Normal file
View File

@ -0,0 +1,24 @@
## 配置 gitea 密钥
`DEPLOY_HOST` SSH HOST
`DEPLOY_PORT` SSH PORT
`DEPLOY_USER` SSH USER
`DEPLOY_PASSWORD` SSH PASSWORD
`DEPLOY_PATH` 部署路径
`DEPLOY_COMMAND` 部署命令
```shell
set -e
TAR_FILE="$DEPLOY_PATH/$ASSET_FILE"
COMPOSE_FILE="$DEPLOY_PATH/docker-compose.yaml" # 或写绝对路径
EXTRACTED_TAR="${TAR_FILE%.gz}"
gzip -dc "$TAR_FILE" > "$EXTRACTED_TAR"
docker load -i "$EXTRACTED_TAR"
rm -f "$EXTRACTED_TAR"
if docker compose version >/dev/null 2>&1; then
IMAGE_TAG="$SAFE_TAG" docker compose -f "$COMPOSE_FILE" up -d --remove-orphans
else
IMAGE_TAG="$SAFE_TAG" docker-compose -f "$COMPOSE_FILE" up -d --remove-orphans
fi
```

61
docker-compose.yml Normal file
View File

@ -0,0 +1,61 @@
version: "3.8"
services:
htknow:
build:
context: .
dockerfile: Dockerfile
container_name: htknow
ports:
- "8080:3000"
volumes:
# Mount data directory for persistent storage
- ./data:/app/data
environment:
- RUST_LOG=info
# 服务器配置
- HTKNOW_SERVER_HOST=0.0.0.0
- HTKNOW_SERVER_PORT=3000
- HTKNOW_SERVER_UPLOAD_LIMIT_MB=500
- HTKNOW_SERVER_PROCESS_INTERVAL_SECS=10
- HTKNOW_BUILD_KNOWLEDGE_GRAPH=false
- "HTKNOW_LANCEDB_COMPACT_CRON=0 0 3 * * *"
# 外部服务地址
- HTKNOW_MINERU_URL=http://192.168.0.46:10001/file_parse
- HTKNOW_REQUEST_TIMEOUT_SECS=600
- HTKNOW_MINERU_MAX_PAGES=0
- HTKNOW_OFFICE_CONVERT_URL=http://192.168.0.46:8003/convert
- HTKNOW_EMBEDDING_URL=http://192.168.0.46:9700/v1/embeddings
- HTKNOW_RERANK_URL=http://192.168.0.46:9600/v1/rerank
# AI 模型配置
- HTKNOW_EMBEDDING_MODEL=bge-m3
- HTKNOW_EMBEDDING_DIM=1024
- HTKNOW_EMBEDDING_BATCH_SIZE=8
- HTKNOW_RERANK_MODEL=bge-rerank
- HTKNOW_RERANK_THRESHOLD=0.1
# 数据库配置
- DATABASE_URL=sqlite://data/app.sqlite
- HTKNOW_DB_MAX_CONNECTIONS=10
- HTKNOW_DB_BUSY_TIMEOUT_MS=5000
# 存储路径配置
- HTKNOW_DATA_DIR=data
- HTKNOW_LANCEDB_PATH=data/lancedb_data
- HTKNOW_TEMP_PATH=data/temp
- HTKNOW_IMAGES_PATH=data/images
# 搜索配置
- HTKNOW_SEARCH_LIMIT=10
- HTKNOW_TANTIVY_INDEX_PATH=data/tantivy_index
- HTKNOW_TANTIVY_MEMORY_MB=50
# LLM 配置（可选，用于知识图谱构建）
# - LLM_API_URL=https://api.openai.com/v1/chat/completions
# - LLM_API_KEY=your-api-key
# - LLM_MODEL=gpt-3.5-turbo
restart: unless-stopped
# Uncomment if you need to connect to other services
# networks:
# - htknow-network
# Uncomment if you need a network
# networks:
# htknow-network:
# driver: bridge

735
docker/apt-fast Normal file
View File

@ -0,0 +1,735 @@
#!/bin/bash
#
# apt-fast v1.10.0
# Use this just like aptitude or apt-get for faster package downloading.
#
# Copyright: 2008-2012 Matt Parnell, http://www.mattparnell.com
# Improvements, maintenance, revisions - 2012, 2017-2019 Dominique Lasserre
#
# You may distribute this file under the terms of the GNU General
# Public License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
shopt -s nullglob
[ -n "$DEBUG" ] && set -xv
# Print colored messages.
# Usage: msg "message text" "message type" "optional: err"
# Message types are 'normal', 'hint' or 'warning'. Warnings and messages with a
# third argument are piped to stderr.
msg(){
msg_options=()
case "$2" in
normal) beginColor="$cGreen";;
hint) beginColor="$cBlue";;
warning) beginColor="$cRed";;
question) beginColor="$cRed"; msg_options=(-n);;
*) beginColor= ;;
esac
if [ -z "$3" ] && [ "$2" != "warning" ]; then
echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}"
else
echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}" >&2
fi
}
# Search for known options and decide if root privileges are needed.
root=1 # default value: we need root privileges
option=
for argument in "$@"; do
case "$argument" in
upgrade | full-upgrade | install | dist-upgrade | build-dep)
option="install"
;;
clean | autoclean)
option="clean"
;;
download)
option="download"
root=0
;;
source)
option="source"
root=0
;;
changelog | show)
root=0
;;
esac
done
# To handle priority of options correctly (environment over config file vars)
# we need to preserve all interesting env variables. As this wouldn't be
# difficult enough we have to preserve complete env vars (especially if value
# ist set (even empty) or not) when changing context (sudo)...
# Set a 'random' string to all unset variables.
TMP_RANDOM="13979853562951413"
TMP_LCK_FILE="${LCK_FILE-${TMP_RANDOM}}"
TMP_DOWNLOADBEFORE="${DOWNLOADBEFORE-${TMP_RANDOM}}"
TMP__APTMGR="${_APTMGR-${TMP_RANDOM}}"
TMP_APTCACHE="${APTCACHE-${TMP_RANDOM}}"
TMP_DLDIR="${DLDIR-${TMP_RANDOM}}"
TMP_DLLIST="${DLLIST-${TMP_RANDOM}}"
TMP__MAXNUM="${MAXNUM-${TMP_RANDOM}}"
TMP__MAXCONPERSRV="${MAXCONPERSRV-${TMP_RANDOM}}"
TMP__SPLITCON="${SPLITCON-${TMP_RANDOM}}"
TMP__MINSPLITSZ=${MINSPLITSZ-${TMP_RANDOM}}
TMP__PIECEALGO=${PIECEALGO-${TMP_RANDOM}}
TMP_aptfast_prefix="${aptfast_prefix-${TMP_RANDOM}}"
TMP_APT_FAST_TIMEOUT="${APT_FAST_TIMEOUT-${TMP_RANDOM}}"
TMP_APT_FAST_APT_AUTH="${APT_FAST_APT_AUTH-${TMP_RANDOM}}"
TMP_VERBOSE_OUTPUT="${VERBOSE_OUTPUT-${TMP_RANDOM}}"
TMP_ftp_proxy="${ftp_proxy-${TMP_RANDOM}}"
TMP_http_proxy="${http_proxy-${TMP_RANDOM}}"
TMP_https_proxy="${https_proxy-${TMP_RANDOM}}"
# Check for proper privileges.
# Call explicitly with environment variables to get them into root conext.
if [ "$root" = 1 ] && [ "$UID" != 0 ]; then
exec sudo DEBUG="$DEBUG" \
LCK_FILE="$TMP_LCK_FILE" \
DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" \
_APTMGR="$TMP__APTMGR" \
APTCACHE="$TMP_APTCACHE" \
DLDIR="$TMP_DLDIR" \
DLLIST="$TMP_DLLIST" \
_MAXNUM="$TMP__MAXNUM" \
_MAXCONPERSRV="$TMP__MAXCONPERSRV" \
_SPLITCON="$TMP__SPLITCON" \
_MINSPLITSZ="$TMP__MINSPLITSZ" \
_PIECEALGO="$TMP__PIECEALGO" \
aptfast_prefix="$TMP_aptfast_prefix" \
APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" \
APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH" \
VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" \
ftp_proxy="$TMP_ftp_proxy" \
http_proxy="$TMP_http_proxy" \
https_proxy="$TMP_https_proxy" \
"$0" "$@"
fi
# Define lockfile.
# Use /tmp as directory because everybody (not only root) has to have write
# permissions.
# We need lock for non-root commands too, because we only have one download
# list file.
LCK_FILE="/tmp/apt-fast"
LCK_FD=99
# Set default package manager, APT cache, temporary download dir,
# temporary download list file, and maximal parallel downloads
_APTMGR='apt-get'
eval "$(apt-config shell APTCACHE Dir::Cache::archives/d)"
# Check if APT config option Dir::Cache::archives::apt-fast-partial is set.
eval "$(apt-config shell apt_fast_partial Dir::Cache::archives::apt-fast-partial/d)"
if [ -z "$apt_fast_partial" ]; then
DLDIR="$(realpath "${APTCACHE}/../apt-fast")"
else
DLDIR="${apt_fast_partial}"
fi
# Check for apt auth files
eval "$(apt-config shell NETRC Dir::Etc::netrc/f)"
eval "$(apt-config shell NETRCDIR Dir::Etc::netrcparts/d)"
APTAUTHFILES=()
if [ -f "$NETRC" ]; then
APTAUTHFILES=("$NETRC")
fi
APTAUTHFILES+=("$NETRCDIR"*)
DLLIST="/tmp/apt-fast.list"
_MAXNUM=5
_MAXCONPERSRV=10
_SPLITCON=8
_MINSPLITSZ="1M"
_PIECEALGO="default"
MIRRORS=()
# Prefix in front of apt-fast output:
aptfast_prefix=
# aptfast_prefix="$(date '+%b %_d %T.%N') apt-fast: "
# Set color variables.
cGreen='\e[0;32m'
cRed='\e[0;31m'
cBlue='\e[0;34m'
endColor='\e[0m'
# Set timout value for apt-fast download confirmation dialog.
# Value is in seconds.
APT_FAST_TIMEOUT=60
# Ask for download confirmation if unset
DOWNLOADBEFORE=
# Enable APT authentication support
APT_FAST_APT_AUTH=1
# Formatted package list in download confirmation if unset
VERBOSE_OUTPUT=
# Download command.
_DOWNLOADER='aria2c --no-conf -c -j ${_MAXNUM} -x ${_MAXCONPERSRV} -s ${_SPLITCON} -i ${DLLIST} --min-split-size=${_MINSPLITSZ} --stream-piece-selector=${_PIECEALGO} --connect-timeout=600 --timeout=600 -m0 --header "Accept: */*"'
# Load config file.
for Prefix in /usr/local/etc /etc; do
CONFFILE=${Prefix}/apt-fast.conf
if [ -e "$CONFFILE" ]; then
source "$CONFFILE"
break
fi
done
# no proxy as default
ftp_proxy=
http_proxy=
https_proxy=
# Now overwrite with preserved values if values were set before (compare with
# 'random' string).
[ "$TMP_LCK_FILE" = "$TMP_RANDOM" ] || LCK_FILE="$TMP_LCK_FILE"
[ "$TMP_DOWNLOADBEFORE" = "$TMP_RANDOM" ] || DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE"
[ "$TMP__APTMGR" = "$TMP_RANDOM" ] || _APTMGR="$TMP__APTMGR"
[ "$TMP_APTCACHE" = "$TMP_RANDOM" ] || APTCACHE="$TMP_APTCACHE"
[ "$TMP_DLDIR" = "$TMP_RANDOM" ] || DLDIR="$TMP_DLDIR"
[ "$TMP_DLLIST" = "$TMP_RANDOM" ] || DLLIST="$TMP_DLLIST"
[ "$TMP__MAXNUM" = "$TMP_RANDOM" ] || _MAXNUM="$TMP__MAXNUM"
[ "$TMP__MAXCONPERSRV" = "$TMP_RANDOM" ] || _MAXCONPERSRV="$TMP__MAXCONPERSRV"
[ "$TMP__SPLITCON" = "$TMP_RANDOM" ] || _SPLITCON="$TMP__SPLITCON"
[ "$TMP__MINSPLITSZ" = "$TMP_RANDOM" ] || _MINSPLITSZ="$TMP__MINSPLITSZ"
[ "$TMP__PIECEALGO" = "$TMP_RANDOM" ] || _PIECEALGO="$TMP__PIECEALGO"
[ "$TMP_aptfast_prefix" = "$TMP_RANDOM" ] || aptfast_prefix="$TMP_aptfast_prefix"
[ "$TMP_APT_FAST_TIMEOUT" = "$TMP_RANDOM" ] || APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT"
[ "$TMP_APT_FAST_APT_AUTH" = "$TMP_RANDOM" ] || APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH"
[ "$TMP_VERBOSE_OUTPUT" = "$TMP_RANDOM" ] || VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT"
[ "$TMP_ftp_proxy" = "$TMP_RANDOM" ] || ftp_proxy="$TMP_ftp_proxy"
[ "$TMP_http_proxy" = "$TMP_RANDOM" ] || http_proxy="$TMP_http_proxy"
[ "$TMP_https_proxy" = "$TMP_RANDOM" ] || https_proxy="$TMP_https_proxy"
# Disable colors if not executed in terminal.
if [ ! -t 1 ]; then
cGreen=
cRed=
cBlue=
endColor=
#FIXME: Time not updated.
[ -z "$aptfast_prefix" ] && aptfast_prefix="[apt-fast $(date +"%T")]"
fi
msg_already_running()
{
msg "apt-fast already running!" "warning"
msg "Verify that all apt-fast processes are finished then remove $LCK_FILE.lock and try again." "hint"
}
# Check if a lock file exists.
if [ -f "$LCK_FILE.lock" ]; then
msg_already_running
exit 1
fi
# create the lock file and lock it, die on failure
_create_lock()
{
eval "exec $LCK_FD>\"$LCK_FILE.lock\""
flock -n $LCK_FD || { msg_already_running; exit 1; }
trap "cleanup_aptfast; exit_cleanup_state" EXIT
trap "cleanup_aptfast; exit 1" INT TERM
}
# unlock and remove the lock file
_remove_lock()
{
flock -u "$LCK_FD" 2>/dev/null
rm -f "$LCK_FILE.lock"
}
# Move download file away so missing permissions won't stop usage.
CLEANUP_STATE=0
cleanup_dllist()
{
if [ -f "$DLLIST" ]
then
if ! mv -- "$DLLIST{,.old}" 2>/dev/null
then
if ! rm -f -- "$DLLIST" 2>/dev/null
then
msg "Could not clean up download list file." "warning"
CLEANUP_STATE=1
fi
fi
fi
}
cleanup_aptfast()
{
local last_exit_code=$?
[ "$CLEANUP_STATE" -eq 0 ] && CLEANUP_STATE=$last_exit_code
cleanup_dllist
_remove_lock
}
exit_cleanup_state()
{
exit $CLEANUP_STATE
}
# decode url string
# translates %xx but must not convert '+' in spaces
urldecode()
{
printf '%b' "${1//%/\\x}"
}
# Check if mirrors are available. And if so add all mirrors to download list.
get_mirrors(){
# Check all mirror lists.
for mirrorstr in "${MIRRORS[@]}"; do
# Build mirrors array from comma separated string.
IFS=", " read -r -a mirrors <<< "$mirrorstr"
# Check for all mirrors if URI of $1 is from mirror. If so add all other
# mirrors to (resmirror) list and break all loops.
for mirror in "${mirrors[@]}"; do
# Real expension.
if [[ "$1" == "$mirror"* ]]; then
filepath="${1#"${mirror}"}"
# Build list for aria download list.
list="${mirrors[*]}"
echo -e "${list// /${filepath}\\t}$filepath\n"
return 0
fi
done
done
# No other mirrors found.
echo "$1"
}
AUTH_INFO_PARSED=()
# Parse apt authentication files.
# Undefined behavior on whitespaces in host, username or password.
prepare_auth(){
if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then
return
fi
for auth_file in "${APTAUTHFILES[@]}"; do
# auth files have netrc syntax, possible multiline entries starting with "machine"
auth_info="$(tr '\n' ' ' < "$auth_file" | sed 's/\(\<machine\>\)/\n\1/g' | sed '1d')"
while IFS= read -r auth; do
machine="$(echo "$auth" | sed 's/.*\<machine\>[ \t]\+\([^ \t]\+\).*/\1/')"
login="$(echo "$auth" | sed 's/.*\<login\>[ \t]\+\([^ \t]\+\).*/\1/')"
password="$(echo "$auth" | sed 's/.*\<password\>[ \t]\+\([^ \t]\+\).*/\1/')"
# if machine does not have protocol, try https://
if ! [[ "$machine" =~ ^.*:// ]]; then
machine="https://$machine"
fi
if [ -z "$machine" ] || [ -z "$login" ] || [ -z "$password" ]; then
msg "Could not parse apt authentication (skipping): $auth ($auth_file)" "warning"
continue
fi
# use space separated string to convert back to array later
AUTH_INFO_PARSED+=("$machine $login $password")
done <<< "$auth_info"
done
}
# Gets URI as parameter and tries to add basic http credentials. Will fail on
# credentials that contain characters that need URL-encoding.
get_auth(){
if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then
echo "$1"
return
fi
for auth_info in "${AUTH_INFO_PARSED[@]}"; do
# convert to array, don't escape variable here
auth_info_arr=($auth_info)
machine="${auth_info_arr[0]}"
# takes first match
if [[ "$1" == "$machine"* ]]; then
login="${auth_info_arr[1]}"
password="${auth_info_arr[2]}"
uri="$(echo "$1" | sed "s|^\([^:]\+://\)|\1$login:$password@|")"
echo "$uri"
return
fi
done
echo "$1"
}
# Globals to save package name, version, size and overall size.
DOWNLOAD_DISPLAY=
DOWNLOAD_SIZE=0
# Get the package URLs.
get_uris(){
if [ ! -d "$(dirname "$DLLIST")" ]
then
if ! mkdir -p -- "$(dirname "$DLLIST")"
then
msg "Could not create download file directory." "warning"
CLEANUP_STATE=1
exit
fi
elif [ -f "$DLLIST" ]; then
if ! rm -f -- "$DLLIST" 2>/dev/null && ! touch -- "$DLLIST" 2>/dev/null
then
msg "Unable to write to download file. Try restarting with root permissions or run 'apt-fast clean' first." "warning"
CLEANUP_STATE=1
exit
fi
fi
# Add header to overwrite file.
echo "# apt-fast mirror list: $(date)" > "$DLLIST"
# NOTE: "aptitude" doesn't have this functionality
# so we use "${_APTMGR}" to get package URI's
case "$(basename "${_APTMGR}")" in
'apt'|'apt-get') uri_mgr="${_APTMGR}";;
*) uri_mgr='apt-get';;
esac
uris_full="$("$uri_mgr" "${APT_SCRIPT_WARNING[@]}" -y --print-uris "$@")"
CLEANUP_STATE="$?"
if [ "$CLEANUP_STATE" -ne 0 ]
then
msg "Package manager quit with exit code." "warning"
exit
fi
prepare_auth
while read -r pkg_uri_info
do
[ -z "$pkg_uri_info" ] && continue
## --print-uris format is:
# 'fileurl' filename filesize checksum_hint:filechecksum
uri="$(get_auth "$(echo "$pkg_uri_info" | cut -d' ' -f1 | tr -d "'")")"
filename="$(echo "$pkg_uri_info" | cut -d' ' -f2)"
filesize="$(echo "$pkg_uri_info" | cut -d' ' -f3)"
checksum_string="$(echo "$pkg_uri_info" | cut -d' ' -f4)"
hash_algo="$(echo "$checksum_string" | cut -d':' -f1)"
checksum="$(echo "$checksum_string" | cut -d':' -f2)"
filename_decoded="$(urldecode "$filename")"
DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY}$(echo "$filename_decoded" | cut -d'_' -f1)"
DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY} $(echo "$filename_decoded" | cut -d'_' -f2)"
DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY} $(echo "$filesize" | numfmt --to=iec-i --suffix=B)\n"
DOWNLOAD_SIZE=$((DOWNLOAD_SIZE + filesize))
## whole uri comes encoded (urlencoded). Filename must NOT be decoded because
# plain aptitude do not decode it when download and install it. Therefore, we
# will have ugly named packages at /var/cache/apt/archives but is the standard
# behavior.
# But package version must be decoded, otherways package=version calls will
# not work.
if [ -n "$HASH_SUPPORTED" ]; then
case "$hash_algo" in
SHA512) [ -z "$SHA512_SUPPORTED" ] && hash_algo= || hash_algo=sha-512 ;;
SHA256) [ -z "$SHA256_SUPPORTED" ] && hash_algo= || hash_algo=sha-256 ;;
SHA1) [ -z "$SHA1_SUPPORTED" ] && hash_algo= || hash_algo=sha-1 ;;
MD5Sum) [ -z "$MD5sum_SUPPORTED" ] && hash_algo= || hash_algo=md5 ;;
*) hash_algo=
esac
# Using apt-cache show package=version to ensure recover single and
# correct package version.
# Warning: assuming that package naming uses '_' as field separator.
# Therefore, this code expects package-name_version_arch.deb Otherways
# below code will fail resoundingly
if [ -z "$hash_algo" ]; then
pkg_name="$(echo "$filename" | cut -d'_' -f1)"
pkg_version="$(echo "$filename" | cut -d'_' -f2)"
pkg_version="$(urldecode "$pkg_version")"
package_info="$(apt-cache show "$pkg_name=$pkg_version")"
patch_checksum=
if [ -n "$SHA512_SUPPORTED" ]; then
patch_checksum="$(echo "$package_info" | grep SHA512 | head -n 1)"
[ -n "$patch_checksum" ] && hash_algo="sha-512"
fi
if [ -z "$patch_checksum" ] && [ -n "$SHA256_SUPPORTED" ]; then
patch_checksum="$(echo "$package_info" | grep SHA256 | head -n 1)"
[ -n "$patch_checksum" ] && hash_algo="sha-256"
fi
if [ -z "$patch_checksum" ] && [ -n "$SHA1_SUPPORTED" ]; then
patch_checksum="$(echo "$package_info" | grep SHA1 | head -n 1)"
[ -n "$patch_checksum" ] && hash_algo="sha-1"
fi
if [ -z "$patch_checksum" ] && [ -n "$MD5sum_SUPPORTED" ]; then
patch_checksum="$(echo "$package_info" | grep MD5sum | head -n 1)"
[ -n "$patch_checksum" ] && hash_algo="md5"
fi
if [ -n "$patch_checksum" ]; then
checksum="$(echo "$patch_checksum" | cut -d' ' -f2)"
else
msg "Couldn't get supported checksum for $pkg_name ($pkg_version)." "warning"
REMOVE_WORKING_MESSAGE=
fi
fi
else
hash_algo=
fi
{
get_mirrors "$uri"
#echo " dir=$DLDIR"
if [ -n "$hash_algo" ]; then
echo " checksum=$hash_algo=$checksum"
fi
echo " out=$filename"
} >> "$DLLIST"
done <<<"$(echo "$uris_full" | grep -E "^'(http(s|)|(s|)ftp)://")"
#cat "$DLLIST"
#LCK_RM
#exit
}
display_downloadfile(){
if [ -n "$VERBOSE_OUTPUT" ]; then
cat "$DLLIST"
else
DISPLAY_SORT_OPTIONS=(-k 1,1)
# Sort output after package download size (decreasing):
#DISPLAY_SORT_OPTIONS=(-k 3,3 -hr)
while read -r line; do
[ -z "$line" ] && continue
pkg="$(echo "$line" | cut -d' ' -f1)"
ver="$(echo "$line" | cut -d' ' -f2)"
size="$(echo "$line" | cut -d' ' -f3)"
printf '%s%-40s %-20s %10s\n' "$aptfast_prefix" "$pkg" "$ver" "$size"
done <<<"$(echo -e "$DOWNLOAD_DISPLAY" | sort "${DISPLAY_SORT_OPTIONS[@]}")"
fi
msg "Download size: $(echo "$DOWNLOAD_SIZE" | numfmt --to=iec-i --suffix=B)" "normal"
}
# Create and insert a PID number to lockfile.
_create_lock
# Make sure aria2c (in general first parameter from _DOWNLOADER) is available.
CMD="$(echo "$_DOWNLOADER" | sed 's/^\s*\([^ ]\+\).*$/\1/')"
if [ ! "$(command -v "$CMD")" ]; then
msg "Command not found: $CMD" "normal" "err"
msg "You must configure $CONFFILE to use aria2c or another supported download manager" "normal" "err"
CLEANUP_STATE=1
exit
fi
# Make sure package manager is available.
if [ ! "$(command -v "$_APTMGR")" ]; then
msg "\`$_APTMGR\` command not available." "warning"
msg "You must configure $CONFFILE to use either apt-get or aptitude." "normal" "err"
CLEANUP_STATE=1
exit
fi
# Disable script warning if apt is used.
APT_SCRIPT_WARNING=()
if [ "$(basename "${_APTMGR}")" == 'apt' ]; then
APT_SCRIPT_WARNING=(-o "Apt::Cmd::Disable-Script-Warning=true")
fi
# Set supported hash algorithms by aria2c (and also by Debian repository).
SHA512_SUPPORTED=
SHA256_SUPPORTED=
SHA1_SUPPORTED=
MD5sum_SUPPORTED=
HASH_SUPPORTED=
if [ "$CMD" == "aria2c" ]; then
for supported_hash in $(LC_ALL=C aria2c -v | sed '/^Hash Algorithms:/!d; s/\(^Hash Algorithms: \|,\)\+//g'); do
case "$supported_hash" in
sha-512) SHA512_SUPPORTED=y; HASH_SUPPORTED=y ;;
sha-256) SHA256_SUPPORTED=y; HASH_SUPPORTED=y ;;
sha-1) SHA1_SUPPORTED=y; HASH_SUPPORTED=y ;;
md5) MD5sum_SUPPORTED=y; HASH_SUPPORTED=y ;;
esac
done
if [ -z "$HASH_SUPPORTED" ]; then
msg "Couldn't find supported checksum algorithm from aria2c. Checksums disabled." "warning"
fi
fi
# Check if "assume yes" switch is enabled and if yes enable $DOWNLOADBEFORE.
# Also check if "download only" switch is enabled.
#TODO: Get real value over APT items APT::Get::Assume-Yes and
# APT::Get::Assume-No .
# Respectively Aptitude::CmdLine::Download-Only and APT::Get::Download-Only.
DOWNLOAD_ONLY=
while true; do
while getopts ":dy-:" optchar; do
case "${optchar}" in
-)
case "${OPTARG}" in
yes | assume-yes) DOWNLOADBEFORE=true ;;
assume-no) DOWNLOADBEFORE= ;;
download-only) DOWNLOAD_ONLY=true ;;
esac
;;
y)
DOWNLOADBEFORE=true
;;
d)
DOWNLOAD_ONLY=true
;;
*)
;;
esac
done
((OPTIND++))
[ $OPTIND -gt $# ] && break
done
# Configure proxies. Use apt values over environment variables.
# Note: If proxy setting is not set, there is no apt-config output.
# Therefore variable doesn't get overriden, which is intended.
# Export the variables to make them available in subshells (aka the
# downloader command).
eval "$(apt-config shell ftp_proxy Acquire::ftp::proxy)"
export ftp_proxy
eval "$(apt-config shell http_proxy Acquire::http::proxy)"
export http_proxy
eval "$(apt-config shell https_proxy Acquire::https::proxy)"
export https_proxy
# aria2 has no socks support (see https://github.com/aria2/aria2/issues/153)
if echo "$http_proxy" | grep -q "^socks5h://" || echo "$https_proxy" | grep -q "^socks5h://"; then
msg "Socks proxy detected. Falling back to ${_APTMGR}" "hint"
"${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@"
exit
fi
# Run actions.
if [ "$option" == "install" ]; then
msg
msg "Working... this may take a while." "normal"
REMOVE_WORKING_MESSAGE=y
get_uris "$@"
[ -t 1 ] && [ -n "$REMOVE_WORKING_MESSAGE" ] && tput cuu 1 && tput el && tput cuu 1
# Test /tmp/apt-fast.list file exists and not just the apt-fast comment line.
# Then download all files from the list.
if [ -f "$DLLIST" ] && [ "$(wc -l "$DLLIST" | cut -d' ' -f1)" -gt 1 ] && [ ! "$DOWNLOADBEFORE" ]; then
display_downloadfile
msg
msg "Do you want to download the packages? [Y/n] " "question"
while ((!updsys)); do
read -r -sn1 -t "$APT_FAST_TIMEOUT" answer || { msg; msg "Timed out." "warning"; CLEANUP_STATE=1; exit; }
case "$answer" in
[JjYy]) result=1; updsys=1 ;;
[Nn]) result=0; updsys=1 ;;
"") result=1; updsys=1 ;;
*) updsys=0 ;;
esac
done
else
result=1
fi
if ((DOWNLOAD_SIZE)); then
msg
# Continue if answer was right or DOWNLOADBEFORE is enabled.
if ((result)); then
if [ -s "$DLLIST" ]; then
# Test if apt-fast directory is present where we put packages.
if [ ! -d "$DLDIR" ]; then
mkdir -p -- "$DLDIR"
fi
cd "$DLDIR" &>/dev/null || { msg; msg "Not able to change into download directory." "warning"; CLEANUP_STATE=1; exit; }
eval "${_DOWNLOADER}" # execute downloadhelper command
if [ "$(find "$DLDIR" -printf . | wc -c)" -gt 1 ]; then
# Delete incomplete/corrupted downloaded files, if any: Not recursive, as we don't expect any dirs to exist within $DLDIR.
# When Aria2c downloads a file and detects it is corrupted, its filename won't be renamed back to its actual name,
# preserving .aria2 file extension, which also indicates when a file hasn't been completely downloaded.
for x in *.aria2; do
rm -f "$x" "${x%.aria2}"
done
# Move all packages to the apt install directory by force to ensure
# already existing debs which may be incomplete are replaced
find . -type f \( -name '*.deb' -o -name '*.ddeb' \) -execdir mv -ft "$APTCACHE" {} \+
fi
cd - &>/dev/null || msg "Failed to change back directory" "warning"
fi
else
CLEANUP_STATE=1
exit
fi
else
[ -t 1 ] && tput el
fi
# different problem resolving for aptitude
if [ -z "$DOWNLOAD_ONLY" ] || [ "$(basename "${_APTMGR}")" == 'aptitude' ]; then
"${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@"
fi
elif [ "$option" == "clean" ]; then
"${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@" && {
if [ -d "$DLDIR" ]; then
find "$DLDIR" -maxdepth 1 -type f -delete
CLEANUP_STATE="$?"
[ -f "$DLLIST" ] && rm -f -- "$DLLIST"* || true
fi
}
elif [ "$option" == "download" ]; then
msg
msg "Working... this may take a while." "normal"
REMOVE_WORKING_MESSAGE=y
get_uris "$@"
[ -t 1 ] && [ -n "$REMOVE_WORKING_MESSAGE" ] && tput cuu 1 && tput el && tput cuu 1
if [ -f "$DLLIST" ] && [ "$(wc -l "$DLLIST" | cut -d' ' -f1)" -gt 1 ]; then
display_downloadfile
eval "${_DOWNLOADER}"
fi
# different problem resolving for aptitude
if [ "$(basename "${_APTMGR}")" == 'aptitude' ]; then
"${_APTMGR}" "$@"
fi
elif [ "$option" == "source" ]; then
msg
msg "Working... this may take a while." "normal"
REMOVE_WORKING_MESSAGE=y
get_uris "$@"
[ -t 1 ] && [ -n "$REMOVE_WORKING_MESSAGE" ] && tput cuu 1 && tput el && tput cuu 1
if [ -f "$DLLIST" ] && [ "$(wc -l "$DLLIST" | cut -d' ' -f1)" -gt 1 ]; then
display_downloadfile
eval "${_DOWNLOADER}"
fi
# We use APT manager here to provide more verbose output. This method is
# slightly slower then extractiong packages manually after download but also
# more hardened (e.g. some options like --compile are available).
"${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@"
# Uncomment following snippet to extract source directly and comment
# both lines before.
#while read srcfile; do
# # extract only .dsc files
# echo "$srcfile" | grep -q '\.dsc$' || continue
# dpkg-source -x "$(basename "$srcfile")"
#done < "$DLLIST"
# Execute package manager directly if unknown options are passed.
else
"${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@"
fi
# After error or all done remove our lockfile (done with EXIT trap)

24
frontend/.gitignore vendored Normal file
View File

@ -0,0 +1,24 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

3
frontend/.vscode/extensions.json vendored Normal file
View File

@ -0,0 +1,3 @@
{
"recommendations": ["Vue.volar"]
}

20
frontend/README.md Normal file
View File

@ -0,0 +1,20 @@
# Frontend
前端使用 `Vue 3 + Vite + Tailwind CSS 4`，构建产物输出到 `frontend/dist`，由后端通过 `rust-embed` 内嵌并提供静态资源服务。
## 开发
```bash
npm ci
npm run dev
```
默认开发代理会把 `/api` 转发到 `http://localhost:3000`。
## 构建
```bash
npm run build
```
构建完成后，后端重新编译时会把 `dist/` 打包进二进制。

13
frontend/index.html Normal file
View File

@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>知识库管理系统</title>
</head>
<body class="bg-gray-50 text-gray-900 antialiased">
<div id="app"></div>
<script type="module" src="/src/main.js"></script>
</body>
</html>

2192
frontend/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

21
frontend/package.json Normal file
View File

@ -0,0 +1,21 @@
{
"name": "frontend",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
},
"dependencies": {
"pdfjs-dist": "^4.2.67",
"vue": "^3.5.24"
},
"devDependencies": {
"@tailwindcss/vite": "^4.1.18",
"@vitejs/plugin-vue": "^6.0.1",
"tailwindcss": "^4.1.18",
"vite": "^7.2.4"
}
}

359
frontend/public/excel-viewer.html Normal file
View File

@ -0,0 +1,359 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Excel 查看器</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
html, body {
width: 100%;
height: 100%;
font-family: system-ui, -apple-system, sans-serif;
background: #f5f5f5;
}
#app {
display: flex;
flex-direction: column;
height: 100%;
}
.header {
background: #fff;
border-bottom: 1px solid #e0e0e0;
padding: 12px 20px;
display: flex;
align-items: center;
justify-content: space-between;
flex-shrink: 0;
}
.header h1 {
font-size: 16px;
font-weight: 600;
color: #333;
}
.header .meta {
font-size: 13px;
color: #666;
}
.sheet-tabs {
display: flex;
background: #fff;
border-bottom: 1px solid #e0e0e0;
padding: 0 20px;
gap: 4px;
flex-shrink: 0;
overflow-x: auto;
}
.sheet-tab {
padding: 10px 20px;
font-size: 14px;
color: #666;
background: none;
border: none;
border-bottom: 2px solid transparent;
cursor: pointer;
white-space: nowrap;
transition: all 0.2s;
}
.sheet-tab:hover {
color: #333;
background: #f5f5f5;
}
.sheet-tab.active {
color: #1a73e8;
border-bottom-color: #1a73e8;
font-weight: 500;
}
.content {
flex: 1;
overflow: auto;
padding: 20px;
}
.table-container {
background: #fff;
border-radius: 8px;
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
overflow-x: auto;
}
table {
width: max-content;
min-width: 100%;
border-collapse: collapse;
font-size: 14px;
}
thead {
background: #f8f9fa;
position: sticky;
top: 0;
z-index: 1;
}
th {
padding: 12px 16px;
text-align: left;
font-weight: 600;
color: #333;
border-bottom: 1px solid #e0e0e0;
white-space: nowrap;
}
td {
padding: 10px 16px;
color: #444;
border-bottom: 1px solid #f0f0f0;
}
tbody tr:hover {
background: #f8f9fa;
}
tbody tr.highlighted {
background: #fff3cd !important;
}
tbody tr.highlighted td {
font-weight: 500;
color: #856404;
}
.row-num {
color: #999;
font-size: 12px;
text-align: center;
width: 50px;
border-right: 1px solid #e0e0e0;
}
.loading, .error {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
height: 100%;
color: #666;
gap: 12px;
}
.error {
color: #c00;
}
.spinner {
width: 32px;
height: 32px;
border: 3px solid #e0e0e0;
border-top-color: #1a73e8;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
.truncated-banner {
background: #fff3cd;
color: #856404;
padding: 10px 20px;
font-size: 13px;
text-align: center;
border-bottom: 1px solid #ffeaa7;
}
</style>
</head>
<body>
<div id="app">
<div class="loading">
<div class="spinner"></div>
<span>加载中...</span>
</div>
</div>
<script>
(function () {
const HEADERS = {
"x-user-id": "user1",
"x-user-name": "42tr",
"x-role": "admin",
};
const params = new URLSearchParams(window.location.search);
const fileId = params.get("file_id") || params.get("fileId");
const sliceId = params.get("slice_id") || params.get("sliceId");
const legacySheet = params.get("sheet_name") || "";
const legacyRow = parseInt(params.get("row_num") || "0", 10);
const app = document.getElementById("app");
const showError = (msg) => {
app.innerHTML = `<div class="error"><div>❌ ${msg}</div></div>`;
};
if (!fileId) {
showError("缺少 file_id 参数");
return;
}
async function loadExcelData() {
const res = await fetch(`/api/v1/knowledge/files/${fileId}/excel-data`, { headers: HEADERS });
if (!res.ok) {
throw new Error(`请求失败: ${res.status}`);
}
return res.json();
}
async function resolveHighlight() {
if (!sliceId) {
return { targetSheet: legacySheet, targetRow: legacyRow };
}
try {
const res = await fetch(
`/api/v1/knowledge/files/${fileId}/slices/${sliceId}/highlight`,
{ headers: HEADERS }
);
if (!res.ok) {
return { targetSheet: "", targetRow: 0 };
}
const data = await res.json();
const pos = data.positions?.[0];
return {
targetSheet: pos?.sheet_name || "",
targetRow: pos?.row_num || 0,
};
} catch (err) {
return { targetSheet: "", targetRow: 0 };
}
}
Promise.all([loadExcelData(), resolveHighlight()])
.then(([data, highlight]) => {
render(data, highlight.targetSheet, highlight.targetRow);
})
.catch((err) => {
showError(err.message || "加载 Excel 数据失败");
});
function render(data, targetSheet, targetRow) {
if (!data.sheets || data.sheets.length === 0) {
showError("Excel 文件为空或没有数据");
return;
}
// 找到目标 sheet，如果没有匹配则默认第一个
let activeIndex = 0;
if (targetSheet) {
const idx = data.sheets.findIndex((s) => s.name === targetSheet);
if (idx >= 0) activeIndex = idx;
}
const renderSheet = (index) => {
const sheet = data.sheets[index];
const tableHtml = buildTable(sheet, targetRow);
document.getElementById("table-area").innerHTML = tableHtml;
// 更新 tab 样式
document.querySelectorAll(".sheet-tab").forEach((tab, i) => {
tab.classList.toggle("active", i === index);
});
// 滚动到高亮行
if (targetRow > 0) {
const highlighted = document.querySelector("tr.highlighted");
if (highlighted) {
highlighted.scrollIntoView({ behavior: "smooth", block: "center" });
}
}
};
const tabsHtml = data.sheets
.map((s, i) => `<button class="sheet-tab ${i === activeIndex ? "active" : ""}" data-index="${i}">${escapeHtml(s.name)}</button>`)
.join("");
app.innerHTML = `
<div class="header">
<h1>${escapeHtml(data.filename)}</h1>
<span class="meta">${data.sheets.length} 个 sheet</span>
</div>
<div class="sheet-tabs">${tabsHtml}</div>
<div class="content">
<div id="table-area"></div>
</div>
`;
// Tab 切换事件
document.querySelectorAll(".sheet-tab").forEach((tab) => {
tab.addEventListener("click", () => {
renderSheet(parseInt(tab.dataset.index, 10));
});
});
renderSheet(activeIndex);
}
function buildTable(sheet, targetRow) {
const truncatedBanner = sheet.truncated
? `<div class="truncated-banner">数据量较大，仅展示前 ${sheet.rows.length} 行</div>`
: "";
const headerHtml = sheet.headers
.map((h) => `<th>${escapeHtml(h)}</th>`)
.join("");
const rowsHtml = sheet.rows
.map((row, idx) => {
const rowNum = idx + 2; // 1-based row number (header is row 1)
const isHighlighted = rowNum === targetRow;
const cells = row
.map((cell) => `<td>${escapeHtml(cell)}</td>`)
.join("");
return `<tr class="${isHighlighted ? "highlighted" : ""}"><td class="row-num">${rowNum}</td>${cells}</tr>`;
})
.join("");
return `
${truncatedBanner}
<div class="table-container">
<table>
<thead>
<tr>
<th class="row-num">#</th>
${headerHtml}
</tr>
</thead>
<tbody>
${rowsHtml}
</tbody>
</table>
</div>
`;
}
function escapeHtml(str) {
if (str == null) return "";
return String(str)
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;");
}
})();
</script>
</body>
</html>

127
frontend/public/pdf-highlight.html Normal file
View File

@ -0,0 +1,127 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>PDF Highlight Viewer</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
html, body {
width: 100%;
height: 100%;
overflow: hidden;
}
#pdf-container {
width: 100%;
height: 100%;
}
#pdf-container iframe {
width: 100%;
height: 100%;
border: none;
}
.loading, .error {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
font-family: system-ui, -apple-system, sans-serif;
color: #666;
font-size: 16px;
}
.error {
color: #c00;
}
</style>
</head>
<body>
<div id="pdf-container">
<div class="loading">Loading PDF...</div>
</div>
<script>
(function () {
const HEADERS = {
"x-user-id": "user1",
"x-user-name": "42tr",
"x-role": "admin",
};
const container = document.getElementById("pdf-container");
const showError = (msg) => {
container.innerHTML = `<div class="error">${msg}</div>`;
};
const params = new URLSearchParams(window.location.search);
const fileId = params.get("file_id") || params.get("fileId");
const sliceId = params.get("slice_id") || params.get("sliceId");
if (!fileId) {
showError("Missing file_id parameter");
return;
}
if (!sliceId) {
showError("Missing slice_id parameter");
return;
}
async function resolveTargetPage() {
try {
const res = await fetch(
`/api/v1/knowledge/files/${fileId}/slices/${sliceId}/highlight-page`,
{ headers: HEADERS }
);
if (!res.ok) {
return null;
}
const data = await res.json();
if (typeof data.page_idx === "number") {
// 后端返回 0-based 页码，PDF 查看器 #page= 需要 1-based
return data.page_idx + 1;
}
} catch (err) {
console.error("Failed to resolve highlight page:", err);
}
return null;
}
const pdfUrl = `/api/v1/knowledge/files/${fileId}/highlighted-pdf?slice_id=${encodeURIComponent(sliceId)}`;
(async () => {
const targetPage = await resolveTargetPage();
// 使用 fetch 获取 PDF（带认证 header），然后创建 blob URL 显示
fetch(pdfUrl, { headers: HEADERS })
.then((res) => {
if (!res.ok) {
throw new Error(`Request failed: ${res.status}`);
}
return res.blob();
})
.then((blob) => {
const blobUrl = URL.createObjectURL(blob);
const iframe = document.createElement("iframe");
iframe.src = targetPage ? `${blobUrl}#page=${targetPage}` : blobUrl;
iframe.title = "PDF Viewer";
container.innerHTML = "";
container.appendChild(iframe);
})
.catch((err) => {
showError(err.message || "Failed to load PDF");
});
})();
})();
</script>
</body>
</html>

1
frontend/public/vite.svg Normal file
View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.5 KiB

322
frontend/src/App.vue Normal file
View File

@ -0,0 +1,322 @@
<script setup>
import { reactive, ref } from 'vue'
import SearchBar from './components/SearchBar.vue'
import SearchResults from './components/SearchResults.vue'
import KnowledgeBaseList from './components/KnowledgeBaseList.vue'
import FileUpload from './components/FileUpload.vue'
import KnowledgeGraph from './components/KnowledgeGraph.vue'
import AdvancedSearchPanel from './components/AdvancedSearchPanel.vue'
import SearchDictionaryManager from './components/SearchDictionaryManager.vue'
import { api } from './api'
const activeTab = ref('search')
const searchResults = ref([])
const isSearching = ref(false)
const advancedState = reactive({
active: false,
running: false,
status: '待开始',
planSteps: [],
results: [],
timeline: [],
debugEvents: [],
error: '',
lastQuery: '',
})
let advancedController = null
const newId = () => `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
const sliceKey = (item) => {
const ids = item.slice_ids || item.sliceIds
if (Array.isArray(ids) && ids.length > 0) {
const normalized = [...new Set(ids)].sort((a, b) => a - b)
return normalized.join('-')
}
if (item.file?.id) {
return `${item.file.id}-${item.step_action || ''}`
}
if (item.content) {
return item.content.slice(0, 40)
}
return item.id || ''
}
const tabs = [
{ id: 'search', name: '搜索', icon: '🔍' },
{ id: 'dictionary', name: '词典', icon: '🧩' },
{ id: 'knowledge', name: '知识库', icon: '📚' },
{ id: 'graph', name: '知识图谱', icon: '🕸️' },
{ id: 'upload', name: '上传', icon: '📤' },
]
const handleSearchResults = (results) => {
searchResults.value = results
}
const handleSearchStart = () => {
isSearching.value = true
}
const handleSearchEnd = () => {
isSearching.value = false
}
const resetAdvancedState = () => {
advancedState.active = true
advancedState.running = true
advancedState.status = '连接中...'
advancedState.planSteps = []
advancedState.results = []
advancedState.timeline = []
advancedState.debugEvents = []
advancedState.error = ''
}
const pushTimeline = (entry) => {
advancedState.timeline.unshift({ id: newId(), time: Date.now(), ...entry })
if (advancedState.timeline.length > 50) {
advancedState.timeline.pop()
}
}
const pushDebug = (entry) => {
advancedState.debugEvents.unshift({ id: newId(), time: Date.now(), ...entry })
if (advancedState.debugEvents.length > 50) {
advancedState.debugEvents.pop()
}
}
const handleAdvancedSearch = ({ query, kbId, options }) => {
if (!query) return
if (advancedController) {
advancedController.cancel()
}
resetAdvancedState()
advancedState.lastQuery = query
pushTimeline({ type: 'start', title: '开始搜索', message: query })
const handlers = {
onStatus: (payload) => {
advancedState.status = payload?.message || payload?.phase || '处理中'
pushTimeline({ type: 'status', title: payload?.phase || '状态', message: payload?.message || '' })
},
onPlan: (payload) => {
const steps = (payload?.steps || []).map((step, index) => ({
...step,
status: 'pending',
details: null,
index: index + 1,
}))
advancedState.planSteps = steps
const message = steps.map((step) => step.comment || step.action || '').filter(Boolean).join(' → ')
pushTimeline({ type: 'plan', title: '执行计划', message: message || '已生成计划' })
},
onStep: (payload) => {
if (payload?.action) {
const target = advancedState.planSteps.find((step) => step.action === payload.action)
if (target) {
target.status = payload.status || 'updated'
target.details = payload.details || null
if (payload.comment) {
target.comment = payload.comment
}
}
}
pushTimeline({
type: 'step',
title: payload?.action ? `步骤 ${payload.action}` : '步骤更新',
message: `${payload?.status || ''} ${payload?.comment || ''}`.trim(),
})
},
onCandidate: (payload) => {
pushDebug({ type: 'candidate', payload })
},
onFiltered: (payload) => {
pushDebug({ type: 'filtered', payload })
},
onResult: (payload) => {
const entry = {
id: newId(),
receivedAt: Date.now(),
...payload,
}
const key = sliceKey(entry)
const existingIdx = advancedState.results.findIndex((item) => sliceKey(item) === key)
if (existingIdx !== -1) {
advancedState.results.splice(existingIdx, 1)
}
advancedState.results.unshift(entry)
if (advancedState.results.length > 30) {
advancedState.results.pop()
}
},
onErrorEvent: (payload) => {
advancedState.error = payload?.message || '服务返回错误'
pushTimeline({ type: 'error', title: '错误', message: advancedState.error })
},
onDone: () => {
advancedState.running = false
advancedState.status = '已完成'
pushTimeline({ type: 'done', title: '完成', message: '高级搜索完成' })
},
onError: (err) => {
advancedState.error = err?.message || '高级搜索失败'
advancedState.running = false
pushTimeline({ type: 'error', title: '连接失败', message: advancedState.error })
},
onFinally: () => {
advancedController = null
},
}
advancedController = api.advancedSearchStream(
{
query,
kbId,
maxSubQueries: options?.maxSteps,
perQueryLimit: options?.docLimit,
contextChars: options?.contextChars,
debug: options?.debug,
},
handlers,
)
}
const stopAdvancedSearch = () => {
if (advancedController) {
advancedController.cancel()
advancedController = null
}
if (advancedState.active) {
advancedState.running = false
advancedState.status = '已停止'
pushTimeline({ type: 'stop', title: '已停止', message: '用户中断搜索' })
}
}
const clearAdvanced = () => {
advancedState.active = false
advancedState.running = false
advancedState.status = '待开始'
advancedState.planSteps = []
advancedState.results = []
advancedState.timeline = []
advancedState.debugEvents = []
advancedState.error = ''
advancedState.lastQuery = ''
}
</script>
<template>
<div class="min-h-screen flex flex-col bg-linear-to-br from-slate-50 to-slate-100">
<!-- Header -->
<header class="bg-white border-b border-slate-200 sticky top-0 z-50">
<div class="max-w-6xl mx-auto px-6 py-4">
<div class="flex items-center justify-between">
<div class="flex items-center gap-3">
<div class="w-10 h-10 bg-linear-to-br from-blue-500 to-indigo-600 rounded-xl flex items-center justify-center">
<span class="text-white text-xl">📖</span>
</div>
<h1 class="text-xl font-semibold text-slate-800">知识库</h1>
</div>
<!-- Navigation -->
<nav class="flex gap-1 bg-slate-100 p-1 rounded-xl">
<button
v-for="tab in tabs"
:key="tab.id"
@click="activeTab = tab.id"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all duration-200',
activeTab === tab.id
? 'bg-white text-slate-900 shadow-sm'
: 'text-slate-600 hover:text-slate-900'
]"
>
<span class="mr-1.5">{{ tab.icon }}</span>
{{ tab.name }}
</button>
</nav>
</div>
</div>
</header>
<!-- Main Content -->
<main class="flex-1 max-w-6xl mx-auto px-6 py-8 w-full">
<!-- Search Tab -->
<div v-if="activeTab === 'search'" class="space-y-6">
<div class="text-center mb-8">
<h2 class="text-3xl font-bold text-slate-800 mb-2">搜索知识库</h2>
<p class="text-slate-500">在所有文档中快速查找您需要的信息</p>
</div>
<SearchBar
@search="handleSearchResults"
@search-start="handleSearchStart"
@search-end="handleSearchEnd"
@advanced-search="handleAdvancedSearch"
/>
<AdvancedSearchPanel
v-if="advancedState.active"
:state="advancedState"
@cancel="stopAdvancedSearch"
@clear="clearAdvanced"
/>
<SearchResults
:results="searchResults"
:loading="isSearching"
/>
</div>
<div v-if="activeTab === 'dictionary'" class="space-y-6">
<div class="text-center mb-8">
<h2 class="text-3xl font-bold text-slate-800 mb-2">词表与同义词</h2>
<p class="text-slate-500">管理搜索词表并发布重建索引，查看重建进度与 ETA</p>
</div>
<SearchDictionaryManager />
</div>
<!-- Knowledge Base Tab -->
<div v-if="activeTab === 'knowledge'" class="space-y-6">
<div class="flex items-center justify-between mb-6">
<div>
<h2 class="text-2xl font-bold text-slate-800">知识库管理</h2>
<p class="text-slate-500 mt-1">管理您的知识库和文档</p>
</div>
</div>
<KnowledgeBaseList />
</div>
<!-- Knowledge Graph Tab -->
<div v-if="activeTab === 'graph'" class="space-y-6">
<div class="text-center mb-8">
<h2 class="text-3xl font-bold text-slate-800 mb-2">知识图谱</h2>
<p class="text-slate-500">探索文档中的实体和关系</p>
</div>
<KnowledgeGraph />
</div>
<!-- Upload Tab -->
<div v-if="activeTab === 'upload'" class="space-y-6">
<div class="text-center mb-8">
<h2 class="text-2xl font-bold text-slate-800 mb-2">上传文档</h2>
<p class="text-slate-500">将文档添加到知识库中</p>
</div>
<FileUpload />
</div>
</main>
<!-- Footer -->
<footer class="border-t border-slate-200 bg-white mt-auto">
<div class="max-w-6xl mx-auto px-6 py-4">
<p class="text-center text-sm text-slate-500">
知识库管理系统 · Powered by Rust + Vue
</p>
</div>
</footer>
</div>
</template>

734
frontend/src/api.js Normal file
View File

@ -0,0 +1,734 @@
const API_BASE = '/api/v1/knowledge'
// 用户认证信息（实际应用中应该从登录获取）
const USER_ID = 'user1'
const USER_NAME = '42tr'
const ROLE = 'admin'
const getHeaders = (contentType = true) => {
const headers = {
'x-user-id': USER_ID,
'x-user-name': USER_NAME,
'x-role': ROLE,
}
if (contentType) {
headers['Content-Type'] = 'application/json'
}
return headers
}
const readErrorMessage = async (response, fallback) => {
try {
const payload = await response.json()
if (payload?.error) return payload.error
if (payload?.message) return payload.message
} catch (_) {
// ignore json parse error
}
return fallback
}
export const api = {
// 搜索
async search(query, kbId = null, fileId = null, options = {}) {
const { advanced = false } = options
let url = `${API_BASE}/search/?query=${encodeURIComponent(query)}`
if (kbId) {
url += `&kb_id=${kbId}`
}
if (fileId) {
url += `&file_id=${fileId}`
}
if (advanced) {
url += '&advanced=true'
}
const response = await fetch(url, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('搜索失败')
const data = await response.json()
return data.results || []
},
advancedSearchStream(params, handlers = {}) {
const {
query,
kbId = null,
fileId = null,
maxSubQueries = 3,
perQueryLimit = 10,
contextChars = 2000,
debug = false,
} = params
const searchParams = new URLSearchParams()
searchParams.set('query', query)
searchParams.set('max_sub_queries', maxSubQueries)
searchParams.set('per_query_limit', perQueryLimit)
searchParams.set('context_chars', contextChars)
if (debug) searchParams.set('debug', 'true')
if (kbId) searchParams.set('kb_id', kbId)
if (fileId) searchParams.set('file_id', fileId)
const controller = new AbortController()
const url = `${API_BASE}/search/advanced/stream?${searchParams.toString()}`
const headers = getHeaders(false)
const decoder = new TextDecoder('utf-8')
const parseEvent = (chunk) => {
const lines = chunk.split('\n')
let eventType = 'message'
let dataLines = []
for (const line of lines) {
const trimmed = line.trim()
if (trimmed.startsWith('event:')) {
eventType = trimmed.slice(6).trim()
} else if (trimmed.startsWith('data:')) {
dataLines.push(trimmed.slice(5).trim())
}
}
const dataStr = dataLines.join('\n')
let parsedData = null
if (dataStr) {
try {
parsedData = JSON.parse(dataStr)
} catch (err) {
parsedData = dataStr
}
}
return { eventType, data: parsedData }
}
const streamPromise = (async () => {
try {
const response = await fetch(url, {
headers,
signal: controller.signal,
})
if (!response.ok) {
throw new Error('高级搜索连接失败')
}
const reader = response.body.getReader()
let buffer = ''
while (true) {
const { value, done } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
let boundary
while ((boundary = buffer.indexOf('\n\n')) !== -1) {
const rawEvent = buffer.slice(0, boundary).trim()
buffer = buffer.slice(boundary + 2)
if (!rawEvent) continue
const { eventType, data } = parseEvent(rawEvent)
switch (eventType) {
case 'status':
handlers.onStatus?.(data)
break
case 'plan':
handlers.onPlan?.(data)
break
case 'step':
handlers.onStep?.(data)
break
case 'candidate':
handlers.onCandidate?.(data)
break
case 'filtered':
handlers.onFiltered?.(data)
break
case 'result':
handlers.onResult?.(data)
break
case 'error':
handlers.onErrorEvent?.(data)
break
case 'done':
handlers.onDone?.()
break
default:
handlers.onMessage?.({ eventType, data })
}
}
}
handlers.onComplete?.()
} catch (err) {
if (controller.signal.aborted) {
handlers.onAbort?.()
} else {
handlers.onError?.(err)
}
} finally {
handlers.onFinally?.()
}
})()
return {
cancel: () => controller.abort(),
finished: streamPromise,
}
},
async searchFull(query, kbId = null, fileId = null) {
let url = `${API_BASE}/search/full?query=${encodeURIComponent(query)}`
if (kbId) {
url += `&kb_id=${kbId}`
}
if (fileId) {
url += `&file_id=${fileId}`
}
const response = await fetch(url, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('全文搜索失败')
const data = await response.json()
return data.results || []
},
async searchImage(file, text = '', kbId = null, fileId = null) {
const formData = new FormData()
formData.append('file', file)
formData.append('text', text)
let url = `${API_BASE}/search/image`
const params = []
if (kbId) {
params.push(`kb_id=${kbId}`)
}
if (fileId) {
params.push(`file_id=${fileId}`)
}
if (params.length > 0) {
url += `?${params.join('&')}`
}
const response = await fetch(url, {
method: 'POST',
headers: getHeaders(false),
body: formData,
})
if (!response.ok) throw new Error('图片搜索失败')
const data = await response.json()
return data.results || []
},
// 词表
async listLexicons(params = {}) {
const searchParams = new URLSearchParams()
if (params.q) searchParams.set('q', params.q)
if (params.enabled === true) searchParams.set('enabled', 'true')
if (params.enabled === false) searchParams.set('enabled', 'false')
if (params.limit) searchParams.set('limit', String(params.limit))
if (params.offset) searchParams.set('offset', String(params.offset))
const query = searchParams.toString()
const url = `${API_BASE}/search/lexicons${query ? `?${query}` : ''}`
const response = await fetch(url, { headers: getHeaders() })
if (!response.ok) throw new Error(await readErrorMessage(response, '获取词表失败'))
return response.json()
},
async createLexicon(data) {
const response = await fetch(`${API_BASE}/search/lexicons`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '创建词条失败'))
return response.json()
},
async updateLexicon(id, data) {
const response = await fetch(`${API_BASE}/search/lexicons/${id}`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '更新词条失败'))
return response.json()
},
async deleteLexicon(id) {
const response = await fetch(`${API_BASE}/search/lexicons/${id}`, {
method: 'DELETE',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '删除词条失败'))
return response.json()
},
async toggleLexiconEnabled(id, enabled) {
const response = await fetch(`${API_BASE}/search/lexicons/${id}/enabled`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify({ enabled }),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '更新词条状态失败'))
return response.json()
},
async reloadLexicon() {
const response = await fetch(`${API_BASE}/search/lexicons/reload`, {
method: 'POST',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '重载词表失败'))
return response.json()
},
async publishLexicon() {
const response = await fetch(`${API_BASE}/search/lexicons/publish`, {
method: 'POST',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '发布词表失败'))
return response.json()
},
// 同义词
async listSynonyms(params = {}) {
const searchParams = new URLSearchParams()
if (params.q) searchParams.set('q', params.q)
if (params.enabled === true) searchParams.set('enabled', 'true')
if (params.enabled === false) searchParams.set('enabled', 'false')
if (params.limit) searchParams.set('limit', String(params.limit))
if (params.offset) searchParams.set('offset', String(params.offset))
const query = searchParams.toString()
const url = `${API_BASE}/search/synonyms${query ? `?${query}` : ''}`
const response = await fetch(url, { headers: getHeaders() })
if (!response.ok) throw new Error(await readErrorMessage(response, '获取同义词失败'))
return response.json()
},
async createSynonym(data) {
const response = await fetch(`${API_BASE}/search/synonyms`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '创建同义词失败'))
return response.json()
},
async updateSynonym(id, data) {
const response = await fetch(`${API_BASE}/search/synonyms/${id}`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '更新同义词失败'))
return response.json()
},
async deleteSynonym(id) {
const response = await fetch(`${API_BASE}/search/synonyms/${id}`, {
method: 'DELETE',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '删除同义词失败'))
return response.json()
},
async toggleSynonymEnabled(id, enabled) {
const response = await fetch(`${API_BASE}/search/synonyms/${id}/enabled`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify({ enabled }),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '更新同义词状态失败'))
return response.json()
},
// 系统
async getIndexRebuildStatus() {
const response = await fetch(`${API_BASE}/system/index/rebuild/status`, {
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '获取重建状态失败'))
return response.json()
},
// 知识库
async getKnowledgeBases(parentId = null) {
let url = `${API_BASE}/knowledge_base/`;
const params = new URLSearchParams();
// A null parentId fetches top-level KBs by default on the backend.
if (parentId) {
params.append('parent_id', parentId);
}
const queryString = params.toString();
if (queryString) {
url += `?${queryString}`;
}
const response = await fetch(url, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取知识库列表失败')
return response.json()
},
async getKnowledgeBase(id) {
const response = await fetch(`${API_BASE}/knowledge_base/${id}`, {
headers: getHeaders(),
});
if (!response.ok) throw new Error('获取知识库详情失败');
return response.json();
},
async getKnowledgeBaseFiles(id, params = {}) {
const searchParams = new URLSearchParams()
if (params.page) searchParams.set('page', String(params.page))
if (params.size) searchParams.set('size', String(params.size))
if (params.filename) searchParams.set('filename', params.filename)
if (params.tag) searchParams.set('tag', params.tag)
const query = searchParams.toString()
const response = await fetch(
`${API_BASE}/knowledge_base/${id}/files${query ? `?${query}` : ''}`,
{ headers: getHeaders() }
)
if (!response.ok) throw new Error(await readErrorMessage(response, '获取知识库文件失败'))
return response.json()
},
async createKnowledgeBase(data) { // data may include { name, description, parent_id }
const response = await fetch(`${API_BASE}/knowledge_base/`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error('创建知识库失败')
return response.json()
},
async updateKnowledgeBase(id, data) { // data may include { name, description, parent_id, is_public }
const response = await fetch(`${API_BASE}/knowledge_base/${id}`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error('更新知识库失败')
return response.json()
},
async deleteKnowledgeBase(id) {
const response = await fetch(`${API_BASE}/knowledge_base/${id}`, {
method: 'DELETE',
headers: getHeaders(),
})
if (!response.ok) throw new Error('删除知识库失败')
return
},
async reparseKnowledgeBases() {
const response = await fetch(`${API_BASE}/knowledge_base/reparse`, {
method: 'POST',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '重新解析失败'))
return response.json()
},
async reparseKnowledgeBase(id) {
const response = await fetch(`${API_BASE}/knowledge_base/${id}/reparse`, {
method: 'POST',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '重新解析指定知识库失败'))
return response.json()
},
async exportKnowledgeBases(kbIds, includeChildren = false) {
const response = await fetch(`${API_BASE}/knowledge_base/export`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify({ kb_ids: kbIds, include_children: includeChildren }),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '导出知识库失败'))
return response.json()
},
// 知识库权限管理
async getKbPermissions(kbId) {
const response = await fetch(`${API_BASE}/knowledge_base/${kbId}/permissions`, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取权限列表失败')
return response.json()
},
async addKbPermission(kbId, userId, permission) {
const response = await fetch(`${API_BASE}/knowledge_base/${kbId}/permissions`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify({ user_id: userId, permission }),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '添加权限失败'))
return response.json()
},
async removeKbPermission(kbId, userId) {
const response = await fetch(`${API_BASE}/knowledge_base/${kbId}/permissions/${userId}`, {
method: 'DELETE',
headers: getHeaders(),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '删除权限失败'))
return response.json()
},
// 文件
async uploadFiles(knowledgeBaseId, files, tags = [], isPublic = false, sliceType = 'smart') {
const formData = new FormData()
if (knowledgeBaseId) {
formData.append('kb_id', knowledgeBaseId)
}
if (tags.length > 0) {
formData.append('tags', JSON.stringify(tags))
}
formData.append('is_public', isPublic ? 'true' : 'false')
formData.append('slice_type', sliceType)
for (const file of files) {
formData.append('file', file)
}
const response = await fetch(`${API_BASE}/files/`, {
method: 'POST',
headers: getHeaders(false),
body: formData,
})
if (!response.ok) throw new Error(await readErrorMessage(response, '上传文件失败'))
return response.json()
},
async deleteFile(id) {
const response = await fetch(`${API_BASE}/files/${id}`, {
method: 'DELETE',
headers: getHeaders(),
})
if (!response.ok) throw new Error('删除文件失败')
return
},
async batchDeleteFiles(ids, options = {}) {
const {
strict = false,
allowProcessing = false,
} = options
const response = await fetch(`${API_BASE}/files/batch-delete`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify({
ids,
strict,
allow_processing: allowProcessing,
}),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '批量删除文件失败'))
return response.json()
},
async getFiles(kbId, tag = null, pagination = {}) {
let url = `${API_BASE}/files/`
const params = []
if (kbId === null) {
// 明确传递null时，获取未分配知识库的文件
params.push('kb_id=null')
} else if (kbId !== undefined) {
// 传递具体的知识库ID
params.push(`kb_id=${kbId}`)
}
if (tag) {
params.push(`tag=${encodeURIComponent(tag)}`)
}
if (pagination.page) params.push(`page=${pagination.page}`)
if (pagination.size) params.push(`size=${pagination.size}`)
if (params.length > 0) {
url += `?${params.join('&')}`
}
const response = await fetch(url, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取文件列表失败')
return response.json()
},
async getFileStats(options = {}) {
const {
kbId = undefined,
includeDescendants = true,
includeUnassigned = true,
} = options
const params = new URLSearchParams()
if (kbId === null) {
params.append('kb_id', 'null')
} else if (kbId !== undefined) {
params.append('kb_id', kbId)
}
if (!includeDescendants) {
params.append('include_descendants', 'false')
}
if (!includeUnassigned) {
params.append('include_unassigned', 'false')
}
const query = params.toString()
const response = await fetch(
`${API_BASE}/files/stats${query ? `?${query}` : ''}`,
{ headers: getHeaders() }
)
if (!response.ok) throw new Error('获取文件状态统计失败')
return response.json()
},
async reparseFailedFiles(options = {}) {
const {
kbId = undefined,
includeDescendants = true,
includeUnassigned = true,
unassignedOnly = false,
} = options
const payload = {
include_descendants: includeDescendants,
include_unassigned: includeUnassigned,
unassigned_only: unassignedOnly,
}
if (kbId !== undefined) {
payload.kb_id = kbId ?? null
}
const response = await fetch(`${API_BASE}/files/reparse-failed`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify(payload),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '重新解析失败文件失败'))
return response.json()
},
async getFile(id) {
const response = await fetch(`${API_BASE}/files/${id}`, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取文件失败')
return response.json()
},
async downloadFile(id) {
const response = await fetch(`${API_BASE}/files/${id}/download`, {
headers: getHeaders(false),
})
if (!response.ok) throw new Error('下载文件失败')
return response.blob()
},
async updateFile(id, data) {
const response = await fetch(`${API_BASE}/files/${id}`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify(data),
})
if (!response.ok) throw new Error('更新文件失败')
return response.json()
},
async moveFile(id, targetKbId) {
const response = await fetch(`${API_BASE}/files/${id}/move`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify({
target_kb_id: targetKbId ?? null,
}),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '移动文件失败'))
return response.json()
},
async getFileSlices(fileId) {
const response = await fetch(`${API_BASE}/files/${fileId}/slices`, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取切片失败')
return response.json()
},
async updateSlices(fileId, slices) {
const response = await fetch(`${API_BASE}/files/${fileId}/slices`, {
method: 'PUT',
headers: getHeaders(),
body: JSON.stringify({ slices }),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '更新切片失败'))
return response.json()
},
// 压缩文件
async getArchiveEntries(id) {
const response = await fetch(`${API_BASE}/files/${id}/archive-entries`, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取压缩文件列表失败')
return response.json()
},
async extractArchive(id, password = null) {
const response = await fetch(`${API_BASE}/files/${id}/archive-extract`, {
method: 'POST',
headers: getHeaders(),
body: JSON.stringify({ password }),
})
if (!response.ok) throw new Error(await readErrorMessage(response, '解压失败'))
return response.json()
},
async downloadArchiveEntry(id, path) {
const encodedPath = encodeURIComponent(path)
const response = await fetch(`${API_BASE}/files/${id}/archive-download?path=${encodedPath}`, {
headers: getHeaders(false),
})
if (!response.ok) throw new Error('下载文件失败')
return response.blob()
},
// 知识图谱
async searchEntities(query = null, entityType = null, kbId = null, limit = 100, fileId = null) {
let url = `${API_BASE}/graph/entities?limit=${limit}`
if (query) {
url += `&q=${encodeURIComponent(query)}`
}
if (entityType) {
url += `&entity_type=${encodeURIComponent(entityType)}`
}
if (kbId) {
url += `&kb_id=${kbId}`
}
if (fileId) {
url += `&file_id=${fileId}`
}
const response = await fetch(url, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('搜索实体失败')
return response.json()
},
async getEntity(id) {
const response = await fetch(`${API_BASE}/graph/entities/${id}`, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取实体详情失败')
return response.json()
},
async getGraphStats(kbId = null, fileId = null) {
let url = `${API_BASE}/graph/stats`
const params = []
if (kbId) {
params.push(`kb_id=${kbId}`)
}
if (fileId) {
params.push(`file_id=${fileId}`)
}
if (params.length > 0) {
url += '?' + params.join('&')
}
const response = await fetch(url, {
headers: getHeaders(),
})
if (!response.ok) throw new Error('获取图谱统计失败')
return response.json()
},
}

1
frontend/src/assets/vue.svg Normal file
View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="37.07" height="36" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 198"><path fill="#41B883" d="M204.8 0H256L128 220.8L0 0h97.92L128 51.2L157.44 0h47.36Z"></path><path fill="#41B883" d="m0 0l128 220.8L256 0h-51.2L128 132.48L50.56 0H0Z"></path><path fill="#35495E" d="M50.56 0L128 133.12L204.8 0h-47.36L128 51.2L97.92 0H50.56Z"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 496 B

205
frontend/src/components/AdvancedSearchPanel.vue Normal file
View File

@ -0,0 +1,205 @@
<script setup>
import { computed, ref } from 'vue'
const props = defineProps({
state: {
type: Object,
required: true,
},
})
const state = props.state
const emit = defineEmits(['cancel', 'clear'])
const showDebug = ref(false)
const hasResults = computed(() => (state?.results || []).length > 0)
const formatTime = (timestamp) => {
if (!timestamp) return ''
return new Date(timestamp).toLocaleTimeString('zh-CN', {
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
})
}
const copyContent = async (text) => {
if (!text) return
try {
await navigator.clipboard.writeText(text)
} catch (err) {
console.error('copy failed', err)
}
}
</script>
<template>
<div class="bg-white border border-slate-200 rounded-2xl p-6 shadow-sm mb-6">
<div class="flex flex-col gap-3 md:flex-row md:items-center md:justify-between">
<div>
<div class="flex items-center gap-2">
<h3 class="text-lg font-semibold text-slate-800">高级搜索流</h3>
<span
class="inline-flex items-center gap-1 text-xs font-medium px-2 py-0.5 rounded-full"
:class="state.running ? 'bg-green-50 text-green-700' : 'bg-slate-100 text-slate-600'"
>
<span class="h-2 w-2 rounded-full" :class="state.running ? 'bg-green-500 animate-pulse' : 'bg-slate-400'"></span>
{{ state.running ? '进行中' : '已结束' }}
</span>
</div>
<p class="text-sm text-slate-500 mt-1">
状态：{{ state.status }}
</p>
</div>
<div class="flex gap-3">
<button
v-if="state.running"
type="button"
class="px-4 py-2 rounded-xl border border-slate-200 text-sm font-medium hover:bg-slate-50"
@click="$emit('cancel')"
>
停止
</button>
<button
type="button"
class="px-4 py-2 rounded-xl border border-slate-200 text-sm font-medium hover:bg-slate-50"
@click="$emit('clear')"
>
清除
</button>
</div>
</div>
<div v-if="state.error" class="mt-4 p-3 rounded-xl bg-rose-50 border border-rose-100 text-rose-700 text-sm">
{{ state.error }}
</div>
<div v-if="state.planSteps?.length" class="mt-4 space-y-2">
<div
v-for="step in state.planSteps"
:key="step.index || step.action"
class="border border-slate-200 rounded-xl px-4 py-2 bg-slate-50"
>
<div class="flex items-center justify-between gap-3">
<div>
<p class="text-sm font-semibold text-slate-800">
步骤 {{ step.index || '?' }} · {{ step.action }}
</p>
<p class="text-xs text-slate-500 mt-0.5">
{{ step.comment || '—' }}
</p>
</div>
<span
class="text-xs font-semibold px-2 py-0.5 rounded-full"
:class="step.status === 'completed'
? 'bg-emerald-50 text-emerald-700'
: step.status === 'skipped'
? 'bg-amber-50 text-amber-700'
: step.status === 'started'
? 'bg-blue-50 text-blue-700'
: 'bg-slate-100 text-slate-600'"
>
{{ step.status || 'pending' }}
</span>
</div>
<pre
v-if="step.details"
class="mt-2 text-xs text-slate-600 bg-white border border-slate-200 rounded-lg p-2 overflow-auto"
>{{ JSON.stringify(step.details, null, 2) }}</pre>
</div>
</div>
<div class="grid gap-6 md:grid-cols-2 mt-6">
<section class="space-y-3">
<h4 class="text-sm font-semibold text-slate-700 flex items-center gap-2">
<span class="w-2 h-2 rounded-full bg-amber-400"></span> 过程记录
</h4>
<div
v-if="state.timeline.length === 0"
class="text-xs text-slate-400 border border-dashed border-slate-200 rounded-xl p-4 text-center"
>
暂无事件
</div>
<ul v-else class="space-y-2 max-h-72 overflow-auto pr-1">
<li
v-for="item in state.timeline"
:key="item.id"
class="rounded-xl border border-slate-100 p-3 bg-slate-50"
>
<div class="flex items-center justify-between text-xs text-slate-500 mb-1">
<span>{{ item.title }}</span>
<span>{{ formatTime(item.time) }}</span>
</div>
<p class="text-sm text-slate-700">
{{ item.message || '-' }}
</p>
</li>
</ul>
</section>
<section class="space-y-3">
<h4 class="text-sm font-semibold text-slate-700 flex items-center gap-2">
<span class="w-2 h-2 rounded-full bg-emerald-400"></span> 相关切片
</h4>
<div
v-if="!hasResults"
class="text-xs text-slate-400 border border-dashed border-slate-200 rounded-xl p-4 text-center"
>
结果将实时显示在这里
</div>
<div v-else class="space-y-4 max-h-80 overflow-auto pr-1">
<article
v-for="item in state.results"
:key="item.id"
class="border border-slate-100 rounded-xl p-4 bg-white shadow-sm"
>
<div class="flex items-start justify-between gap-3">
<div>
<p class="text-sm font-semibold text-slate-800">{{ item.file?.filename || '未知文件' }}</p>
<p class="text-xs text-slate-500 mt-1">
步骤：{{ item.step_action || '—' }}
</p>
</div>
<button
class="text-xs text-blue-600 hover:text-blue-800"
type="button"
@click="copyContent(item.content)"
>
复制
</button>
</div>
<p class="text-xs text-slate-500 mt-2">
相关度：{{ item.judge_score?.toFixed(2) ?? '-' }} · {{ item.judge_reason }}
</p>
<p v-if="item.refine_reason" class="text-xs text-slate-500 mt-1">
筛选说明：{{ item.refine_reason }}
</p>
<pre class="mt-3 text-sm text-slate-700 bg-slate-50 rounded-lg p-3 max-h-40 overflow-auto whitespace-pre-wrap">{{ item.content }}</pre>
</article>
</div>
</section>
</div>
<div v-if="state.debugEvents.length" class="mt-6 border-t border-dashed border-slate-200 pt-4">
<button
type="button"
class="text-xs text-slate-500 hover:text-slate-700 flex items-center gap-1"
@click="showDebug = !showDebug"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7" />
</svg>
{{ showDebug ? '隐藏调试事件' : '显示调试事件' }}
</button>
<div v-if="showDebug" class="mt-3 max-h-48 overflow-auto bg-slate-50 rounded-xl p-3 text-xs text-slate-600">
<div
v-for="event in state.debugEvents"
:key="event.id"
class="mb-2 border-b border-slate-200 pb-2 last:border-none last:pb-0"
>
<p class="font-semibold">[{{ event.type }}] {{ formatTime(event.time) }}</p>
<pre class="whitespace-pre-wrap">{{ JSON.stringify(event.payload, null, 2) }}</pre>
</div>
</div>
</div>
</div>
</template>

78
frontend/src/components/ArchiveTreeNode.vue Normal file
View File

@ -0,0 +1,78 @@
<script setup>
const props = defineProps({
node: { type: Object, required: true },
level: { type: Number, default: 0 },
expandedDirs: { type: Object, required: true },
downloading: { type: Object, required: true }
})
const emit = defineEmits(['toggle', 'download'])
const isExpanded = (path) => props.expandedDirs.has(path)
const sortedChildren = (node) => {
const children = Object.values(node.children || {})
return children.sort((a, b) => {
if (a.isDir !== b.isDir) return a.isDir ? -1 : 1
return a.name.localeCompare(b.name)
})
}
const formatSize = (bytes) => {
if (bytes == null) return '-'
if (bytes < 1024) return bytes + ' B'
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB'
if (bytes < 1024 * 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MB'
return (bytes / (1024 * 1024 * 1024)).toFixed(1) + ' GB'
}
</script>
<template>
<div>
<div
v-for="child in sortedChildren(node)"
:key="child.path"
>
<!-- 目录 -->
<div
v-if="child.isDir"
class="flex items-center gap-2 py-1.5 px-2 hover:bg-slate-50 rounded cursor-pointer select-none"
:style="{ paddingLeft: (level * 16 + 8) + 'px' }"
@click="$emit('toggle', child.path)"
>
<span class="text-amber-500 text-sm">{{ isExpanded(child.path) ? '📂' : '📁' }}</span>
<span class="text-sm text-slate-700">{{ child.name }}</span>
</div>
<!-- 文件 -->
<div
v-else
class="flex items-center gap-2 py-1.5 px-2 hover:bg-slate-50 rounded cursor-pointer group"
:style="{ paddingLeft: (level * 16 + 8) + 'px' }"
>
<span class="text-slate-400 text-sm">📄</span>
<span class="text-sm text-slate-700 flex-1 truncate">{{ child.name }}</span>
<span class="text-xs text-slate-400 mr-2">{{ formatSize(child.size) }}</span>
<button
@click.stop="$emit('download', child.path)"
:disabled="downloading.has(child.path)"
class="opacity-0 group-hover:opacity-100 p-1 text-slate-400 hover:text-emerald-500 hover:bg-emerald-50 rounded transition-all text-xs"
title="下载"
>
{{ downloading.has(child.path) ? '⏳' : '⬇️' }}
</button>
</div>
<!-- 递归渲染子目录 -->
<ArchiveTreeNode
v-if="child.isDir && isExpanded(child.path)"
:node="child"
:level="level + 1"
:expanded-dirs="expandedDirs"
:downloading="downloading"
@toggle="$emit('toggle', $event)"
@download="$emit('download', $event)"
/>
</div>
</div>
</template>

260
frontend/src/components/ArchiveViewer.vue Normal file
View File

@ -0,0 +1,260 @@
<script setup>
import { ref, computed, watch } from 'vue'
import { api } from '../api'
import ArchiveTreeNode from './ArchiveTreeNode.vue'
const props = defineProps({
file: {
type: Object,
required: true
}
})
const emit = defineEmits(['close'])
const loading = ref(false)
const entries = ref([])
const needsPassword = ref(false)
const password = ref('')
const error = ref('')
const downloading = ref(new Set())
const expandedDirs = ref(new Set())
const isArchive = computed(() => {
if (!props.file?.filename) return false
const lower = props.file.filename.toLowerCase()
return /\.(zip|7z|tar|tgz|tar\.gz|tar\.bz2|tar\.xz)$/i.test(lower)
})
// 构建文件树
const fileTree = computed(() => {
const root = { name: '', children: {}, isDir: true, path: '' }
for (const entry of entries.value) {
const parts = entry.entry_path.split('/')
let current = root
for (let i = 0; i < parts.length; i++) {
const part = parts[i]
if (!part) continue
const isLast = i === parts.length - 1
const isDir = isLast ? entry.is_directory : true
const fullPath = parts.slice(0, i + 1).join('/')
if (!current.children[part]) {
current.children[part] = {
name: part,
children: {},
isDir,
path: fullPath,
size: isLast && !isDir ? entry.size : null,
entry
}
}
current = current.children[part]
}
}
return root
})
const isExpanded = (path) => expandedDirs.value.has(path)
const toggleDir = (path) => {
if (expandedDirs.value.has(path)) {
expandedDirs.value.delete(path)
} else {
expandedDirs.value.add(path)
}
}
const loadEntries = async () => {
console.log('[ArchiveViewer] loadEntries start, file_id=' + props.file.id)
loading.value = true
error.value = ''
try {
const data = await api.getArchiveEntries(props.file.id)
console.log('[ArchiveViewer] getArchiveEntries returned', data.length, 'entries')
entries.value = data
needsPassword.value = false
if (entries.value.length === 0) {
console.log('[ArchiveViewer] entries empty, triggering extractArchive')
await extractArchive()
} else {
console.log('[ArchiveViewer] entries loaded from db:', entries.value.map(e => e.entry_path))
}
} catch (e) {
console.error('[ArchiveViewer] getArchiveEntries failed:', e)
error.value = e.message || '加载失败'
} finally {
loading.value = false
console.log('[ArchiveViewer] loadEntries done, final entries.length=', entries.value.length)
}
}
const extractArchive = async () => {
console.log('[ArchiveViewer] extractArchive start, password_provided=' + !!password.value)
loading.value = true
error.value = ''
try {
const result = await api.extractArchive(props.file.id, password.value || null)
console.log('[ArchiveViewer] extractArchive response:', result)
if (result.needs_password) {
needsPassword.value = true
entries.value = []
} else {
needsPassword.value = false
entries.value = result.entries || []
console.log('[ArchiveViewer] extracted', entries.value.length, 'entries:', entries.value.map(e => e.entry_path))
for (const entry of entries.value) {
const parts = entry.entry_path.split('/')
if (parts.length > 1) {
expandedDirs.value.add(parts[0])
}
}
}
} catch (e) {
console.error('[ArchiveViewer] extractArchive failed:', e)
error.value = e.message || '解压失败'
} finally {
loading.value = false
console.log('[ArchiveViewer] extractArchive done')
}
}
const handlePasswordSubmit = async () => {
if (!password.value.trim()) {
error.value = '请输入密码'
return
}
await extractArchive()
}
const downloadEntry = async (path) => {
if (downloading.value.has(path)) return
downloading.value.add(path)
try {
const blob = await api.downloadArchiveEntry(props.file.id, path)
const url = window.URL.createObjectURL(blob)
const link = document.createElement('a')
const filename = path.split('/').pop() || 'download'
link.href = url
link.download = filename
document.body.appendChild(link)
link.click()
link.remove()
window.URL.revokeObjectURL(url)
} catch (e) {
alert('下载失败：' + (e?.message || '未知错误'))
} finally {
downloading.value.delete(path)
}
}
watch(() => props.file, () => {
if (isArchive.value) {
loadEntries()
}
}, { immediate: true })
</script>
<template>
<Teleport to="body">
<div class="fixed inset-0 z-50 flex items-center justify-center p-4">
<div @click="emit('close')" class="absolute inset-0 bg-black/50 backdrop-blur-sm"></div>
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-2xl max-h-[80vh] flex flex-col">
<!-- Header -->
<div class="flex items-center justify-between p-5 border-b border-slate-100">
<div class="flex items-center gap-3">
<span class="text-2xl">📦</span>
<div>
<h3 class="text-lg font-semibold text-slate-800">压缩文件内容</h3>
<p class="text-xs text-slate-400">{{ file.filename }}</p>
</div>
</div>
<button
@click="emit('close')"
class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg transition-all"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<!-- Content -->
<div class="flex-1 overflow-auto p-4">
<!-- Loading -->
<div v-if="loading" class="flex items-center justify-center py-12">
<div class="animate-spin w-6 h-6 border-2 border-blue-500 border-t-transparent rounded-full mr-3"></div>
<span class="text-slate-500">{{ needsPassword ? '正在解压...' : '正在加载...' }}</span>
</div>
<!-- Password Input -->
<div v-else-if="needsPassword" class="py-8 px-4">
<div class="text-center mb-6">
<span class="text-4xl mb-3 block">🔐</span>
<h4 class="text-lg font-medium text-slate-700 mb-1">压缩文件已加密</h4>
<p class="text-sm text-slate-400">请输入解压密码</p>
</div>
<div class="max-w-xs mx-auto">
<input
v-model="password"
@keyup.enter="handlePasswordSubmit"
type="password"
placeholder="输入密码..."
class="w-full px-4 py-2.5 border border-slate-200 rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500 mb-3"
/>
<button
@click="handlePasswordSubmit"
class="w-full py-2.5 bg-blue-500 text-white rounded-xl font-medium hover:bg-blue-600 transition-all"
>
确认解压
</button>
</div>
</div>
<!-- Error -->
<div v-else-if="error" class="py-8 text-center">
<span class="text-3xl mb-2 block">⚠️</span>
<p class="text-red-500 text-sm">{{ error }}</p>
<button
@click="extractArchive"
class="mt-4 px-4 py-2 bg-slate-100 text-slate-600 rounded-lg hover:bg-slate-200 transition-all text-sm"
>
重试
</button>
</div>
<!-- File Tree -->
<div v-else-if="entries.length > 0">
<div class="mb-2 px-2 text-xs text-slate-400 flex items-center justify-between">
<span>共 {{ entries.filter(e => !e.is_directory).length }} 个文件</span>
<button
@click="extractArchive"
class="text-blue-500 hover:text-blue-600 transition-all"
>
🔄 重新解压
</button>
</div>
<div class="border border-slate-100 rounded-xl overflow-hidden">
<ArchiveTreeNode
:node="fileTree"
:level="0"
:expanded-dirs="expandedDirs"
:downloading="downloading"
@toggle="toggleDir"
@download="downloadEntry"
/>
</div>
</div>
<!-- Empty -->
<div v-else class="py-12 text-center">
<span class="text-3xl mb-2 block">📂</span>
<p class="text-slate-400 text-sm">压缩包为空</p>
</div>
</div>
</div>
</div>
</Teleport>
</template>

223
frontend/src/components/CreateKnowledgeBase.vue Normal file
View File

@ -0,0 +1,223 @@
<script setup>
import { ref, defineProps } from 'vue'
import { api } from '../api'
const props = defineProps({
parentId: {
type: Number,
default: null
}
})
const emit = defineEmits(['created'])
const showModal = ref(false)
const name = ref('')
const description = ref('')
const kbType = ref('analysis')
const isPublic = ref(false)
const creating = ref(false)
const error = ref('')
const handleCreate = async () => {
if (!name.value.trim()) {
error.value = '请输入知识库名称'
return
}
creating.value = true
error.value = ''
try {
await api.createKnowledgeBase({
name: name.value.trim(),
description: description.value.trim(),
kb_type: kbType.value,
parent_id: props.parentId,
is_public: isPublic.value
})
showModal.value = false
name.value = ''
description.value = ''
kbType.value = 'analysis'
isPublic.value = false
emit('created')
} catch (e) {
error.value = e.message
} finally {
creating.value = false
}
}
const closeModal = () => {
showModal.value = false
name.value = ''
description.value = ''
kbType.value = 'analysis'
error.value = ''
}
</script>
<template>
<div>
<button
@click="showModal = true"
class="px-4 py-2.5 bg-gradient-to-r from-blue-500 to-indigo-600 text-white rounded-xl font-medium hover:from-blue-600 hover:to-indigo-700 transition-all duration-200 shadow-sm flex items-center gap-2"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 4v16m8-8H4" />
</svg>
新建知识库
</button>
<!-- Modal -->
<Teleport to="body">
<div
v-if="showModal"
class="fixed inset-0 z-50 flex items-center justify-center p-4"
>
<!-- Backdrop -->
<div
@click="closeModal"
class="absolute inset-0 bg-black/50 backdrop-blur-sm"
></div>
<!-- Modal Content -->
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-md p-6">
<div class="flex items-center justify-between mb-5">
<h3 class="text-lg font-semibold text-slate-800">新建知识库</h3>
<button
@click="closeModal"
class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg transition-all"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<div class="space-y-4">
<div>
<label class="block text-sm font-medium text-slate-700 mb-1.5">
名称 <span class="text-red-500">*</span>
</label>
<input
v-model="name"
type="text"
placeholder="输入知识库名称"
class="w-full px-4 py-3 bg-slate-50 border border-slate-200 rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent"
/>
</div>
<div>
<label class="block text-sm font-medium text-slate-700 mb-1.5">
描述
</label>
<textarea
v-model="description"
placeholder="输入知识库描述（可选）"
rows="3"
class="w-full px-4 py-3 bg-slate-50 border border-slate-200 rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent resize-none"
></textarea>
</div>
<div>
<label class="block text-sm font-medium text-slate-700 mb-2">
可见性
</label>
<div class="flex gap-4">
<label
:class="[
'flex-1 flex items-center gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
!isPublic ? 'border-blue-500 bg-blue-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="isPublic" :value="false" class="w-4 h-4 text-blue-500" />
<div>
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🔒</span>
<span class="font-medium text-slate-800">私有</span>
</div>
<p class="text-xs text-slate-500">仅自己可见</p>
</div>
</label>
<label
:class="[
'flex-1 flex items-center gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
isPublic ? 'border-green-500 bg-green-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="isPublic" :value="true" class="w-4 h-4 text-green-500" />
<div>
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🌐</span>
<span class="font-medium text-slate-800">公开</span>
</div>
<p class="text-xs text-slate-500">所有人可见</p>
</div>
</label>
</div>
</div>
<div>
<label class="block text-sm font-medium text-slate-700 mb-2">
类型
</label>
<div class="flex gap-4">
<label
:class="[
'flex-1 flex items-center gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
kbType === 'analysis' ? 'border-indigo-500 bg-indigo-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="kbType" value="analysis" class="w-4 h-4 text-indigo-500" />
<div>
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🧠</span>
<span class="font-medium text-slate-800">分析型</span>
</div>
<p class="text-xs text-slate-500">解析并切片，支持搜索与知识图谱</p>
</div>
</label>
<label
:class="[
'flex-1 flex items-center gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
kbType === 'storage' ? 'border-amber-500 bg-amber-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="kbType" value="storage" class="w-4 h-4 text-amber-500" />
<div>
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🗄️</span>
<span class="font-medium text-slate-800">存储型</span>
</div>
<p class="text-xs text-slate-500">仅存储文件，不进行解析</p>
</div>
</label>
</div>
</div>
<p v-if="error" class="text-sm text-red-500">{{ error }}</p>
<div class="flex gap-3 pt-2">
<button
@click="closeModal"
class="flex-1 py-3 bg-slate-100 text-slate-700 rounded-xl font-medium hover:bg-slate-200 transition-colors"
>
取消
</button>
<button
@click="handleCreate"
:disabled="creating"
class="flex-1 py-3 bg-gradient-to-r from-blue-500 to-indigo-600 text-white rounded-xl font-medium hover:from-blue-600 hover:to-indigo-700 transition-all disabled:opacity-50 disabled:cursor-not-allowed"
>
<span v-if="creating">创建中...</span>
<span v-else>创建</span>
</button>
</div>
</div>
</div>
</div>
</Teleport>
</div>
</template>

194
frontend/src/components/EntityDetail.vue Normal file
View File

@ -0,0 +1,194 @@
<script setup>
import { ref, onMounted } from 'vue'
import { api } from '../api.js'
const props = defineProps({
entityId: {
type: Number,
required: true
}
})
const emit = defineEmits(['close'])
const entityDetail = ref(null)
const loading = ref(false)
const entityTypeMap = {
'person': { label: '人物', icon: '👤', color: 'blue' },
'organization': { label: '组织', icon: '🏢', color: 'purple' },
'location': { label: '地点', icon: '📍', color: 'green' },
'date': { label: '日期', icon: '📅', color: 'orange' },
'product': { label: '产品', icon: '📦', color: 'pink' },
'technology': { label: '技术', icon: '⚡', color: 'cyan' },
'concept': { label: '概念', icon: '💡', color: 'yellow' },
'api': { label: 'API', icon: '🔌', color: 'indigo' },
}
const relationTypeMap = {
'cooccurs': { label: '共现', icon: '🔗', color: 'blue' },
'isa': { label: '是一种', icon: '📌', color: 'purple' },
'partof': { label: '部分', icon: '🧩', color: 'green' },
'hasproperty': { label: '具有属性', icon: '⚙️', color: 'orange' },
'dependson': { label: '依赖于', icon: '🔄', color: 'pink' },
'relatedto': { label: '相关', icon: '↔️', color: 'cyan' },
'contains': { label: '包含', icon: '📦', color: 'indigo' },
'mentionedin': { label: '提及于', icon: '📝', color: 'yellow' },
}
const getEntityTypeInfo = (type) => {
return entityTypeMap[type] || { label: type, icon: '📌', color: 'gray' }
}
const getRelationTypeInfo = (type) => {
return relationTypeMap[type] || { label: type, icon: '🔗', color: 'gray' }
}
const loadEntityDetail = async () => {
loading.value = true
try {
entityDetail.value = await api.getEntity(props.entityId)
} catch (error) {
console.error('加载实体详情失败:', error)
} finally {
loading.value = false
}
}
const handleClose = () => {
emit('close')
}
onMounted(() => {
loadEntityDetail()
})
</script>
<template>
<div class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4" @click.self="handleClose">
<div class="bg-white rounded-2xl max-w-4xl w-full max-h-[90vh] overflow-hidden shadow-2xl">
<!-- 头部 -->
<div class="px-6 py-4 border-b border-slate-200 flex items-center justify-between bg-gradient-to-r from-blue-50 to-indigo-50">
<div class="flex items-center gap-3">
<div v-if="entityDetail" :class="`w-12 h-12 bg-${getEntityTypeInfo(entityDetail.entity.entity_type).color}-100 rounded-xl flex items-center justify-center`">
<span class="text-2xl">{{ getEntityTypeInfo(entityDetail.entity.entity_type).icon }}</span>
</div>
<div>
<h2 class="text-xl font-bold text-slate-800">
{{ entityDetail?.entity.name || '加载中...' }}
</h2>
<p v-if="entityDetail" class="text-sm text-slate-500">
{{ getEntityTypeInfo(entityDetail.entity.entity_type).label }}
</p>
</div>
</div>
<button
@click="handleClose"
class="w-8 h-8 rounded-lg hover:bg-slate-200 transition-colors flex items-center justify-center"
>
✕
</button>
</div>
<!-- 内容 -->
<div class="overflow-y-auto max-h-[calc(90vh-80px)]">
<div v-if="loading" class="p-8 text-center">
<div class="inline-block animate-spin rounded-full h-8 w-8 border-b-2 border-blue-500"></div>
<p class="mt-2 text-slate-500">加载中...</p>
</div>
<div v-else-if="entityDetail" class="p-6 space-y-6">
<!-- 基本信息 -->
<div class="bg-slate-50 rounded-xl p-4">
<h3 class="text-sm font-semibold text-slate-700 mb-3">基本信息</h3>
<div class="grid grid-cols-2 gap-4">
<div>
<p class="text-xs text-slate-500">实体ID</p>
<p class="text-sm font-medium text-slate-800">{{ entityDetail.entity.id }}</p>
</div>
<div>
<p class="text-xs text-slate-500">实体类型</p>
<p class="text-sm font-medium text-slate-800">{{ getEntityTypeInfo(entityDetail.entity.entity_type).label }}</p>
</div>
<div v-if="entityDetail.entity.file_id">
<p class="text-xs text-slate-500">来源文件ID</p>
<p class="text-sm font-medium text-slate-800">{{ entityDetail.entity.file_id }}</p>
</div>
<div v-if="entityDetail.entity.kb_id">
<p class="text-xs text-slate-500">知识库ID</p>
<p class="text-sm font-medium text-slate-800">{{ entityDetail.entity.kb_id }}</p>
</div>
</div>
</div>
<!-- 关联实体 -->
<div v-if="entityDetail.neighbors && entityDetail.neighbors.length > 0">
<h3 class="text-sm font-semibold text-slate-700 mb-3 flex items-center gap-2">
<span>🔗</span>
<span>关联实体 ({{ entityDetail.neighbors.length }})</span>
</h3>
<div class="space-y-2">
<div
v-for="neighbor in entityDetail.neighbors"
:key="neighbor.entity.id"
class="bg-white border border-slate-200 rounded-lg p-4 hover:border-blue-300 transition-colors"
>
<div class="flex items-center gap-3">
<div :class="`w-10 h-10 bg-${getEntityTypeInfo(neighbor.entity.entity_type).color}-100 rounded-lg flex items-center justify-center flex-shrink-0`">
<span class="text-lg">{{ getEntityTypeInfo(neighbor.entity.entity_type).icon }}</span>
</div>
<div class="flex-1">
<div class="flex items-center gap-2">
<h4 class="font-medium text-slate-800">{{ neighbor.entity.name }}</h4>
<span :class="`px-2 py-0.5 text-xs rounded-full bg-${getEntityTypeInfo(neighbor.entity.entity_type).color}-100 text-${getEntityTypeInfo(neighbor.entity.entity_type).color}-700`">
{{ getEntityTypeInfo(neighbor.entity.entity_type).label }}
</span>
</div>
<div class="flex items-center gap-2 mt-1">
<span :class="`px-2 py-0.5 text-xs rounded-full bg-${getRelationTypeInfo(neighbor.relation_type).color}-100 text-${getRelationTypeInfo(neighbor.relation_type).color}-700 flex items-center gap-1`">
<span>{{ getRelationTypeInfo(neighbor.relation_type).icon }}</span>
<span>{{ getRelationTypeInfo(neighbor.relation_type).label }}</span>
</span>
<span class="text-xs text-slate-400">
{{ neighbor.direction === 'outgoing' ? '→ 出边' : '← 入边' }}
</span>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- 文档提及 -->
<div v-if="entityDetail.mentions && entityDetail.mentions.length > 0">
<h3 class="text-sm font-semibold text-slate-700 mb-3 flex items-center gap-2">
<span>📝</span>
<span>文档提及 ({{ entityDetail.mentions.length }})</span>
</h3>
<div class="space-y-2">
<div
v-for="mention in entityDetail.mentions"
:key="mention.slice_id"
class="bg-slate-50 rounded-lg p-4 border border-slate-200"
>
<div class="flex items-start gap-2 mb-2">
<span class="text-xs font-medium text-blue-600">📄 {{ mention.filename }}</span>
<span class="text-xs text-slate-400">切片ID: {{ mention.slice_id }}</span>
</div>
<p class="text-sm text-slate-700 leading-relaxed">
{{ mention.context }}
</p>
</div>
</div>
</div>
<!-- 空状态 -->
<div v-if="(!entityDetail.neighbors || entityDetail.neighbors.length === 0) && (!entityDetail.mentions || entityDetail.mentions.length === 0)" class="text-center py-8 text-slate-400">
<span class="text-4xl block mb-2">📭</span>
暂无关联信息
</div>
</div>
</div>
</div>
</div>
</template>

128
frontend/src/components/ExportRecordPanel.vue Normal file
View File

@ -0,0 +1,128 @@
<script setup>
import { ref, computed, onMounted } from 'vue'
const STORAGE_KEY = 'htknow_export_records'
const MAX_RECORDS = 50
const props = defineProps({
records: {
type: Array,
default: () => [],
},
})
const emit = defineEmits(['clear'])
const expanded = ref(false)
const formatTime = (iso) => {
const d = new Date(iso)
return d.toLocaleString('zh-CN', {
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
})
}
const formatKbNames = (names) => {
if (!names || names.length === 0) return '-'
if (names.length <= 2) return names.join('、')
return `${names[0]}、${names[1]} 等 ${names.length} 个`
}
const formatSize = (num) => {
if (num >= 10000) return `${(num / 10000).toFixed(1)}万`
return String(num)
}
const handleClear = () => {
if (!confirm('确定要清空所有导出记录吗？')) return
emit('clear')
}
const copyPath = async (path) => {
try {
await navigator.clipboard.writeText(path)
alert('路径已复制到剪贴板')
} catch {
alert('复制失败')
}
}
</script>
<template>
<div class="mt-8 border border-slate-200 rounded-xl bg-white overflow-hidden">
<button
@click="expanded = !expanded"
class="w-full px-5 py-3 flex items-center justify-between hover:bg-slate-50 transition-colors"
>
<div class="flex items-center gap-2">
<svg class="w-5 h-5 text-slate-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 17v-2m3 2v-4m3 4v-6m2 10H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
</svg>
<span class="font-medium text-slate-700">导出记录</span>
<span class="text-xs text-slate-400 bg-slate-100 px-2 py-0.5 rounded-full">{{ records.length }}</span>
</div>
<div class="flex items-center gap-2">
<button
v-if="records.length > 0"
@click.stop="handleClear"
class="text-xs text-slate-400 hover:text-red-500 px-2 py-1 rounded transition-colors"
>
清空
</button>
<svg
class="w-5 h-5 text-slate-400 transition-transform duration-200"
:class="expanded ? 'rotate-180' : ''"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7" />
</svg>
</div>
</button>
<div v-if="expanded" class="border-t border-slate-100">
<div v-if="records.length === 0" class="px-5 py-8 text-center text-slate-400 text-sm">
暂无导出记录
</div>
<div v-else class="divide-y divide-slate-100 max-h-96 overflow-y-auto">
<div
v-for="record in records"
:key="record.id"
class="px-5 py-3 hover:bg-slate-50 transition-colors"
>
<div class="flex items-start justify-between gap-3">
<div class="flex-1 min-w-0">
<div class="flex items-center gap-2 mb-1">
<span class="text-xs text-slate-400">{{ formatTime(record.timestamp) }}</span>
<span class="text-xs px-1.5 py-0.5 rounded bg-blue-50 text-blue-600 border border-blue-100">
{{ record.kbCount || record.kb_ids?.length || 0 }} 个知识库
</span>
</div>
<div class="text-sm text-slate-700 truncate" :title="record.kbNames?.join('、')">
{{ formatKbNames(record.kbNames) }}
</div>
<div class="flex items-center gap-3 mt-1.5 text-xs text-slate-400">
<span v-if="record.fileCount">📄 {{ formatSize(record.fileCount) }} 文件</span>
<span v-if="record.sliceCount">📑 {{ formatSize(record.sliceCount) }} 切片</span>
<span v-if="record.tantivyDocCount">🔍 {{ formatSize(record.tantivyDocCount) }} 索引</span>
</div>
</div>
<button
@click="copyPath(record.exportPath)"
class="shrink-0 p-1.5 text-slate-400 hover:text-blue-500 hover:bg-blue-50 rounded-lg transition-colors"
title="复制导出路径"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z" />
</svg>
</button>
</div>
</div>
</div>
</div>
</div>
</template>

554
frontend/src/components/FileCard.vue Normal file
View File

@ -0,0 +1,554 @@
<script setup>
import { ref, computed } from 'vue'
import { api } from '../api'
import FileSlices from './FileSlices.vue'
import FileGraph from './FileGraph.vue'
import KnowledgeBaseSelector from './KnowledgeBaseSelector.vue'
import ArchiveViewer from './ArchiveViewer.vue'
const props = defineProps({
file: {
type: Object,
required: true
},
kbType: {
type: String,
default: null
},
highlighted: {
type: Boolean,
default: false
}
})
const emit = defineEmits(['updated', 'deleted'])
const showSlices = ref(false)
const showSettings = ref(false)
const showGraph = ref(false)
const showTagsEditor = ref(false)
const showMoveKbSelector = ref(false)
const showArchive = ref(false)
const selectedSliceType = ref(props.file.slice_type || 'paragraph')
const updating = ref(false)
const downloading = ref(false)
const moving = ref(false)
const editingTags = ref([])
const newTag = ref('')
const sliceTypes = [
{ value: 'smart', label: '智能切片', desc: '根据文档结构智能切分（推荐）' },
{ value: 'fixed', label: '固定长度', desc: '每 8000 字符一个切片，重叠 100 字' },
]
const isStorageKb = computed(() => props.kbType === 'storage')
const isArchive = computed(() => {
if (!props.file?.filename) return false
const lower = props.file.filename.toLowerCase()
return /\.(zip|7z|tar|tgz|tar\.gz|tar\.bz2|tar\.xz)$/i.test(lower)
})
const statusInfo = computed(() => {
switch (props.file.status) {
case 0:
return { text: '待处理', color: 'bg-amber-100 text-amber-700', icon: '⏳' }
case 2:
return { text: '处理中', color: 'bg-blue-100 text-blue-700', icon: '⚙️' }
case 1:
return { text: '已完成', color: 'bg-green-100 text-green-700', icon: '✓' }
case 3:
return isArchive.value
? { text: '压缩文件', color: 'bg-purple-100 text-purple-700', icon: '📦' }
: { text: '不解析', color: 'bg-amber-100 text-amber-700', icon: '🗄️' }
case -1:
return { text: '处理失败', color: 'bg-red-100 text-red-700', icon: '✗' }
default:
return { text: '未知', color: 'bg-slate-100 text-slate-700', icon: '?' }
}
})
const publicInfo = computed(() => {
return {
isPublic: props.file.is_public,
text: props.file.is_public ? '公开' : '私有',
color: props.file.is_public ? 'bg-green-50 text-green-600 border-green-200' : 'bg-slate-50 text-slate-600 border-slate-200',
icon: props.file.is_public ? '🌐' : '🔒'
}
})
const sliceTypeLabel = computed(() => {
if (isStorageKb.value) {
return '存储模式'
}
const type = sliceTypes.find(t => t.value === props.file.slice_type)
return type ? type.label : '智能切片'
})
const fileTags = computed(() => {
if (!props.file.tags) return []
try {
return JSON.parse(props.file.tags)
} catch {
return []
}
})
const formatDate = (timestamp) => {
if (!timestamp) return '-'
return new Date(timestamp * 1000).toLocaleString('zh-CN')
}
const handleUpdateSliceType = async () => {
const currentSliceType = props.file.slice_type || 'smart'
const isSameSliceType = selectedSliceType.value === currentSliceType
if (isSameSliceType) {
const shouldReparse = confirm('切片方式未变化，是否重新解析该文件？')
if (!shouldReparse) return
}
updating.value = true
try {
await api.updateFile(props.file.id, { slice_type: selectedSliceType.value })
showSettings.value = false
emit('updated')
} catch (e) {
alert('更新失败：' + e.message)
} finally {
updating.value = false
}
}
const handleDelete = async () => {
if (!confirm('确定要删除这个文件吗？')) return
try {
await api.deleteFile(props.file.id)
emit('deleted')
} catch (e) {
alert('删除失败：' + e.message)
}
}
const handleDownload = async () => {
if (downloading.value) return
downloading.value = true
try {
const blob = await api.downloadFile(props.file.id)
const url = window.URL.createObjectURL(blob)
const link = document.createElement('a')
link.href = url
link.download = props.file.filename || 'download'
document.body.appendChild(link)
link.click()
link.remove()
window.URL.revokeObjectURL(url)
} catch (e) {
alert('下载失败：' + (e?.message || '未知错误'))
} finally {
downloading.value = false
}
}
const openTagsEditor = () => {
editingTags.value = [...fileTags.value]
newTag.value = ''
showTagsEditor.value = true
}
const addTag = () => {
const tag = newTag.value.trim()
if (tag && !editingTags.value.includes(tag)) {
editingTags.value.push(tag)
newTag.value = ''
}
}
const removeTag = (index) => {
editingTags.value.splice(index, 1)
}
const handleSaveTags = async () => {
updating.value = true
try {
await api.updateFile(props.file.id, { tags: editingTags.value })
showTagsEditor.value = false
emit('updated')
} catch (e) {
alert('更新标签失败：' + e.message)
} finally {
updating.value = false
}
}
const handleTogglePublic = async () => {
const newPublic = !publicInfo.value.isPublic
if (!confirm(`确定要将文件设置为${newPublic ? '公开' : '私有'}吗？`)) return
updating.value = true
try {
await api.updateFile(props.file.id, { is_public: newPublic })
emit('updated')
} catch (e) {
alert('更新失败：' + e.message)
} finally {
updating.value = false
}
}
const openMoveSelector = () => {
if (moving.value || props.file.status === 2) return
showMoveKbSelector.value = true
}
const handleMoveToKb = async (kb) => {
const targetKbId = kb?.id ?? null
const currentKbId = props.file.kb_id ?? null
if (targetKbId === currentKbId) {
alert('文件已在当前知识库中')
return
}
const targetName = kb?.name || '未分配知识库'
if (!confirm(`确定要将文件移动到「${targetName}」吗？移动后会重新进入解析流程。`)) return
moving.value = true
try {
await api.moveFile(props.file.id, targetKbId)
emit('updated')
} catch (e) {
alert('移动失败：' + e.message)
} finally {
moving.value = false
}
}
</script>
<template>
<div
:class="[
'bg-white rounded-xl border transition-all duration-300',
highlighted
? 'border-blue-500 ring-4 ring-blue-100 shadow-md'
: 'border-slate-200 hover:border-slate-300'
]"
>
<!-- File Info -->
<div class="p-4">
<div class="flex items-start gap-4">
<!-- File Icon -->
<div class="w-10 h-10 bg-linear-to-br from-amber-100 to-orange-100 rounded-lg flex items-center justify-center shrink-0">
<span class="text-lg">{{ isArchive ? '📦' : '📄' }}</span>
</div>
<!-- File Details -->
<div class="flex-1 min-w-0">
<div class="flex items-center gap-2 mb-1">
<h3 class="font-medium text-slate-800 truncate">{{ file.filename }}</h3>
<span :class="['px-2 py-0.5 text-xs rounded-full', statusInfo.color]">
{{ statusInfo.icon }} {{ statusInfo.text }}
</span>
<span :class="['px-2 py-0.5 text-xs rounded-full border', publicInfo.color]">
{{ publicInfo.icon }} {{ publicInfo.text }}
</span>
</div>
<div class="flex items-center gap-4 text-xs text-slate-400">
<span>切片方式：{{ sliceTypeLabel }}</span>
<span>创建时间：{{ formatDate(file.created_at) }}</span>
</div>
<!-- Tags -->
<div v-if="fileTags.length > 0" class="flex items-center gap-2 mt-2 flex-wrap">
<span
v-for="tag in fileTags"
:key="tag"
class="px-2 py-0.5 text-xs bg-blue-50 text-blue-600 rounded-full border border-blue-100"
>
🏷️ {{ tag }}
</span>
</div>
<!-- Error Log -->
<div v-if="file.status === -1 && file.log" class="mt-2 p-2 bg-red-50 rounded-lg">
<p class="text-xs text-red-600">{{ file.log }}</p>
</div>
</div>
<!-- Actions -->
<div class="flex items-center gap-1">
<button
@click="handleTogglePublic"
class="p-2 text-slate-400 hover:text-green-500 hover:bg-green-50 rounded-lg transition-all"
:title="publicInfo.isPublic ? '设置为私有' : '设置为公开'"
>
<svg v-if="publicInfo.isPublic" class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 15v2m-6 4h12a2 2 0 002-2v-6a2 2 0 00-2-2H6a2 2 0 00-2 2v6a2 2 0 002 2zm10-10V7a4 4 0 00-8 0v4h8z" />
</svg>
<svg v-else class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3.055 11H5a2 2 0 012 2v1a2 2 0 002 2 2 2 0 012 2v2.945M8 3.935V5.5A2.5 2.5 0 0010.5 8h.5a2 2 0 012 2 2 2 0 104 0 2 2 0 012-2h1.064M15 20.488V18a2 2 0 012-2h3.064M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</button>
<button
@click="openTagsEditor"
class="p-2 text-slate-400 hover:text-blue-500 hover:bg-blue-50 rounded-lg transition-all"
title="管理标签"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 7h.01M7 3h5c.512 0 1.024.195 1.414.586l7 7a2 2 0 010 2.828l-7 7a2 2 0 01-2.828 0l-7-7A1.994 1.994 0 013 12V7a4 4 0 014-4z" />
</svg>
</button>
<button
v-if="file.status === 1 && !isStorageKb && !isArchive"
@click="showGraph = true"
class="p-2 text-slate-400 hover:text-purple-500 hover:bg-purple-50 rounded-lg transition-all"
title="查看知识图谱"
>
<svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<circle cx="12" cy="12" r="3" />
<circle cx="4" cy="8" r="2" />
<circle cx="20" cy="8" r="2" />
<circle cx="4" cy="16" r="2" />
<circle cx="20" cy="16" r="2" />
<line x1="6" y1="8" x2="9" y2="10" />
<line x1="18" y1="8" x2="15" y2="10" />
<line x1="6" y1="16" x2="9" y2="14" />
<line x1="18" y1="16" x2="15" y2="14" />
</svg>
</button>
<button
v-if="file.status === 1 && !isStorageKb && !isArchive"
@click="showSlices = true"
class="p-2 text-slate-400 hover:text-blue-500 hover:bg-blue-50 rounded-lg transition-all"
title="查看切片"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 10h16M4 14h16M4 18h16" />
</svg>
</button>
<button
v-if="isArchive"
@click="showArchive = true"
class="p-2 text-slate-400 hover:text-purple-500 hover:bg-purple-50 rounded-lg transition-all"
title="查看内容"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 19a2 2 0 01-2-2V7a2 2 0 012-2h4l2 2h4a2 2 0 012 2v1M5 19h14a2 2 0 002-2v-5a2 2 0 00-2-2H9a2 2 0 00-2 2v5a2 2 0 01-2 2z" />
</svg>
</button>
<button
v-if="!isStorageKb && !isArchive"
@click="showSettings = true; selectedSliceType = file.slice_type || 'smart'"
class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg transition-all"
title="修改切片方式"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.065 2.572c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.572 1.065c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.065-2.572c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065z" />
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z" />
</svg>
</button>
<button
@click="openMoveSelector"
:disabled="moving || file.status === 2"
class="p-2 text-slate-400 hover:text-indigo-500 hover:bg-indigo-50 rounded-lg transition-all disabled:opacity-50 disabled:cursor-not-allowed"
:title="file.status === 2 ? '处理中不可移动' : (moving ? '移动中...' : '移动到其他知识库')"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3 7h5l2 2h11v8a2 2 0 01-2 2H3V7z" />
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 13h6m0 0l-2-2m2 2l-2 2" />
</svg>
</button>
<button
@click="handleDownload"
:disabled="downloading"
class="p-2 text-slate-400 hover:text-emerald-500 hover:bg-emerald-50 rounded-lg transition-all disabled:opacity-50 disabled:cursor-not-allowed"
:title="downloading ? '下载中...' : '下载'"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 3v12m0 0l4-4m-4 4l-4-4M4 17v2a2 2 0 002 2h12a2 2 0 002-2v-2" />
</svg>
</button>
<button
@click="handleDelete"
class="p-2 text-slate-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-all"
title="删除"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" />
</svg>
</button>
</div>
</div>
</div>
<!-- Modals -->
<Teleport to="body">
<!-- Tags Editor Modal -->
<div v-if="showTagsEditor" class="fixed inset-0 z-50 flex items-center justify-center p-4">
<div @click="showTagsEditor = false" class="absolute inset-0 bg-black/50 backdrop-blur-sm"></div>
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-md p-6">
<div class="flex items-center justify-between mb-5">
<h3 class="text-lg font-semibold text-slate-800">管理标签</h3>
<button @click="showTagsEditor = false" class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<!-- Add Tag Input -->
<div class="mb-4">
<div class="flex gap-2">
<input
v-model="newTag"
@keyup.enter="addTag"
type="text"
placeholder="输入标签名称..."
class="flex-1 px-3 py-2 border border-slate-200 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
<button
@click="addTag"
class="px-4 py-2 bg-blue-500 text-white rounded-lg font-medium hover:bg-blue-600 transition-all"
>
添加
</button>
</div>
</div>
<!-- Tags List -->
<div class="mb-6">
<p class="text-sm text-slate-600 mb-2">当前标签：</p>
<div v-if="editingTags.length === 0" class="text-center py-8 text-slate-400">
<svg class="w-12 h-12 mx-auto mb-2 opacity-50" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 7h.01M7 3h5c.512 0 1.024.195 1.414.586l7 7a2 2 0 010 2.828l-7 7a2 2 0 01-2.828 0l-7-7A1.994 1.994 0 013 12V7a4 4 0 014-4z" />
</svg>
<p>暂无标签</p>
</div>
<div v-else class="flex flex-wrap gap-2">
<div
v-for="(tag, index) in editingTags"
:key="index"
class="flex items-center gap-1 px-3 py-1.5 bg-blue-50 text-blue-600 rounded-lg border border-blue-100 group"
>
<span class="text-sm">{{ tag }}</span>
<button
@click="removeTag(index)"
class="ml-1 p-0.5 text-blue-400 hover:text-red-500 hover:bg-red-50 rounded transition-all"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
</div>
</div>
<div class="flex gap-3">
<button
@click="showTagsEditor = false"
class="flex-1 py-3 bg-slate-100 text-slate-700 rounded-xl font-medium hover:bg-slate-200"
>
取消
</button>
<button
@click="handleSaveTags"
:disabled="updating"
class="flex-1 py-3 bg-linear-to-r from-blue-500 to-indigo-600 text-white rounded-xl font-medium hover:from-blue-600 hover:to-indigo-700 disabled:opacity-50"
>
{{ updating ? '保存中...' : '保存' }}
</button>
</div>
</div>
</div>
<!-- Slice Settings Modal -->
<div v-if="showSettings" class="fixed inset-0 z-50 flex items-center justify-center p-4">
<div @click="showSettings = false" class="absolute inset-0 bg-black/50 backdrop-blur-sm"></div>
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-md p-6">
<div class="flex items-center justify-between mb-5">
<h3 class="text-lg font-semibold text-slate-800">修改切片方式</h3>
<button @click="showSettings = false" class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<div class="space-y-2 mb-6">
<label
v-for="type in sliceTypes"
:key="type.value"
:class="[
'flex items-center gap-3 p-3 rounded-xl border-2 cursor-pointer transition-all',
selectedSliceType === type.value
? 'border-blue-500 bg-blue-50'
: 'border-slate-200 hover:border-slate-300'
]"
>
<input
type="radio"
v-model="selectedSliceType"
:value="type.value"
class="w-4 h-4 text-blue-500"
/>
<div>
<p class="font-medium text-slate-800">{{ type.label }}</p>
<p class="text-xs text-slate-500">{{ type.desc }}</p>
</div>
</label>
</div>
<div class="bg-amber-50 border border-amber-200 rounded-xl p-3 mb-6">
<p class="text-sm text-amber-700">
<span class="font-medium">注意：</span>修改切片方式将删除现有切片并重新处理文件
</p>
</div>
<div class="flex gap-3">
<button
@click="showSettings = false"
class="flex-1 py-3 bg-slate-100 text-slate-700 rounded-xl font-medium hover:bg-slate-200"
>
取消
</button>
<button
@click="handleUpdateSliceType"
:disabled="updating"
class="flex-1 py-3 bg-linear-to-r from-blue-500 to-indigo-600 text-white rounded-xl font-medium hover:from-blue-600 hover:to-indigo-700 disabled:opacity-50"
>
{{ updating ? '更新中...' : '确认修改' }}
</button>
</div>
</div>
</div>
</Teleport>
<!-- Slices Modal -->
<FileSlices
v-if="showSlices"
:file="file"
@close="showSlices = false"
/>
<!-- Graph Modal -->
<FileGraph
v-if="showGraph"
:file="file"
@close="showGraph = false"
/>
<KnowledgeBaseSelector
v-if="showMoveKbSelector"
:show="showMoveKbSelector"
@close="showMoveKbSelector = false"
@select="handleMoveToKb"
/>
<!-- Archive Viewer Modal -->
<ArchiveViewer
v-if="showArchive"
:file="file"
@close="showArchive = false"
/>
</div>
</template>

123
frontend/src/components/FileGraph.vue Normal file
View File

@ -0,0 +1,123 @@
<script setup>
import { ref, onMounted, computed } from 'vue'
import { api } from '../api.js'
import GraphVisualization from './GraphVisualization.vue'
import EntityDetail from './EntityDetail.vue'
const props = defineProps({
file: {
type: Object,
required: true
}
})
const emit = defineEmits(['close'])
const stats = ref(null)
const loading = ref(true)
const selectedEntity = ref(null)
const showEntityDetail = ref(false)
const loadStats = async () => {
loading.value = true
try {
stats.value = await api.getGraphStats(null, props.file.id)
} catch (error) {
console.error('加载图谱统计失败:', error)
} finally {
loading.value = false
}
}
const handleEntityClick = (entity) => {
selectedEntity.value = entity
showEntityDetail.value = true
}
const closeEntityDetail = () => {
showEntityDetail.value = false
selectedEntity.value = null
}
onMounted(() => {
loadStats()
})
</script>
<template>
<Teleport to="body">
<div class="fixed inset-0 z-50 flex items-center justify-center p-4">
<!-- Backdrop -->
<div @click="emit('close')" class="absolute inset-0 bg-black/50 backdrop-blur-sm"></div>
<!-- Modal -->
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-6xl max-h-[90vh] overflow-hidden flex flex-col">
<!-- Header -->
<div class="px-6 py-4 border-b border-slate-200 flex items-center justify-between flex-shrink-0">
<div class="flex items-center gap-3">
<div class="w-10 h-10 bg-gradient-to-br from-purple-100 to-indigo-100 rounded-lg flex items-center justify-center">
<span class="text-xl">🕸️</span>
</div>
<div>
<h2 class="text-lg font-semibold text-slate-800">文件知识图谱</h2>
<p class="text-sm text-slate-500">{{ file.filename }}</p>
</div>
</div>
<button
@click="emit('close')"
class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg transition-all"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<!-- Stats -->
<div v-if="stats" class="px-6 py-3 border-b border-slate-100 bg-slate-50 flex-shrink-0">
<div class="flex items-center gap-6 text-sm">
<div class="flex items-center gap-2">
<span class="text-blue-500">🔵</span>
<span class="text-slate-600">实体节点:</span>
<span class="font-semibold text-slate-800">{{ stats.node_count }}</span>
</div>
<div class="flex items-center gap-2">
<span class="text-purple-500">↔️</span>
<span class="text-slate-600">关系边:</span>
<span class="font-semibold text-slate-800">{{ stats.edge_count }}</span>
</div>
<div class="flex items-center gap-2">
<span class="text-green-500">📊</span>
<span class="text-slate-600">实体类型:</span>
<span class="font-semibold text-slate-800">{{ Object.keys(stats.entity_types || {}).length }}</span>
</div>
<div class="flex items-center gap-2">
<span class="text-orange-500">🔗</span>
<span class="text-slate-600">关系类型:</span>
<span class="font-semibold text-slate-800">{{ Object.keys(stats.relation_types || {}).length }}</span>
</div>
</div>
</div>
<!-- Graph Visualization -->
<div class="flex-1 overflow-auto p-4">
<div v-if="stats && stats.node_count === 0" class="flex items-center justify-center h-96">
<div class="text-center text-slate-400">
<span class="text-4xl block mb-2">🕸️</span>
<p>此文件暂无知识图谱数据</p>
<p class="text-sm mt-1">文件处理完成后将自动生成知识图谱</p>
</div>
</div>
<GraphVisualization v-else :file-id="file.id" :max-nodes="100" />
</div>
</div>
</div>
<!-- Entity Detail -->
<EntityDetail
v-if="showEntityDetail && selectedEntity"
:entity-id="selectedEntity.id"
@close="closeEntityDetail"
/>
</Teleport>
</template>

270
frontend/src/components/FileSlices.vue Normal file
View File

@ -0,0 +1,270 @@
<script setup>
import { ref, onMounted } from 'vue'
import { api } from '../api'
const props = defineProps({
file: {
type: Object,
required: true
}
})
const emit = defineEmits(['close'])
const slices = ref([])
const loading = ref(true)
const error = ref('')
const expandedSlice = ref(null)
const editingSliceId = ref(null)
const editingContent = ref('')
const saving = ref(false)
const loadSlices = async () => {
loading.value = true
error.value = ''
try {
slices.value = await api.getFileSlices(props.file.id)
} catch (e) {
error.value = e.message
} finally {
loading.value = false
}
}
const toggleExpand = (id) => {
if (editingSliceId.value !== null) return
expandedSlice.value = expandedSlice.value === id ? null : id
}
const isOfficeFile = (filename) => {
if (!filename) return false
return /\.(pdf|doc|docx|ppt|pptx|xlsx|xls)$/i.test(filename)
}
const isExcelFile = (filename) => {
if (!filename) return false
return /\.(xlsx|xls)$/i.test(filename)
}
const canHighlight = () => isOfficeFile(props.file?.filename)
const openViewer = (slice) => {
if (!canHighlight() || editingSliceId.value !== null) return
const fileId = String(props.file.id)
const sliceId = String(slice.id)
// Excel 文件 → 表格预览器
if (isExcelFile(props.file?.filename)) {
const params = new URLSearchParams({
file_id: fileId,
slice_id: sliceId,
})
window.open(`/excel-viewer.html?${params.toString()}`, '_blank', 'noopener')
return
}
// PDF/Word/PPT → PDF 高亮
const params = new URLSearchParams({
file_id: fileId,
slice_id: sliceId,
})
const url = `/pdf-highlight.html?${params.toString()}`
window.open(url, '_blank', 'noopener')
}
const truncateText = (text, maxLength = 200) => {
if (!text || text.length <= maxLength) return text
return text.substring(0, maxLength) + '...'
}
const startEdit = (slice) => {
if (editingSliceId.value !== null && editingSliceId.value !== slice.id) {
if (!confirm('当前有未保存的修改，是否放弃？')) return
}
editingSliceId.value = slice.id
editingContent.value = slice.content
expandedSlice.value = slice.id
}
const cancelEdit = () => {
editingSliceId.value = null
editingContent.value = ''
}
const saveEdit = async (slice) => {
const content = editingContent.value.trim()
if (!content) {
alert('切片内容不能为空')
return
}
saving.value = true
try {
const updatedSlices = await api.updateSlices(props.file.id, [{ id: slice.id, content }])
const updated = updatedSlices.find((s) => s.id === slice.id)
if (updated) {
const idx = slices.value.findIndex((s) => s.id === slice.id)
if (idx !== -1) {
slices.value[idx] = updated
}
}
editingSliceId.value = null
editingContent.value = ''
} catch (e) {
alert('保存失败：' + e.message)
} finally {
saving.value = false
}
}
onMounted(() => {
loadSlices()
})
</script>
<template>
<Teleport to="body">
<div class="fixed inset-0 z-50 flex items-center justify-center p-4">
<div @click="emit('close')" class="absolute inset-0 bg-black/50 backdrop-blur-sm"></div>
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-3xl max-h-[80vh] flex flex-col">
<!-- Header -->
<div class="flex items-center justify-between p-6 border-b border-slate-100">
<div>
<h3 class="text-lg font-semibold text-slate-800">文件切片</h3>
<p class="text-sm text-slate-500">{{ file.filename }}</p>
</div>
<button @click="emit('close')" class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<!-- Content -->
<div class="flex-1 overflow-y-auto p-6">
<!-- Loading -->
<div v-if="loading" class="flex justify-center py-12">
<div class="flex items-center gap-3 text-slate-500">
<svg class="animate-spin h-5 w-5" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
<span>加载切片...</span>
</div>
</div>
<!-- Error -->
<div v-else-if="error" class="text-center py-12">
<p class="text-red-500 mb-4">{{ error }}</p>
<button @click="loadSlices" class="px-4 py-2 bg-slate-100 text-slate-700 rounded-lg hover:bg-slate-200">
重试
</button>
</div>
<!-- Empty -->
<div v-else-if="slices.length === 0" class="text-center py-12">
<p class="text-slate-500">暂无切片数据</p>
</div>
<!-- Slices List -->
<div v-else class="space-y-3">
<p class="text-sm text-slate-500 mb-4">共 {{ slices.length }} 个切片</p>
<div
v-for="(slice, index) in slices"
:key="slice.id"
class="bg-slate-50 rounded-xl border border-slate-200 overflow-hidden"
>
<div class="p-4 hover:bg-slate-100 transition-colors">
<div class="flex items-start gap-3">
<span class="shrink-0 w-8 h-8 bg-white border border-slate-200 rounded-lg flex items-center justify-center text-sm font-medium text-slate-600">
{{ index + 1 }}
</span>
<div class="flex-1 min-w-0">
<!-- View mode -->
<div v-if="editingSliceId !== slice.id">
<p
class="text-sm text-slate-700 whitespace-pre-wrap wrap-break-words cursor-pointer"
@click="toggleExpand(slice.id)"
>
{{ expandedSlice === slice.id ? slice.content : truncateText(slice.content) }}
</p>
<p v-if="slice.content && slice.content.length > 200" class="mt-2 text-xs text-blue-500">
{{ expandedSlice === slice.id ? '点击收起' : '点击展开全部' }}
</p>
</div>
<!-- Edit mode -->
<div v-else class="space-y-3">
<textarea
v-model="editingContent"
rows="6"
class="w-full p-3 text-sm text-slate-700 bg-white border border-slate-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 resize-y"
:disabled="saving"
placeholder="请输入切片内容"
></textarea>
<div class="flex gap-3">
<button
@click="cancelEdit"
:disabled="saving"
class="flex-1 py-2 px-4 bg-slate-100 text-slate-700 rounded-lg hover:bg-slate-200 disabled:opacity-50"
>
取消
</button>
<button
@click="saveEdit(slice)"
:disabled="saving"
class="flex-1 py-2 px-4 bg-linear-to-r from-blue-500 to-indigo-600 text-white rounded-lg hover:from-blue-600 hover:to-indigo-700 disabled:opacity-50"
>
{{ saving ? '保存中...' : '保存' }}
</button>
</div>
</div>
<!-- Actions -->
<div class="mt-3 flex flex-wrap items-center gap-2">
<button
v-if="editingSliceId !== slice.id"
@click="startEdit(slice)"
class="inline-flex items-center gap-1 text-xs font-medium text-blue-700 bg-blue-50 px-2.5 py-1 rounded-full hover:bg-blue-100"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15.232 5.232l3.536 3.536m-2.036-5.036a2.5 2.5 0 113.536 3.536L6.5 21.036H3v-3.572L16.732 3.732z" />
</svg>
<span>编辑</span>
</button>
<button
v-if="canHighlight() && editingSliceId !== slice.id"
class="inline-flex items-center gap-1 text-xs font-medium text-amber-700 bg-amber-50 px-2.5 py-1 rounded-full hover:bg-amber-100"
@click="openViewer(slice)"
>
<span>定位原文</span>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5l7 7-7 7" />
</svg>
</button>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Footer -->
<div class="p-4 border-t border-slate-100">
<button
@click="emit('close')"
class="w-full py-3 bg-slate-100 text-slate-700 rounded-xl font-medium hover:bg-slate-200"
>
关闭
</button>
</div>
</div>
</div>
</Teleport>
</template>

202
frontend/src/components/FileStatusSummary.vue Normal file
View File

@ -0,0 +1,202 @@
<script setup>
import { computed, ref } from 'vue'
const props = defineProps({
stats: {
type: Object,
default: () => ({
total: 0,
pending: 0,
processing: 0,
completed: 0,
skipped: 0,
failed: 0,
unknown: 0,
}),
},
loading: {
type: Boolean,
default: false,
},
retryFailedLoading: {
type: Boolean,
default: false,
},
error: {
type: String,
default: '',
},
title: {
type: String,
default: '文件状态概览',
},
subtitle: {
type: String,
default: '',
},
})
const emit = defineEmits(['retry', 'reparse-failed', 'locate-file'])
const normalizedStats = computed(() => ({
total: props.stats?.total ?? 0,
pending: props.stats?.pending ?? 0,
processing: props.stats?.processing ?? 0,
completed: props.stats?.completed ?? 0,
skipped: props.stats?.skipped ?? 0,
failed: props.stats?.failed ?? 0,
unknown: props.stats?.unknown ?? 0,
}))
const cardConfigs = [
{ key: 'pending', label: '待处理', bar: 'bg-amber-300', chip: 'bg-amber-50 text-amber-700' },
{ key: 'processing', label: '处理中', bar: 'bg-blue-300', chip: 'bg-blue-50 text-blue-700' },
{ key: 'completed', label: '已完成', bar: 'bg-emerald-400', chip: 'bg-emerald-50 text-emerald-700' },
{ key: 'failed', label: '处理失败', bar: 'bg-red-400', chip: 'bg-red-50 text-red-700' },
{ key: 'skipped', label: '不解析', bar: 'bg-slate-300', chip: 'bg-slate-50 text-slate-600' },
{ key: 'unknown', label: '未知状态', bar: 'bg-slate-200', chip: 'bg-slate-50 text-slate-600' },
]
const cards = computed(() => {
return cardConfigs.filter((card) => card.key !== 'unknown' || normalizedStats.value.unknown > 0).map((card) => ({
...card,
value: normalizedStats.value[card.key] ?? 0,
}))
})
const hasData = computed(() => normalizedStats.value.total > 0)
const hasFailedFiles = computed(() => normalizedStats.value.failed > 0)
const processingFiles = computed(() => props.stats?.processing_files ?? [])
const failedFiles = computed(() => props.stats?.failed_files ?? [])
const hoveredCard = ref(null)
const selectedCard = ref(null)
const previewFiles = computed(() => {
const key = hoveredCard.value || selectedCard.value
if (key === 'processing') return processingFiles.value
if (key === 'failed') return failedFiles.value
return []
})
const activePreviewCard = computed(() => hoveredCard.value || selectedCard.value)
const previewTitle = computed(() => activePreviewCard.value === 'failed' ? '最近处理失败的文件' : '正在处理的文件')
const canPreview = (card) => {
return (card.key === 'processing' && processingFiles.value.length > 0)
|| (card.key === 'failed' && failedFiles.value.length > 0)
}
const togglePreview = (card) => {
if (!canPreview(card)) return
selectedCard.value = selectedCard.value === card.key ? null : card.key
}
const getPercent = (value) => {
if (!normalizedStats.value.total) return 0
return Math.round((value / normalizedStats.value.total) * 100)
}
const formatTimestamp = (timestamp) => {
if (!timestamp) return '-'
return new Date(timestamp * 1000).toLocaleString('zh-CN')
}
</script>
<template>
<div class="bg-white border border-slate-200 rounded-2xl p-3 shadow-sm">
<div class="flex flex-wrap md:flex-nowrap items-center gap-3 text-sm">
<div class="flex flex-col gap-1 min-w-[180px]">
<span class="text-xs font-semibold text-slate-500 uppercase tracking-wide">{{ title }}</span>
<span v-if="subtitle" class="text-[11px] text-slate-400">{{ subtitle }}</span>
<span class="text-base font-semibold text-slate-900">
{{ normalizedStats.total.toLocaleString() }} <span class="ml-1 text-xs text-slate-400">文件总数</span>
</span>
<button
v-if="!error && !loading && hasFailedFiles"
class="mt-1 w-fit px-2.5 py-1 text-xs font-medium rounded-lg border border-red-200 bg-red-50 text-red-700 hover:bg-red-100 disabled:opacity-60 disabled:cursor-not-allowed"
:disabled="retryFailedLoading"
@click="emit('reparse-failed')"
>
{{ retryFailedLoading ? '提交中...' : `重新解析失败文件 (${normalizedStats.failed})` }}
</button>
</div>
<div class="flex-1 min-w-0">
<div
v-if="error"
class="px-3 py-2 rounded-xl bg-red-50 text-red-600 text-xs flex items-center justify-between"
>
<span class="truncate">{{ error }}</span>
<button class="text-red-600 underline" @click="emit('retry')">重试</button>
</div>
<div v-else>
<div v-if="loading" class="flex items-center gap-2 overflow-hidden">
<div v-for="i in 4" :key="`stats-skeleton-${i}`" class="h-10 flex-1 rounded-full bg-slate-100 animate-pulse" />
</div>
<div
v-else-if="hasData"
class="flex items-center gap-2 overflow-x-auto scrollbar-thin scrollbar-thumb-slate-300"
>
<div
v-for="card in cards"
:key="card.key"
:class="[
'shrink-0 px-3 py-2 rounded-2xl border border-slate-100 bg-slate-50 text-xs flex flex-col gap-0.5',
canPreview(card)
? 'cursor-pointer'
: ''
]"
@mouseenter="hoveredCard = card.key"
@mouseleave="hoveredCard = null"
@click="togglePreview(card)"
>
<span class="text-[11px] text-slate-500 font-medium">{{ card.label }}</span>
<span class="text-base font-semibold text-slate-900 leading-none">
{{ card.value }}
<span class="ml-1 text-[11px] text-slate-400">{{ getPercent(card.value) }}%</span>
</span>
</div>
</div>
<div v-else class="text-xs text-slate-400 px-3 py-2">
暂无文件统计数据
</div>
</div>
</div>
</div>
<div
v-if="previewFiles.length"
class="mt-3 bg-white border border-slate-200 rounded-2xl shadow-sm p-3 text-xs text-slate-600 w-full"
@mouseleave="hoveredCard = null"
>
<div class="flex items-center justify-between text-[11px] uppercase tracking-wide text-slate-400 mb-2">
<span>{{ previewTitle }}</span>
<span>最近 {{ previewFiles.length }} 项</span>
</div>
<ul class="space-y-1">
<li
v-for="file in previewFiles"
:key="file.id"
class="border-b border-slate-100 last:border-0"
>
<button
type="button"
class="group flex w-full items-center justify-between gap-3 rounded-md px-1 py-1.5 text-left hover:bg-blue-50 focus:outline-none focus:ring-2 focus:ring-blue-300"
:title="`前往 ${file.kb_path || file.kb_name || '未分配知识库'} 定位文件`"
@click="emit('locate-file', file)"
>
<span class="min-w-0 flex-1">
<span class="block font-medium text-slate-800 truncate group-hover:text-blue-700">{{ file.filename }}</span>
<span class="block text-[11px] text-slate-400 group-hover:text-blue-500">
位置：{{ file.kb_path || file.kb_name || '未分配知识库' }} · {{ activePreviewCard === 'failed' ? '失败时间' : '更新时间' }} {{ formatTimestamp(file.updated_at) }}
</span>
</span>
<span class="shrink-0 text-slate-300 group-hover:text-blue-500" aria-hidden="true">›</span>
</button>
</li>
</ul>
</div>
</div>
</template>

322
frontend/src/components/FileUpload.vue Normal file
View File

@ -0,0 +1,322 @@
<script setup>
import { ref, computed } from 'vue'
import { api } from '../api'
import KnowledgeBaseSelector from './KnowledgeBaseSelector.vue'
const selectedKb = ref({ id: null, name: '不分配到知识库', kb_type: 'analysis' })
const showKbSelector = ref(false)
const files = ref([])
const uploading = ref(false)
const uploadStatus = ref('')
const uploadError = ref('')
const isDragging = ref(false)
const tags = ref([])
const newTag = ref('')
const isPublic = ref(false)
const sliceType = ref('smart')
const isStorageKb = computed(() => selectedKb.value?.kb_type === 'storage')
const handleKbSelect = (kb) => {
if (kb) {
selectedKb.value = kb
} else {
selectedKb.value = { id: null, name: '不分配到知识库', kb_type: 'analysis' }
}
}
const handleFileSelect = (e) => {
const selectedFiles = Array.from(e.target.files || [])
files.value = [...files.value, ...selectedFiles]
}
const handleDrop = (e) => {
isDragging.value = false
const droppedFiles = Array.from(e.dataTransfer.files || [])
files.value = [...files.value, ...droppedFiles]
}
const removeFile = (index) => {
files.value.splice(index, 1)
}
const formatFileSize = (bytes) => {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]
}
const addTag = () => {
const tag = newTag.value.trim()
if (tag && !tags.value.includes(tag)) {
tags.value.push(tag)
newTag.value = ''
}
}
const removeTag = (index) => {
tags.value.splice(index, 1)
}
const handleUpload = async () => {
if (files.value.length === 0) return
uploading.value = true
uploadStatus.value = ''
uploadError.value = ''
try {
await api.uploadFiles(selectedKb.value?.id, files.value, tags.value, isPublic.value, sliceType.value)
uploadStatus.value = `成功上传 ${files.value.length} 个文件到 "${selectedKb.value.name}"`
files.value = []
tags.value = []
isPublic.value = false
sliceType.value = 'smart'
} catch (e) {
uploadError.value = e.message
} finally {
uploading.value = false
}
}
</script>
<template>
<div class="max-w-2xl mx-auto">
<!-- Knowledge Base Select -->
<div class="bg-white rounded-xl p-5 border border-slate-200 mb-4">
<label class="block text-sm font-medium text-slate-700 mb-2">上传到知识库（可选）</label>
<button
@click="showKbSelector = true"
class="w-full px-4 py-3 bg-slate-50 border border-slate-200 rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent text-left"
>
<span class="font-mono text-blue-600">{{ selectedKb.name }}</span>
</button>
<p class="mt-2 text-xs text-slate-500">
点击以上选择文件要上传到的知识库层级
</p>
</div>
<KnowledgeBaseSelector
:show="showKbSelector"
@close="showKbSelector = false"
@select="handleKbSelect"
/>
<!-- Tags Input -->
<div class="bg-white rounded-xl p-5 border border-slate-200 mb-4">
<label class="block text-sm font-medium text-slate-700 mb-2">文件标签（可选）</label>
<div class="flex gap-2 mb-3">
<input
v-model="newTag"
@keyup.enter="addTag"
type="text"
placeholder="输入标签名称..."
class="flex-1 px-3 py-2 border border-slate-200 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 text-sm"
/>
<button
@click="addTag"
class="px-4 py-2 bg-blue-500 text-white rounded-lg text-sm font-medium hover:bg-blue-600 transition-all"
>
添加
</button>
</div>
<div v-if="tags.length > 0" class="flex flex-wrap gap-2">
<div
v-for="(tag, index) in tags"
:key="index"
class="flex items-center gap-1 px-3 py-1.5 bg-blue-50 text-blue-600 rounded-lg border border-blue-100 text-sm"
>
<span>{{ tag }}</span>
<button
@click="removeTag(index)"
class="ml-1 p-0.5 text-blue-400 hover:text-red-500 hover:bg-red-50 rounded transition-all"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
</div>
<p class="mt-2 text-xs text-slate-500">
为上传的文件添加标签，方便分类和搜索
</p>
</div>
<!-- Public/Private Toggle -->
<div class="bg-white rounded-xl p-5 border border-slate-200 mb-4">
<label class="block text-sm font-medium text-slate-700 mb-3">文件可见性</label>
<div class="flex gap-4">
<label
:class="[
'flex-1 flex items-center gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
!isPublic ? 'border-blue-500 bg-blue-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="isPublic" :value="false" class="w-4 h-4 text-blue-500" />
<div>
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🔒</span>
<span class="font-medium text-slate-800">私有</span>
</div>
<p class="text-xs text-slate-500">仅自己可见</p>
</div>
</label>
<label
:class="[
'flex-1 flex items-center gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
isPublic ? 'border-green-500 bg-green-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="isPublic" :value="true" class="w-4 h-4 text-green-500" />
<div>
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🌐</span>
<span class="font-medium text-slate-800">公开</span>
</div>
<p class="text-xs text-slate-500">所有人可见</p>
</div>
</label>
</div>
</div>
<!-- Slice Type Selection -->
<div v-if="isStorageKb" class="bg-white rounded-xl p-5 border border-slate-200 mb-4">
<label class="block text-sm font-medium text-slate-700 mb-3">切片方式</label>
<div class="flex items-center gap-3 p-4 rounded-xl border border-amber-200 bg-amber-50 text-amber-700 text-sm">
<span class="text-lg">🗄️</span>
<div>
<p class="font-medium">存储型知识库不进行解析</p>
<p class="text-xs text-amber-600">文件将直接保存，不会生成切片或知识图谱</p>
</div>
</div>
</div>
<div v-else class="bg-white rounded-xl p-5 border border-slate-200 mb-4">
<label class="block text-sm font-medium text-slate-700 mb-3">切片方式</label>
<div class="space-y-3">
<label
:class="[
'flex items-start gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
sliceType === 'smart' ? 'border-purple-500 bg-purple-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="sliceType" value="smart" class="mt-1 w-4 h-4 text-purple-500" />
<div class="flex-1">
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">🧠</span>
<span class="font-medium text-slate-800">智能切片（推荐）</span>
</div>
<p class="text-xs text-slate-500 leading-relaxed">
根据文档结构智能切分。PDF文件会保留标题层级，每个切片包含其所在章节标题，内容超过8000字时自动按句子切分
</p>
</div>
</label>
<label
:class="[
'flex items-start gap-3 p-4 rounded-xl border-2 cursor-pointer transition-all',
sliceType === 'fixed' ? 'border-orange-500 bg-orange-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<input type="radio" v-model="sliceType" value="fixed" class="mt-1 w-4 h-4 text-orange-500" />
<div class="flex-1">
<div class="flex items-center gap-2 mb-1">
<span class="text-lg">📏</span>
<span class="font-medium text-slate-800">固定长度切片</span>
</div>
<p class="text-xs text-slate-500 leading-relaxed">
按固定字数（8000字）切分，切片之间重叠100字以保证上下文连贯性
</p>
</div>
</label>
</div>
</div>
<!-- Drop Zone -->
<div
@dragover.prevent="isDragging = true"
@dragleave.prevent="isDragging = false"
@drop.prevent="handleDrop"
:class="[
'bg-white rounded-xl border-2 border-dashed p-8 text-center transition-all duration-200',
isDragging ? 'border-blue-500 bg-blue-50' : 'border-slate-200 hover:border-slate-300'
]"
>
<div class="w-16 h-16 bg-slate-100 rounded-full flex items-center justify-center mx-auto mb-4">
<svg class="w-8 h-8 text-slate-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12" />
</svg>
</div>
<p class="text-slate-600 mb-2">拖拽文件到这里，或</p>
<label class="inline-block px-4 py-2 bg-slate-100 text-slate-700 rounded-lg hover:bg-slate-200 cursor-pointer transition-colors">
选择文件
<input type="file" multiple @change="handleFileSelect" class="hidden" />
</label>
<p class="mt-3 text-xs text-slate-400">支持 PDF、Word、TXT、Markdown 等格式</p>
</div>
<!-- File List -->
<div v-if="files.length > 0" class="mt-4 bg-white rounded-xl border border-slate-200 overflow-hidden">
<div class="p-4 border-b border-slate-100 flex items-center justify-between">
<span class="text-sm font-medium text-slate-700">待上传文件 ({{ files.length }})</span>
<button @click="files = []" class="text-xs text-slate-500 hover:text-red-500">
清空
</button>
</div>
<ul class="divide-y divide-slate-100">
<li
v-for="(file, index) in files"
:key="index"
class="px-4 py-3 flex items-center justify-between hover:bg-slate-50"
>
<div class="flex items-center gap-3 min-w-0">
<div class="w-8 h-8 bg-slate-100 rounded-lg flex items-center justify-center shrink-0">
<span class="text-sm">📄</span>
</div>
<div class="min-w-0">
<p class="text-sm text-slate-700 truncate">{{ file.name }}</p>
<p class="text-xs text-slate-400">{{ formatFileSize(file.size) }}</p>
</div>
</div>
<button
@click="removeFile(index)"
class="p-1.5 text-slate-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-all"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</li>
</ul>
</div>
<!-- Upload Button -->
<button
@click="handleUpload"
:disabled="files.length === 0 || uploading"
:class="[
'w-full mt-4 py-4 rounded-xl font-medium transition-all duration-200',
files.length === 0 || uploading
? 'bg-slate-100 text-slate-400 cursor-not-allowed'
: 'bg-linear-to-r from-blue-500 to-indigo-600 text-white hover:from-blue-600 hover:to-indigo-700 shadow-sm'
]"
>
<span v-if="uploading" class="flex items-center justify-center gap-2">
<svg class="animate-spin h-5 w-5" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
上传中...
</span>
<span v-else>上传文件</span>
</button>
<!-- Status Messages -->
<p v-if="uploadStatus" class="mt-4 text-center text-green-600 text-sm">
{{ uploadStatus }}
</p>
<p v-if="uploadError" class="mt-4 text-center text-red-500 text-sm">
{{ uploadError }}
</p>
</div>
</template>

1121
frontend/src/components/GraphVisualization.vue Normal file
View File

File diff suppressed because it is too large Load Diff

175
frontend/src/components/KbPermissionModal.vue Normal file
View File

@ -0,0 +1,175 @@
<script setup>
import { ref, computed, watch } from 'vue'
import { api } from '../api'
const props = defineProps({
kb: {
type: Object,
required: true,
},
show: {
type: Boolean,
default: false,
},
})
const emit = defineEmits(['close'])
const permissions = ref([])
const loading = ref(false)
const saving = ref(false)
const error = ref('')
const newUserId = ref('')
const newPermission = ref('viewer')
const permissionOptions = [
{ value: 'viewer', label: '👁️ 只读', desc: '可查看、搜索、下载' },
{ value: 'editor', label: '✏️ 可写', desc: '可上传文件、重新解析' },
{ value: 'admin', label: '⚙️ 管理员', desc: '可修改属性、删除、管理权限' },
]
const permissionLabel = (perm) => {
const opt = permissionOptions.find(o => o.value === perm)
return opt ? opt.label : perm
}
const loadPermissions = async () => {
if (!props.kb?.id) return
loading.value = true
error.value = ''
try {
permissions.value = await api.getKbPermissions(props.kb.id)
} catch (e) {
error.value = e.message
} finally {
loading.value = false
}
}
const addPermission = async () => {
if (!newUserId.value.trim()) {
error.value = '请输入用户ID'
return
}
saving.value = true
error.value = ''
try {
await api.addKbPermission(props.kb.id, newUserId.value.trim(), newPermission.value)
newUserId.value = ''
newPermission.value = 'viewer'
await loadPermissions()
} catch (e) {
error.value = e.message
} finally {
saving.value = false
}
}
const removePermission = async (userId) => {
if (!confirm(`确定要删除用户「${userId}」的权限吗？`)) return
try {
await api.removeKbPermission(props.kb.id, userId)
await loadPermissions()
} catch (e) {
error.value = e.message
}
}
watch(() => props.show, (val) => {
if (val) loadPermissions()
})
</script>
<template>
<Teleport to="body">
<div
v-if="show"
class="fixed inset-0 z-50 flex items-center justify-center p-4"
>
<div class="absolute inset-0 bg-black/50 backdrop-blur-sm" @click="emit('close')"></div>
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-lg p-6">
<div class="flex items-center justify-between mb-5">
<h3 class="text-lg font-semibold text-slate-800">
📋 「{{ kb.name }}」权限管理
</h3>
<button @click="emit('close')" class="p-2 text-slate-400 hover:text-slate-600 hover:bg-slate-100 rounded-lg transition-all">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<div class="mb-4">
<p class="text-sm text-slate-500">
创建者：<span class="font-medium text-slate-700">{{ kb.user_id }}</span>
<span class="ml-2 px-2 py-0.5 text-xs rounded-full bg-purple-50 text-purple-600 border border-purple-200">⚙️ 管理员</span>
</p>
</div>
<!-- Add new permission -->
<div class="bg-slate-50 rounded-xl p-4 mb-4">
<label class="block text-sm font-medium text-slate-700 mb-2">添加成员</label>
<div class="flex gap-2">
<input
v-model="newUserId"
type="text"
placeholder="用户ID"
class="flex-1 px-3 py-2 bg-white border border-slate-200 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
<select
v-model="newPermission"
class="px-3 py-2 bg-white border border-slate-200 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500"
>
<option v-for="opt in permissionOptions" :key="opt.value" :value="opt.value">
{{ opt.label }}
</option>
</select>
<button
@click="addPermission"
:disabled="saving"
class="px-4 py-2 bg-blue-500 text-white rounded-lg hover:bg-blue-600 transition-colors disabled:opacity-50"
>
{{ saving ? '...' : '添加' }}
</button>
</div>
</div>
<!-- Error -->
<p v-if="error" class="text-sm text-red-500 mb-3">{{ error }}</p>
<!-- Permission list -->
<div class="max-h-64 overflow-y-auto">
<div v-if="loading" class="text-center py-4 text-slate-500">加载中...</div>
<div v-else-if="permissions.length === 0" class="text-center py-4 text-slate-400">暂无其他成员权限</div>
<div v-else class="space-y-2">
<div
v-for="perm in permissions"
:key="perm.user_id"
class="flex items-center justify-between p-3 bg-slate-50 rounded-lg"
>
<div class="flex items-center gap-3">
<div class="w-8 h-8 bg-blue-100 rounded-full flex items-center justify-center text-blue-600 font-medium text-sm">
{{ perm.user_id.charAt(0).toUpperCase() }}
</div>
<div>
<p class="text-sm font-medium text-slate-800">{{ perm.user_id }}</p>
<p class="text-xs text-slate-500">{{ permissionLabel(perm.permission) }}</p>
</div>
</div>
<button
@click="removePermission(perm.user_id)"
class="p-1.5 text-slate-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-all"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" />
</svg>
</button>
</div>
</div>
</div>
</div>
</div>
</Teleport>
</template>

164
frontend/src/components/KnowledgeBaseDetail.vue Normal file
View File

@ -0,0 +1,164 @@
<script setup>
import { ref, onMounted, watch } from 'vue'
import { api } from '../api'
import FileCard from './FileCard.vue'
import Pagination from './Pagination.vue'
const props = defineProps({
kb: {
type: Object,
required: true
}
})
const emit = defineEmits(['back'])
const files = ref([])
const totalFiles = ref(0)
const currentPage = ref(1)
const pageSize = ref(10)
const loading = ref(true)
const error = ref('')
const loadFiles = async () => {
loading.value = true
error.value = ''
try {
const result = await api.getKnowledgeBaseFiles(props.kb.id, {
page: currentPage.value,
size: pageSize.value,
})
files.value = result.items || []
totalFiles.value = result.total || 0
} catch (e) {
error.value = e.message
} finally {
loading.value = false
}
}
const handleFileUpdated = () => {
loadFiles()
}
const handleFileDeleted = () => {
loadFiles()
}
onMounted(() => {
loadFiles()
})
watch(() => props.kb.id, () => {
currentPage.value = 1
loadFiles()
})
</script>
<template>
<div>
<!-- Header -->
<div class="flex items-center gap-4 mb-6">
<button
@click="emit('back')"
class="p-2 text-slate-500 hover:text-slate-700 hover:bg-slate-100 rounded-lg transition-all"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 19l-7-7 7-7" />
</svg>
</button>
<div class="flex items-center gap-3">
<div class="w-12 h-12 bg-linear-to-br from-blue-100 to-indigo-100 rounded-xl flex items-center justify-center">
<span class="text-2xl">📚</span>
</div>
<div>
<h2 class="text-xl font-bold text-slate-800 flex items-center gap-2">
{{ kb.name }}
<span :class="[
'px-2 py-0.5 text-xs rounded-full border',
kb.is_public ? 'bg-green-50 text-green-600 border-green-200' : 'bg-slate-50 text-slate-600 border-slate-200'
]">
{{ kb.is_public ? '🌐 公开' : '🔒 私有' }}
</span>
<span :class="[
'px-2 py-0.5 text-xs rounded-full border',
kb.kb_type === 'storage' ? 'bg-amber-50 text-amber-600 border-amber-200' : 'bg-indigo-50 text-indigo-600 border-indigo-200'
]">
{{ kb.kb_type === 'storage' ? '🗄️ 存储型' : '🧠 分析型' }}
</span>
<span
v-if="kb.current_user_permission"
class="px-2 py-0.5 text-xs rounded-full border"
:class="{
'bg-purple-50 text-purple-600 border-purple-200': kb.current_user_permission === 'admin',
'bg-blue-50 text-blue-600 border-blue-200': kb.current_user_permission === 'editor',
'bg-slate-50 text-slate-500 border-slate-200': kb.current_user_permission === 'viewer'
}"
>
{{ kb.current_user_permission === 'admin' ? '⚙️ 管理员' : kb.current_user_permission === 'editor' ? '✏️ 可写' : '👁️ 只读' }}
</span>
</h2>
<p class="text-sm text-slate-500">{{ kb.description || '暂无描述' }}</p>
</div>
</div>
</div>
<!-- Loading -->
<div v-if="loading" class="flex justify-center py-12">
<div class="flex items-center gap-3 text-slate-500">
<svg class="animate-spin h-5 w-5" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
<span>加载文件列表...</span>
</div>
</div>
<!-- Error -->
<div v-else-if="error" class="text-center py-12">
<p class="text-red-500 mb-4">{{ error }}</p>
<button @click="loadFiles" class="px-4 py-2 bg-slate-100 text-slate-700 rounded-lg hover:bg-slate-200">
重试
</button>
</div>
<!-- Empty -->
<div v-else-if="files.length === 0" class="text-center py-12">
<div class="w-16 h-16 bg-slate-100 rounded-full flex items-center justify-center mx-auto mb-4">
<span class="text-3xl">📄</span>
</div>
<p class="text-slate-500 mb-2">暂无文件</p>
<p class="text-sm text-slate-400">请在「上传」页面添加文件到此知识库</p>
</div>
<!-- File List -->
<div v-else class="space-y-3">
<div class="flex items-center justify-between mb-4">
<p class="text-sm text-slate-500">共 {{ totalFiles }} 个文件</p>
<button @click="loadFiles" class="text-sm text-blue-500 hover:text-blue-600 flex items-center gap-1">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
刷新
</button>
</div>
<FileCard
v-for="file in files"
:key="file.id"
:file="file"
:kb-type="kb.kb_type"
@updated="handleFileUpdated"
@deleted="handleFileDeleted"
/>
<Pagination
v-if="totalFiles > 0"
v-model:page="currentPage"
v-model:size="pageSize"
:total="totalFiles"
@change="loadFiles"
/>
</div>
</div>
</template>

793
frontend/src/components/KnowledgeBaseList.vue Normal file
View File

@ -0,0 +1,793 @@
<script setup>
import { ref, onMounted, computed, nextTick } from 'vue'
import { api } from '../api'
import FileCard from './FileCard.vue'
import CreateKnowledgeBase from './CreateKnowledgeBase.vue'
import FileStatusSummary from './FileStatusSummary.vue'
import ExportRecordPanel from './ExportRecordPanel.vue'
import KbPermissionModal from './KbPermissionModal.vue'
import Pagination from './Pagination.vue'
import { setCurrentKb } from '../store'
// Reactive state for the current view
const currentKb = ref(null) // The KB we are currently inside, null for root
const childrenKbs = ref([])
const files = ref([])
const breadcrumbs = ref([])
const loading = ref(true)
const error = ref('')
const reparseLoading = ref(false)
const reparseFailedLoading = ref(false)
const currentKbReparseLoading = ref(false)
const childKbReparseLoading = ref({})
const priorityDrafts = ref({})
const prioritySaving = ref({})
const locatedFileId = ref(null)
// Pagination / filter state for KB file list
const currentPage = ref(1)
const pageSize = ref(10)
const totalFiles = ref(0)
const fileFilterName = ref('')
const fileFilterTag = ref('')
// Permission modal state
const showPermissionModal = ref(false)
const permissionModalKb = ref(null)
const openPermissionModal = (kb) => {
permissionModalKb.value = kb
showPermissionModal.value = true
}
// Export state
const selectedKbs = ref(new Map())
const exportLoading = ref(false)
const exportIncludeChildren = ref(false)
const exportRecords = ref([])
const EXPORT_RECORDS_KEY = 'htknow_export_records'
const loadExportRecords = () => {
try {
const raw = localStorage.getItem(EXPORT_RECORDS_KEY)
if (raw) exportRecords.value = JSON.parse(raw)
} catch {
exportRecords.value = []
}
}
const saveExportRecords = () => {
localStorage.setItem(EXPORT_RECORDS_KEY, JSON.stringify(exportRecords.value))
}
const addExportRecord = (result) => {
const manifest = result.manifest || {}
const record = {
id: Date.now(),
timestamp: new Date().toISOString(),
exportPath: result.export_path || '',
kbNames: manifest.kb_names || [],
kbCount: manifest.kb_ids?.length || 0,
kb_ids: manifest.kb_ids || [],
fileCount: manifest.file_count || 0,
sliceCount: manifest.slice_count || 0,
tantivyDocCount: manifest.tantivy_doc_count || 0,
lancedbRowCount: manifest.lancedb_row_count || 0,
}
exportRecords.value.unshift(record)
if (exportRecords.value.length > 50) {
exportRecords.value = exportRecords.value.slice(0, 50)
}
saveExportRecords()
}
const clearExportRecords = () => {
exportRecords.value = []
localStorage.removeItem(EXPORT_RECORDS_KEY)
}
const selectedKbIds = computed(() => Array.from(selectedKbs.value.keys()))
const selectedKbCount = computed(() => selectedKbs.value.size)
const hasSelectedKbs = computed(() => selectedKbCount.value > 0)
const selectedKbNames = computed(() => Array.from(selectedKbs.value.values()).map(kb => kb.name))
const selectedKbPreview = computed(() => {
if (selectedKbNames.value.length === 0) return ''
if (selectedKbNames.value.length <= 3) return selectedKbNames.value.join('、')
return `${selectedKbNames.value.slice(0, 3).join('、')} 等 ${selectedKbNames.value.length} 个`
})
const toggleKbSelection = (kb) => {
const next = new Map(selectedKbs.value)
if (next.has(kb.id)) {
next.delete(kb.id)
} else {
next.set(kb.id, { id: kb.id, name: kb.name })
}
selectedKbs.value = next
}
const selectAllKbs = () => {
const currentPageIds = childrenKbs.value.map(kb => kb.id)
const allSelected = currentPageIds.length > 0 && currentPageIds.every(id => selectedKbs.value.has(id))
const next = new Map(selectedKbs.value)
if (allSelected) {
currentPageIds.forEach(id => next.delete(id))
} else {
childrenKbs.value.forEach(kb => next.set(kb.id, { id: kb.id, name: kb.name }))
}
selectedKbs.value = next
}
const clearSelectedKbs = () => {
selectedKbs.value = new Map()
}
const handleExport = async () => {
if (selectedKbCount.value === 0) return
const ids = selectedKbIds.value
const names = selectedKbNames.value
const label = names.length <= 2 ? names.join('、') : `${names[0]} 等 ${names.length} 个`
if (!confirm(`确定要导出「${label}」${exportIncludeChildren.value ? '（含子知识库）' : ''}吗？`)) return
exportLoading.value = true
try {
const result = await api.exportKnowledgeBases(ids, exportIncludeChildren.value)
addExportRecord(result)
alert(`导出成功！\n路径：${result.export_path}`)
clearSelectedKbs()
} catch (e) {
alert('导出失败：' + e.message)
} finally {
exportLoading.value = false
}
}
const createEmptyStats = () => ({
total: 0,
pending: 0,
processing: 0,
completed: 0,
skipped: 0,
failed: 0,
unknown: 0,
processing_files: [],
failed_files: [],
})
const stats = ref(createEmptyStats())
const statsLoading = ref(true)
const statsError = ref('')
const statsSubtitle = computed(() => {
if (currentKb.value && currentKb.value.id !== null) {
return `覆盖 ${currentKb.value.name} 及其子知识库`
}
return '覆盖所有知识库（含未分配文件）'
})
const getCurrentKbId = () => {
return currentKb.value?.id ?? null
}
const fetchStats = async (kbId) => {
statsLoading.value = true
statsError.value = ''
try {
const params = {}
if (kbId === null || kbId === undefined) {
// 全局统计，后台默认包含未分配文件
} else {
params.kbId = kbId
params.includeDescendants = true
}
stats.value = await api.getFileStats(params)
} catch (e) {
statsError.value = e?.message || '加载统计失败'
} finally {
statsLoading.value = false
}
}
const loadKbContent = async (kbId) => {
const targetId = kbId ?? null
loading.value = true
error.value = ''
try {
let newCurrentKb;
if (targetId === null) {
// Root view: fetch top-level KBs and unassigned files
const [topLevelKbs, unassignedFiles] = await Promise.all([
api.getKnowledgeBases(null),
api.getFiles(null, null, { page: currentPage.value, size: pageSize.value })
]);
childrenKbs.value = topLevelKbs
files.value = unassignedFiles.items || []
totalFiles.value = unassignedFiles.total || 0
newCurrentKb = { id: null, name: '所有知识库', kb_type: null }
breadcrumbs.value = []
} else {
// Inside a specific KB
const [data, filesData] = await Promise.all([
api.getKnowledgeBase(targetId),
api.getKnowledgeBaseFiles(targetId, {
page: currentPage.value,
size: pageSize.value,
filename: fileFilterName.value || undefined,
tag: fileFilterTag.value || undefined,
})
])
childrenKbs.value = data.children_kbs || []
files.value = filesData.items || []
totalFiles.value = filesData.total || 0
newCurrentKb = { id: data.id, name: data.name, description: data.description, kb_type: data.kb_type }
breadcrumbs.value = data.path || []
}
const nextPriorityDrafts = {}
for (const kb of childrenKbs.value) {
nextPriorityDrafts[kb.id] = Number.isInteger(kb.parse_priority) ? kb.parse_priority : 50
}
priorityDrafts.value = nextPriorityDrafts
currentKb.value = newCurrentKb;
setCurrentKb(newCurrentKb); // Update global store
await fetchStats(targetId)
} catch (e) {
error.value = e.message
} finally {
loading.value = false
}
}
// --- Navigation ---
const navigateToKb = (kbId) => {
currentPage.value = 1
fileFilterName.value = ''
fileFilterTag.value = ''
loadKbContent(kbId)
}
const applyFileFilters = () => {
currentPage.value = 1
loadKbContent(getCurrentKbId())
}
const handleLocateFile = async (file) => {
if (!file?.id) return
locatedFileId.value = null
currentPage.value = 1
fileFilterName.value = ''
fileFilterTag.value = ''
await loadKbContent(file.kb_id ?? null)
await nextTick()
const target = document.getElementById(`file-card-${file.id}`)
if (!target) {
alert('文件所在知识库已打开，但未找到该文件，文件可能已被移动或删除')
return
}
locatedFileId.value = file.id
target.scrollIntoView({ behavior: 'smooth', block: 'center' })
window.setTimeout(() => {
if (locatedFileId.value === file.id) locatedFileId.value = null
}, 3000)
}
// --- Event Handlers ---
const handleKbCreated = () => {
loadKbContent(getCurrentKbId())
}
const handleDeleteKb = async (e, kbId) => {
e.stopPropagation()
if (!confirm('确定要删除这个知识库及其所有内容吗？此操作不可逆！')) return
try {
await api.deleteKnowledgeBase(kbId)
await loadKbContent(getCurrentKbId()) // Refresh current view
} catch (e) {
alert('删除失败：' + e.message)
}
}
const handleFileAction = () => {
loadKbContent(getCurrentKbId())
}
const handleTogglePublic = async (e, kbId, currentPublic) => {
e.stopPropagation()
const newPublic = !currentPublic
if (!confirm(`确定要将知识库设置为${newPublic ? '公开' : '私有'}吗？`)) return
try {
await api.updateKnowledgeBase(kbId, { is_public: newPublic })
await loadKbContent(getCurrentKbId())
} catch (e) {
alert('更新失败：' + e.message)
}
}
const handleReparse = async () => {
if (!confirm('确定要重新解析所有知识库及未分配文件吗？')) return
reparseLoading.value = true
try {
const result = await api.reparseKnowledgeBases()
const count = result?.file_count ?? 0
alert(`已提交重新解析任务，共 ${count} 个文件`)
await loadKbContent(getCurrentKbId())
} catch (e) {
alert('重新解析失败：' + e.message)
} finally {
reparseLoading.value = false
}
}
const handleReparseFailedFiles = async () => {
const failedCount = stats.value?.failed ?? 0
if (failedCount <= 0) {
alert('当前范围内没有处理失败的文件')
return
}
const isRootScope = !currentKb.value || currentKb.value.id === null
const scopeLabel = isRootScope
? '所有知识库及未分配文件中的失败文件'
: `「${currentKb.value.name || '当前知识库'}」及其子知识库中的失败文件`
if (!confirm(`确定要重新解析${scopeLabel}吗？`)) return
reparseFailedLoading.value = true
try {
const result = isRootScope
? await api.reparseFailedFiles()
: await api.reparseFailedFiles({
kbId: currentKb.value.id,
includeDescendants: true,
})
const count = result?.file_count ?? 0
alert(count > 0 ? `已提交 ${count} 个失败文件重新解析` : '当前范围内没有可重新解析的失败文件')
await loadKbContent(getCurrentKbId())
} catch (e) {
alert('重新解析失败文件失败：' + e.message)
} finally {
reparseFailedLoading.value = false
}
}
const submitKbReparse = async (kbId, kbName) => {
const result = await api.reparseKnowledgeBase(kbId)
const kbCount = result?.kb_count ?? 0
const fileCount = result?.file_count ?? 0
alert(`已提交「${kbName}」重新解析任务（含子知识库），覆盖 ${kbCount} 个知识库、${fileCount} 个文件`)
}
const handleReparseCurrentKb = async () => {
if (!currentKb.value || currentKb.value.id === null) return
if (currentKb.value.kb_type === 'storage') {
alert('存储型知识库不参与解析')
return
}
const kbName = currentKb.value.name || '当前知识库'
if (!confirm(`确定要重新解析「${kbName}」及其子知识库吗？`)) return
currentKbReparseLoading.value = true
try {
await submitKbReparse(currentKb.value.id, kbName)
await loadKbContent(getCurrentKbId())
} catch (e) {
alert('重新解析失败：' + e.message)
} finally {
currentKbReparseLoading.value = false
}
}
const handleReparseChildKb = async (e, kb) => {
e.stopPropagation()
if (!kb || kb.kb_type === 'storage') {
alert('存储型知识库不参与解析')
return
}
if (childKbReparseLoading.value[kb.id]) return
if (!confirm(`确定要重新解析「${kb.name}」及其子知识库吗？`)) return
childKbReparseLoading.value[kb.id] = true
try {
await submitKbReparse(kb.id, kb.name)
await loadKbContent(getCurrentKbId())
} catch (e) {
alert('重新解析失败：' + e.message)
} finally {
childKbReparseLoading.value[kb.id] = false
}
}
const handleSaveParsePriority = async (e, kb) => {
e.stopPropagation()
if (!kb || kb.kb_type === 'storage') return
const raw = priorityDrafts.value[kb.id]
const value = Number(raw)
if (!Number.isInteger(value) || value < 0 || value > 100) {
alert('解析优先级必须是 0 到 100 的整数')
return
}
if (value === kb.parse_priority) {
return
}
prioritySaving.value[kb.id] = true
try {
await api.updateKnowledgeBase(kb.id, { parse_priority: value })
kb.parse_priority = value
} catch (err) {
alert('保存优先级失败：' + (err?.message || '未知错误'))
priorityDrafts.value[kb.id] = Number.isInteger(kb.parse_priority) ? kb.parse_priority : 50
} finally {
prioritySaving.value[kb.id] = false
}
}
// Expose refresh method to parent component
defineExpose({
refresh: () => loadKbContent(getCurrentKbId()),
})
// Initial load
onMounted(() => {
loadKbContent(null)
loadExportRecords()
})
</script>
<template>
<div>
<!-- Header with Breadcrumbs and Create Button -->
<div class="flex items-center justify-between gap-4 mb-6">
<nav class="flex items-center text-sm text-slate-500">
<span @click="navigateToKb(null)" class="hover:text-blue-500 cursor-pointer">主目录</span>
<template v-for="crumb in breadcrumbs" :key="crumb.id">
<span class="mx-2">/</span>
<span @click="navigateToKb(crumb.id)" class="hover:text-blue-500 cursor-pointer">{{ crumb.name }}</span>
</template>
<template v-if="currentKb && currentKb.id !== null">
<span class="mx-2">/</span>
<span class="font-semibold text-slate-700">{{ currentKb.name }}</span>
</template>
</nav>
<div class="flex items-center gap-3">
<button
v-if="currentKb && currentKb.id === null"
@click="handleReparse"
:disabled="reparseLoading"
title="重新解析所有知识库及未分配文件"
:class="[
'px-4 py-2.5 rounded-xl font-medium transition-all duration-200 border flex items-center gap-2',
reparseLoading
? 'bg-slate-100 text-slate-400 border-slate-200 cursor-not-allowed'
: 'bg-white text-slate-700 border-slate-200 hover:border-blue-300 hover:text-blue-600'
]"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v6h6M20 20v-6h-6M5 19a9 9 0 0014-7M19 5a9 9 0 00-14 7" />
</svg>
{{ reparseLoading ? '解析中...' : '全部重新解析' }}
</button>
<button
v-if="currentKb && currentKb.id !== null && currentKb.kb_type !== 'storage'"
@click="handleReparseCurrentKb"
:disabled="currentKbReparseLoading"
title="重新解析当前知识库及子知识库"
:class="[
'px-4 py-2.5 rounded-xl font-medium transition-all duration-200 border flex items-center gap-2',
currentKbReparseLoading
? 'bg-slate-100 text-slate-400 border-slate-200 cursor-not-allowed'
: 'bg-white text-slate-700 border-slate-200 hover:border-blue-300 hover:text-blue-600'
]"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v6h6M20 20v-6h-6M5 19a9 9 0 0014-7M19 5a9 9 0 00-14 7" />
</svg>
{{ currentKbReparseLoading ? '解析中...' : '重新解析当前知识库' }}
</button>
<!-- Export Controls -->
<div v-if="childrenKbs.length > 0 || hasSelectedKbs" class="flex items-center gap-2"
:class="hasSelectedKbs ? 'opacity-100' : 'opacity-60'"
>
<button
v-if="childrenKbs.length > 0"
@click="selectAllKbs"
type="button"
class="px-3 py-2 text-sm rounded-xl border border-slate-200 bg-white text-slate-600 hover:border-blue-300 hover:text-blue-600 transition-all duration-200"
>
{{
childrenKbs.length > 0 && childrenKbs.every(kb => selectedKbs.has(kb.id))
? '取消本层全选'
: '全选本层'
}}
</button>
<label class="flex items-center gap-1.5 text-sm text-slate-600 cursor-pointer select-none"
@click.stop
>
<input
v-model="exportIncludeChildren"
type="checkbox"
class="w-4 h-4 rounded border-slate-300 text-blue-600 focus:ring-blue-500"
/>
含子知识库
</label>
<button
@click="handleExport"
:disabled="!hasSelectedKbs || exportLoading"
:class="[
'px-4 py-2.5 rounded-xl font-medium transition-all duration-200 border flex items-center gap-2',
!hasSelectedKbs || exportLoading
? 'bg-slate-100 text-slate-400 border-slate-200 cursor-not-allowed'
: 'bg-blue-50 text-blue-700 border-blue-200 hover:bg-blue-100 hover:border-blue-300'
]"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M9 19l3 3m0 0l3-3m-3 3V10" />
</svg>
{{ exportLoading ? '导出中...' : `导出选中 (${selectedKbCount})` }}
</button>
<button
v-if="hasSelectedKbs"
@click="clearSelectedKbs"
type="button"
class="px-3 py-2 text-sm rounded-xl border border-slate-200 bg-white text-slate-500 hover:border-red-300 hover:text-red-600 transition-all duration-200"
>
清空已选
</button>
</div>
<CreateKnowledgeBase :parent-id="currentKb?.id" @created="handleKbCreated" />
</div>
</div>
<div
v-if="hasSelectedKbs"
class="mb-4 flex items-center justify-between gap-3 rounded-xl border border-blue-200 bg-blue-50 px-4 py-3 text-sm text-blue-800"
>
<div class="min-w-0">
<span class="font-medium">已选 {{ selectedKbCount }} 个知识库：</span>
<span class="truncate">{{ selectedKbPreview }}</span>
</div>
<button
type="button"
@click="clearSelectedKbs"
class="shrink-0 rounded-lg border border-blue-200 bg-white px-3 py-1.5 text-blue-700 hover:border-blue-300"
>
清空
</button>
</div>
<FileStatusSummary
class="mb-4"
:stats="stats"
:loading="statsLoading"
:retry-failed-loading="reparseFailedLoading"
:error="statsError"
:title="currentKb && currentKb.id !== null ? '知识库文件状态' : '全局文件状态'"
:subtitle="statsSubtitle"
@retry="fetchStats(getCurrentKbId())"
@reparse-failed="handleReparseFailedFiles"
@locate-file="handleLocateFile"
/>
<!-- Loading -->
<div v-if="loading" class="text-center py-12">
<p>加载中...</p>
</div>
<!-- Error -->
<div v-else-if="error" class="text-center py-12 text-red-500">
<p>错误: {{ error }}</p>
<button @click="loadKbContent(getCurrentKbId())">重试</button>
</div>
<!-- Empty State -->
<div v-else-if="childrenKbs.length === 0 && files.length === 0" class="text-center py-12">
<div class="w-16 h-16 bg-slate-100 rounded-full flex items-center justify-center mx-auto mb-4">
<span class="text-3xl">🗂️</span>
</div>
<p class="text-slate-500">这个知识库是空的</p>
</div>
<!-- Grid for KBs and Files -->
<div v-else>
<!-- Child KBs -->
<div v-if="childrenKbs.length > 0" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<div
v-for="kb in childrenKbs"
:key="`kb-${kb.id}`"
@click="navigateToKb(kb.id)"
class="bg-white rounded-xl p-5 border border-slate-200 hover:border-blue-300 hover:shadow-md transition-all duration-200 group cursor-pointer relative"
>
<!-- Selection checkbox -->
<div class="absolute top-3 left-3 z-10" @click.stop>
<input
type="checkbox"
:checked="selectedKbs.has(kb.id)"
@change="toggleKbSelection(kb)"
class="w-5 h-5 rounded border-slate-300 text-blue-600 focus:ring-blue-500 cursor-pointer"
/>
</div>
<div class="flex items-start justify-between mb-3 pl-8">
<div class="w-12 h-12 bg-linear-to-br from-blue-100 to-indigo-100 rounded-xl flex items-center justify-center">
<span class="text-2xl">📚</span>
</div>
<div class="flex items-center gap-1">
<span
v-if="kb.current_user_permission"
class="px-2 py-0.5 text-xs rounded-full border"
:class="{
'bg-purple-50 text-purple-600 border-purple-200': kb.current_user_permission === 'admin',
'bg-blue-50 text-blue-600 border-blue-200': kb.current_user_permission === 'editor',
'bg-slate-50 text-slate-500 border-slate-200': kb.current_user_permission === 'viewer'
}"
>
{{ kb.current_user_permission === 'admin' ? '⚙️ 管理员' : kb.current_user_permission === 'editor' ? '✏️ 可写' : '👁️ 只读' }}
</span>
<button
v-if="kb.current_user_permission === 'admin'"
@click="(e) => { e.stopPropagation(); openPermissionModal(kb) }"
class="opacity-0 group-hover:opacity-100 p-2 text-slate-400 hover:text-purple-500 hover:bg-purple-50 rounded-lg transition-all"
title="权限管理"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 4.354a4 4 0 110 5.292M15 21H3v-1a6 6 0 0112 0v1zm0 0h6v-1a6 6 0 00-9-5.197M13 7a4 4 0 11-8 0 4 4 0 018 0z" />
</svg>
</button>
<button
v-if="kb.current_user_permission === 'admin'"
@click="(e) => handleTogglePublic(e, kb.id, kb.is_public)"
:class="[
'opacity-0 group-hover:opacity-100 p-2 rounded-lg transition-all',
kb.is_public ? 'text-green-500 hover:bg-green-50' : 'text-slate-500 hover:bg-slate-100'
]"
:title="kb.is_public ? '设置为私有' : '设置为公开'"
>
<svg v-if="kb.is_public" class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 15v2m-6 4h12a2 2 0 002-2v-6a2 2 0 00-2-2H6a2 2 0 00-2 2v6a2 2 0 002 2zm10-10V7a4 4 0 00-8 0v4h8z" />
</svg>
<svg v-else class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3.055 11H5a2 2 0 012 2v1a2 2 0 012 2 2 2 0 012 2v2.945M8 3.935V5.5A2.5 2.5 0 0010.5 8h.5a2 2 0 012 2 2 2 0 104 0 2 2 0 012-2h1.064M15 20.488V18a2 2 0 012-2h3.064M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</button>
<button
v-if="kb.current_user_permission === 'editor' || kb.current_user_permission === 'admin'"
@click="(e) => handleReparseChildKb(e, kb)"
:disabled="kb.kb_type === 'storage' || childKbReparseLoading[kb.id]"
class="opacity-0 group-hover:opacity-100 p-2 text-slate-400 hover:text-blue-500 hover:bg-blue-50 rounded-lg transition-all disabled:opacity-40 disabled:cursor-not-allowed"
:title="kb.kb_type === 'storage' ? '存储型知识库不参与解析' : (childKbReparseLoading[kb.id] ? '解析中...' : '重新解析该知识库')"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v6h6M20 20v-6h-6M5 19a9 9 0 0014-7M19 5a9 9 0 00-14 7" />
</svg>
</button>
<button
v-if="kb.current_user_permission === 'admin'"
@click="(e) => handleDeleteKb(e, kb.id)"
class="opacity-0 group-hover:opacity-100 p-2 text-slate-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-all"
title="删除"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" />
</svg>
</button>
</div>
</div>
<h3 class="font-semibold text-slate-800 mb-1 flex items-center gap-2">
{{ kb.name }}
<span :class="[
'px-2 py-0.5 text-xs rounded-full border',
kb.is_public ? 'bg-green-50 text-green-600 border-green-200' : 'bg-slate-50 text-slate-600 border-slate-200'
]">
{{ kb.is_public ? '🌐 公开' : '🔒 私有' }}
</span>
<span :class="[
'px-2 py-0.5 text-xs rounded-full border',
kb.kb_type === 'storage' ? 'bg-amber-50 text-amber-600 border-amber-200' : 'bg-indigo-50 text-indigo-600 border-indigo-200'
]">
{{ kb.kb_type === 'storage' ? '🗄️ 存储型' : '🧠 分析型' }}
</span>
</h3>
<p class="text-sm text-slate-500 line-clamp-2 mb-3">{{ kb.description || '暂无描述' }}</p>
<div class="mb-3 p-2 rounded-lg border border-slate-200 bg-slate-50" @click.stop>
<div class="flex items-center justify-between gap-2">
<label class="text-xs text-slate-600">解析优先级 (0-100)</label>
<span class="text-xs text-slate-400" v-if="kb.kb_type === 'storage'">存储型不参与解析</span>
</div>
<div class="mt-2 flex items-center gap-2">
<input
v-model.number="priorityDrafts[kb.id]"
type="number"
min="0"
max="100"
step="1"
:disabled="kb.kb_type === 'storage' || prioritySaving[kb.id] || (kb.current_user_permission !== 'editor' && kb.current_user_permission !== 'admin')"
class="w-24 px-2 py-1 text-sm border border-slate-200 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-slate-100 disabled:text-slate-400"
/>
<button
type="button"
:disabled="kb.kb_type === 'storage' || prioritySaving[kb.id] || (kb.current_user_permission !== 'editor' && kb.current_user_permission !== 'admin')"
@click="(e) => handleSaveParsePriority(e, kb)"
class="px-2.5 py-1 text-xs font-medium rounded-md border border-slate-200 bg-white text-slate-700 hover:border-blue-300 hover:text-blue-600 disabled:bg-slate-100 disabled:text-slate-400 disabled:border-slate-200"
>
{{ prioritySaving[kb.id] ? '保存中...' : '保存' }}
</button>
</div>
</div>
<div class="flex items-center justify-between text-xs text-slate-400">
<span>{{ kb.children_kb_count || 0 }} 个子知识库</span>
<span>{{ kb.file_count || 0 }} 个文件</span>
</div>
</div>
</div>
<!-- Files -->
<div class="mt-6 space-y-3" v-if="files.length > 0 || currentKb?.id !== null">
<div class="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 mb-4">
<h3 class="text-lg font-semibold text-slate-700">文件 <span class="text-sm font-normal text-slate-500">（共 {{ totalFiles }} 个）</span></h3>
<div v-if="currentKb?.id !== null" class="flex items-center gap-2">
<input
v-model="fileFilterName"
type="text"
placeholder="文件名搜索"
@keyup.enter="applyFileFilters"
class="px-2.5 py-1.5 text-sm border border-slate-200 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 w-40"
/>
<input
v-model="fileFilterTag"
type="text"
placeholder="标签筛选"
@keyup.enter="applyFileFilters"
class="px-2.5 py-1.5 text-sm border border-slate-200 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 w-32"
/>
<button
type="button"
@click="applyFileFilters"
class="px-3 py-1.5 text-sm font-medium rounded-md bg-blue-600 text-white hover:bg-blue-700"
>
搜索
</button>
</div>
</div>
<FileCard
v-for="file in files"
:key="`file-${file.id}`"
:id="`file-card-${file.id}`"
:file="file"
:kb-type="currentKb?.kb_type"
:highlighted="locatedFileId === file.id"
@updated="handleFileAction"
@deleted="handleFileAction"
/>
<Pagination
v-if="totalFiles > 0"
v-model:page="currentPage"
v-model:size="pageSize"
:total="totalFiles"
@change="loadKbContent(currentKb?.id ?? null)"
/>
</div>
</div>
<!-- Export Records -->
<ExportRecordPanel
:records="exportRecords"
@clear="clearExportRecords"
/>
<!-- Permission Modal -->
<KbPermissionModal
:kb="permissionModalKb || {}"
:show="showPermissionModal"
@close="showPermissionModal = false"
/>
</div>
</template>

118
frontend/src/components/KnowledgeBaseSelector.vue Normal file
View File

@ -0,0 +1,118 @@
<script setup>
import { ref, onMounted } from 'vue';
import { api } from '../api';
const props = defineProps({
show: Boolean,
});
const emit = defineEmits(['close', 'select']);
const currentParent = ref(null);
const breadcrumbs = ref([]);
const childrenKbs = ref([]);
const loading = ref(true);
const currentKbInfo = ref(null);
const loadPath = async (kbId) => {
if (!kbId) {
breadcrumbs.value = [];
currentKbInfo.value = null;
return null;
}
try {
const data = await api.getKnowledgeBase(kbId);
breadcrumbs.value = [...data.path, { id: data.id, name: data.name }];
currentKbInfo.value = data;
return data;
} catch (e) {
console.error('Failed to load path', e);
currentKbInfo.value = null;
}
return null;
};
const loadChildren = async (parentId) => {
loading.value = true;
currentParent.value = parentId;
if (parentId) {
await loadPath(parentId);
} else {
breadcrumbs.value = [];
currentKbInfo.value = null;
}
try {
childrenKbs.value = await api.getKnowledgeBases(parentId);
} catch (e) {
console.error('Failed to load knowledge bases', e);
} finally {
loading.value = false;
}
};
const selectKb = (kb) => {
emit('select', kb);
emit('close');
};
const navigate = (kbId) => {
loadChildren(kbId);
};
onMounted(() => {
loadChildren(null);
});
</script>
<template>
<Teleport to="body">
<div v-if="show" class="fixed inset-0 z-50 flex items-center justify-center p-4 bg-black/50 backdrop-blur-sm" @click.self="emit('close')">
<div class="relative bg-white rounded-2xl shadow-xl w-full max-w-lg p-6">
<h3 class="text-lg font-semibold text-slate-800 mb-4">Select a Knowledge Base</h3>
<!-- Breadcrumbs -->
<nav class="text-sm text-slate-500 mb-4 flex items-center gap-2">
<span @click="navigate(null)" class="cursor-pointer hover:text-blue-500">Root</span>
<template v-for="crumb in breadcrumbs" :key="crumb.id">
<span>/</span>
<span @click="navigate(crumb.id)" class="cursor-pointer hover:text-blue-500">{{ crumb.name }}</span>
</template>
</nav>
<!-- List -->
<div class="min-h-50 max-h-100 overflow-y-auto">
<div v-if="loading" class="text-center p-8">Loading...</div>
<ul v-else class="divide-y divide-slate-100">
<li v-for="kb in childrenKbs" :key="kb.id" class="p-3 flex justify-between items-center group">
<div class="flex items-center gap-2">
<span class="text-slate-700">{{ kb.name }}</span>
<span :class="[
'px-2 py-0.5 text-xs rounded-full border',
kb.kb_type === 'storage' ? 'bg-amber-50 text-amber-600 border-amber-200' : 'bg-indigo-50 text-indigo-600 border-indigo-200'
]">
{{ kb.kb_type === 'storage' ? '🗄️ 存储型' : '🧠 分析型' }}
</span>
</div>
<div class="flex items-center gap-2">
<button @click="selectKb(kb)" class="text-sm text-blue-500 hover:underline">Select</button>
<button @click="navigate(kb.id)" class="text-sm text-slate-500 hover:underline opacity-50 group-hover:opacity-100">Open</button>
</div>
</li>
<li v-if="!loading && childrenKbs.length === 0" class="text-center text-slate-400 p-8">No sub-folders.</li>
</ul>
</div>
<!-- Current Folder Selection -->
<div class="mt-4 pt-4 border-t border-slate-200">
<button
@click="selectKb(currentParent ? {id: currentParent, name: breadcrumbs[breadcrumbs.length-1]?.name || 'Root', kb_type: currentKbInfo?.kb_type} : null)"
class="w-full py-2 bg-slate-100 rounded-lg hover:bg-slate-200"
>
Select current folder: <span class="font-semibold">{{ currentParent ? (breadcrumbs[breadcrumbs.length-1]?.name || 'Root') : 'None (Top Level)' }}</span>
</button>
</div>
</div>
</div>
</Teleport>
</template>

424
frontend/src/components/KnowledgeGraph.vue Normal file
View File

@ -0,0 +1,424 @@
<script setup>
import { ref, onMounted, computed, watch } from 'vue'
import { api } from '../api.js'
import EntityDetail from './EntityDetail.vue'
import GraphVisualization from './GraphVisualization.vue'
const entities = ref([])
const stats = ref(null)
const loading = ref(false)
const searchQuery = ref('')
const selectedEntityType = ref('')
const selectedEntity = ref(null)
const showEntityDetail = ref(false)
const viewMode = ref('list') // 'list' or 'graph'
// 范围过滤
const scopeType = ref('all') // 'all', 'kb', 'file'
const knowledgeBases = ref([])
const files = ref([])
const selectedKbId = ref(null)
const selectedFileId = ref(null)
// 实体类型映射
const entityTypeMap = {
'person': { label: '人物', icon: '👤', color: 'blue' },
'organization': { label: '组织', icon: '🏢', color: 'purple' },
'location': { label: '地点', icon: '📍', color: 'green' },
'date': { label: '日期', icon: '📅', color: 'orange' },
'product': { label: '产品', icon: '📦', color: 'pink' },
'technology': { label: '技术', icon: '⚡', color: 'cyan' },
'concept': { label: '概念', icon: '💡', color: 'yellow' },
'api': { label: 'API', icon: '🔌', color: 'indigo' },
'document': { label: '文档', icon: '📄', color: 'gray' },
'chapter': { label: '章节', icon: '📑', color: 'slate' },
'table': { label: '表格', icon: '📊', color: 'teal' },
'image': { label: '图片', icon: '🖼️', color: 'rose' },
}
const entityTypes = computed(() => {
if (!stats.value) return []
return Object.entries(stats.value.entity_types || {}).map(([type, count]) => ({
type,
count,
...entityTypeMap[type] || { label: type, icon: '📌', color: 'gray' }
}))
})
const filteredEntities = computed(() => {
return entities.value
})
// 当前有效的过滤参数
const currentKbId = computed(() => scopeType.value === 'kb' ? selectedKbId.value : null)
const currentFileId = computed(() => scopeType.value === 'file' ? selectedFileId.value : null)
// 加载知识库列表
const loadKnowledgeBases = async () => {
try {
knowledgeBases.value = await api.getKnowledgeBases()
} catch (error) {
console.error('加载知识库列表失败:', error)
}
}
// 加载文件列表
const loadFiles = async () => {
try {
const result = await api.getFiles(undefined, null, { size: 10000 })
files.value = result.items || []
} catch (error) {
console.error('加载文件列表失败:', error)
}
}
const loadStats = async () => {
try {
stats.value = await api.getGraphStats(currentKbId.value, currentFileId.value)
} catch (error) {
console.error('加载统计失败:', error)
}
}
const loadEntities = async () => {
loading.value = true
try {
entities.value = await api.searchEntities(
searchQuery.value || null,
selectedEntityType.value || null,
currentKbId.value,
100,
currentFileId.value
)
} catch (error) {
console.error('加载实体失败:', error)
} finally {
loading.value = false
}
}
const handleSearch = () => {
loadStats()
loadEntities()
}
const handleScopeChange = () => {
// 切换范围时重置选择
if (scopeType.value === 'all') {
selectedKbId.value = null
selectedFileId.value = null
} else if (scopeType.value === 'kb') {
selectedFileId.value = null
if (knowledgeBases.value.length > 0 && !selectedKbId.value) {
selectedKbId.value = knowledgeBases.value[0].id
}
} else if (scopeType.value === 'file') {
selectedKbId.value = null
if (files.value.length > 0 && !selectedFileId.value) {
selectedFileId.value = files.value[0].id
}
}
loadStats()
loadEntities()
}
const handleEntityTypeFilter = (type) => {
selectedEntityType.value = selectedEntityType.value === type ? '' : type
loadEntities()
}
const handleEntityClick = (entity) => {
selectedEntity.value = entity
showEntityDetail.value = true
}
const closeEntityDetail = () => {
showEntityDetail.value = false
selectedEntity.value = null
}
const getEntityTypeInfo = (type) => {
return entityTypeMap[type] || { label: type, icon: '📌', color: 'gray' }
}
const formatDate = (timestamp) => {
return new Date(timestamp * 1000).toLocaleString('zh-CN')
}
// 监听知识库和文件选择变化
watch(selectedKbId, () => {
if (scopeType.value === 'kb') {
loadStats()
loadEntities()
}
})
watch(selectedFileId, () => {
if (scopeType.value === 'file') {
loadStats()
loadEntities()
}
})
onMounted(() => {
loadKnowledgeBases()
loadFiles()
loadStats()
loadEntities()
})
</script>
<template>
<div class="space-y-6">
<!-- 范围选择 -->
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<h3 class="text-sm font-semibold text-slate-700 mb-4">数据范围</h3>
<div class="flex flex-wrap items-center gap-4">
<!-- 范围类型选择 -->
<div class="flex gap-1 bg-slate-100 p-1 rounded-lg">
<button
@click="scopeType = 'all'; handleScopeChange()"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all',
scopeType === 'all' ? 'bg-white text-slate-900 shadow-sm' : 'text-slate-600 hover:text-slate-900'
]"
>
🌐 全部
</button>
<button
@click="scopeType = 'kb'; handleScopeChange()"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all',
scopeType === 'kb' ? 'bg-white text-slate-900 shadow-sm' : 'text-slate-600 hover:text-slate-900'
]"
>
📚 知识库
</button>
<button
@click="scopeType = 'file'; handleScopeChange()"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all',
scopeType === 'file' ? 'bg-white text-slate-900 shadow-sm' : 'text-slate-600 hover:text-slate-900'
]"
>
📄 文件
</button>
</div>
<!-- 知识库选择 -->
<select
v-if="scopeType === 'kb'"
v-model="selectedKbId"
class="px-4 py-2 border border-slate-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent bg-white min-w-[200px]"
>
<option :value="null" disabled>选择知识库...</option>
<option v-for="kb in knowledgeBases" :key="kb.id" :value="kb.id">
{{ kb.name }}
</option>
</select>
<!-- 文件选择 -->
<select
v-if="scopeType === 'file'"
v-model="selectedFileId"
class="px-4 py-2 border border-slate-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent bg-white min-w-[200px]"
>
<option :value="null" disabled>选择文件...</option>
<option v-for="file in files" :key="file.id" :value="file.id">
{{ file.filename }}
</option>
</select>
<!-- 当前范围提示 -->
<div class="text-sm text-slate-500">
<span v-if="scopeType === 'all'">显示所有知识图谱数据</span>
<span v-else-if="scopeType === 'kb' && selectedKbId">
已选择知识库: <strong class="text-slate-700">{{ knowledgeBases.find(kb => kb.id === selectedKbId)?.name }}</strong>
</span>
<span v-else-if="scopeType === 'file' && selectedFileId">
已选择文件: <strong class="text-slate-700">{{ files.find(f => f.id === selectedFileId)?.filename }}</strong>
</span>
</div>
</div>
</div>
<!-- 统计卡片 -->
<div v-if="stats" class="grid grid-cols-1 md:grid-cols-4 gap-4">
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<div class="flex items-center gap-3">
<div class="w-12 h-12 bg-blue-100 rounded-lg flex items-center justify-center">
<span class="text-2xl">🔵</span>
</div>
<div>
<p class="text-sm text-slate-500">实体节点</p>
<p class="text-2xl font-bold text-slate-800">{{ stats.node_count }}</p>
</div>
</div>
</div>
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<div class="flex items-center gap-3">
<div class="w-12 h-12 bg-purple-100 rounded-lg flex items-center justify-center">
<span class="text-2xl">↔️</span>
</div>
<div>
<p class="text-sm text-slate-500">关系边</p>
<p class="text-2xl font-bold text-slate-800">{{ stats.edge_count }}</p>
</div>
</div>
</div>
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<div class="flex items-center gap-3">
<div class="w-12 h-12 bg-green-100 rounded-lg flex items-center justify-center">
<span class="text-2xl">📊</span>
</div>
<div>
<p class="text-sm text-slate-500">实体类型</p>
<p class="text-2xl font-bold text-slate-800">{{ Object.keys(stats.entity_types || {}).length }}</p>
</div>
</div>
</div>
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<div class="flex items-center gap-3">
<div class="w-12 h-12 bg-orange-100 rounded-lg flex items-center justify-center">
<span class="text-2xl">🔗</span>
</div>
<div>
<p class="text-sm text-slate-500">关系类型</p>
<p class="text-2xl font-bold text-slate-800">{{ Object.keys(stats.relation_types || {}).length }}</p>
</div>
</div>
</div>
</div>
<!-- 实体类型筛选 -->
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<h3 class="text-sm font-semibold text-slate-700 mb-4">实体类型</h3>
<div class="flex flex-wrap gap-2">
<button
v-for="entityType in entityTypes"
:key="entityType.type"
@click="handleEntityTypeFilter(entityType.type)"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all duration-200 flex items-center gap-2',
selectedEntityType === entityType.type
? `bg-${entityType.color}-100 text-${entityType.color}-700 border-2 border-${entityType.color}-300`
: 'bg-slate-50 text-slate-600 border-2 border-transparent hover:bg-slate-100'
]"
>
<span>{{ entityType.icon }}</span>
<span>{{ entityType.label }}</span>
<span class="ml-1 px-2 py-0.5 bg-white rounded-full text-xs">{{ entityType.count }}</span>
</button>
</div>
</div>
<!-- 搜索框 -->
<div class="bg-white rounded-xl p-6 border border-slate-200 shadow-sm">
<div class="flex gap-3">
<input
v-model="searchQuery"
@keyup.enter="handleSearch"
type="text"
placeholder="搜索实体名称..."
class="flex-1 px-4 py-3 border border-slate-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent"
/>
<button
@click="handleSearch"
class="px-6 py-3 bg-blue-500 text-white rounded-lg font-medium hover:bg-blue-600 transition-colors"
>
🔍 搜索
</button>
<div class="flex gap-1 bg-slate-100 p-1 rounded-lg">
<button
@click="viewMode = 'list'"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all',
viewMode === 'list' ? 'bg-white text-slate-900 shadow-sm' : 'text-slate-600 hover:text-slate-900'
]"
>
📋 列表
</button>
<button
@click="viewMode = 'graph'"
:class="[
'px-4 py-2 rounded-lg text-sm font-medium transition-all',
viewMode === 'graph' ? 'bg-white text-slate-900 shadow-sm' : 'text-slate-600 hover:text-slate-900'
]"
>
🕸️ 图形
</button>
</div>
</div>
</div>
<!-- 图形可视化视图 -->
<GraphVisualization
v-if="viewMode === 'graph'"
:kb-id="currentKbId"
:file-id="currentFileId"
:query="searchQuery || null"
:entity-type="selectedEntityType || null"
/>
<!-- 实体列表 -->
<div v-if="viewMode === 'list'" class="bg-white rounded-xl border border-slate-200 shadow-sm overflow-hidden">
<div class="px-6 py-4 border-b border-slate-200 bg-slate-50">
<h3 class="font-semibold text-slate-800">
实体列表
<span class="text-sm text-slate-500 ml-2">({{ filteredEntities.length }} 个实体)</span>
</h3>
</div>
<div v-if="loading" class="p-8 text-center">
<div class="inline-block animate-spin rounded-full h-8 w-8 border-b-2 border-blue-500"></div>
<p class="mt-2 text-slate-500">加载中...</p>
</div>
<div v-else-if="filteredEntities.length === 0" class="p-8 text-center text-slate-500">
<span class="text-4xl mb-2 block">🔍</span>
暂无实体数据
</div>
<div v-else class="divide-y divide-slate-100">
<div
v-for="entity in filteredEntities"
:key="entity.id"
@click="handleEntityClick(entity)"
class="p-4 hover:bg-slate-50 transition-colors cursor-pointer"
>
<div class="flex items-center justify-between">
<div class="flex items-center gap-3 flex-1">
<div :class="`w-10 h-10 bg-${getEntityTypeInfo(entity.entity_type).color}-100 rounded-lg flex items-center justify-center`">
<span class="text-xl">{{ getEntityTypeInfo(entity.entity_type).icon }}</span>
</div>
<div class="flex-1">
<h4 class="font-medium text-slate-800">{{ entity.name }}</h4>
<div class="flex items-center gap-2 mt-1">
<span :class="`px-2 py-0.5 text-xs rounded-full bg-${getEntityTypeInfo(entity.entity_type).color}-100 text-${getEntityTypeInfo(entity.entity_type).color}-700`">
{{ getEntityTypeInfo(entity.entity_type).label }}
</span>
<span class="text-xs text-slate-400">
{{ formatDate(entity.created_at) }}
</span>
</div>
</div>
</div>
<div class="text-slate-400">
→
</div>
</div>
</div>
</div>
</div>
<!-- 实体详情弹窗 -->
<EntityDetail
v-if="showEntityDetail && selectedEntity"
:entity-id="selectedEntity.id"
@close="closeEntityDetail"
/>
</div>
</template>

113
frontend/src/components/Pagination.vue Normal file
View File

@ -0,0 +1,113 @@
<script setup>
import { computed } from 'vue'
const props = defineProps({
page: {
type: Number,
required: true,
},
size: {
type: Number,
required: true,
},
total: {
type: Number,
required: true,
},
sizes: {
type: Array,
default: () => [10, 20, 50],
},
})
const emit = defineEmits(['update:page', 'update:size', 'change'])
const totalPages = computed(() => Math.max(1, Math.ceil(props.total / props.size)))
const visiblePages = computed(() => {
const current = props.page
const last = totalPages.value
if (last <= 7) {
return Array.from({ length: last }, (_, i) => i + 1)
}
if (current <= 4) {
return [1, 2, 3, 4, 5, '...', last]
}
if (current >= last - 3) {
return [1, '...', last - 4, last - 3, last - 2, last - 1, last]
}
return [1, '...', current - 1, current, current + 1, '...', last]
})
function setPage(p) {
if (p === '...' || p < 1 || p > totalPages.value || p === props.page) return
emit('update:page', p)
emit('change')
}
function prev() {
if (props.page > 1) setPage(props.page - 1)
}
function next() {
if (props.page < totalPages.value) setPage(props.page + 1)
}
function onSizeChange(event) {
emit('update:size', Number(event.target.value))
emit('update:page', 1)
emit('change')
}
</script>
<template>
<div v-if="total > 0" class="flex flex-col sm:flex-row items-center justify-between gap-3 mt-6">
<div class="text-sm text-slate-500">
共 {{ total }} 条，第 {{ page }} / {{ totalPages }} 页
</div>
<div class="flex items-center gap-2">
<select
:value="size"
@change="onSizeChange"
class="px-2 py-1.5 text-sm border border-slate-200 rounded-md bg-white text-slate-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
>
<option v-for="s in sizes" :key="s" :value="s">{{ s }} 条/页</option>
</select>
<button
type="button"
:disabled="page <= 1"
@click="prev"
class="px-3 py-1.5 text-sm border border-slate-200 rounded-md bg-white text-slate-700 hover:bg-slate-50 disabled:opacity-50 disabled:cursor-not-allowed"
>
上一页
</button>
<button
v-for="p in visiblePages"
:key="String(p) + '-' + page"
type="button"
:disabled="p === '...'"
@click="() => setPage(p)"
:class="[
'px-3 py-1.5 text-sm border rounded-md min-w-[2.5rem]',
p === page
? 'bg-blue-600 border-blue-600 text-white'
: 'bg-white border-slate-200 text-slate-700 hover:bg-slate-50',
p === '...' && 'cursor-default opacity-60'
]"
>
{{ p }}
</button>
<button
type="button"
:disabled="page >= totalPages"
@click="next"
class="px-3 py-1.5 text-sm border border-slate-200 rounded-md bg-white text-slate-700 hover:bg-slate-50 disabled:opacity-50 disabled:cursor-not-allowed"
>
下一页
</button>
</div>
</div>
</template>

330
frontend/src/components/SearchBar.vue Normal file
View File

@ -0,0 +1,330 @@
<script setup>
import { computed, ref, watch } from 'vue'
import { api } from '../api'
import { currentKb } from '../store' // Import global store
import KnowledgeBaseSelector from './KnowledgeBaseSelector.vue' // Import selector
const emit = defineEmits(['search', 'search-start', 'search-end', 'advanced-search'])
const query = ref('')
const error = ref('')
const searchMode = ref('full')
const imageFile = ref(null)
const fileInput = ref(null)
const advancedOptions = ref({
maxSteps: 3,
docLimit: 10,
contextChars: 2000,
debug: false,
})
const sliceOptions = ref({
useAdvancedFlow: false,
})
const canSubmit = computed(() => {
if (searchMode.value === 'image') {
return Boolean(imageFile.value)
}
return Boolean(query.value.trim())
})
// Local state for search scope
const localSelectedKb = ref({ id: null, name: '所有知识库' })
const showKbSelector = ref(false)
// Initialize local scope with global context on mount/visibility
// And keep it in sync when global context changes
watch(currentKb, (newGlobalKb) => {
localSelectedKb.value = newGlobalKb
}, { immediate: true }) // immediate: true ensures it runs on initial component setup
const handleKbSelect = (kb) => {
if (kb) {
localSelectedKb.value = kb
} else {
localSelectedKb.value = { id: null, name: '所有知识库' }
}
showKbSelector.value = false // Close modal after selection
}
const handleSearch = async () => {
if (searchMode.value === 'advanced') {
if (!query.value.trim()) {
error.value = '请输入搜索内容'
return
}
emit('advanced-search', {
query: query.value.trim(),
kbId: localSelectedKb.value?.id,
options: { ...advancedOptions.value },
})
return
}
if (searchMode.value === 'image') {
if (!imageFile.value) {
error.value = '请先选择图片'
emit('search', [])
return
}
} else if (!query.value.trim()) {
return
}
error.value = ''
emit('search-start')
try {
let results = []
if (searchMode.value === 'image') {
results = await api.searchImage(imageFile.value, query.value, localSelectedKb.value?.id)
} else {
results =
searchMode.value === 'slice'
? await api.search(query.value, localSelectedKb.value?.id, null, { advanced: sliceOptions.value.useAdvancedFlow })
: await api.searchFull(query.value, localSelectedKb.value?.id)
}
emit('search', results)
} catch (e) {
error.value = e.message
emit('search', [])
} finally {
emit('search-end')
}
}
const handleImageChange = (e) => {
const file = e.target.files?.[0] || null
imageFile.value = file
}
const clearImage = () => {
imageFile.value = null
if (fileInput.value) {
fileInput.value.value = ''
}
}
const handleKeydown = (e) => {
if (e.key === 'Enter') {
handleSearch()
}
}
</script>
<template>
<div class="max-w-4xl mx-auto space-y-4">
<!-- Search Scope Selection -->
<div class="rounded-2xl border border-slate-200/80 bg-white/90 p-4 shadow-sm">
<label class="block text-xs font-semibold tracking-wide text-slate-600">搜索范围</label>
<button
@click="showKbSelector = true"
class="mt-2 w-full px-4 py-3 bg-white border border-slate-200 rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent text-left flex justify-between items-center hover:border-blue-300 transition-colors"
>
<span class="font-medium text-slate-700">{{ localSelectedKb.name }}</span>
<svg class="w-5 h-5 text-slate-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 9l4-4 4 4m0 6l-4 4-4-4" />
</svg>
</button>
<p class="mt-3 text-xs text-slate-500">
当前搜索将在 <span class="font-medium text-blue-600">{{ localSelectedKb.name }}</span> 及其子知识库中进行。
</p>
</div>
<!-- Search Mode Selection -->
<div class="rounded-2xl border border-slate-200/80 bg-white/90 p-4 shadow-sm">
<label class="block text-xs font-semibold tracking-wide text-slate-600 mb-2">搜索模式</label>
<div class="grid grid-cols-2 sm:grid-cols-4 gap-2 rounded-xl border border-slate-200 bg-slate-50 p-1.5">
<button
type="button"
class="px-4 py-2.5 text-sm rounded-lg transition-all duration-200"
:class="searchMode === 'full' ? 'bg-slate-900 text-white shadow-sm' : 'text-slate-600 hover:bg-white hover:text-slate-900'"
@click="searchMode = 'full'"
>
文件搜索
</button>
<button
type="button"
class="px-4 py-2.5 text-sm rounded-lg transition-all duration-200"
:class="searchMode === 'slice' ? 'bg-slate-900 text-white shadow-sm' : 'text-slate-600 hover:bg-white hover:text-slate-900'"
@click="searchMode = 'slice'"
>
切片搜索
</button>
<button
type="button"
class="px-4 py-2.5 text-sm rounded-lg transition-all duration-200"
:class="searchMode === 'image' ? 'bg-slate-900 text-white shadow-sm' : 'text-slate-600 hover:bg-white hover:text-slate-900'"
@click="searchMode = 'image'"
>
图片搜索
</button>
<button
type="button"
class="px-4 py-2.5 text-sm rounded-lg transition-all duration-200"
:class="searchMode === 'advanced' ? 'bg-slate-900 text-white shadow-sm' : 'text-slate-600 hover:bg-white hover:text-slate-900'"
@click="searchMode = 'advanced'"
>
高级搜索
</button>
</div>
<p class="mt-3 text-xs text-slate-500 leading-5">
文件搜索返回高亮片段；切片搜索返回命中的切片内容；图片搜索支持以图搜图；高级搜索以 SSE 流返回更长上下文。
</p>
</div>
<!-- Knowledge Base Selector Modal -->
<KnowledgeBaseSelector
:show="showKbSelector"
@close="showKbSelector = false"
@select="handleKbSelect"
/>
<!-- 图片搜索上传 -->
<div v-if="searchMode === 'image'" class="space-y-4 rounded-2xl border border-slate-200/80 bg-white/90 p-4 shadow-sm">
<div class="flex flex-wrap items-center gap-3">
<input
ref="fileInput"
type="file"
accept="image/*"
class="hidden"
@change="handleImageChange"
/>
<button
type="button"
class="px-4 py-2.5 bg-white border border-slate-200 rounded-xl text-sm font-medium text-slate-700 hover:border-blue-300 transition-colors"
@click="fileInput && fileInput.click()"
>
选择图片
</button>
<span class="text-sm text-slate-600 truncate max-w-xs sm:max-w-sm">
{{ imageFile ? imageFile.name : '未选择图片' }}
</span>
<button
v-if="imageFile"
type="button"
class="text-xs text-slate-500 hover:text-slate-700"
@click="clearImage"
>
清除
</button>
</div>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-4 flex items-center pointer-events-none">
<svg class="h-5 w-5 text-slate-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
</svg>
</div>
<input
v-model="query"
@keydown="handleKeydown"
type="text"
placeholder="图片描述（可选）..."
class="w-full h-16 pl-12 pr-28 sm:pr-36 bg-white border border-slate-200 rounded-2xl text-lg shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent transition-all duration-200"
/>
<button
@click="handleSearch"
:disabled="!canSubmit"
class="absolute right-2 top-1/2 -translate-y-1/2 min-w-[88px] sm:min-w-[108px] px-5 py-2.5 bg-linear-to-r from-blue-500 to-indigo-600 text-white rounded-xl font-medium hover:from-blue-600 hover:to-indigo-700 transition-all duration-200 shadow-sm disabled:opacity-60 disabled:cursor-not-allowed"
>
搜索
</button>
</div>
</div>
<!-- 文本搜索输入框 -->
<div v-else class="space-y-4">
<div v-if="searchMode === 'advanced'" class="grid grid-cols-1 md:grid-cols-2 gap-4 bg-white border border-slate-200 rounded-2xl p-4">
<div>
<label class="block text-xs font-medium text-slate-600 mb-2">计划步骤数</label>
<input
type="number"
min="1"
max="8"
v-model.number="advancedOptions.maxSteps"
class="w-full px-3 py-2 border border-slate-200 rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
</div>
<div>
<label class="block text-xs font-medium text-slate-600 mb-2">每步处理文档</label>
<input
type="number"
min="1"
max="20"
v-model.number="advancedOptions.docLimit"
class="w-full px-3 py-2 border border-slate-200 rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
</div>
<div>
<label class="block text-xs font-medium text-slate-600 mb-2">上下文单侧字数</label>
<input
type="number"
min="200"
max="6000"
step="100"
v-model.number="advancedOptions.contextChars"
class="w-full px-3 py-2 border border-slate-200 rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
</div>
<div class="flex items-center gap-3 mt-6">
<label class="text-xs font-medium text-slate-600">输出调试信息</label>
<button
type="button"
class="relative inline-flex h-6 w-11 items-center rounded-full transition-colors"
:class="advancedOptions.debug ? 'bg-blue-600' : 'bg-slate-300'"
@click="advancedOptions.debug = !advancedOptions.debug"
>
<span
class="inline-block h-5 w-5 transform rounded-full bg-white shadow transition-transform"
:class="advancedOptions.debug ? 'translate-x-5' : 'translate-x-1'"
/>
</button>
</div>
</div>
<div v-else-if="searchMode === 'slice'" class="bg-white border border-slate-200 rounded-2xl p-4">
<div class="flex items-center justify-between gap-4">
<div>
<p class="text-sm font-semibold text-slate-700">切片高级流程</p>
<p class="text-xs text-slate-500 mt-1 leading-5">开启后会走高级搜索判定流程，但返回仍是普通切片搜索结果格式。</p>
</div>
<button
type="button"
class="relative inline-flex h-6 w-11 items-center rounded-full transition-colors"
:class="sliceOptions.useAdvancedFlow ? 'bg-blue-600' : 'bg-slate-300'"
@click="sliceOptions.useAdvancedFlow = !sliceOptions.useAdvancedFlow"
>
<span
class="inline-block h-5 w-5 transform rounded-full bg-white shadow transition-transform"
:class="sliceOptions.useAdvancedFlow ? 'translate-x-5' : 'translate-x-1'"
/>
</button>
</div>
</div>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-4 flex items-center pointer-events-none">
<svg class="h-5 w-5 text-slate-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
</svg>
</div>
<input
v-model="query"
@keydown="handleKeydown"
type="text"
placeholder="输入关键词搜索..."
class="w-full h-16 pl-12 pr-28 sm:pr-36 bg-white border border-slate-200 rounded-2xl text-lg shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent transition-all duration-200"
/>
<button
@click="handleSearch"
:disabled="!canSubmit"
class="absolute right-2 top-1/2 -translate-y-1/2 min-w-[88px] sm:min-w-[108px] px-5 py-2.5 bg-linear-to-r from-blue-500 to-indigo-600 text-white rounded-xl font-medium hover:from-blue-600 hover:to-indigo-700 transition-all duration-200 shadow-sm disabled:opacity-60 disabled:cursor-not-allowed"
>
搜索
</button>
</div>
</div>
<p v-if="error" class="rounded-xl border border-red-200 bg-red-50 px-4 py-2 text-center text-sm text-red-600">{{ error }}</p>
</div>
</template>

667
frontend/src/components/SearchDictionaryManager.vue Normal file
View File

@ -0,0 +1,667 @@
<script setup>
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
import { api } from '../api'
const loading = ref(false)
const statusLoading = ref(false)
const busy = ref(false)
const publishLoading = ref(false)
const error = ref('')
const success = ref('')
const lexiconQuery = ref('')
const lexiconEnabledFilter = ref('all')
const lexicons = ref([])
const lexiconTotal = ref(0)
const newLexiconTerm = ref('')
const newLexiconFreq = ref('')
const newLexiconTag = ref('')
const newLexiconEnabled = ref(true)
const synonymQuery = ref('')
const synonymEnabledFilter = ref('all')
const synonyms = ref([])
const synonymTotal = ref(0)
const newSynonymTerm = ref('')
const newSynonymValue = ref('')
const newSynonymWeight = ref('1')
const newSynonymBidirectional = ref(true)
const newSynonymEnabled = ref(true)
const rebuildStatus = ref({
job_id: null,
status: 'idle',
phase: '',
total_docs: 0,
processed_docs: 0,
progress_pct: 0,
elapsed_secs: null,
eta_secs: null,
updated_at: null,
started_at: null,
finished_at: null,
error: null,
})
let statusPollTimer = null
const resolveEnabledFilter = (value) => {
if (value === 'enabled') return true
if (value === 'disabled') return false
return undefined
}
const progressPercent = computed(() => {
const pct = Number(rebuildStatus.value?.progress_pct || 0)
if (Number.isNaN(pct)) return 0
return Math.max(0, Math.min(100, pct))
})
const progressText = computed(() => {
const total = Number(rebuildStatus.value?.total_docs || 0)
const processed = Number(rebuildStatus.value?.processed_docs || 0)
if (total <= 0) return '0 / 0'
return `${Math.min(processed, total)} / ${total}`
})
const isRebuilding = computed(() => rebuildStatus.value?.status === 'running')
const statusBadgeClass = computed(() => {
if (rebuildStatus.value?.status === 'running') return 'bg-amber-50 text-amber-700 border-amber-200'
if (rebuildStatus.value?.status === 'completed') return 'bg-emerald-50 text-emerald-700 border-emerald-200'
if (rebuildStatus.value?.status === 'failed') return 'bg-red-50 text-red-700 border-red-200'
return 'bg-slate-50 text-slate-600 border-slate-200'
})
const formatSecs = (value) => {
if (value === null || value === undefined) return '-'
const total = Math.max(0, Number(value))
if (total < 60) return `${total}s`
const minutes = Math.floor(total / 60)
const seconds = total % 60
if (minutes < 60) return `${minutes}m ${seconds}s`
const hours = Math.floor(minutes / 60)
const remMinutes = minutes % 60
return `${hours}h ${remMinutes}m`
}
const formatTime = (value) => {
if (!value) return '-'
const date = new Date(Number(value) * 1000)
if (Number.isNaN(date.getTime())) return '-'
return date.toLocaleString()
}
const clearTips = () => {
error.value = ''
success.value = ''
}
const refreshStatus = async () => {
statusLoading.value = true
try {
rebuildStatus.value = await api.getIndexRebuildStatus()
} catch (e) {
error.value = e?.message || '获取重建状态失败'
} finally {
statusLoading.value = false
}
}
const refreshLexicons = async () => {
const data = await api.listLexicons({
q: lexiconQuery.value.trim() || undefined,
enabled: resolveEnabledFilter(lexiconEnabledFilter.value),
limit: 100,
offset: 0,
})
lexicons.value = data?.items || []
lexiconTotal.value = Number(data?.total || 0)
}
const refreshSynonyms = async () => {
const data = await api.listSynonyms({
q: synonymQuery.value.trim() || undefined,
enabled: resolveEnabledFilter(synonymEnabledFilter.value),
limit: 100,
offset: 0,
})
synonyms.value = data?.items || []
synonymTotal.value = Number(data?.total || 0)
}
const refreshAll = async () => {
loading.value = true
clearTips()
try {
await Promise.all([refreshLexicons(), refreshSynonyms(), refreshStatus()])
} catch (e) {
error.value = e?.message || '加载失败'
} finally {
loading.value = false
}
}
const submitLexicon = async () => {
if (busy.value) return
clearTips()
const term = String(newLexiconTerm.value ?? '').trim()
if (!term) {
error.value = '词条不能为空'
return
}
const freqRaw = String(newLexiconFreq.value ?? '').trim()
let freq
if (freqRaw) {
freq = Number(freqRaw)
if (!Number.isInteger(freq) || freq <= 0) {
error.value = '词频必须是正整数'
return
}
}
busy.value = true
try {
const payload = {
term,
enabled: newLexiconEnabled.value,
}
if (freq) payload.freq = freq
const tag = String(newLexiconTag.value ?? '').trim()
if (tag) payload.tag = tag
await api.createLexicon(payload)
newLexiconTerm.value = ''
newLexiconFreq.value = ''
newLexiconTag.value = ''
newLexiconEnabled.value = true
success.value = '词条已保存（待发布后生效）'
await refreshLexicons()
} catch (e) {
error.value = e?.message || '新增词条失败'
} finally {
busy.value = false
}
}
const editLexicon = async (item) => {
if (busy.value) return
clearTips()
const term = window.prompt('词条', item.term)
if (term === null) return
const freqInput = window.prompt('词频（留空表示不修改）', item.freq ?? '')
if (freqInput === null) return
const tagInput = window.prompt('词性（留空表示清空）', item.tag ?? '')
if (tagInput === null) return
const nextEnabled = window.confirm('点击“确定”启用，点击“取消”停用。')
const payload = {
term: term.trim(),
enabled: nextEnabled,
tag: tagInput,
}
const freqValue = String(freqInput).trim()
if (freqValue) {
const parsed = Number(freqValue)
if (!Number.isInteger(parsed) || parsed <= 0) {
error.value = '词频必须是正整数'
return
}
payload.freq = parsed
}
busy.value = true
try {
await api.updateLexicon(item.id, payload)
success.value = '词条已更新（待发布后生效）'
await refreshLexicons()
} catch (e) {
error.value = e?.message || '更新词条失败'
} finally {
busy.value = false
}
}
const toggleLexicon = async (item) => {
if (busy.value) return
clearTips()
busy.value = true
try {
await api.toggleLexiconEnabled(item.id, !item.enabled)
success.value = '词条状态已更新（待发布后生效）'
await refreshLexicons()
} catch (e) {
error.value = e?.message || '更新状态失败'
} finally {
busy.value = false
}
}
const removeLexicon = async (item) => {
if (busy.value) return
if (!window.confirm(`确定删除词条 "${item.term}" 吗？`)) return
clearTips()
busy.value = true
try {
await api.deleteLexicon(item.id)
success.value = '词条已删除（待发布后生效）'
await refreshLexicons()
} catch (e) {
error.value = e?.message || '删除词条失败'
} finally {
busy.value = false
}
}
const submitSynonym = async () => {
if (busy.value) return
clearTips()
const term = newSynonymTerm.value.trim()
const synonym = newSynonymValue.value.trim()
if (!term || !synonym) {
error.value = '原词和同义词不能为空'
return
}
const weight = Number(newSynonymWeight.value)
if (!Number.isFinite(weight) || weight <= 0) {
error.value = '权重必须大于 0'
return
}
busy.value = true
try {
await api.createSynonym({
term,
synonym,
weight,
bidirectional: newSynonymBidirectional.value,
enabled: newSynonymEnabled.value,
})
newSynonymTerm.value = ''
newSynonymValue.value = ''
newSynonymWeight.value = '1'
newSynonymBidirectional.value = true
newSynonymEnabled.value = true
success.value = '同义词已保存（查询立即生效）'
await refreshSynonyms()
} catch (e) {
error.value = e?.message || '新增同义词失败'
} finally {
busy.value = false
}
}
const editSynonym = async (item) => {
if (busy.value) return
clearTips()
const term = window.prompt('原词', item.term)
if (term === null) return
const synonym = window.prompt('同义词', item.synonym)
if (synonym === null) return
const weightInput = window.prompt('权重', item.weight)
if (weightInput === null) return
const weight = Number(weightInput)
if (!Number.isFinite(weight) || weight <= 0) {
error.value = '权重必须大于 0'
return
}
const bidirectional = window.confirm('点击“确定”设为双向；点击“取消”设为单向。')
const enabled = window.confirm('点击“确定”启用；点击“取消”停用。')
busy.value = true
try {
await api.updateSynonym(item.id, {
term: term.trim(),
synonym: synonym.trim(),
weight,
bidirectional,
enabled,
})
success.value = '同义词已更新'
await refreshSynonyms()
} catch (e) {
error.value = e?.message || '更新同义词失败'
} finally {
busy.value = false
}
}
const toggleSynonym = async (item) => {
if (busy.value) return
clearTips()
busy.value = true
try {
await api.toggleSynonymEnabled(item.id, !item.enabled)
success.value = '同义词状态已更新'
await refreshSynonyms()
} catch (e) {
error.value = e?.message || '更新状态失败'
} finally {
busy.value = false
}
}
const removeSynonym = async (item) => {
if (busy.value) return
if (!window.confirm(`确定删除同义词 "${item.term} -> ${item.synonym}" 吗？`)) return
clearTips()
busy.value = true
try {
await api.deleteSynonym(item.id)
success.value = '同义词已删除'
await refreshSynonyms()
} catch (e) {
error.value = e?.message || '删除同义词失败'
} finally {
busy.value = false
}
}
const publishLexicon = async () => {
if (publishLoading.value) return
clearTips()
publishLoading.value = true
try {
const resp = await api.publishLexicon()
success.value = `发布成功，已启动重建任务 #${resp.job_id}`
await refreshStatus()
} catch (e) {
error.value = e?.message || '发布失败'
} finally {
publishLoading.value = false
}
}
const startStatusPolling = () => {
if (statusPollTimer) return
statusPollTimer = setInterval(async () => {
try {
await refreshStatus()
} catch (_) {
// ignore in background polling
}
}, 3000)
}
const stopStatusPolling = () => {
if (statusPollTimer) {
clearInterval(statusPollTimer)
statusPollTimer = null
}
}
onMounted(async () => {
await refreshAll()
startStatusPolling()
})
onBeforeUnmount(() => {
stopStatusPolling()
})
</script>
<template>
<div class="space-y-6">
<div class="bg-white rounded-2xl border border-slate-200 p-6">
<div class="flex flex-wrap items-center justify-between gap-3">
<div>
<h3 class="text-xl font-semibold text-slate-800">词表发布与索引重建</h3>
<p class="text-slate-500 text-sm mt-1">词表改动在“发布”后生效，发布会暂停解析并重建索引。</p>
</div>
<div class="flex items-center gap-2">
<button
@click="refreshAll"
:disabled="loading || busy || publishLoading"
class="px-4 py-2 rounded-lg border border-slate-200 text-slate-600 hover:text-slate-900 hover:border-slate-300 disabled:opacity-50 disabled:cursor-not-allowed"
>
刷新
</button>
<button
@click="publishLexicon"
:disabled="publishLoading || isRebuilding"
class="px-4 py-2 rounded-lg bg-blue-600 text-white hover:bg-blue-700 disabled:opacity-50 disabled:cursor-not-allowed"
>
{{ publishLoading ? '发布中...' : '发布并重建索引' }}
</button>
</div>
</div>
<div class="mt-4 grid grid-cols-1 md:grid-cols-3 gap-3">
<div class="rounded-xl border border-slate-200 p-3">
<div class="text-xs text-slate-400 mb-1">状态</div>
<span :class="['inline-flex px-2.5 py-1 rounded-full border text-xs font-medium', statusBadgeClass]">
{{ rebuildStatus.status || 'idle' }}
</span>
</div>
<div class="rounded-xl border border-slate-200 p-3">
<div class="text-xs text-slate-400 mb-1">阶段</div>
<div class="text-sm text-slate-700">{{ rebuildStatus.phase || '-' }}</div>
</div>
<div class="rounded-xl border border-slate-200 p-3">
<div class="text-xs text-slate-400 mb-1">任务 ID</div>
<div class="text-sm text-slate-700">{{ rebuildStatus.job_id ?? '-' }}</div>
</div>
</div>
<div class="mt-4">
<div class="flex justify-between text-xs text-slate-500 mb-1">
<span>进度 {{ progressText }}</span>
<span>{{ progressPercent.toFixed(1) }}%</span>
</div>
<div class="w-full h-2 bg-slate-100 rounded-full overflow-hidden">
<div class="h-full bg-linear-to-r from-blue-500 to-indigo-600" :style="{ width: `${progressPercent}%` }" />
</div>
</div>
<div class="mt-3 grid grid-cols-2 md:grid-cols-4 gap-2 text-xs">
<div class="rounded-lg bg-slate-50 px-3 py-2">
<div class="text-slate-400">已运行</div>
<div class="text-slate-700">{{ formatSecs(rebuildStatus.elapsed_secs) }}</div>
</div>
<div class="rounded-lg bg-slate-50 px-3 py-2">
<div class="text-slate-400">ETA</div>
<div class="text-slate-700">{{ formatSecs(rebuildStatus.eta_secs) }}</div>
</div>
<div class="rounded-lg bg-slate-50 px-3 py-2">
<div class="text-slate-400">开始时间</div>
<div class="text-slate-700">{{ formatTime(rebuildStatus.started_at) }}</div>
</div>
<div class="rounded-lg bg-slate-50 px-3 py-2">
<div class="text-slate-400">最近更新</div>
<div class="text-slate-700">{{ formatTime(rebuildStatus.updated_at) }}</div>
</div>
</div>
<p v-if="statusLoading" class="mt-3 text-xs text-slate-500">状态刷新中...</p>
<p v-if="rebuildStatus.error" class="mt-3 text-sm text-red-600 bg-red-50 border border-red-100 rounded-lg p-3">
{{ rebuildStatus.error }}
</p>
<p v-if="success" class="mt-3 text-sm text-emerald-700 bg-emerald-50 border border-emerald-100 rounded-lg p-3">
{{ success }}
</p>
<p v-if="error" class="mt-3 text-sm text-red-700 bg-red-50 border border-red-100 rounded-lg p-3">
{{ error }}
</p>
</div>
<div class="grid grid-cols-1 xl:grid-cols-2 gap-6">
<section class="bg-white rounded-2xl border border-slate-200 p-6 space-y-4">
<div class="flex items-center justify-between">
<h3 class="text-lg font-semibold text-slate-800">词表管理</h3>
<span class="text-xs text-slate-500">共 {{ lexiconTotal }} 条</span>
</div>
<div class="grid grid-cols-1 md:grid-cols-4 gap-2">
<input
v-model="lexiconQuery"
type="text"
class="md:col-span-2 px-3 py-2 border border-slate-200 rounded-lg text-sm"
placeholder="搜索 term"
/>
<select v-model="lexiconEnabledFilter" class="px-3 py-2 border border-slate-200 rounded-lg text-sm">
<option value="all">全部</option>
<option value="enabled">仅启用</option>
<option value="disabled">仅停用</option>
</select>
<button
@click="refreshLexicons"
class="px-3 py-2 rounded-lg border border-slate-200 text-slate-600 hover:text-slate-900 text-sm"
>
筛选
</button>
</div>
<div class="grid grid-cols-1 md:grid-cols-5 gap-2">
<input v-model="newLexiconTerm" type="text" class="px-3 py-2 border border-slate-200 rounded-lg text-sm" placeholder="term" />
<input v-model="newLexiconFreq" type="number" min="1" class="px-3 py-2 border border-slate-200 rounded-lg text-sm" placeholder="freq" />
<input v-model="newLexiconTag" type="text" class="px-3 py-2 border border-slate-200 rounded-lg text-sm" placeholder="tag" />
<label class="px-3 py-2 border border-slate-200 rounded-lg text-sm flex items-center gap-2">
<input v-model="newLexiconEnabled" type="checkbox" />
启用
</label>
<button
@click="submitLexicon"
:disabled="busy"
class="px-3 py-2 rounded-lg bg-slate-900 text-white text-sm hover:bg-slate-800 disabled:opacity-50"
>
新增
</button>
</div>
<div class="border border-slate-200 rounded-xl overflow-hidden">
<table class="w-full text-sm">
<thead class="bg-slate-50 text-slate-500">
<tr>
<th class="text-left px-3 py-2">term</th>
<th class="text-left px-3 py-2">freq</th>
<th class="text-left px-3 py-2">tag</th>
<th class="text-left px-3 py-2">状态</th>
<th class="text-right px-3 py-2">操作</th>
</tr>
</thead>
<tbody>
<tr v-if="lexicons.length === 0">
<td colspan="5" class="px-3 py-6 text-center text-slate-400">暂无词表数据</td>
</tr>
<tr v-for="item in lexicons" :key="item.id" class="border-t border-slate-100">
<td class="px-3 py-2 text-slate-800 break-all">{{ item.term }}</td>
<td class="px-3 py-2 text-slate-600">{{ item.freq ?? '-' }}</td>
<td class="px-3 py-2 text-slate-600">{{ item.tag || '-' }}</td>
<td class="px-3 py-2">
<span
:class="[
'px-2 py-0.5 rounded-full text-xs border',
item.enabled ? 'bg-emerald-50 text-emerald-700 border-emerald-200' : 'bg-slate-50 text-slate-500 border-slate-200'
]"
>
{{ item.enabled ? '启用' : '停用' }}
</span>
</td>
<td class="px-3 py-2">
<div class="flex justify-end gap-1">
<button @click="editLexicon(item)" class="px-2 py-1 text-xs border border-slate-200 rounded hover:bg-slate-50">编辑</button>
<button @click="toggleLexicon(item)" class="px-2 py-1 text-xs border border-slate-200 rounded hover:bg-slate-50">
{{ item.enabled ? '停用' : '启用' }}
</button>
<button @click="removeLexicon(item)" class="px-2 py-1 text-xs border border-red-200 text-red-600 rounded hover:bg-red-50">删除</button>
</div>
</td>
</tr>
</tbody>
</table>
</div>
</section>
<section class="bg-white rounded-2xl border border-slate-200 p-6 space-y-4">
<div class="flex items-center justify-between">
<h3 class="text-lg font-semibold text-slate-800">同义词管理</h3>
<span class="text-xs text-slate-500">共 {{ synonymTotal }} 条</span>
</div>
<div class="grid grid-cols-1 md:grid-cols-4 gap-2">
<input
v-model="synonymQuery"
type="text"
class="md:col-span-2 px-3 py-2 border border-slate-200 rounded-lg text-sm"
placeholder="搜索 term/synonym"
/>
<select v-model="synonymEnabledFilter" class="px-3 py-2 border border-slate-200 rounded-lg text-sm">
<option value="all">全部</option>
<option value="enabled">仅启用</option>
<option value="disabled">仅停用</option>
</select>
<button
@click="refreshSynonyms"
class="px-3 py-2 rounded-lg border border-slate-200 text-slate-600 hover:text-slate-900 text-sm"
>
筛选
</button>
</div>
<div class="grid grid-cols-1 md:grid-cols-6 gap-2">
<input v-model="newSynonymTerm" type="text" class="px-3 py-2 border border-slate-200 rounded-lg text-sm" placeholder="term" />
<input v-model="newSynonymValue" type="text" class="px-3 py-2 border border-slate-200 rounded-lg text-sm" placeholder="synonym" />
<input v-model="newSynonymWeight" type="number" min="0.01" step="0.01" class="px-3 py-2 border border-slate-200 rounded-lg text-sm" placeholder="weight" />
<label class="px-3 py-2 border border-slate-200 rounded-lg text-sm flex items-center gap-2">
<input v-model="newSynonymBidirectional" type="checkbox" />
双向
</label>
<label class="px-3 py-2 border border-slate-200 rounded-lg text-sm flex items-center gap-2">
<input v-model="newSynonymEnabled" type="checkbox" />
启用
</label>
<button
@click="submitSynonym"
:disabled="busy"
class="px-3 py-2 rounded-lg bg-slate-900 text-white text-sm hover:bg-slate-800 disabled:opacity-50"
>
新增
</button>
</div>
<div class="border border-slate-200 rounded-xl overflow-hidden">
<table class="w-full text-sm">
<thead class="bg-slate-50 text-slate-500">
<tr>
<th class="text-left px-3 py-2">term</th>
<th class="text-left px-3 py-2">synonym</th>
<th class="text-left px-3 py-2">权重</th>
<th class="text-left px-3 py-2">双向</th>
<th class="text-left px-3 py-2">状态</th>
<th class="text-right px-3 py-2">操作</th>
</tr>
</thead>
<tbody>
<tr v-if="synonyms.length === 0">
<td colspan="6" class="px-3 py-6 text-center text-slate-400">暂无同义词数据</td>
</tr>
<tr v-for="item in synonyms" :key="item.id" class="border-t border-slate-100">
<td class="px-3 py-2 text-slate-800 break-all">{{ item.term }}</td>
<td class="px-3 py-2 text-slate-700 break-all">{{ item.synonym }}</td>
<td class="px-3 py-2 text-slate-600">{{ item.weight }}</td>
<td class="px-3 py-2 text-slate-600">{{ item.bidirectional ? '是' : '否' }}</td>
<td class="px-3 py-2">
<span
:class="[
'px-2 py-0.5 rounded-full text-xs border',
item.enabled ? 'bg-emerald-50 text-emerald-700 border-emerald-200' : 'bg-slate-50 text-slate-500 border-slate-200'
]"
>
{{ item.enabled ? '启用' : '停用' }}
</span>
</td>
<td class="px-3 py-2">
<div class="flex justify-end gap-1">
<button @click="editSynonym(item)" class="px-2 py-1 text-xs border border-slate-200 rounded hover:bg-slate-50">编辑</button>
<button @click="toggleSynonym(item)" class="px-2 py-1 text-xs border border-slate-200 rounded hover:bg-slate-50">
{{ item.enabled ? '停用' : '启用' }}
</button>
<button @click="removeSynonym(item)" class="px-2 py-1 text-xs border border-red-200 text-red-600 rounded hover:bg-red-50">删除</button>
</div>
</td>
</tr>
</tbody>
</table>
</div>
</section>
</div>
</div>
</template>

166
frontend/src/components/SearchResults.vue Normal file
View File

@ -0,0 +1,166 @@
<script setup>
const formatDate = (timestamp) => {
if (!timestamp) return '-'
return new Date(timestamp * 1000).toLocaleString('zh-CN', {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit'
})
}
const isImageFile = (filename) => {
if (!filename) return false
return /\.(jpg|jpeg|png|gif|bmp|webp|tiff|tif|svg|ico|avif|heic|heif)$/i.test(filename)
}
const isOfficeFile = (filename) => {
if (!filename) return false
return /\.(pdf|doc|docx|ppt|pptx|xlsx|xls)$/i.test(filename)
}
const isExcelFile = (filename) => {
if (!filename) return false
return /\.(xlsx|xls)$/i.test(filename)
}
const getFileEmoji = (filename) => (isImageFile(filename) ? '🖼️' : '📄')
const canHighlight = (result) => isOfficeFile(result.file?.filename)
const openViewer = (result) => {
if (!canHighlight(result)) return
const fileId = String(result.file?.id || '')
const sliceId = String(result.id)
// Excel 文件 → 表格预览器
if (isExcelFile(result.file?.filename)) {
const params = new URLSearchParams({
file_id: fileId,
slice_id: sliceId,
})
window.open(`/excel-viewer.html?${params.toString()}`, '_blank', 'noopener')
return
}
// PDF/Word/PPT → PDF 高亮
const params = new URLSearchParams({
file_id: fileId,
slice_id: sliceId,
})
const url = `/pdf-highlight.html?${params.toString()}`
window.open(url, '_blank', 'noopener')
}
defineProps({
results: {
type: Array,
default: () => []
},
loading: {
type: Boolean,
default: false
}
})
</script>
<template>
<div class="max-w-3xl mx-auto">
<!-- Loading State -->
<div v-if="loading" class="flex justify-center py-12">
<div class="flex items-center gap-3 text-slate-500">
<svg class="animate-spin h-5 w-5" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
<span>搜索中...</span>
</div>
</div>
<!-- Empty State -->
<div v-else-if="results.length === 0" class="text-center py-12">
<div class="w-16 h-16 bg-slate-100 rounded-full flex items-center justify-center mx-auto mb-4">
<svg class="w-8 h-8 text-slate-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.663 17h4.673M12 3v1m6.364 1.636l-.707.707M21 12h-1M4 12H3m3.343-5.657l-.707-.707m2.828 9.9a5 5 0 117.072 0l-.548.547A3.374 3.374 0 0014 18.469V19a2 2 0 11-4 0v-.531c0-.895-.356-1.754-.988-2.386l-.548-.547z" />
</svg>
</div>
<p class="text-slate-500">输入关键词开始搜索</p>
</div>
<!-- Results -->
<div v-else class="space-y-4">
<p class="text-sm text-slate-500 mb-4">找到 {{ results.length }} 个结果</p>
<div
v-for="(result, index) in results"
:key="index"
class="bg-white rounded-xl p-5 border border-slate-200 hover:border-slate-300 hover:shadow-md transition-all duration-200"
:class="canHighlight(result) ? 'cursor-pointer' : 'cursor-default'"
@click="openViewer(result)"
>
<div class="flex items-start gap-4">
<div class="w-10 h-10 bg-linear-to-br from-amber-100 to-orange-100 rounded-lg flex items-center justify-center shrink-0">
<span class="text-lg">{{ getFileEmoji(result.file?.filename) }}</span>
</div>
<div class="flex-1 min-w-0">
<h3 class="font-semibold text-slate-800 mb-1 truncate">
{{ result.file?.filename || '未命名文档' }}
</h3>
<p
v-if="result.snippet"
class="text-slate-600 text-sm line-clamp-2 mb-2 search-snippet"
v-html="result.snippet"
></p>
<p v-else class="text-slate-600 text-sm line-clamp-2 mb-2">
{{ result.content || (isImageFile(result.file?.filename) ? '图片匹配结果' : '无内容预览') }}
</p>
<div class="flex items-center gap-4 text-xs text-slate-400">
<span class="flex items-center gap-1">
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z" />
</svg>
得分: {{ Number.isFinite(result.score) ? result.score.toFixed(3) : '-' }}
</span>
<span v-if="result.kb?.name" class="flex items-center gap-1">
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 19a2 2 0 01-2-2V7a2 2 0 012-2h4l2 2h4a2 2 0 012 2v1M5 19h14a2 2 0 002-2v-5a2 2 0 00-2-2H9a2 2 0 00-2 2v5a2 2 0 01-2 2z" />
</svg>
{{ result.kb.name }}
</span>
<span v-if="result.file?.created_at" class="flex items-center gap-1">
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
上传时间: {{ formatDate(result.file.created_at) }}
</span>
</div>
<div v-if="canHighlight(result)" class="mt-3">
<button
class="inline-flex items-center gap-1 text-xs font-medium text-amber-700 bg-amber-50 px-2.5 py-1 rounded-full hover:bg-amber-100"
@click.stop="openViewer(result)"
>
<span>定位原文</span>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5l7 7-7 7" />
</svg>
</button>
</div>
</div>
</div>
</div>
</div>
</div>
</template>
<style scoped>
.search-snippet :deep(b) {
font-weight: 600;
color: #b45309;
background-color: rgba(251, 191, 36, 0.25);
padding: 0 2px;
border-radius: 2px;
}
</style>

5
frontend/src/main.js Normal file
View File

@ -0,0 +1,5 @@
import { createApp } from 'vue'
import './style.css'
import App from './App.vue'
createApp(App).mount('#app')

10
frontend/src/store.js Normal file
View File

@ -0,0 +1,10 @@
import { ref } from 'vue'
// This is a simple reactive store for cross-component state.
// For larger applications, consider using Pinia.
export const currentKb = ref({ id: null, name: '所有知识库' });
export const setCurrentKb = (kb) => {
currentKb.value = kb;
};

1
frontend/src/style.css Normal file
View File

@ -0,0 +1 @@
@import "tailwindcss";

25
frontend/vite.config.js Normal file
View File

@ -0,0 +1,25 @@
import { defineConfig } from 'vite'
import vue from '@vitejs/plugin-vue'
import tailwindcss from '@tailwindcss/vite'
export default defineConfig({
plugins: [vue(), tailwindcss()],
optimizeDeps: {
include: ['pdfjs-dist'],
},
resolve: {
dedupe: ['pdfjs-dist'],
},
build: {
outDir: 'dist',
emptyOutDir: true,
},
server: {
proxy: {
'/api': {
target: 'http://localhost:3000',
changeOrigin: true,
},
},
},
})

106
src/api/common.rs Normal file
View File

@ -0,0 +1,106 @@
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
use crate::{
AuthUser, api::{
error::{ApiError, ApiResult}, knowledge_base
}
};
/// 要求当前用户为 admin，否则返回 BadRequest。
pub fn ensure_admin(auth_user: &AuthUser) -> ApiResult<()> {
if auth_user.is_admin() { Ok(()) } else { Err(ApiError::BadRequest("admin role required".to_string())) }
}
/// 校验用户能否访问指定知识库（owner / public / 显式授权）。不存在或无权限返回 NotFound。
pub async fn ensure_kb_accessible(pool: &SqlitePool, kb_id: i64, user_id: &str, is_admin: bool) -> ApiResult<()> {
let perm = knowledge_base::get_kb_permission(pool, kb_id, user_id, is_admin).await;
if perm.is_none() {
return Err(ApiError::NotFound("Knowledge base not found or permission denied".to_string()));
}
Ok(())
}
/// 校验用户是否为指定知识库的 editor / admin。无权限返回 Forbidden。
pub async fn ensure_kb_editor_or_admin(pool: &SqlitePool, kb_id: i64, auth_user: &AuthUser) -> ApiResult<()> {
let perm = knowledge_base::get_kb_permission(pool, kb_id, &auth_user.user_id, auth_user.is_admin()).await;
if !knowledge_base::meets_requirement(perm.as_deref(), "editor") {
return Err(ApiError::Forbidden("Permission denied. Requires editor or admin.".to_string()));
}
Ok(())
}
/// 追加知识库访问过滤条件（不含别名）。
/// 生成 `AND (user_id = ? OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ?))`
pub fn push_kb_access_filter<'a>(qb: &mut QueryBuilder<'a, Sqlite>, user_id: &'a str) {
qb.push(" AND (user_id = ");
qb.push_bind(user_id);
qb.push(" OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ");
qb.push_bind(user_id);
qb.push(")");
qb.push(")");
}
/// 追加知识库访问过滤条件（带 `kb.` 别名）。
/// 生成 `WHERE (kb.user_id = ? OR kb.is_public = 1 OR kb.id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ?))`
pub fn push_kb_access_filter_where<'a>(qb: &mut QueryBuilder<'a, Sqlite>, user_id: &'a str) {
qb.push(" WHERE (kb.user_id = ");
qb.push_bind(user_id);
qb.push(" OR kb.is_public = 1 OR kb.id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ");
qb.push_bind(user_id);
qb.push(")");
qb.push(")");
}
/// 追加文件访问过滤条件。
/// 生成 `(f.user_id = ? OR f.is_public = 1)`；调用方需自行在前面加 `AND `。
pub fn push_file_access_filter<'a>(qb: &mut QueryBuilder<'a, Sqlite>, user_id: &'a str, alias: Option<&'a str>) {
let col_prefix = alias.map(|a| format!("{}.", a)).unwrap_or_default();
qb.push("(");
qb.push(format!("{}user_id", col_prefix));
qb.push(" = ");
qb.push_bind(user_id);
qb.push(format!(" OR {}is_public = 1", col_prefix));
qb.push(")");
}
/// 收集某个知识库的所有后代知识库 ID（包含自身）。
/// 若 `exclude_storage` 为 true，则结果中过滤掉 `kb_type = 'storage'` 的节点。
pub async fn collect_kb_descendant_ids(
pool: &SqlitePool, root_kb_id: i64, exclude_storage: bool,
) -> Result<Vec<i64>, sqlx::Error> {
let rows: Vec<(i64,)> = if exclude_storage {
sqlx::query_as(
r#"
WITH RECURSIVE descendants AS (
SELECT id, kb_type FROM knowledge_bases WHERE id = ?
UNION ALL
SELECT kb.id, kb.kb_type
FROM knowledge_bases kb
INNER JOIN descendants d ON kb.parent_id = d.id
)
SELECT id FROM descendants WHERE kb_type != ?;
"#,
)
.bind(root_kb_id)
.bind(knowledge_base::KB_TYPE_STORAGE)
.fetch_all(pool)
.await?
} else {
sqlx::query_as(
r#"
WITH RECURSIVE descendants AS (
SELECT id FROM knowledge_bases WHERE id = ?
UNION ALL
SELECT kb.id
FROM knowledge_bases kb
INNER JOIN descendants d ON kb.parent_id = d.id
)
SELECT id FROM descendants;
"#,
)
.bind(root_kb_id)
.fetch_all(pool)
.await?
};
Ok(rows.into_iter().map(|(id,)| id).collect())
}

125
src/api/error.rs Normal file
View File

@ -0,0 +1,125 @@
use std::{fmt, num::ParseIntError, result};
use axum::{
Json, http::StatusCode, response::{IntoResponse, Response}
};
use serde::Serialize;
use sqlx;
/// Unified API error type for handlers to return.
///
/// Example:
/// - `Result<T, ApiError>` is aliased as `ApiResult<T>` below.
/// - Convert database errors with `ApiError::from(sqlx::Error)`.
#[derive(Debug)]
pub enum ApiError {
BadRequest(String),
// Unauthorized(String),
Forbidden(String),
NotFound(String),
Database(sqlx::Error),
System(std::io::Error),
Internal(String),
}
impl fmt::Display for ApiError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ApiError::BadRequest(msg) => write!(f, "BadRequest: {}", msg),
// ApiError::Unauthorized(msg) => write!(f, "Unauthorized: {}", msg),
ApiError::Forbidden(msg) => write!(f, "Forbidden: {}", msg),
ApiError::NotFound(msg) => write!(f, "NotFound: {}", msg),
ApiError::Database(e) => write!(f, "Database error: {}", e),
ApiError::System(e) => write!(f, "System error: {}", e),
ApiError::Internal(msg) => write!(f, "Internal error: {}", msg),
}
}
}
impl std::error::Error for ApiError {}
impl From<sqlx::Error> for ApiError {
fn from(e: sqlx::Error) -> Self {
ApiError::Database(e)
}
}
impl From<std::io::Error> for ApiError {
fn from(e: std::io::Error) -> Self {
ApiError::System(e)
}
}
impl From<axum::extract::multipart::MultipartError> for ApiError {
fn from(e: axum::extract::multipart::MultipartError) -> Self {
ApiError::Internal(format!("Multipart error: {}", e))
}
}
impl From<ParseIntError> for ApiError {
fn from(e: ParseIntError) -> Self {
ApiError::BadRequest(format!("Int error: {}", e))
}
}
impl From<anyhow::Error> for ApiError {
fn from(e: anyhow::Error) -> Self {
ApiError::Internal(format!("Internal error: {}", e))
}
}
impl From<serde_json::Error> for ApiError {
fn from(e: serde_json::Error) -> Self {
ApiError::BadRequest(format!("JSON error: {}", e))
}
}
/// Standardized JSON body returned for errors.
#[derive(Serialize)]
struct ErrorBody {
code: u16,
error: String,
// optional: you can add `details: Option<String>` if you want more info
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
// Map each error variant to an HTTP status code and message.
let (status, message) = match &self {
ApiError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg.clone()),
// ApiError::Unauthorized(msg) => (StatusCode::UNAUTHORIZED, msg.clone()),
ApiError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.clone()),
ApiError::NotFound(msg) => (StatusCode::NOT_FOUND, msg.clone()),
ApiError::Database(e) => {
// Log detailed error server-side but avoid leaking internals to clients.
log::error!("Database error: {}", e);
(StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".into())
}
ApiError::System(e) => {
// Log detailed error server-side but avoid leaking internals to clients.
log::error!("System error: {}", e);
(StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".into())
}
ApiError::Internal(msg) => {
log::error!("Internal error: {}", msg);
(StatusCode::INTERNAL_SERVER_ERROR, msg.clone())
}
};
let body = ErrorBody { code: status.as_u16(), error: message };
(status, Json(body)).into_response()
}
}
/// Convenience alias for handlers' return type.
pub type ApiResult<T> = result::Result<T, ApiError>;
impl ApiError {
/// Helper to create a generic internal error (also logs).
pub fn internal<M: Into<String>>(msg: M) -> Self {
let msg = msg.into();
log::error!("Internal error created: {}", msg);
ApiError::Internal(msg)
}
}

3245
src/api/file.rs Normal file
View File

File diff suppressed because it is too large Load Diff

377
src/api/graph.rs Normal file
View File

@ -0,0 +1,377 @@
use std::collections::HashMap;
use axum::{
Extension, extract::{Path, Query, State}, response::Json
};
use serde::{Deserialize, Serialize};
use sqlx::SqlitePool;
use utoipa::{IntoParams, ToSchema};
use crate::{AuthUser, api::error::ApiError};
// Type aliases for complex SQL row types
#[allow(clippy::type_complexity)]
type EntityRow = (i64, String, String, Option<String>, Option<i64>, Option<i64>, i64);
#[allow(clippy::type_complexity)]
type NeighborRow = (i64, String, String, Option<String>, Option<i64>, Option<i64>, i64, String);
type MentionRow = (i64, String, i64, String);
/// 实体信息
#[derive(Debug, Serialize, ToSchema)]
pub struct EntityInfo {
pub id: i64,
pub name: String,
pub entity_type: String,
pub properties: HashMap<String, String>,
pub file_id: Option<i64>,
pub kb_id: Option<i64>,
pub created_at: i64,
}
/// 实体搜索参数
#[derive(Debug, Deserialize, IntoParams)]
pub struct EntitySearchParams {
pub q: Option<String>, // 搜索关键词
pub entity_type: Option<String>, // 实体类型筛选
pub kb_id: Option<i64>, // 知识库ID筛选
pub file_id: Option<i64>, // 文件ID筛选
pub limit: Option<i64>, // 返回数量限制
}
/// 实体查询
#[utoipa::path(
get,
path = "/api/v1/knowledge/graph/entities",
operation_id = "graph_search_entities",
tag = "graph",
params(EntitySearchParams),
responses(
(status = 200, description = "成功返回实体列表", body = Vec<EntityInfo>),
(status = 400, description = "请求参数错误")
)
)]
pub async fn search_entities(
Query(params): Query<EntitySearchParams>, State(pool): State<SqlitePool>, Extension(auth_user): Extension<AuthUser>,
) -> Result<Json<Vec<EntityInfo>>, ApiError> {
let is_admin = auth_user.is_admin();
// Check KB permission if kb_id is specified
if let Some(kb_id) = params.kb_id {
let perm = crate::api::knowledge_base::get_kb_permission(&pool, kb_id, &auth_user.user_id, is_admin).await;
if !crate::api::knowledge_base::meets_requirement(perm.as_deref(), "viewer") {
return Err(ApiError::Forbidden("Permission denied.".to_string()));
}
}
let mut sql =
"SELECT id, name, entity_type, properties, file_id, kb_id, created_at FROM graph_nodes WHERE 1=1".to_string();
// 构建查询条件
if params.q.is_some() {
sql.push_str(" AND name LIKE '%' || ? || '%'");
}
if params.entity_type.is_some() {
sql.push_str(" AND entity_type = ?");
}
if params.kb_id.is_some() {
sql.push_str(" AND kb_id = ?");
}
if params.file_id.is_some() {
sql.push_str(" AND file_id = ?");
}
sql.push_str(" ORDER BY created_at DESC");
if let Some(limit) = params.limit {
sql.push_str(&format!(" LIMIT {}", limit));
} else {
sql.push_str(" LIMIT 100");
}
// 执行查询
let mut query = sqlx::query_as::<_, (i64, String, String, Option<String>, Option<i64>, Option<i64>, i64)>(&sql);
if let Some(ref q) = params.q {
query = query.bind(q);
}
if let Some(ref entity_type) = params.entity_type {
query = query.bind(entity_type);
}
if let Some(kb_id) = params.kb_id {
query = query.bind(kb_id);
}
if let Some(file_id) = params.file_id {
query = query.bind(file_id);
}
let rows = query.fetch_all(&pool).await?;
// Preload accessible KB ids for filtering
let allowed_kb_ids: std::collections::HashSet<i64> = if is_admin {
std::collections::HashSet::new()
} else {
crate::api::knowledge_base::get_user_viewable_kb_ids(&pool, &auth_user.user_id, false)
.await
.into_iter()
.collect()
};
let entities: Vec<EntityInfo> = rows
.into_iter()
.filter_map(|(id, name, entity_type, properties_json, file_id, kb_id, created_at)| {
// Filter by KB permission
if let Some(kid) = kb_id
&& !is_admin
&& !allowed_kb_ids.contains(&kid)
{
return None;
}
let properties: HashMap<String, String> = if let Some(ref json) = properties_json {
serde_json::from_str(json).unwrap_or_default()
} else {
HashMap::new()
};
Some(EntityInfo { id, name, entity_type, properties, file_id, kb_id, created_at })
})
.collect();
Ok(Json(entities))
}
/// 实体详情（包含邻居和提及）
#[derive(Debug, Serialize, ToSchema)]
pub struct EntityDetail {
pub entity: EntityInfo,
pub neighbors: Vec<NeighborInfo>,
pub mentions: Vec<MentionInfo>,
}
#[derive(Debug, Serialize, ToSchema)]
pub struct NeighborInfo {
pub entity: EntityInfo,
pub relation_type: String,
pub direction: String, // "outgoing" or "incoming"
}
#[derive(Debug, Serialize, ToSchema)]
pub struct MentionInfo {
pub slice_id: i64,
pub context: String,
pub file_id: i64,
pub filename: String,
}
/// 获取实体详情
#[utoipa::path(
get,
path = "/api/v1/knowledge/graph/entities/{id}",
operation_id = "graph_get_entity",
tag = "graph",
params(
("id" = i64, Path, description = "实体 ID")
),
responses(
(status = 200, description = "成功返回实体详情", body = EntityDetail),
(status = 404, description = "实体不存在")
)
)]
pub async fn get_entity(
Path(id): Path<i64>, State(pool): State<SqlitePool>, Extension(auth_user): Extension<AuthUser>,
) -> Result<Json<EntityDetail>, ApiError> {
let is_admin = auth_user.is_admin();
// 查询实体基本信息
let entity_sql =
"SELECT id, name, entity_type, properties, file_id, kb_id, created_at FROM graph_nodes WHERE id = ?";
let entity_row: Option<EntityRow> = sqlx::query_as(entity_sql).bind(id).fetch_optional(&pool).await?;
let entity_row = entity_row.ok_or_else(|| ApiError::Internal("Entity not found".to_string()))?;
// Check permission on the entity's KB
if let Some(kb_id) = entity_row.5 {
let perm = crate::api::knowledge_base::get_kb_permission(&pool, kb_id, &auth_user.user_id, is_admin).await;
if !crate::api::knowledge_base::meets_requirement(perm.as_deref(), "viewer") {
return Err(ApiError::Forbidden("Permission denied.".to_string()));
}
}
let properties: HashMap<String, String> =
if let Some(ref json) = entity_row.3 { serde_json::from_str(json).unwrap_or_default() } else { HashMap::new() };
let entity = EntityInfo {
id: entity_row.0,
name: entity_row.1,
entity_type: entity_row.2,
properties,
file_id: entity_row.4,
kb_id: entity_row.5,
created_at: entity_row.6,
};
// 出边、入边、提及三条查询相互独立，并发执行
let outgoing_sql = "SELECT n.id, n.name, n.entity_type, n.properties, n.file_id, n.kb_id, n.created_at, e.relation_type \
FROM graph_edges e \
INNER JOIN graph_nodes n ON e.target_node_id = n.id \
WHERE e.source_node_id = ? LIMIT 50";
let incoming_sql = "SELECT n.id, n.name, n.entity_type, n.properties, n.file_id, n.kb_id, n.created_at, e.relation_type \
FROM graph_edges e \
INNER JOIN graph_nodes n ON e.source_node_id = n.id \
WHERE e.target_node_id = ? LIMIT 50";
let mentions_sql = "SELECT m.slice_id, m.context, s.file_id, f.filename \
FROM entity_mentions m \
INNER JOIN slices s ON m.slice_id = s.id \
INNER JOIN files f ON s.file_id = f.id \
WHERE m.node_id = ? LIMIT 20";
let (outgoing_rows, incoming_rows, mentions_rows): (Vec<NeighborRow>, Vec<NeighborRow>, Vec<MentionRow>) = tokio::try_join!(
sqlx::query_as(outgoing_sql).bind(id).fetch_all(&pool),
sqlx::query_as(incoming_sql).bind(id).fetch_all(&pool),
sqlx::query_as(mentions_sql).bind(id).fetch_all(&pool),
)?;
let mut neighbors = Vec::new();
for row in outgoing_rows {
let properties: HashMap<String, String> =
if let Some(ref json) = row.3 { serde_json::from_str(json).unwrap_or_default() } else { HashMap::new() };
neighbors.push(NeighborInfo {
entity: EntityInfo {
id: row.0,
name: row.1,
entity_type: row.2,
properties,
file_id: row.4,
kb_id: row.5,
created_at: row.6,
},
relation_type: row.7,
direction: "outgoing".to_string(),
});
}
for row in incoming_rows {
let properties: HashMap<String, String> =
if let Some(ref json) = row.3 { serde_json::from_str(json).unwrap_or_default() } else { HashMap::new() };
neighbors.push(NeighborInfo {
entity: EntityInfo {
id: row.0,
name: row.1,
entity_type: row.2,
properties,
file_id: row.4,
kb_id: row.5,
created_at: row.6,
},
relation_type: row.7,
direction: "incoming".to_string(),
});
}
let mentions: Vec<MentionInfo> = mentions_rows
.into_iter()
.map(|(slice_id, context, file_id, filename)| MentionInfo { slice_id, context, file_id, filename })
.collect();
Ok(Json(EntityDetail { entity, neighbors, mentions }))
}
/// 图统计信息
#[derive(Debug, Serialize, ToSchema)]
pub struct GraphStats {
pub node_count: i64,
pub edge_count: i64,
pub entity_types: HashMap<String, i64>,
pub relation_types: HashMap<String, i64>,
}
#[derive(Debug, Deserialize, IntoParams)]
pub struct StatsParams {
pub kb_id: Option<i64>,
pub file_id: Option<i64>,
}
/// 获取图统计信息
#[utoipa::path(
get,
path = "/api/v1/knowledge/graph/stats",
operation_id = "graph_get_graph_stats",
tag = "graph",
params(StatsParams),
responses(
(status = 200, description = "成功返回图统计信息", body = GraphStats),
(status = 400, description = "请求参数错误")
)
)]
pub async fn get_graph_stats(
Query(params): Query<StatsParams>, State(pool): State<SqlitePool>, Extension(auth_user): Extension<AuthUser>,
) -> Result<Json<GraphStats>, ApiError> {
let is_admin = auth_user.is_admin();
// Check KB permission if kb_id is specified
if let Some(kb_id) = params.kb_id {
let perm = crate::api::knowledge_base::get_kb_permission(&pool, kb_id, &auth_user.user_id, is_admin).await;
if !crate::api::knowledge_base::meets_requirement(perm.as_deref(), "viewer") {
return Err(ApiError::Forbidden("Permission denied.".to_string()));
}
}
// 构建 WHERE 子句
let where_clause = if let Some(file_id) = params.file_id {
format!("WHERE file_id = {}", file_id)
} else if let Some(kb_id) = params.kb_id {
format!("WHERE kb_id = {}", kb_id)
} else {
"".to_string()
};
// 节点数量
let node_count_sql = format!("SELECT COUNT(*) FROM graph_nodes {}", where_clause);
let node_count: (i64,) = sqlx::query_as(&node_count_sql).fetch_one(&pool).await?;
// 边数量
let edge_count_sql = if let Some(file_id) = params.file_id {
format!("SELECT COUNT(*) FROM graph_edges WHERE file_id = {}", file_id)
} else if let Some(kb_id) = params.kb_id {
format!(
"SELECT COUNT(*) FROM graph_edges e \
INNER JOIN graph_nodes n ON e.source_node_id = n.id \
WHERE n.kb_id = {}",
kb_id
)
} else {
"SELECT COUNT(*) FROM graph_edges".to_string()
};
let edge_count: (i64,) = sqlx::query_as(&edge_count_sql).fetch_one(&pool).await?;
// 实体类型分布
let entity_types_sql =
format!("SELECT entity_type, COUNT(*) FROM graph_nodes {} GROUP BY entity_type", where_clause);
let entity_types_rows: Vec<(String, i64)> = sqlx::query_as(&entity_types_sql).fetch_all(&pool).await?;
let entity_types: HashMap<String, i64> = entity_types_rows.into_iter().collect();
// 关系类型分布
let relation_types_sql = if let Some(file_id) = params.file_id {
format!(
"SELECT relation_type, COUNT(*) FROM graph_edges \
WHERE file_id = {} GROUP BY relation_type",
file_id
)
} else if let Some(kb_id) = params.kb_id {
format!(
"SELECT e.relation_type, COUNT(*) FROM graph_edges e \
INNER JOIN graph_nodes n ON e.source_node_id = n.id \
WHERE n.kb_id = {} GROUP BY e.relation_type",
kb_id
)
} else {
"SELECT relation_type, COUNT(*) FROM graph_edges GROUP BY relation_type".to_string()
};
let relation_types_rows: Vec<(String, i64)> = sqlx::query_as(&relation_types_sql).fetch_all(&pool).await?;
let relation_types: HashMap<String, i64> = relation_types_rows.into_iter().collect();
Ok(Json(GraphStats { node_count: node_count.0, edge_count: edge_count.0, entity_types, relation_types }))
}

1818
src/api/knowledge_base.rs Normal file
View File

File diff suppressed because it is too large Load Diff

254
src/api/mod.rs Normal file
View File

@ -0,0 +1,254 @@
use axum::{
Extension, Router,
routing::{delete, get, post, put},
};
use sqlx::SqlitePool;
use utoipa::OpenApi;
mod common;
mod error;
mod file;
mod graph;
mod knowledge_base;
mod search;
mod system;
// 重新导出 File 类型供其他模块使用
pub use file::File;
pub(crate) use file::{
FILE_COLS_NO_CONTENT, backfill_missing_image_meta_for_files, collect_image_paths_for_files,
collect_image_raw_paths_for_files, effective_parse_file_id, find_reusable_parsed_file, remove_image_files,
resolve_image_storage_path, update_file_custom_image_meta,
};
use crate::search::SearchEngine;
/// OpenAPI 文档定义
#[derive(OpenApi)]
#[openapi(
paths(
// Knowledge Base
knowledge_base::list,
knowledge_base::create,
knowledge_base::get,
knowledge_base::get_files,
knowledge_base::get_tags,
knowledge_base::update,
knowledge_base::reparse,
knowledge_base::reparse_by_id,
knowledge_base::delete,
knowledge_base::tree,
knowledge_base::batch_export_kb,
knowledge_base::list_permissions,
knowledge_base::add_permission,
knowledge_base::remove_permission,
// File
file::upload,
file::list,
file::stats,
file::get,
file::update,
file::move_to_kb,
file::batch_delete,
file::reparse_failed,
file::delete,
file::get_slices,
file::update_slices,
file::get_image_by_filename,
file::download,
file::get_highlighted_pdf,
file::get_slice_highlight,
file::get_slice_highlight_page,
file::excel_data,
file::archive_entries,
file::archive_extract,
file::archive_download,
// Search
search::search,
search::search_full,
search::search_summary,
search::search_with_graph,
search::search_image,
search::search_image_by_text,
search::advanced_search_stream,
search::list_lexicons,
search::create_lexicon,
search::update_lexicon,
search::delete_lexicon,
search::toggle_lexicon_enabled,
search::reload_lexicon,
search::publish_lexicon,
search::list_synonyms,
search::create_synonym,
search::update_synonym,
search::delete_synonym,
search::toggle_synonym_enabled,
// Graph
graph::search_entities,
graph::get_entity,
graph::get_graph_stats,
// System
system::heap_profile,
system::heap_profile_pdf,
system::lancedb_compact,
system::index_force_merge,
system::index_rebuild_status,
),
components(
schemas(
knowledge_base::Knowledge,
knowledge_base::KnowledgeResponse,
knowledge_base::KnowledgeDetailResponse,
knowledge_base::KnowledgeBaseFilesResponse,
knowledge_base::KnowledgeBaseTagsResponse,
knowledge_base::TagFileCount,
knowledge_base::KnowledgeTreeFile,
knowledge_base::KnowledgeTreeNode,
knowledge_base::KnowledgeCreateReq,
knowledge_base::KnowledgeUpdateReq,
knowledge_base::ReparseKnowledgeBaseResponse,
knowledge_base::ExportKbResponse,
knowledge_base::BatchExportKbRequest,
knowledge_base::KbPermissionItem,
knowledge_base::KbPermissionCreateReq,
crate::export::ExportManifest,
file::File,
file::FileListResponse,
file::UpdateFileReq,
file::MoveFileReq,
file::BatchDeleteFilesReq,
file::BatchDeleteFilesResp,
file::BatchDeleteSkippedItem,
file::BatchDeleteCleanupFailedItem,
file::ReparseFailedFilesReq,
file::ReparseFailedFilesResp,
file::Slice,
file::UpdateSliceItem,
file::UpdateSlicesReq,
file::SlicePosition,
file::SliceHighlight,
file::SliceHighlightPage,
file::FileStatusBreakdown,
file::ExcelData,
file::ExcelSheetData,
crate::archive::ArchiveEntry,
crate::archive::ExtractResult,
search::SearchResult,
search::SearchResultItem,
search::FullSearchResult,
search::FullSearchResultItem,
search::SummarySearchResult,
search::SummarySearchResultItem,
search::FileInfo,
search::KbInfo,
search::LexiconItem,
search::LexiconListResponse,
search::CreateLexiconReq,
search::UpdateLexiconReq,
search::ToggleLexiconReq,
search::DeleteLexiconResponse,
search::ReloadLexiconResponse,
search::PublishLexiconResponse,
search::SynonymItem,
search::SynonymListResponse,
search::CreateSynonymReq,
search::UpdateSynonymReq,
search::ToggleSynonymReq,
search::DeleteSynonymResponse,
graph::EntityInfo,
graph::EntityDetail,
graph::NeighborInfo,
graph::MentionInfo,
graph::GraphStats,
system::MemoryUsage,
system::HeapProfileStatus,
system::LanceDbCompactStats,
system::TantivyForceMergeIndexStats,
system::TantivyForceMergeResponse,
system::IndexRebuildStatus,
)
),
tags(
(name = "knowledge_base", description = "知识库管理接口"),
(name = "file", description = "文件管理接口"),
(name = "search", description = "搜索接口"),
(name = "graph", description = "知识图谱接口"),
(name = "system", description = "系统监控接口")
),
info(
title = "HTKnow API",
version = "0.1.0",
description = "HTKnow 知识库管理系统 API 文档",
),
)]
pub struct ApiDoc;
/// 获取 OpenAPI 文档
pub fn openapi() -> utoipa::openapi::OpenApi {
ApiDoc::openapi()
}
pub fn app(pool: SqlitePool, search_engine: SearchEngine) -> Router {
let knowledge_router = Router::new()
.route("/", get(knowledge_base::list).post(knowledge_base::create))
.route("/reparse", post(knowledge_base::reparse))
.route("/{id}/reparse", post(knowledge_base::reparse_by_id))
.route("/{id}/files", get(knowledge_base::get_files))
.route("/{id}/tags", get(knowledge_base::get_tags))
.route("/export", post(knowledge_base::batch_export_kb))
.route("/tree", get(knowledge_base::tree))
.route("/{id}", get(knowledge_base::get).put(knowledge_base::update).delete(knowledge_base::delete))
.route("/{id}/permissions", get(knowledge_base::list_permissions).post(knowledge_base::add_permission))
.route("/{id}/permissions/{user_id}", delete(knowledge_base::remove_permission));
let file_router = Router::new()
.route("/", get(file::list).post(file::upload))
.route("/batch-delete", post(file::batch_delete))
.route("/reparse-failed", post(file::reparse_failed))
.route("/stats", get(file::stats))
.route("/images/{filename}", get(file::get_image_by_filename))
.route("/{id}", get(file::get).put(file::update).delete(file::delete))
.route("/{id}/move", put(file::move_to_kb))
.route("/{id}/slices", get(file::get_slices).put(file::update_slices))
.route("/{id}/slices/{slice_id}/highlight", get(file::get_slice_highlight))
.route("/{id}/slices/{slice_id}/highlight-page", get(file::get_slice_highlight_page))
.route("/{id}/download", get(file::download))
.route("/{id}/highlighted-pdf", get(file::get_highlighted_pdf))
.route("/{id}/excel-data", get(file::excel_data))
.route("/{id}/archive-entries", get(file::archive_entries))
.route("/{id}/archive-extract", post(file::archive_extract))
.route("/{id}/archive-download", get(file::archive_download));
let search_router = Router::new()
.route("/", get(search::search))
.route("/full", get(search::search_full))
.route("/summary", get(search::search_summary))
.route("/graph", get(search::search_with_graph))
.route("/image", post(search::search_image))
.route("/image-by-text", get(search::search_image_by_text))
.route("/lexicons", get(search::list_lexicons).post(search::create_lexicon))
.route("/lexicons/reload", post(search::reload_lexicon))
.route("/lexicons/publish", post(search::publish_lexicon))
.route("/lexicons/{id}", put(search::update_lexicon).delete(search::delete_lexicon))
.route("/lexicons/{id}/enabled", put(search::toggle_lexicon_enabled))
.route("/synonyms", get(search::list_synonyms).post(search::create_synonym))
.route("/synonyms/{id}", put(search::update_synonym).delete(search::delete_synonym))
.route("/synonyms/{id}/enabled", put(search::toggle_synonym_enabled))
.route("/advanced/stream", get(search::advanced_search_stream));
let graph_router = Router::new()
.route("/entities", get(graph::search_entities))
.route("/entities/{id}", get(graph::get_entity))
.route("/stats", get(graph::get_graph_stats));
let system_router = Router::new()
.route("/heap", get(system::heap_profile))
.route("/heap/pdf", get(system::heap_profile_pdf))
.route("/lancedb/compact", post(system::lancedb_compact))
.route("/index/force-merge", post(system::index_force_merge))
.route("/index/rebuild/status", get(system::index_rebuild_status));
Router::new()
.nest("/knowledge_base/", knowledge_router)
.nest("/files/", file_router)
.nest("/search/", search_router)
.nest("/graph/", graph_router)
.nest("/system/", system_router)
.with_state(pool)
.layer(Extension(search_engine))
}

2960
src/api/search.rs Normal file
View File

File diff suppressed because it is too large Load Diff

518
src/api/system.rs Normal file
View File

@ -0,0 +1,518 @@
use axum::{Extension, Json, extract::State, response::Response};
use chrono::Utc;
use serde::Serialize;
use sqlx::SqlitePool;
use utoipa::ToSchema;
use crate::{
AuthUser, api::{
common, error::{ApiError, ApiResult}
}, search::{SearchEngine, tantivy_engine::ForceMergeStats}
};
/// 进程内存占用
#[derive(Debug, Serialize, ToSchema)]
pub struct MemoryUsage {
/// 进程 ID
pub pid: u32,
/// 常驻内存（RSS），单位：字节
pub rss_bytes: u64,
/// 虚拟内存，单位：字节
pub virtual_bytes: u64,
}
/// 堆分析状态
#[derive(Debug, Serialize, ToSchema)]
pub struct HeapProfileStatus {
/// jemalloc 版本
pub jemalloc_version: Option<String>,
/// 是否启用 heap profiling（opt.prof）
pub prof_enabled: Option<bool>,
/// 是否开启采样（prof.active）
pub prof_active: Option<bool>,
/// jemalloc 构建时默认配置
pub malloc_conf: Option<String>,
/// 运行时环境变量 MALLOC_CONF
pub env_malloc_conf: Option<String>,
/// 状态读取失败的提示
pub warnings: Vec<String>,
}
/// LanceDB compact 统计信息
#[derive(Debug, Serialize, ToSchema)]
pub struct LanceDbCompactStats {
/// 删除行数（本次 compact 清理）
pub deleted_rows: u64,
/// compact 前被标记删除的行数
pub deleted_rows_before: u64,
/// compact 后被标记删除的行数
pub deleted_rows_after: u64,
/// compact 前总行数
pub total_rows_before: u64,
/// compact 后总行数
pub total_rows_after: u64,
/// compact 前磁盘大小（字节）
pub size_before_bytes: u64,
/// compact 后磁盘大小（字节）
pub size_after_bytes: u64,
}
#[derive(Debug, Serialize, ToSchema)]
pub struct IndexRebuildStatus {
/// 任务 ID（无任务时为 null）
pub job_id: Option<i64>,
/// 任务状态：idle/running/completed/failed
pub status: String,
/// 当前阶段描述
pub phase: String,
/// 总文档数
pub total_docs: i64,
/// 已处理文档数
pub processed_docs: i64,
/// 进度百分比（0-100）
pub progress_pct: f32,
/// 已运行秒数
pub elapsed_secs: Option<i64>,
/// 预计剩余秒数（ETA）
pub eta_secs: Option<i64>,
/// 任务开始时间（秒级时间戳）
pub started_at: Option<i64>,
/// 最近更新时间（秒级时间戳）
pub updated_at: Option<i64>,
/// 任务结束时间（秒级时间戳）
pub finished_at: Option<i64>,
/// 失败时错误信息
pub error: Option<String>,
}
/// 单个 Tantivy 索引 force merge 统计信息
#[derive(Debug, Serialize, ToSchema)]
pub struct TantivyForceMergeIndexStats {
/// 索引名称：index/full_index
pub index: String,
/// merge 前 segment 数
pub before_segments: usize,
/// merge 后 segment 数
pub after_segments: usize,
/// merge 前存活文档数
pub before_docs: u64,
/// merge 后存活文档数
pub after_docs: u64,
/// merge 前已删除文档数
pub before_deleted_docs: u64,
/// merge 后已删除文档数
pub after_deleted_docs: u64,
/// GC 删除的不再被索引引用的文件数
pub gc_deleted_files: usize,
/// GC 删除失败的文件数
pub gc_failed_files: usize,
/// 是否因为 segment 少于 2 个而跳过
pub skipped: bool,
/// 单个索引耗时（毫秒）
pub duration_ms: u128,
}
/// Tantivy force merge 响应
#[derive(Debug, Serialize, ToSchema)]
pub struct TantivyForceMergeResponse {
/// 普通切片索引统计
pub index: TantivyForceMergeIndexStats,
/// 全文索引统计
pub full_index: TantivyForceMergeIndexStats,
/// 总耗时（毫秒）
pub total_duration_ms: u128,
}
#[derive(Debug, sqlx::FromRow)]
struct IndexRebuildStatusRow {
id: i64,
status: String,
phase: String,
total_docs: i64,
processed_docs: i64,
started_at: Option<i64>,
updated_at: Option<i64>,
finished_at: Option<i64>,
error: Option<String>,
}
/// 生成当前进程堆内存快照（jemalloc heap profile）
#[utoipa::path(
get,
path = "/api/v1/knowledge/system/heap",
operation_id = "system_heap_profile",
tag = "system",
responses(
(status = 200, description = "成功返回堆分析快照", body = Vec<u8>, content_type = "application/octet-stream"),
(status = 400, description = "未启用堆分析"),
(status = 500, description = "生成堆分析快照失败")
)
)]
pub async fn heap_profile() -> ApiResult<Response> {
heap_profile_impl().await
}
/// 生成堆内存分析 PDF 报告（使用 jeprof）
#[utoipa::path(
get,
path = "/api/v1/knowledge/system/heap/pdf",
operation_id = "system_heap_profile_pdf",
tag = "system",
responses(
(status = 200, description = "成功返回堆分析 PDF 报告", body = Vec<u8>, content_type = "application/pdf"),
(status = 400, description = "未启用堆分析或 jeprof 不可用"),
(status = 500, description = "生成堆分析 PDF 失败")
)
)]
pub async fn heap_profile_pdf() -> ApiResult<Response> {
heap_profile_pdf_impl().await
}
/// 主动触发 LanceDB compact
#[utoipa::path(
post,
path = "/api/v1/knowledge/system/lancedb/compact",
operation_id = "system_lancedb_compact",
tag = "system",
responses(
(status = 200, description = "LanceDB compact 成功", body = LanceDbCompactStats),
(status = 500, description = "LanceDB compact 失败")
),
security(
("x-user-id" = []),
("x-role" = [])
)
)]
pub async fn lancedb_compact(
Extension(search_engine): Extension<SearchEngine>,
) -> ApiResult<Json<LanceDbCompactStats>> {
let stats = search_engine
.compact_lancedb()
.await
.map_err(|e| ApiError::internal(format!("LanceDB compact failed: {}", e)))?;
Ok(Json(LanceDbCompactStats {
deleted_rows: stats.deleted_rows,
deleted_rows_before: stats.deleted_rows_before,
deleted_rows_after: stats.deleted_rows_after,
total_rows_before: stats.total_rows_before,
total_rows_after: stats.total_rows_after,
size_before_bytes: stats.size_before_bytes,
size_after_bytes: stats.size_after_bytes,
}))
}
/// 强制合并 Tantivy segment，减少索引碎片
#[utoipa::path(
post,
path = "/api/v1/knowledge/system/index/force-merge",
operation_id = "system_index_force_merge",
tag = "system",
responses(
(status = 200, description = "Tantivy force merge 成功", body = TantivyForceMergeResponse),
(status = 400, description = "权限不足"),
(status = 500, description = "Tantivy force merge 失败")
),
security(
("x-user-id" = []),
("x-role" = [])
)
)]
pub async fn index_force_merge(
Extension(search_engine): Extension<SearchEngine>, Extension(auth_user): Extension<AuthUser>,
) -> ApiResult<Json<TantivyForceMergeResponse>> {
common::ensure_admin(&auth_user)?;
let start = std::time::Instant::now();
let (index_stats, full_index_stats) = search_engine
.force_merge_tantivy_indexes()
.await
.map_err(|e| ApiError::internal(format!("Tantivy force merge failed: {}", e)))?;
Ok(Json(TantivyForceMergeResponse {
index: force_merge_index_stats("index", index_stats),
full_index: force_merge_index_stats("full_index", full_index_stats),
total_duration_ms: start.elapsed().as_millis(),
}))
}
/// 查询索引重建进度与 ETA
#[utoipa::path(
get,
path = "/api/v1/knowledge/system/index/rebuild/status",
operation_id = "system_index_rebuild_status",
tag = "system",
responses(
(status = 200, description = "成功返回索引重建状态", body = IndexRebuildStatus)
),
security(
("x-user-id" = []),
("x-role" = [])
)
)]
pub async fn index_rebuild_status(
State(pool): State<SqlitePool>, Extension(auth_user): Extension<AuthUser>,
) -> ApiResult<Json<IndexRebuildStatus>> {
common::ensure_admin(&auth_user)?;
let row: Option<IndexRebuildStatusRow> = sqlx::query_as(
"SELECT id, status, phase, total_docs, processed_docs, started_at, updated_at, finished_at, error
FROM index_rebuild_jobs
ORDER BY CASE WHEN status = 'running' THEN 0 ELSE 1 END, id DESC
LIMIT 1",
)
.fetch_optional(&pool)
.await?;
let Some(row) = row else {
return Ok(Json(IndexRebuildStatus {
job_id: None,
status: "idle".to_string(),
phase: String::new(),
total_docs: 0,
processed_docs: 0,
progress_pct: 0.0,
elapsed_secs: None,
eta_secs: None,
started_at: None,
updated_at: None,
finished_at: None,
error: None,
}));
};
let total_docs = row.total_docs.max(0);
let processed_docs = row.processed_docs.clamp(0, total_docs.max(0));
let progress_pct = if total_docs > 0 {
((processed_docs as f64 / total_docs as f64) * 100.0).clamp(0.0, 100.0) as f32
} else if row.status == "completed" {
100.0
} else {
0.0
};
let now = Utc::now().timestamp();
let elapsed_secs = match row.started_at {
Some(start) => match row.status.as_str() {
"running" => Some((now - start).max(0)),
_ => row.finished_at.map(|finish| (finish - start).max(0)).or(Some((now - start).max(0))),
},
None => None,
};
let eta_secs = if row.status == "running" && total_docs > processed_docs {
match elapsed_secs {
Some(elapsed) if elapsed > 0 && processed_docs > 0 => {
let speed = processed_docs as f64 / elapsed as f64;
if speed > 0.0 {
let remaining = (total_docs - processed_docs) as f64;
Some((remaining / speed).ceil() as i64)
} else {
None
}
}
_ => None,
}
} else if row.status == "completed" {
Some(0)
} else {
None
};
Ok(Json(IndexRebuildStatus {
job_id: Some(row.id),
status: row.status,
phase: row.phase,
total_docs,
processed_docs,
progress_pct,
elapsed_secs,
eta_secs,
started_at: row.started_at,
updated_at: row.updated_at,
finished_at: row.finished_at,
error: row.error,
}))
}
fn force_merge_index_stats(index: &str, stats: ForceMergeStats) -> TantivyForceMergeIndexStats {
TantivyForceMergeIndexStats {
index: index.to_string(),
before_segments: stats.before_segments,
after_segments: stats.after_segments,
before_docs: stats.before_docs,
after_docs: stats.after_docs,
before_deleted_docs: stats.before_deleted_docs,
after_deleted_docs: stats.after_deleted_docs,
gc_deleted_files: stats.gc_deleted_files,
gc_failed_files: stats.gc_failed_files,
skipped: stats.skipped,
duration_ms: stats.duration_ms,
}
}
#[cfg(feature = "profiling")]
async fn heap_profile_impl() -> ApiResult<Response> {
use std::{
ffi::CString, io::Seek, time::{SystemTime, UNIX_EPOCH}
};
use axum::{
body::Body, http::{HeaderValue, header}
};
use tempfile::NamedTempFile;
use tikv_jemalloc_ctl::raw;
use tokio_util::io::ReaderStream;
let pid = std::process::id();
let ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|e| ApiError::internal(format!("SystemTime error: {}", e)))?
.as_secs();
let filename = format!("htknow.heap.{}.{}.heap", pid, ts);
// 将堆 dump 写入临时文件后立即删除路径，只保留 fd 用于流式返回。
let temp = NamedTempFile::new().map_err(|e| ApiError::internal(format!("create temp file failed: {}", e)))?;
let path = temp.path().to_path_buf();
let path_str = path.to_string_lossy().into_owned();
let c_path =
CString::new(path_str.clone()).map_err(|_| ApiError::internal("Failed to create C string for path"))?;
let ptr = c_path.as_ptr();
unsafe {
raw::write(b"prof.dump\0", ptr).map_err(|e| ApiError::internal(format!("jemalloc heap dump failed: {}", e)))?;
}
let mut std_file = temp.into_file();
std_file
.seek(std::io::SeekFrom::Start(0))
.map_err(|e| ApiError::internal(format!("seek temp file failed: {}", e)))?;
let file = tokio::fs::File::from_std(std_file);
let len = file.metadata().await.map_err(|e| ApiError::internal(format!("metadata failed: {}", e)))?.len();
let stream = ReaderStream::new(file);
let mut response = Response::new(Body::from_stream(stream));
let headers = response.headers_mut();
headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream"));
headers.insert(
header::CONTENT_LENGTH,
HeaderValue::from_str(&len.to_string())
.map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?,
);
let disposition = format!("attachment; filename=\"{}\"", filename);
headers.insert(
header::CONTENT_DISPOSITION,
HeaderValue::from_str(&disposition).map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?,
);
Ok(response)
}
#[cfg(not(feature = "profiling"))]
async fn heap_profile_impl() -> ApiResult<Response> {
Err(ApiError::BadRequest("Heap profiling is only available with 'profiling' feature enabled.".to_string()))
}
#[cfg(feature = "profiling")]
async fn heap_profile_pdf_impl() -> ApiResult<Response> {
use std::{
ffi::CString, process::Stdio, time::{SystemTime, UNIX_EPOCH}
};
use axum::{
body::Body, http::{HeaderValue, header}
};
use tempfile::NamedTempFile;
use tikv_jemalloc_ctl::raw;
use tokio::{io::AsyncSeekExt, process::Command};
use tokio_util::io::ReaderStream;
let pid = std::process::id();
let ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|e| ApiError::internal(format!("SystemTime error: {}", e)))?
.as_secs();
// 生成 heap dump 文件（路径需要保持到 jeprof 执行完毕）
let heap_temp = NamedTempFile::new().map_err(|e| ApiError::internal(format!("create heap temp failed: {}", e)))?;
let heap_path = heap_temp.path().to_path_buf();
let heap_path_str = heap_path.to_string_lossy().into_owned();
let c_path = CString::new(heap_path_str.clone())
.map_err(|_| ApiError::internal("Failed to create C string for heap path"))?;
log::info!("Generating heap dump: {}", heap_path_str);
let ptr = c_path.as_ptr();
unsafe {
raw::write(b"prof.dump\0", ptr).map_err(|e| ApiError::internal(format!("jemalloc heap dump failed: {}", e)))?;
}
// 生成 PDF 文件（用 NamedTempFile 承接 jeprof 标准输出）
let pdf_temp = NamedTempFile::new().map_err(|e| ApiError::internal(format!("create pdf temp failed: {}", e)))?;
let pdf_std = pdf_temp.into_file();
let mut pdf_file = tokio::fs::File::from_std(pdf_std);
// 获取当前二进制文件路径
let binary_path =
std::env::current_exe().map_err(|e| ApiError::internal(format!("Failed to get binary path: {}", e)))?;
let binary_path_str = binary_path.to_string_lossy();
log::info!("Running jeprof: binary={}, heap={}, output=pdf_stream", binary_path_str, heap_path_str);
let mut child = Command::new("jeprof")
.arg("--show_bytes")
.arg("--pdf")
.arg(binary_path_str.as_ref())
.arg(&heap_path_str)
.stdout(Stdio::piped())
.spawn()
.map_err(|e| ApiError::internal(format!("Failed to execute jeprof (make sure jeprof is installed): {}", e)))?;
let mut stdout = child.stdout.take().ok_or_else(|| ApiError::internal("jeprof stdout unavailable"))?;
tokio::io::copy(&mut stdout, &mut pdf_file)
.await
.map_err(|e| ApiError::internal(format!("Failed to copy jeprof output: {}", e)))?;
let status = child.wait().await.map_err(|e| ApiError::internal(format!("jeprof process wait failed: {}", e)))?;
// jeprof 已完成，heap dump 临时文件可以清理
drop(heap_temp);
if !status.success() {
let code = status.code().unwrap_or(-1);
log::error!("jeprof failed with exit code: {}", code);
return Err(ApiError::internal(format!("jeprof execution failed with exit code: {}", code)));
}
pdf_file
.seek(std::io::SeekFrom::Start(0))
.await
.map_err(|e| ApiError::internal(format!("seek pdf temp failed: {}", e)))?;
let len = pdf_file.metadata().await.map_err(|e| ApiError::internal(format!("pdf metadata failed: {}", e)))?.len();
if len == 0 {
return Err(ApiError::internal("jeprof generated empty PDF"));
}
let pdf_filename = format!("htknow.heap.{}.{}.pdf", pid, ts);
log::info!("Successfully generated heap profile PDF: {} bytes", len);
let stream = ReaderStream::new(pdf_file);
let mut response = Response::new(Body::from_stream(stream));
let headers = response.headers_mut();
headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/pdf"));
headers.insert(
header::CONTENT_LENGTH,
HeaderValue::from_str(&len.to_string())
.map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?,
);
let disposition = format!("attachment; filename=\"{}\"", pdf_filename);
headers.insert(
header::CONTENT_DISPOSITION,
HeaderValue::from_str(&disposition).map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?,
);
Ok(response)
}
#[cfg(not(feature = "profiling"))]
async fn heap_profile_pdf_impl() -> ApiResult<Response> {
Err(ApiError::BadRequest("Heap profiling is only available with 'profiling' feature enabled.".to_string()))
}

631
src/archive.rs Normal file
View File

@ -0,0 +1,631 @@
//! 压缩文件解压模块
//!
//! 支持 ZIP、TAR（及其 GZ/BZ2/XZ 变体）格式的解压。
//! 7Z 和 RAR 格式暂不支持。
use std::{
io::{Read, Write}, path::Path
};
use encoding_rs::GB18030;
use serde::Serialize;
use tempfile::NamedTempFile;
use utoipa::ToSchema;
/// 智能解码文件名：先尝试 UTF-8，发现乱码时回退到 GB18030（兼容 GBK/GB2312）
fn decode_filename(raw: &[u8]) -> String {
// 先尝试 UTF-8
if let Ok(utf8) = std::str::from_utf8(raw) {
// 如果 UTF-8 解码成功且不包含替换字符，直接使用
if !utf8.contains('\u{FFFD}') {
return utf8.to_string();
}
}
// 尝试 GB18030（兼容 GBK/GB2312）
let (decoded, _, had_errors) = GB18030.decode(raw);
if !had_errors {
return decoded.to_string();
}
// 回退：UTF-8 lossy
String::from_utf8_lossy(raw).to_string()
}
/// 判断原始文件名是否表示目录（以 / 或 \ 结尾）
fn is_dir_from_raw(raw: &[u8]) -> bool {
raw.ends_with(b"/") || raw.ends_with(b"\\")
}
/// 压缩包内单个文件条目
#[derive(Debug, Clone, Serialize, sqlx::FromRow, ToSchema)]
pub struct ArchiveEntry {
pub id: Option<i64>,
pub file_id: i64,
pub entry_path: String,
pub size: Option<i64>,
pub is_directory: bool,
}
/// 解压结果
#[derive(Debug, Serialize, ToSchema)]
pub struct ExtractResult {
pub entries: Vec<ArchiveEntry>,
pub needs_password: bool,
}
/// 压缩文件处理错误
#[derive(Debug, thiserror::Error)]
pub enum ArchiveError {
#[error("密码错误或需要密码")]
PasswordRequired,
#[error("无效的密码")]
InvalidPassword,
#[error("不支持的压缩格式: {0}")]
UnsupportedFormat(String),
#[error("IO 错误: {0}")]
Io(#[from] std::io::Error),
#[error("ZIP 错误: {0}")]
Zip(String),
#[error("解压大小超过限制: 最大 {max_mb}MB, 实际约 {actual_mb}MB")]
SizeLimitExceeded { max_mb: u64, actual_mb: u64 },
#[error("文件数量超过限制: 最大 {max}, 实际 {actual}")]
FileCountExceeded { max: usize, actual: usize },
#[error("{0}")]
Other(String),
}
/// 解压安全限制
pub struct ExtractLimits {
/// 最大解压后总大小（字节），默认 1GB
pub max_total_size: u64,
/// 最大文件数量，默认 10000
pub max_file_count: usize,
/// 单个文件最大大小（字节），默认 100MB
pub max_file_size: u64,
}
impl Default for ExtractLimits {
fn default() -> Self {
Self {
max_total_size: 1_073_741_824, // 1GB
max_file_count: 10_000,
max_file_size: 104_857_600, // 100MB
}
}
}
/// 支持的压缩格式
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArchiveFormat {
Zip,
SevenZ,
Tar,
TarGz,
Tgz,
TarBz2,
TarXz,
Unknown,
}
impl ArchiveFormat {
/// 根据文件名识别压缩格式（大小写不敏感）
pub fn from_filename(filename: &str) -> Self {
let lower = filename.to_lowercase();
if lower.ends_with(".zip") {
ArchiveFormat::Zip
} else if lower.ends_with(".7z") {
ArchiveFormat::SevenZ
} else if lower.ends_with(".tar.gz") {
ArchiveFormat::TarGz
} else if lower.ends_with(".tgz") {
ArchiveFormat::Tgz
} else if lower.ends_with(".tar.bz2") {
ArchiveFormat::TarBz2
} else if lower.ends_with(".tar.xz") {
ArchiveFormat::TarXz
} else if lower.ends_with(".tar") {
ArchiveFormat::Tar
} else {
ArchiveFormat::Unknown
}
}
/// 是否为已识别的压缩格式（含暂不解压的 7Z）
pub fn is_archive(self) -> bool {
!matches!(self, ArchiveFormat::Unknown)
}
/// 当前是否可直接解压
pub fn is_supported(self) -> bool {
matches!(self, ArchiveFormat::Zip) || self.is_tar_variant()
}
/// 是否为 tar 变体
pub fn is_tar_variant(self) -> bool {
matches!(
self,
ArchiveFormat::Tar
| ArchiveFormat::TarGz
| ArchiveFormat::Tgz
| ArchiveFormat::TarBz2
| ArchiveFormat::TarXz
)
}
/// 格式描述文本
pub fn desc(self) -> &'static str {
match self {
ArchiveFormat::Zip => "ZIP",
ArchiveFormat::SevenZ => "7Z",
ArchiveFormat::Tar => "TAR",
ArchiveFormat::TarGz | ArchiveFormat::Tgz => "TAR.GZ",
ArchiveFormat::TarBz2 => "TAR.BZ2",
ArchiveFormat::TarXz => "TAR.XZ",
ArchiveFormat::Unknown => "未知",
}
}
}
/// 判断文件是否为支持的压缩格式
pub fn is_archive_file(filename: &str) -> bool {
ArchiveFormat::from_filename(filename).is_archive()
}
/// 将压缩文件解压到指定目录
///
/// `filename` 为原始文件名（含扩展名），用于判断压缩格式。
/// 返回解压后的文件列表
pub fn extract_archive(
src_path: &str, dest_dir: &str, filename: &str, password: Option<&str>, file_id: i64,
) -> Result<Vec<ArchiveEntry>, ArchiveError> {
let limits = ExtractLimits::default();
extract_archive_with_limits(src_path, dest_dir, filename, password, file_id, &limits)
}
/// 带限制的解压
pub fn extract_archive_with_limits(
src_path: &str, dest_dir: &str, filename: &str, password: Option<&str>, file_id: i64, limits: &ExtractLimits,
) -> Result<Vec<ArchiveEntry>, ArchiveError> {
let format = ArchiveFormat::from_filename(filename);
if !format.is_archive() {
return Err(ArchiveError::UnsupportedFormat(format.desc().to_string()));
}
// 创建目标目录
std::fs::create_dir_all(dest_dir)?;
match format {
ArchiveFormat::Zip => extract_zip(src_path, dest_dir, password, file_id, limits),
ArchiveFormat::SevenZ => Err(ArchiveError::UnsupportedFormat("7Z 格式暂不支持".to_string())),
_ if format.is_tar_variant() => extract_tar(src_path, dest_dir, file_id, limits),
_ => Err(ArchiveError::UnsupportedFormat(format.desc().to_string())),
}
}
/// 读取压缩包内单个文件内容到临时文件
///
/// `filename` 为原始文件名（含扩展名），用于判断压缩格式。
/// 返回的 `NamedTempFile` 需要由调用方在合适时机消费/关闭。
pub fn read_archive_entry(
src_path: &str, filename: &str, entry_path: &str, password: Option<&str>,
) -> Result<NamedTempFile, ArchiveError> {
let format = ArchiveFormat::from_filename(filename);
match format {
ArchiveFormat::Zip => read_zip_entry(src_path, entry_path, password),
ArchiveFormat::SevenZ => Err(ArchiveError::UnsupportedFormat("7Z 格式暂不支持".to_string())),
_ if format.is_tar_variant() => read_tar_entry(src_path, entry_path),
_ => Err(ArchiveError::UnsupportedFormat(format.desc().to_string())),
}
}
// ============================================================================
// ZIP 解压
// ============================================================================
fn extract_zip(
src_path: &str, dest_dir: &str, password: Option<&str>, file_id: i64, limits: &ExtractLimits,
) -> Result<Vec<ArchiveEntry>, ArchiveError> {
let file = std::fs::File::open(src_path)?;
let mut archive = match zip::ZipArchive::new(file) {
Ok(a) => a,
Err(e) => return Err(ArchiveError::Zip(e.to_string())),
};
let mut entries = Vec::new();
let mut total_size: u64 = 0;
let password_bytes = password.map(|p| p.as_bytes());
log::info!("ZIP archive has {} entries, file_id={}", archive.len(), file_id);
for i in 0..archive.len() {
// 先检查文件是否加密
let is_encrypted = {
let file = archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))?;
file.encrypted()
};
let mut file_entry = if is_encrypted {
if let Some(pw) = password_bytes {
archive.by_index_decrypt(i, pw).map_err(|e| ArchiveError::Zip(e.to_string()))?
} else {
return Err(ArchiveError::PasswordRequired);
}
} else {
archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))?
};
let raw_bytes = file_entry.name_raw();
let raw_name = decode_filename(raw_bytes);
// 统一路径分隔符为 /，并处理连续分隔符和首尾分隔符
let name: String =
raw_name.replace('\\', "/").split('/').filter(|s| !s.is_empty()).collect::<Vec<_>>().join("/");
if name.is_empty() {
log::info!(
"ZIP entry {}: raw={:?} utf8={:?} -> skipped (empty after normalization)",
i,
raw_bytes,
raw_name
);
continue;
}
let is_dir = is_dir_from_raw(raw_bytes);
let name = if is_dir && name.ends_with('/') { name.trim_end_matches('/').to_string() } else { name };
let size = file_entry.size();
log::info!(
"ZIP entry {}: raw={:?} decoded={:?} -> normalized={}, is_dir={}, size={}",
i,
raw_bytes,
raw_name,
name,
is_dir,
size
);
// 安全检查：路径遍历
if name.contains("..") {
log::warn!("Skipping ZIP entry with path traversal: {}", name);
continue;
}
// 大小限制检查
if !is_dir {
if size > limits.max_file_size {
return Err(ArchiveError::SizeLimitExceeded {
max_mb: limits.max_file_size / 1_048_576,
actual_mb: size / 1_048_576,
});
}
total_size += size;
if total_size > limits.max_total_size {
return Err(ArchiveError::SizeLimitExceeded {
max_mb: limits.max_total_size / 1_048_576,
actual_mb: total_size / 1_048_576,
});
}
}
// 数量限制检查
if entries.len() >= limits.max_file_count {
return Err(ArchiveError::FileCountExceeded { max: limits.max_file_count, actual: entries.len() });
}
let out_path = Path::new(dest_dir).join(&name);
if is_dir {
std::fs::create_dir_all(&out_path)?;
} else {
if let Some(parent) = out_path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut out_file = std::fs::File::create(&out_path)?;
std::io::copy(&mut file_entry, &mut out_file)?;
}
entries.push(ArchiveEntry {
id: None,
file_id,
entry_path: name,
size: Some(size as i64),
is_directory: is_dir,
});
}
log::info!("ZIP extraction completed: {} entries, file_id={}", entries.len(), file_id);
Ok(entries)
}
fn read_zip_entry(src_path: &str, entry_path: &str, password: Option<&str>) -> Result<NamedTempFile, ArchiveError> {
let file = std::fs::File::open(src_path)?;
let mut archive = match zip::ZipArchive::new(file) {
Ok(a) => a,
Err(e) => return Err(ArchiveError::Zip(e.to_string())),
};
let password_bytes = password.map(|p| p.as_bytes());
let normalized_target = entry_path.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string();
// 遍历所有条目，用解码后的文件名匹配（处理 GBK 编码中文文件名）
for i in 0..archive.len() {
let file_ref = archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))?;
let decoded_name = decode_filename(file_ref.name_raw());
let normalized_name = decoded_name.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string();
if normalized_name == normalized_target {
let is_encrypted = file_ref.encrypted();
drop(file_ref);
let mut file_entry = if is_encrypted {
if let Some(pw) = password_bytes {
archive.by_index_decrypt(i, pw).map_err(|e| ArchiveError::Zip(e.to_string()))?
} else {
return Err(ArchiveError::PasswordRequired);
}
} else {
archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))?
};
let mut temp = NamedTempFile::new()?;
std::io::copy(&mut file_entry, &mut temp)?;
temp.flush()?;
return Ok(temp);
}
}
Err(ArchiveError::Other(format!("文件不存在: {}", entry_path)))
}
// ============================================================================
// TAR 解压（含 GZ/BZ2/XZ 变体）
// ============================================================================
fn open_tar_reader(src_path: &str) -> Result<Box<dyn Read>, ArchiveError> {
let file = std::fs::File::open(src_path)?;
let lower = src_path.to_lowercase();
if lower.ends_with(".tar") {
Ok(Box::new(file))
} else if lower.ends_with(".tar.gz") || lower.ends_with(".tgz") {
Ok(Box::new(flate2::read::GzDecoder::new(file)))
} else if lower.ends_with(".tar.bz2") {
Ok(Box::new(bzip2::read::BzDecoder::new(file)))
} else if lower.ends_with(".tar.xz") {
Ok(Box::new(xz2::read::XzDecoder::new(file)))
} else {
Err(ArchiveError::UnsupportedFormat("TAR 变体".to_string()))
}
}
fn extract_tar(
src_path: &str, dest_dir: &str, file_id: i64, limits: &ExtractLimits,
) -> Result<Vec<ArchiveEntry>, ArchiveError> {
let reader = open_tar_reader(src_path)?;
let mut archive = tar::Archive::new(reader);
let mut entries = Vec::new();
let mut total_size: u64 = 0;
for entry_result in archive.entries()? {
let mut entry = entry_result?;
let raw_path = entry.path_bytes();
let decoded = decode_filename(&raw_path);
let name = decoded.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string();
if name.is_empty() {
continue;
}
let is_dir = entry.header().entry_type().is_dir() || decoded.ends_with('/') || decoded.ends_with('\\');
let name = if is_dir && name.ends_with('/') { name.trim_end_matches('/').to_string() } else { name };
let size = entry.size();
// 安全检查
if name.contains("..") {
log::warn!("Skipping TAR entry with path traversal: {}", name);
continue;
}
// 大小限制
if !is_dir {
if size > limits.max_file_size {
return Err(ArchiveError::SizeLimitExceeded {
max_mb: limits.max_file_size / 1_048_576,
actual_mb: size / 1_048_576,
});
}
total_size += size;
if total_size > limits.max_total_size {
return Err(ArchiveError::SizeLimitExceeded {
max_mb: limits.max_total_size / 1_048_576,
actual_mb: total_size / 1_048_576,
});
}
}
if entries.len() >= limits.max_file_count {
return Err(ArchiveError::FileCountExceeded { max: limits.max_file_count, actual: entries.len() });
}
let out_path = Path::new(dest_dir).join(&name);
if is_dir {
std::fs::create_dir_all(&out_path)?;
} else {
if let Some(parent) = out_path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut out_file = std::fs::File::create(&out_path)?;
std::io::copy(&mut entry, &mut out_file)?;
}
entries.push(ArchiveEntry {
id: None,
file_id,
entry_path: name,
size: Some(size as i64),
is_directory: is_dir,
});
}
Ok(entries)
}
fn read_tar_entry(src_path: &str, entry_path: &str) -> Result<NamedTempFile, ArchiveError> {
let reader = open_tar_reader(src_path)?;
let mut archive = tar::Archive::new(reader);
let normalized_target = entry_path.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string();
for entry_result in archive.entries()? {
let mut entry = entry_result?;
let raw_path = entry.path_bytes();
let decoded = decode_filename(&raw_path);
let name = decoded.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string();
if name == normalized_target {
let mut temp = NamedTempFile::new()?;
std::io::copy(&mut entry, &mut temp)?;
temp.flush()?;
return Ok(temp);
}
}
Err(ArchiveError::Other(format!("文件不存在: {}", entry_path)))
}
// ============================================================================
// 辅助函数
// ============================================================================
/// 清理压缩文件解压目录
pub fn cleanup_archive_dir(archives_path: &str, file_id: i64) -> std::io::Result<()> {
let dir = Path::new(archives_path).join(file_id.to_string());
if dir.exists() {
std::fs::remove_dir_all(&dir)?;
}
Ok(())
}
/// 获取压缩文件条目的本地文件系统路径
pub fn resolve_archive_entry_path(archives_path: &str, file_id: i64, entry_path: &str) -> Option<std::path::PathBuf> {
let base = Path::new(archives_path).join(file_id.to_string());
let resolved = base.join(entry_path);
// 安全检查：确保解析后的路径在 base 目录下
let canonical_base = base.canonicalize().unwrap_or(base.clone());
let canonical_resolved = match resolved.canonicalize() {
Ok(p) => p,
Err(_) => resolved.clone(),
};
if !canonical_resolved.starts_with(&canonical_base) {
return None;
}
Some(resolved)
}
#[cfg(test)]
mod tests {
use std::io::Write;
use super::*;
#[test]
fn test_is_archive_file() {
assert!(is_archive_file("test.zip"));
assert!(is_archive_file("test.ZIP"));
assert!(is_archive_file("test.tar.gz"));
assert!(is_archive_file("test.tgz"));
assert!(is_archive_file("test.tar.bz2"));
assert!(is_archive_file("test.tar.xz"));
assert!(is_archive_file("test.7z"));
assert!(!is_archive_file("test.pdf"));
assert!(!is_archive_file("test.txt"));
}
#[test]
fn test_extract_zip_with_backslash_paths() {
// 创建测试 ZIP（Windows 风格路径）
let zip_path = "/tmp/test_htknow_zip.zip";
let extract_dir = "/tmp/test_htknow_extract";
// 清理
let _ = std::fs::remove_file(zip_path);
let _ = std::fs::remove_dir_all(extract_dir);
{
let file = std::fs::File::create(zip_path).unwrap();
let mut zip = zip::ZipWriter::new(file);
let options = zip::write::SimpleFileOptions::default();
zip.start_file("folder\\file1.txt", options).unwrap();
zip.write_all(b"hello world 1").unwrap();
zip.start_file("folder\\sub\\file2.txt", options).unwrap();
zip.write_all(b"hello world 2").unwrap();
zip.start_file("root.txt", options).unwrap();
zip.write_all(b"hello root").unwrap();
zip.finish().unwrap();
}
let entries = extract_archive(zip_path, extract_dir, "test.zip", None, 1).unwrap();
// 应该解压出 3 个文件条目（目录条目会自动从路径推断）
assert!(
entries.len() >= 3,
"Expected at least 3 entries, got {}: {:?}",
entries.len(),
entries.iter().map(|e| &e.entry_path).collect::<Vec<_>>()
);
// 检查路径统一使用了 /
let paths: Vec<String> = entries.iter().map(|e| e.entry_path.clone()).collect();
assert!(paths.iter().any(|p| p == "folder/file1.txt"), "Missing folder/file1.txt in {:?}", paths);
assert!(paths.iter().any(|p| p == "folder/sub/file2.txt"), "Missing folder/sub/file2.txt in {:?}", paths);
assert!(paths.iter().any(|p| p == "root.txt"), "Missing root.txt in {:?}", paths);
// 没有路径应该包含反斜杠
for p in &paths {
assert!(!p.contains('\\'), "Path should not contain backslash: {}", p);
}
// 检查文件内容
let content1 = std::fs::read_to_string(format!("{}/folder/file1.txt", extract_dir)).unwrap();
assert_eq!(content1, "hello world 1");
let content2 = std::fs::read_to_string(format!("{}/folder/sub/file2.txt", extract_dir)).unwrap();
assert_eq!(content2, "hello world 2");
// 清理
let _ = std::fs::remove_file(zip_path);
let _ = std::fs::remove_dir_all(extract_dir);
}
#[test]
fn test_extract_zip_with_forward_slash_paths() {
let zip_path = "/tmp/test_htknow_zip2.zip";
let extract_dir = "/tmp/test_htknow_extract2";
let _ = std::fs::remove_file(zip_path);
let _ = std::fs::remove_dir_all(extract_dir);
{
let file = std::fs::File::create(zip_path).unwrap();
let mut zip = zip::ZipWriter::new(file);
let options = zip::write::SimpleFileOptions::default();
zip.start_file("docs/report.pdf", options).unwrap();
zip.write_all(b"fake pdf").unwrap();
zip.start_file("data/info.txt", options).unwrap();
zip.write_all(b"info").unwrap();
zip.finish().unwrap();
}
let entries = extract_archive(zip_path, extract_dir, "test.zip", None, 2).unwrap();
assert!(entries.len() >= 2, "Expected at least 2 entries, got {}", entries.len());
let paths: Vec<String> = entries.iter().map(|e| e.entry_path.clone()).collect();
assert!(paths.iter().any(|p| p == "docs/report.pdf"));
assert!(paths.iter().any(|p| p == "data/info.txt"));
let _ = std::fs::remove_file(zip_path);
let _ = std::fs::remove_dir_all(extract_dir);
}
}

516
src/config.rs Normal file
View File

@ -0,0 +1,516 @@
//! 配置管理模块
//!
//! 支持从环境变量读取配置，预留 etcd 支持接口，并支持实时更新。
use std::sync::{Arc, RwLock};
use once_cell::sync::Lazy;
/// 全局配置单例
///
/// 内层存 `Arc<AppConfig>`，`get()` 只克隆 Arc（廉价），避免每次深拷贝整个配置结构体。
/// 保留外层 `RwLock` 以支持未来的配置热更新（替换内层 Arc 即可）。
static CONFIG: Lazy<RwLock<Arc<AppConfig>>> = Lazy::new(|| RwLock::new(Arc::new(AppConfig::from_env())));
/// 获取当前配置的只读快照（克隆 Arc，零深拷贝）
pub fn get() -> Arc<AppConfig> {
CONFIG.read().unwrap().clone()
}
// 重新加载配置（用于热更新）
// pub fn reload() {
// let new_config = AppConfig::from_env();
// let mut config = CONFIG.write().unwrap();
// *config = new_config;
// info!("Configuration reloaded");
// }
// 使用新配置更新（用于 etcd watch 等场景）
// pub fn update(new_config: AppConfig) {
// let mut config = CONFIG.write().unwrap();
// *config = new_config;
// info!("Configuration updated");
// }
/// 应用配置
#[derive(Debug, Clone)]
pub struct AppConfig {
pub server: ServerConfig,
pub services: ServicesConfig,
pub ai: AIConfig,
pub database: DatabaseConfig,
pub storage: StorageConfig,
pub search: SearchConfig,
pub slice: SliceConfig,
pub llm: LLMConfig,
}
/// 服务器配置
#[derive(Debug, Clone)]
pub struct ServerConfig {
/// 监听地址
pub host: String,
/// 监听端口
pub port: u16,
/// 上传文件大小限制（MB）
pub upload_limit_mb: usize,
/// 文件处理间隔（秒）
pub process_interval_secs: u64,
/// 文件处理并发数
pub process_concurrency: usize,
/// LanceDB 自动压缩 cron 表达式（off/disabled/0 表示禁用）
pub lancedb_compact_cron: String,
/// 是否启用后台文件解析
pub parse_enabled: bool,
/// 是否启用重复文件复用
pub reuse_duplicate_files: bool,
/// 文件解析后是否构建知识图谱
pub build_knowledge_graph: bool,
}
/// 外部服务配置
#[derive(Debug, Clone)]
pub struct ServicesConfig {
/// MinerU PDF 解析服务地址
pub mineru_url: String,
/// 外部接口请求超时（秒）
pub request_timeout_secs: u64,
/// MinerU 单次解析 PDF 最大页数（0 表示不限制）
pub mineru_max_pages: usize,
/// Office 文档转 PDF 服务地址
pub office_convert_url: String,
/// 自定义解析服务地址（配置后仅 Word/PPT/PDF 走该服务）
pub custom_parse_url: Option<String>,
/// 自定义解析复用服务地址（仅输入 pdf_contents）
pub custom_parse_reuse_url: Option<String>,
/// 音频转写服务地址
pub audio_transcription_url: String,
/// 音频转写服务 API Key（可选）
pub audio_transcription_key: Option<String>,
/// Embedding 服务地址
pub embedding_url: String,
/// 图片 Embedding 服务地址（可选，未配置时不进行图片 embedding）
pub image_embedding_url: Option<String>,
/// Rerank 服务地址
pub rerank_url: String,
/// 图片文本化服务地址（可选）
pub image_parse_url: Option<String>,
/// 图片文本化服务超时（秒）
pub image_parse_timeout_secs: u64,
/// 图片文本化并发数
pub image_parse_concurrency: usize,
}
/// AI 模型配置
#[derive(Debug, Clone)]
pub struct AIConfig {
/// Embedding 模型名称
pub embedding_model: String,
/// Embedding 向量维度
pub embedding_dim: i32,
/// 图片 Embedding 向量维度
pub image_embedding_dim: i32,
/// Rerank 模型名称
pub rerank_model: String,
/// Rerank 分数阈值
pub rerank_threshold: f32,
/// Embedding 批量请求批次大小
pub embedding_batch_size: usize,
}
/// 数据库配置
#[derive(Debug, Clone)]
pub struct DatabaseConfig {
/// 数据库连接 URL
pub url: String,
/// 最大连接数
pub max_connections: u32,
/// 最小空闲连接数（保持热连接，避免负载波动时反复建连）
pub min_connections: u32,
/// 忙超时（毫秒）
pub busy_timeout_ms: u64,
/// 是否初始化默认知识库
pub init_default_kbs: bool,
}
/// 存储路径配置
#[derive(Debug, Clone)]
pub struct StorageConfig {
/// LanceDB 存储路径
pub lancedb_path: String,
/// 临时目录路径
pub temp_path: String,
/// 图片保存路径
pub images_path: String,
/// 转换后的 PDF 保存路径
pub pdf_path: String,
/// 上传文件保存路径
pub files_path: String,
/// 压缩文件解压后保存路径
pub archives_path: String,
/// 文件解析后完整文本的保存路径
pub contents_path: String,
/// 文件切片正文（按源文件聚合）的保存路径
pub slice_contents_path: String,
}
/// 搜索配置
#[derive(Debug, Clone)]
pub struct SearchConfig {
/// 搜索结果限制数
pub limit: usize,
/// Tantivy 索引路径
pub tantivy_index_path: String,
/// Tantivy 全文索引路径
pub tantivy_full_index_path: String,
/// Tantivy 索引内存（MB）
pub tantivy_memory_mb: usize,
/// Tantivy 重建批次大小
pub tantivy_rebuild_batch_size: usize,
/// LanceDB 从 SQLite 重建时的批次大小
pub lancedb_rebuild_batch_size: usize,
/// 文本 embedding 请求超时（秒）
pub embedding_timeout_secs: u64,
/// rerank 请求超时（秒）
pub rerank_timeout_secs: u64,
/// 是否启用同义词扩展
pub synonym_enabled: bool,
/// 同义词默认权重因子
pub synonym_boost: f32,
/// 每个词最多扩展的同义词数
pub max_synonyms_per_term: usize,
/// 单次查询最多扩展的同义词总数
pub max_total_synonyms: usize,
/// 高亮页码选择阈值：首选页的内容字数（按 pdf_content 中文本长度计算）少于该值时，优先使用第二页（如果存在）
pub highlight_page_min_chars: usize,
}
/// 切片配置
#[derive(Debug, Clone)]
pub struct SliceConfig {
/// 智能切片每个切片的最大字数
pub smart_slice_max_chars: usize,
/// 固定长度切片的重叠字数
pub fixed_slice_overlap_chars: usize,
}
/// LLM 配置
#[derive(Debug, Clone)]
pub struct LLMConfig {
/// LLM API URL
pub api_url: Option<String>,
/// LLM API Key
pub api_key: Option<String>,
/// LLM 模型名称
pub model: String,
}
impl AppConfig {
/// 从环境变量加载配置
pub fn from_env() -> Self {
Self {
server: ServerConfig::from_env(),
services: ServicesConfig::from_env(),
ai: AIConfig::from_env(),
database: DatabaseConfig::from_env(),
storage: StorageConfig::from_env(),
search: SearchConfig::from_env(),
slice: SliceConfig::from_env(),
llm: LLMConfig::from_env(),
}
}
}
impl ServerConfig {
fn from_env() -> Self {
Self {
host: env_or("HTKNOW_SERVER_HOST", "0.0.0.0"),
port: env_or_parse("HTKNOW_SERVER_PORT", 3000),
upload_limit_mb: env_or_parse("HTKNOW_SERVER_UPLOAD_LIMIT_MB", 500),
process_interval_secs: env_or_parse("HTKNOW_SERVER_PROCESS_INTERVAL_SECS", 10),
process_concurrency: env_or_parse("HTKNOW_SERVER_PROCESS_CONCURRENCY", 1),
lancedb_compact_cron: env_or("HTKNOW_LANCEDB_COMPACT_CRON", "0 0 3 * * *"),
parse_enabled: env_or_parse("HTKNOW_PARSE_ENABLED", true),
reuse_duplicate_files: env_or_parse("HTKNOW_REUSE_DUPLICATE_FILES", true),
build_knowledge_graph: env_or_parse("HTKNOW_BUILD_KNOWLEDGE_GRAPH", false),
}
}
}
impl ServicesConfig {
fn from_env() -> Self {
Self {
mineru_url: env_or("HTKNOW_MINERU_URL", "http://192.168.0.46:10001/file_parse"),
request_timeout_secs: env_or_parse("HTKNOW_REQUEST_TIMEOUT_SECS", 600),
mineru_max_pages: env_or_parse("HTKNOW_MINERU_MAX_PAGES", 50),
office_convert_url: env_or("HTKNOW_OFFICE_CONVERT_URL", "http://192.168.0.46:8003/convert"),
custom_parse_url: env_optional("HTKNOW_CUSTOM_PARSE_URL"),
custom_parse_reuse_url: env_optional("HTKNOW_CUSTOM_PARSE_REUSE_URL"),
audio_transcription_url: env_or(
"HTKNOW_AUDIO_TRANSCRIPTION_URL",
"http://192.168.0.46:59805/api/v1/audio/transcriptions",
),
audio_transcription_key: std::env::var("HTKNOW_AUDIO_TRANSCRIPTION_KEY").ok(),
embedding_url: env_or("HTKNOW_EMBEDDING_URL", "http://222.190.139.186:59700/v1/embeddings"),
image_embedding_url: env_optional("HTKNOW_IMAGE_EMBEDDING_URL"),
rerank_url: env_or("HTKNOW_RERANK_URL", "http://222.190.139.186:59600/v1/rerank"),
image_parse_url: env_optional("HTKNOW_IMAGE_PARSE_URL"),
image_parse_timeout_secs: env_or_parse("HTKNOW_IMAGE_PARSE_TIMEOUT_SECS", 120),
image_parse_concurrency: env_or_parse("HTKNOW_IMAGE_PARSE_CONCURRENCY", 5),
}
}
}
impl AIConfig {
fn from_env() -> Self {
let embedding_dim = env_or_parse("HTKNOW_EMBEDDING_DIM", 1024);
Self {
embedding_model: env_or("HTKNOW_EMBEDDING_MODEL", "bge-m3"),
embedding_dim,
image_embedding_dim: env_or_parse("HTKNOW_IMAGE_EMBEDDING_DIM", 2048),
rerank_model: env_or("HTKNOW_RERANK_MODEL", "bge-rerank"),
rerank_threshold: env_or_parse("HTKNOW_RERANK_THRESHOLD", 0.1),
embedding_batch_size: env_or_parse("HTKNOW_EMBEDDING_BATCH_SIZE", 8),
}
}
}
impl DatabaseConfig {
fn from_env() -> Self {
Self {
url: env_or("DATABASE_URL", "sqlite://data/app.sqlite"),
// 文件型 SQLite 写操作串行，连接数过高只会加剧锁竞争；WAL 下读可并发，16 足够。
max_connections: env_or_parse("HTKNOW_DB_MAX_CONNECTIONS", 16),
min_connections: env_or_parse("HTKNOW_DB_MIN_CONNECTIONS", 2),
busy_timeout_ms: env_or_parse("HTKNOW_DB_BUSY_TIMEOUT_MS", 5000),
init_default_kbs: env_or_parse("HTKNOW_DB_INIT_DEFAULT_KBS", true),
}
}
}
impl StorageConfig {
fn from_env() -> Self {
let data_dir = env_or("HTKNOW_DATA_DIR", "data");
Self {
lancedb_path: env_or("HTKNOW_LANCEDB_PATH", &format!("{}/lancedb_data", data_dir)),
temp_path: env_or("HTKNOW_TEMP_PATH", &format!("{}/temp", data_dir)),
images_path: env_or("HTKNOW_IMAGES_PATH", &format!("{}/images", data_dir)),
pdf_path: env_or("HTKNOW_PDF_PATH", &format!("{}/pdfs", data_dir)),
files_path: env_or("HTKNOW_FILES_PATH", &format!("{}/files", data_dir)),
archives_path: env_or("HTKNOW_ARCHIVES_PATH", &format!("{}/archives", data_dir)),
contents_path: env_or("HTKNOW_CONTENTS_PATH", &format!("{}/contents", data_dir)),
slice_contents_path: env_or("HTKNOW_SLICE_CONTENTS_PATH", &format!("{}/slice_contents", data_dir)),
}
}
}
impl SearchConfig {
fn from_env() -> Self {
let data_dir = env_or("HTKNOW_DATA_DIR", "data");
Self {
limit: env_or_parse("HTKNOW_SEARCH_LIMIT", 10),
tantivy_index_path: env_or("HTKNOW_TANTIVY_INDEX_PATH", &format!("{}/tantivy_index", data_dir)),
tantivy_full_index_path: env_or(
"HTKNOW_TANTIVY_FULL_INDEX_PATH",
&format!("{}/tantivy_full_index", data_dir),
),
tantivy_memory_mb: env_or_parse("HTKNOW_TANTIVY_MEMORY_MB", 50),
tantivy_rebuild_batch_size: env_or_parse("HTKNOW_SEARCH_TANTIVY_REBUILD_BATCH_SIZE", 100),
lancedb_rebuild_batch_size: env_or_parse("HTKNOW_SEARCH_LANCEDB_REBUILD_BATCH_SIZE", 1000),
embedding_timeout_secs: env_or_parse("HTKNOW_SEARCH_EMBEDDING_TIMEOUT_SECS", 30),
rerank_timeout_secs: env_or_parse("HTKNOW_SEARCH_RERANK_TIMEOUT_SECS", 20),
synonym_enabled: env_or_parse("HTKNOW_SEARCH_SYNONYM_ENABLED", true),
synonym_boost: env_or_parse("HTKNOW_SEARCH_SYNONYM_BOOST", 0.7),
max_synonyms_per_term: env_or_parse("HTKNOW_SEARCH_MAX_SYNONYMS_PER_TERM", 5),
max_total_synonyms: env_or_parse("HTKNOW_SEARCH_MAX_TOTAL_SYNONYMS", 30),
highlight_page_min_chars: std::env::var("HTKNOW_HIGHLIGHT_PAGE_MIN_CHARS")
.ok()
.and_then(|v| v.parse().ok())
.or_else(|| std::env::var("HTKNOW_HIGHLIGHT_PAGE_MIN_POSITIONS").ok().and_then(|v| v.parse().ok()))
.unwrap_or(20),
}
}
}
impl SliceConfig {
fn from_env() -> Self {
Self {
smart_slice_max_chars: env_or_parse("HTKNOW_SMART_SLICE_MAX_CHARS", 8000),
fixed_slice_overlap_chars: env_or_parse("HTKNOW_FIXED_SLICE_OVERLAP_CHARS", 100),
}
}
}
impl LLMConfig {
fn from_env() -> Self {
Self {
api_url: std::env::var("LLM_API_URL").ok(),
api_key: std::env::var("LLM_API_KEY").ok(),
model: env_or("LLM_MODEL", "gpt-3.5-turbo"),
}
}
/// 检查 LLM 是否已启用
pub fn is_enabled(&self) -> bool {
self.api_url.is_some()
}
}
/// 从环境变量读取字符串，如果不存在则返回默认值
fn env_or(key: &str, default: &str) -> String {
std::env::var(key).unwrap_or_else(|_| default.to_string())
}
/// 从环境变量读取字符串，空字符串视为未配置
fn env_optional(key: &str) -> Option<String> {
std::env::var(key).ok().and_then(|value| {
let trimmed = value.trim();
if trimmed.is_empty() { None } else { Some(trimmed.to_string()) }
})
}
/// 从环境变量读取并解析值，如果不存在或解析失败则返回默认值
fn env_or_parse<T: std::str::FromStr>(key: &str, default: T) -> T {
std::env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
}
// ============================================================================
// etcd 支持（可选功能）
// ============================================================================
/// 配置加载器 trait
#[allow(dead_code)]
pub trait ConfigLoader: Send+Sync {
/// 加载配置
fn load(&self) -> anyhow::Result<AppConfig>;
}
/// 环境变量配置加载器
#[allow(dead_code)]
pub struct EnvConfigLoader;
impl ConfigLoader for EnvConfigLoader {
fn load(&self) -> anyhow::Result<AppConfig> {
Ok(AppConfig::from_env())
}
}
/// etcd 配置加载器（预留接口）
#[cfg(feature = "etcd")]
#[allow(dead_code)]
pub struct EtcdConfigLoader {
pub endpoints: Vec<String>,
pub prefix: String,
}
#[cfg(feature = "etcd")]
#[allow(dead_code)]
impl EtcdConfigLoader {
pub fn new(endpoints: Vec<String>, prefix: String) -> Self {
Self { endpoints, prefix }
}
/// 启动 watch 监听配置变化
pub async fn watch<F>(&self, _callback: F) -> anyhow::Result<()>
where
F: Fn(AppConfig)+Send+Sync+'static, {
// TODO: 实现 etcd watch 逻辑
// 1. 连接 etcd
// 2. 监听 prefix 下的 key 变化
// 3. 当配置变化时调用 callback
unimplemented!("etcd watch not implemented yet")
}
}
#[cfg(feature = "etcd")]
impl ConfigLoader for EtcdConfigLoader {
fn load(&self) -> anyhow::Result<AppConfig> {
// TODO: 实现从 etcd 读取配置
// 1. 连接 etcd
// 2. 读取 prefix 下的所有 key
// 3. 解析并构建 AppConfig
// 4. 如果 etcd 中没有配置，回退到环境变量
unimplemented!("etcd config loading not implemented yet")
}
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use once_cell::sync::Lazy;
use super::*;
static ENV_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
#[test]
fn test_default_config() {
let _guard = ENV_LOCK.lock().unwrap();
unsafe {
std::env::remove_var("HTKNOW_PARSE_ENABLED");
std::env::remove_var("HTKNOW_BUILD_KNOWLEDGE_GRAPH");
std::env::remove_var("HTKNOW_REQUEST_TIMEOUT_SECS");
}
let config = AppConfig::from_env();
// 验证默认值
assert_eq!(config.server.host, "0.0.0.0");
assert_eq!(config.server.port, 3000);
assert_eq!(config.server.upload_limit_mb, 500);
assert_eq!(config.ai.embedding_model, "bge-m3");
assert_eq!(config.ai.embedding_dim, 1024);
assert_eq!(config.ai.image_embedding_dim, 2048);
assert_eq!(config.services.request_timeout_secs, 600);
assert_eq!(config.search.embedding_timeout_secs, 30);
assert_eq!(config.search.rerank_timeout_secs, 20);
assert!(config.server.parse_enabled);
assert!(!config.server.build_knowledge_graph);
assert_eq!(config.search.highlight_page_min_chars, 20);
}
#[test]
fn test_get_config() {
let config = get();
assert!(!config.server.host.is_empty());
}
#[test]
fn test_parse_enabled_env_override() {
let _guard = ENV_LOCK.lock().unwrap();
unsafe {
std::env::set_var("HTKNOW_PARSE_ENABLED", "false");
}
let config = AppConfig::from_env();
assert!(!config.server.parse_enabled);
unsafe {
std::env::remove_var("HTKNOW_PARSE_ENABLED");
}
}
#[test]
fn test_build_knowledge_graph_env_override() {
let _guard = ENV_LOCK.lock().unwrap();
unsafe {
std::env::set_var("HTKNOW_BUILD_KNOWLEDGE_GRAPH", "true");
}
let config = AppConfig::from_env();
assert!(config.server.build_knowledge_graph);
unsafe {
std::env::remove_var("HTKNOW_BUILD_KNOWLEDGE_GRAPH");
}
}
#[test]
fn test_request_timeout_env_override() {
let _guard = ENV_LOCK.lock().unwrap();
unsafe {
std::env::set_var("HTKNOW_REQUEST_TIMEOUT_SECS", "120");
}
let config = AppConfig::from_env();
assert_eq!(config.services.request_timeout_secs, 120);
unsafe {
std::env::remove_var("HTKNOW_REQUEST_TIMEOUT_SECS");
}
}
}

1061
src/db.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1800
src/export.rs Normal file
View File

File diff suppressed because it is too large Load Diff

62
src/file_content.rs Normal file
View File

@ -0,0 +1,62 @@
use std::path::PathBuf;
use anyhow::Context;
use crate::config;
/// 返回指定 file_id 对应的 content 文件路径。
fn content_path(file_id: i64) -> PathBuf {
PathBuf::from(&config::get().storage.contents_path).join(format!("{}.txt", file_id))
}
/// 读取文件的完整 content。
///
/// - 文件存在：返回 `Some(内容)`。
/// - 文件不存在：返回 `None`（对应数据库中 `content IS NULL` 的语义）。
/// - 其他 IO 错误：返回 `Err`。
pub async fn read(file_id: i64) -> anyhow::Result<Option<String>> {
let path = content_path(file_id);
match tokio::fs::read_to_string(&path).await {
Ok(content) => Ok(Some(content)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e).with_context(|| format!("failed to read content file {}", path.display())),
}
}
/// 写入文件的完整 content。
///
/// 使用临时文件 + rename 保证原子性，避免崩溃后出现半写文件。
pub async fn write(file_id: i64, content: &str) -> anyhow::Result<()> {
if content.is_empty() {
// 空内容与 NULL 语义统一：删除文件，避免留下空文件。
return delete(file_id).await;
}
let path = content_path(file_id);
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("failed to create contents directory {}", parent.display()))?;
}
let tmp_path = path.with_extension("txt.tmp");
tokio::fs::write(&tmp_path, content.as_bytes())
.await
.with_context(|| format!("failed to write tmp content file {}", tmp_path.display()))?;
tokio::fs::rename(&tmp_path, &path)
.await
.with_context(|| format!("failed to rename content file to {}", path.display()))?;
Ok(())
}
/// 删除文件的 content。
///
/// 文件不存在时视为已成功删除。
pub async fn delete(file_id: i64) -> anyhow::Result<()> {
let path = content_path(file_id);
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).with_context(|| format!("failed to delete content file {}", path.display())),
}
}

44
src/frontend.rs Normal file
View File

@ -0,0 +1,44 @@
use axum::{
Router, body::Body, http::{StatusCode, Uri, header}, response::{IntoResponse, Response}, routing::get
};
use rust_embed::Embed;
#[derive(Embed)]
#[folder = "frontend/dist/"]
struct Assets;
async fn static_handler(uri: Uri) -> impl IntoResponse {
let path = uri.path().trim_start_matches('/');
// 如果路径为空，返回 index.html
let path = if path.is_empty() { "index.html" } else { path };
serve_file(path)
}
async fn index_handler() -> impl IntoResponse {
serve_file("index.html")
}
fn serve_file(path: &str) -> Response {
if path == "favicon.ico" {
return Response::builder().status(StatusCode::NO_CONTENT).body(Body::empty()).unwrap();
}
match Assets::get(path) {
Some(content) => {
let mime = mime_guess::from_path(path).first_or_octet_stream().to_string();
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, mime)
.body(Body::from(content.data.into_owned()))
.unwrap()
}
None => Response::builder().status(StatusCode::NOT_FOUND).body(Body::from("Not Found")).unwrap(),
}
}
pub fn router() -> Router {
Router::new().route("/", get(index_handler)).route("/{*path}", get(static_handler))
}

299
src/graph/graph_manager.rs Normal file
View File

@ -0,0 +1,299 @@
use std::collections::HashMap;
use anyhow::Result;
use petgraph::graph::{DiGraph, NodeIndex};
use sqlx::SqlitePool;
use super::{Edge, Entity, EntityType, Node, Relation, RelationType};
/// 知识图谱管理器
pub struct KnowledgeGraph {
graph: DiGraph<Node, Edge>,
node_index: HashMap<String, NodeIndex>,
node_id_to_index: HashMap<i64, NodeIndex>,
pool: SqlitePool,
kb_id: Option<i64>,
}
impl KnowledgeGraph {
/// 从数据库加载图
pub async fn load_from_db(pool: SqlitePool, kb_id: Option<i64>) -> Result<Self> {
let mut graph = DiGraph::new();
let mut node_index = HashMap::new();
let mut node_id_to_index = HashMap::new();
// 加载节点
let nodes: Vec<(i64, String, String, Option<String>)> = if let Some(kb_id) = kb_id {
sqlx::query_as("SELECT id, name, entity_type, properties FROM graph_nodes WHERE kb_id = ?")
.bind(kb_id)
.fetch_all(&pool)
.await?
} else {
sqlx::query_as("SELECT id, name, entity_type, properties FROM graph_nodes").fetch_all(&pool).await?
};
for (id, name, entity_type_str, properties_json) in nodes {
let entity_type = EntityType::Custom(entity_type_str);
let properties: HashMap<String, String> = if let Some(json) = properties_json {
serde_json::from_str(&json).unwrap_or_default()
} else {
HashMap::new()
};
let node = Node { id, name: name.clone(), entity_type, properties };
let idx = graph.add_node(node);
node_index.insert(name, idx);
node_id_to_index.insert(id, idx);
}
// 加载边（使用 n1.kb_id 避免歧义）
let edges: Vec<(i64, i64, i64, String, Option<String>, f32)> = if let Some(kb_id) = kb_id {
sqlx::query_as(
"SELECT e.id, e.source_node_id, e.target_node_id, e.relation_type, e.properties, e.weight \
FROM graph_edges e \
INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id \
INNER JOIN graph_nodes n2 ON e.target_node_id = n2.id \
WHERE n1.kb_id = ?",
)
.bind(kb_id)
.fetch_all(&pool)
.await?
} else {
sqlx::query_as(
"SELECT e.id, e.source_node_id, e.target_node_id, e.relation_type, e.properties, e.weight \
FROM graph_edges e \
INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id \
INNER JOIN graph_nodes n2 ON e.target_node_id = n2.id",
)
.fetch_all(&pool)
.await?
};
for (id, source_id, target_id, relation_type_str, properties_json, weight) in edges {
if let (Some(&source_idx), Some(&target_idx)) =
(node_id_to_index.get(&source_id), node_id_to_index.get(&target_id))
{
let relation_type = RelationType::Custom(relation_type_str);
let properties: HashMap<String, String> = if let Some(json) = properties_json {
serde_json::from_str(&json).unwrap_or_default()
} else {
HashMap::new()
};
let edge = Edge { id, relation_type, weight, properties };
graph.add_edge(source_idx, target_idx, edge);
}
}
Ok(Self { graph, node_index, node_id_to_index, pool, kb_id })
}
/// 添加节点（在给定事务连接上执行，调用方负责提交）
async fn add_node(&mut self, entity: &Entity, conn: &mut sqlx::SqliteConnection) -> Result<NodeIndex> {
if let Some(&idx) = self.node_index.get(&entity.name) {
return Ok(idx);
}
let entity_type_str = entity.entity_type.as_str();
let properties_json = serde_json::to_string(&entity.properties)?;
let sql = "INSERT INTO graph_nodes (name, entity_type, properties, file_id, kb_id) \
VALUES (?, ?, ?, ?, ?) \
ON CONFLICT(name, entity_type, kb_id) DO UPDATE SET \
properties = excluded.properties, \
updated_at = strftime('%s','now') \
RETURNING id";
let id: (i64,) = sqlx::query_as(sql)
.bind(&entity.name)
.bind(entity_type_str)
.bind(&properties_json)
.bind(entity.file_id)
.bind(entity.kb_id.or(self.kb_id))
.fetch_one(&mut *conn)
.await?;
let node = Node::from_entity(entity, id.0);
let idx = self.graph.add_node(node);
self.node_index.insert(entity.name.clone(), idx);
self.node_id_to_index.insert(id.0, idx);
Ok(idx)
}
/// 添加边（在给定事务连接上执行，调用方负责提交）
async fn add_edge(
&mut self, source_name: &str, target_name: &str, relation: &Relation, conn: &mut sqlx::SqliteConnection,
) -> Result<Option<petgraph::graph::EdgeIndex>> {
let source_idx = match self.node_index.get(source_name) {
Some(&idx) => idx,
None => return Ok(None),
};
let target_idx = match self.node_index.get(target_name) {
Some(&idx) => idx,
None => return Ok(None),
};
let source_id = self.graph[source_idx].id;
let target_id = self.graph[target_idx].id;
let relation_type_str = relation.relation_type.as_str();
let properties_json = serde_json::to_string(&relation.properties)?;
let sql = "INSERT INTO graph_edges (source_node_id, target_node_id, relation_type, properties, weight, file_id) \
VALUES (?, ?, ?, ?, ?, ?) \
RETURNING id";
let id: (i64,) = sqlx::query_as(sql)
.bind(source_id)
.bind(target_id)
.bind(relation_type_str)
.bind(&properties_json)
.bind(relation.weight)
.bind(relation.file_id)
.fetch_one(&mut *conn)
.await?;
let edge = Edge::from_relation(relation, id.0);
let edge_idx = self.graph.add_edge(source_idx, target_idx, edge);
Ok(Some(edge_idx))
}
/// 增量更新：所有实体/关系的写入在单个事务内完成，避免逐条提交的往返开销
pub async fn incremental_update(&mut self, entities: Vec<Entity>, relations: Vec<Relation>) -> Result<()> {
let mut tx = self.pool.begin().await?;
for entity in &entities {
self.add_node(entity, &mut tx).await?;
}
for relation in &relations {
self.add_edge(&relation.source_name, &relation.target_name, relation, &mut tx).await?;
}
tx.commit().await?;
Ok(())
}
/// 不加载整个内存图，直接在数据库中做增量 upsert（用于后台文件处理）。
pub async fn incremental_update_direct(
pool: SqlitePool, kb_id: Option<i64>, entities: Vec<Entity>, relations: Vec<Relation>,
) -> Result<()> {
let mut tx = pool.begin().await?;
let mut name_to_id: HashMap<String, i64> = HashMap::with_capacity(entities.len());
for entity in entities {
let entity_type_str = entity.entity_type.as_str();
let properties_json = serde_json::to_string(&entity.properties)?;
let id: (i64,) = sqlx::query_as(
"INSERT INTO graph_nodes (name, entity_type, properties, file_id, kb_id) \
VALUES (?, ?, ?, ?, ?) \
ON CONFLICT(name, entity_type, kb_id) DO UPDATE SET \
properties = excluded.properties, \
updated_at = strftime('%s','now') \
RETURNING id",
)
.bind(&entity.name)
.bind(entity_type_str)
.bind(&properties_json)
.bind(entity.file_id)
.bind(entity.kb_id.or(kb_id))
.fetch_one(&mut *tx)
.await?;
name_to_id.insert(entity.name, id.0);
}
for relation in relations {
let Some(&source_id) = name_to_id.get(&relation.source_name) else {
continue;
};
let Some(&target_id) = name_to_id.get(&relation.target_name) else {
continue;
};
let relation_type_str = relation.relation_type.as_str();
let properties_json = serde_json::to_string(&relation.properties)?;
sqlx::query(
"INSERT INTO graph_edges (source_node_id, target_node_id, relation_type, properties, weight, file_id) \
VALUES (?, ?, ?, ?, ?, ?)",
)
.bind(source_id)
.bind(target_id)
.bind(relation_type_str)
.bind(&properties_json)
.bind(relation.weight)
.bind(relation.file_id)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(())
}
/// 保存图快照
pub async fn save_snapshot(&self) -> Result<()> {
let node_count = self.graph.node_count() as i64;
let edge_count = self.graph.edge_count() as i64;
let delete_sql = "DELETE FROM graph_snapshots WHERE kb_id IS ?";
sqlx::query(delete_sql).bind(self.kb_id).execute(&self.pool).await?;
let insert_sql = "INSERT INTO graph_snapshots (kb_id, graph_data, node_count, edge_count) \
VALUES (?, ?, ?, ?)";
sqlx::query(insert_sql)
.bind(self.kb_id)
.bind(Vec::<u8>::new())
.bind(node_count)
.bind(edge_count)
.execute(&self.pool)
.await?;
Ok(())
}
/// 不加载内存图，直接统计数据库中当前 KB 的节点/边数量并保存快照。
pub async fn save_snapshot_direct(pool: &SqlitePool, kb_id: Option<i64>) -> Result<()> {
let (node_count,): (i64,) = if let Some(kb_id) = kb_id {
sqlx::query_as("SELECT COUNT(*) FROM graph_nodes WHERE kb_id = ?").bind(kb_id).fetch_one(pool).await?
} else {
sqlx::query_as("SELECT COUNT(*) FROM graph_nodes").fetch_one(pool).await?
};
let (edge_count,): (i64,) = if let Some(kb_id) = kb_id {
sqlx::query_as(
"SELECT COUNT(*) FROM graph_edges e \
INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id \
WHERE n1.kb_id = ?",
)
.bind(kb_id)
.fetch_one(pool)
.await?
} else {
sqlx::query_as(
"SELECT COUNT(*) FROM graph_edges e \
INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id",
)
.fetch_one(pool)
.await?
};
let mut tx = pool.begin().await?;
sqlx::query("DELETE FROM graph_snapshots WHERE kb_id IS ?").bind(kb_id).execute(&mut *tx).await?;
sqlx::query(
"INSERT INTO graph_snapshots (kb_id, graph_data, node_count, edge_count) \
VALUES (?, ?, ?, ?)",
)
.bind(kb_id)
.bind(Vec::<u8>::new())
.bind(node_count)
.bind(edge_count)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
}

315
src/graph/llm_extractor.rs Normal file
View File

@ -0,0 +1,315 @@
use std::time::Duration;
use anyhow::{Result, anyhow};
use once_cell::sync::Lazy;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use super::{Entity, EntityType, Relation, RelationType};
use crate::config;
/// 复用的 LLM HTTP client（连接池）。
static LLM_HTTP_CLIENT: Lazy<Client> = Lazy::new(|| {
#[cfg(test)]
let builder = Client::builder().timeout(Duration::from_secs(120)).no_proxy();
#[cfg(not(test))]
let builder = Client::builder().timeout(Duration::from_secs(120));
builder.build().expect("build llm extractor http client")
});
/// LLM API响应格式（OpenAI 兼容格式）
#[derive(Debug, Deserialize)]
struct LLMApiResponse {
choices: Vec<Choice>,
}
#[derive(Debug, Deserialize)]
struct Choice {
message: ResponseMessage,
}
#[derive(Debug, Deserialize)]
struct ResponseMessage {
content: String,
}
/// 知识图谱提取结果（完全由LLM定义）
#[derive(Debug, Deserialize)]
struct KnowledgeGraphResponse {
#[serde(default)]
entities: Vec<LLMEntity>,
#[serde(default)]
relations: Vec<LLMRelation>,
}
#[derive(Debug, Deserialize)]
struct LLMEntity {
name: String,
#[serde(rename = "type")]
entity_type: String,
#[serde(default)]
description: Option<String>,
}
#[derive(Debug, Deserialize)]
struct LLMRelation {
source: String,
target: String,
#[serde(rename = "type")]
relation_type: String,
#[serde(default)]
description: Option<String>,
}
/// LLM请求格式（OpenAI 兼容格式）
#[derive(Debug, Serialize)]
struct LLMRequest {
model: String,
messages: Vec<Message>,
#[serde(skip_serializing_if = "Option::is_none")]
max_tokens: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
temperature: Option<f32>,
}
#[derive(Debug, Serialize)]
struct Message {
role: String,
content: String,
}
/// LLM知识图谱提取器（完全由LLM生成实体和关系）
pub struct LLMGraphExtractor {
client: Client,
api_url: String,
api_key: Option<String>,
model: String,
enabled: bool,
}
impl LLMGraphExtractor {
/// 从配置创建
pub fn from_env() -> Self {
let cfg = config::get();
let llm_config = &cfg.llm;
let enabled = llm_config.is_enabled();
let api_url = llm_config.api_url.clone().unwrap_or_default();
let api_key = llm_config.api_key.clone();
let model = llm_config.model.clone();
// 复用全局 client，避免每次构造都新建连接池
let client = LLM_HTTP_CLIENT.clone();
Self { client, api_url, api_key, model, enabled }
}
/// 检查是否已启用
pub fn is_enabled(&self) -> bool {
self.enabled
}
/// 提取知识图谱（实体和关系一次性提取，类型完全由LLM定义）
pub async fn extract_knowledge_graph(&self, text: &str, context: &str) -> Result<(Vec<Entity>, Vec<Relation>)> {
if !self.enabled {
return Err(anyhow!("LLM未启用"));
}
if text.trim().is_empty() {
return Ok((Vec::new(), Vec::new()));
}
let prompt = self.build_extraction_prompt(text, context);
let response = self.call_llm(&prompt).await?;
// 转换实体
let entities: Vec<Entity> = response
.entities
.into_iter()
.map(|e| {
// 实体类型完全由LLM定义，直接使用Custom类型
let entity_type = EntityType::Custom(e.entity_type);
let mut entity = Entity::new(e.name, entity_type);
if let Some(desc) = e.description {
entity = entity.with_property("description".to_string(), desc);
}
entity
})
.collect();
// 转换关系
let relations: Vec<Relation> = response
.relations
.into_iter()
.map(|r| {
// 关系类型完全由LLM定义，直接使用Custom类型
let relation_type = RelationType::Custom(r.relation_type);
let mut relation = Relation::new(r.source, r.target, relation_type);
if let Some(desc) = r.description {
relation = relation.with_property("description".to_string(), desc);
}
relation
})
.collect();
Ok((entities, relations))
}
/// 构建知识图谱提取提示词
fn build_extraction_prompt(&self, text: &str, context: &str) -> String {
format!(
r#"你是一个专业的知识图谱构建专家。请仔细阅读以下文本，从中提取实体和实体之间的关系，构建知识图谱。
文档信息: {}
文本内容:
{}
## 任务要求
### 1. 实体提取
请识别文本中的所有重要实体，包括但不限于：
- 人名、组织名、公司名
- 技术名词、编程语言、框架、工具
- 项目名称、产品名称
- 地点、时间
- 专业术语、概念
- 其他你认为重要的实体
对于每个实体，请自定义一个最合适的类型名称（用中文），如：
- 人物、公司、技术、框架、编程语言、数据库、工具、项目、地点、时间段、概念、技能、职位 等
### 2. 关系提取
请识别实体之间的所有有意义的关系。关系类型请自定义（用中文），可以包括但不限于：
- 任职于、工作于、属于
- 使用、掌握、熟悉、精通
- 负责、开发、设计、实现
- 包含、组成、依赖
- 位于、发生于
- 关联、相关
- 其他你认为合适的关系类型
### 3. 输出要求
请严格按照以下JSON格式输出，不要输出任何其他内容：
```json
{{
"entities": [
{{
"name": "实体名称",
"type": "实体类型（自定义中文类型）",
"description": "简短描述（可选）"
}}
],
"relations": [
{{
"source": "源实体名称",
"target": "目标实体名称",
"type": "关系类型（自定义中文类型）",
"description": "关系描述（可选）"
}}
]
}}
```
### 4. 注意事项
- 实体名称要准确，与文本中的表述一致
- 关系的source和target必须是entities中存在的实体名称
- 尽量提取所有有意义的实体和关系，不要遗漏
- 避免提取过于笼统或无意义的实体（如"系统"、"项目"等太泛化的词）
- 关系要基于文本内容，不要臆测
- 只输出JSON，不要有任何其他文字
请开始提取："#,
context, text
)
}
/// 调用LLM API
async fn call_llm(&self, prompt: &str) -> Result<KnowledgeGraphResponse> {
let request = LLMRequest {
model: self.model.clone(),
messages: vec![Message { role: "user".to_string(), content: prompt.to_string() }],
max_tokens: Some(4000),
temperature: Some(0.1), // 低温度，更确定性的输出
};
let mut req_builder = self.client.post(&self.api_url).json(&request);
if let Some(ref key) = self.api_key {
req_builder = req_builder.header("Authorization", format!("Bearer {}", key));
}
let response = req_builder.send().await.map_err(|e| anyhow!("LLM API请求失败: {}", e))?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(anyhow!("LLM API返回错误 {}: {}", status, body));
}
let body = response.text().await.map_err(|e| anyhow!("读取响应失败: {}", e))?;
// 解析 OpenAI 格式的响应
let api_response: LLMApiResponse =
serde_json::from_str(&body).map_err(|e| anyhow!("解析LLM API响应失败: {}. 响应内容: {}", e, body))?;
// 从第一个 choice 的 message 中提取内容
let content =
api_response.choices.first().ok_or_else(|| anyhow!("LLM响应中没有choices"))?.message.content.clone();
// 尝试清理可能的markdown代码块标记
let clean_content = self.clean_json_response(&content);
// 解析 JSON 内容
serde_json::from_str::<KnowledgeGraphResponse>(&clean_content)
.map_err(|e| anyhow!("解析知识图谱JSON失败: {}. 内容: {}", e, clean_content))
}
/// 清理LLM响应中可能的markdown代码块标记
fn clean_json_response(&self, content: &str) -> String {
let content = content.trim();
// 移除可能的 ```json 和 ``` 标记
let content = if content.starts_with("```json") {
content.strip_prefix("```json").unwrap_or(content)
} else if content.starts_with("```") {
content.strip_prefix("```").unwrap_or(content)
} else {
content
};
let content = if content.ends_with("```") { content.strip_suffix("```").unwrap_or(content) } else { content };
content.trim().to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_llm_graph_extractor_creation() {
let extractor = LLMGraphExtractor::from_env();
// 如果没有设置环境变量，应该是禁用状态
// assert!(!extractor.is_enabled());
println!("LLM enabled: {}", extractor.is_enabled());
}
#[test]
fn test_clean_json_response() {
let extractor = LLMGraphExtractor::from_env();
let input = r#"```json
{"entities": [], "relations": []}
```"#;
let cleaned = extractor.clean_json_response(input);
assert_eq!(cleaned, r#"{"entities": [], "relations": []}"#);
let input2 = r#"{"entities": [], "relations": []}"#;
let cleaned2 = extractor.clean_json_response(input2);
assert_eq!(cleaned2, r#"{"entities": [], "relations": []}"#);
}
}

121
src/graph/mod.rs Normal file
View File

@ -0,0 +1,121 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
pub mod graph_manager;
pub mod llm_extractor;
/// 实体类型枚举
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum EntityType {
// 领域特定（完全由LLM定义）
Custom(String),
}
impl EntityType {
/// 转换为字符串
pub fn as_str(&self) -> String {
match self {
EntityType::Custom(s) => s.clone(),
}
}
}
/// 关系类型枚举
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum RelationType {
// 自定义（完全由LLM定义）
Custom(String),
}
impl RelationType {
/// 转换为字符串
pub fn as_str(&self) -> String {
match self {
RelationType::Custom(s) => s.clone(),
}
}
}
/// 实体结构
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entity {
pub name: String,
pub entity_type: EntityType,
pub properties: HashMap<String, String>,
pub file_id: Option<i64>,
pub kb_id: Option<i64>,
}
impl Entity {
pub fn new(name: String, entity_type: EntityType) -> Self {
Self { name, entity_type, properties: HashMap::new(), file_id: None, kb_id: None }
}
pub fn with_property(mut self, key: String, value: String) -> Self {
self.properties.insert(key, value);
self
}
}
/// 关系结构
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Relation {
pub source_name: String,
pub target_name: String,
pub relation_type: RelationType,
pub properties: HashMap<String, String>,
pub weight: f32,
pub file_id: Option<i64>,
}
impl Relation {
pub fn new(source_name: String, target_name: String, relation_type: RelationType) -> Self {
Self { source_name, target_name, relation_type, properties: HashMap::new(), weight: 1.0, file_id: None }
}
pub fn with_property(mut self, key: String, value: String) -> Self {
self.properties.insert(key, value);
self
}
}
/// 图节点（用于petgraph）
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Node {
pub id: i64,
pub name: String,
pub entity_type: EntityType,
pub properties: HashMap<String, String>,
}
impl Node {
pub fn from_entity(entity: &Entity, id: i64) -> Self {
Self {
id,
name: entity.name.clone(),
entity_type: entity.entity_type.clone(),
properties: entity.properties.clone(),
}
}
}
/// 图边（用于petgraph）
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Edge {
pub id: i64,
pub relation_type: RelationType,
pub weight: f32,
pub properties: HashMap<String, String>,
}
impl Edge {
pub fn from_relation(relation: &Relation, id: i64) -> Self {
Self {
id,
relation_type: relation.relation_type.clone(),
weight: relation.weight,
properties: relation.properties.clone(),
}
}
}

129
src/image_description.rs Normal file
View File

@ -0,0 +1,129 @@
//! 图片描述持久化模块
//!
//! 将外部图片文本化接口返回的描述和原始响应保存到 SQLite，
//! 支持索引重建/恢复时无需再次调用外部服务。
use std::collections::HashMap;
use anyhow::{Context, Result};
use sqlx::{FromRow, QueryBuilder, Sqlite, SqlitePool};
#[derive(Debug, Clone, FromRow)]
pub struct ImageDescription {
pub id: i64,
pub file_id: i64,
pub image_filename: String,
pub description: String,
pub jpg_filename: Option<String>,
pub raw_response: String,
pub source: String,
pub created_at: i64,
pub updated_at: i64,
}
/// 保存或更新单条图片描述
pub async fn save(
pool: &SqlitePool, file_id: i64, image_filename: &str, description: &str, raw_response: &str, source: &str, jpg_filename: Option<&str>,
) -> Result<()> {
sqlx::query(
"INSERT INTO image_descriptions (file_id, image_filename, description, raw_response, source, jpg_filename)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(file_id, image_filename) DO UPDATE SET
description = excluded.description,
raw_response = excluded.raw_response,
source = excluded.source,
jpg_filename = excluded.jpg_filename,
updated_at = strftime('%s','now')",
)
.bind(file_id)
.bind(image_filename)
.bind(description)
.bind(raw_response)
.bind(source)
.bind(jpg_filename)
.execute(pool)
.await
.with_context(|| format!("failed to save image_description for file_id={} filename={}", file_id, image_filename))?;
Ok(())
}
/// 按文件 ID + 图片文件名查询
pub async fn get(pool: &SqlitePool, file_id: i64, image_filename: &str) -> Result<Option<ImageDescription>> {
let row = sqlx::query_as::<_, ImageDescription>(
"SELECT id, file_id, image_filename, description, raw_response, source, jpg_filename, created_at, updated_at
FROM image_descriptions WHERE file_id = ? AND image_filename = ?",
)
.bind(file_id)
.bind(image_filename)
.fetch_optional(pool)
.await
.with_context(|| format!("failed to get image_description for file_id={} filename={}", file_id, image_filename))?;
Ok(row)
}
/// 删除指定图片的已缓存描述。用于清理旧版本误写入的服务错误文本。
pub async fn delete(pool: &SqlitePool, file_id: i64, image_filename: &str) -> Result<()> {
sqlx::query("DELETE FROM image_descriptions WHERE file_id = ? AND image_filename = ?")
.bind(file_id)
.bind(image_filename)
.execute(pool)
.await
.with_context(|| format!("failed to delete image_description for file_id={} filename={}", file_id, image_filename))?;
Ok(())
}
/// 按文件 ID 列出所有图片描述（filename -> description）
pub async fn list_by_file(pool: &SqlitePool, file_id: i64) -> Result<HashMap<String, String>> {
let rows: Vec<ImageDescription> = sqlx::query_as::<_, ImageDescription>(
"SELECT id, file_id, image_filename, description, raw_response, source, jpg_filename, created_at, updated_at
FROM image_descriptions WHERE file_id = ?",
)
.bind(file_id)
.fetch_all(pool)
.await
.with_context(|| format!("failed to list image_descriptions for file_id={}", file_id))?;
Ok(rows.into_iter().map(|row| (row.image_filename, row.description)).collect())
}
#[derive(Debug, sqlx::FromRow)]
struct FileImageDescriptionRow {
file_id: i64,
image_filename: String,
description: String,
jpg_filename: Option<String>,
}
/// 批量查询多个文件的图片描述，返回 (file_id, image_filename) -> description 映射。
pub async fn list_by_file_ids(pool: &SqlitePool, file_ids: &[i64]) -> Result<HashMap<(i64, String), (String, Option<String>)>> {
if file_ids.is_empty() {
return Ok(HashMap::new());
}
let mut query_builder: QueryBuilder<'_, Sqlite> =
QueryBuilder::new("SELECT file_id, image_filename, description, jpg_filename FROM image_descriptions WHERE file_id IN (");
let mut separated = query_builder.separated(", ");
for file_id in file_ids {
separated.push_bind(file_id);
}
separated.push_unseparated(")");
let rows: Vec<FileImageDescriptionRow> = query_builder.build_query_as().fetch_all(pool).await?;
Ok(rows.into_iter().map(|row| ((row.file_id, row.image_filename), (row.description, row.jpg_filename))).collect())
}
/// 复制源文件的所有图片描述记录到目标文件（旧版解析结果复用场景）
pub async fn copy_for_file(pool: &SqlitePool, src_file_id: i64, dst_file_id: i64) -> Result<()> {
let rows: Vec<ImageDescription> = sqlx::query_as::<_, ImageDescription>(
"SELECT id, file_id, image_filename, description, raw_response, source, jpg_filename, created_at, updated_at
FROM image_descriptions WHERE file_id = ?",
)
.bind(src_file_id)
.fetch_all(pool)
.await
.with_context(|| format!("failed to copy image_descriptions from file_id={}", src_file_id))?;
for row in rows {
save(pool, dst_file_id, &row.image_filename, &row.description, &row.raw_response, &row.source, row.jpg_filename.as_deref()).await?;
}
Ok(())
}

239
src/image_parse.rs Normal file
View File

@ -0,0 +1,239 @@
//! 外部图片文本化服务客户端
//!
//! 当配置了 `HTKNOW_IMAGE_PARSE_URL` 时，处理器会把遇到的每张图片发送到该服务，
//! 获取文本化描述并持久化。本模块负责构造请求、解析多种可能的响应格式。
use std::time::Duration;
use anyhow::{Context, Result};
use base64::{Engine, engine::general_purpose::STANDARD};
use once_cell::sync::Lazy;
use reqwest::Client;
use serde::Serialize;
use serde_json::Value;
use crate::config;
static HTTP_CLIENT: Lazy<Client> = Lazy::new(Client::new);
#[derive(Debug, Serialize)]
struct ParseImageRequest {
#[serde(skip_serializing_if = "Option::is_none")]
content: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
original_filename: Option<String>,
filename: String,
image_base64: String,
}
/// 图片文本化结果
#[derive(Debug, Clone)]
pub struct ImageParseResponse {
/// 提取后的描述文本
pub description: String,
/// 服务返回的原始响应（JSON 或纯文本），用于恢复/排查
pub raw_response: String,
/// 解析服务返回的 JPG 文件名
pub jpg_filename: Option<String>,
}
/// 从本地图片文件调用文本化服务
pub async fn parse_image_file(
path: &std::path::Path, filename: &str, surrounding_content: Option<&str>,
) -> Result<ImageParseResponse> {
let bytes =
tokio::fs::read(path).await.with_context(|| format!("failed to read image file: {}", path.display()))?;
let base64 = STANDARD.encode(&bytes);
parse_image_base64(filename, &base64, surrounding_content, None).await
}
/// 从 base64 编码的图片内容调用文本化服务
pub async fn parse_image_base64(
filename: &str, image_base64: &str, surrounding_content: Option<&str>, original_filename: Option<&str>,
) -> Result<ImageParseResponse> {
let cfg = config::get();
let url =
cfg.services.image_parse_url.as_deref().ok_or_else(|| anyhow::anyhow!("image parse URL is not configured"))?;
// 兼容 data:image/xxx;base64, 前缀以及空白字符
let payload = image_base64
.find("base64,")
.map(|idx| &image_base64[idx + "base64,".len()..])
.unwrap_or(image_base64)
.chars()
.filter(|c| !c.is_whitespace())
.collect();
let request = ParseImageRequest {
content: surrounding_content.map(|s| s.to_string()),
original_filename: original_filename.map(|s| s.to_string()),
filename: filename.to_string(),
image_base64: payload,
};
let response = HTTP_CLIENT
.post(url)
.timeout(Duration::from_secs(cfg.services.image_parse_timeout_secs))
.json(&request)
.send()
.await
.with_context(|| format!("image parse request failed: url={}, filename={}", url, filename))?;
if !response.status().is_success() {
let status = response.status();
let text = response.text().await.unwrap_or_default();
anyhow::bail!("image parse API error: {} - {}", status, text);
}
let raw_response = response.text().await.unwrap_or_default();
if is_error_response(&raw_response) {
anyhow::bail!("image parse API returned an error response: {}", raw_response.trim());
}
let description = extract_description(&raw_response);
let jpg_filename = extract_string_field(&raw_response, "jpg_filename");
Ok(ImageParseResponse { description, raw_response, jpg_filename })
}
/// 判断文本化服务是否在 HTTP 200 中返回了业务错误。
///
/// 部分服务会用 `Error: ...` 或 JSON 的 `error` 字段表示失败。此类响应不能作为图片描述写入索引。
pub fn is_error_response(raw_response: &str) -> bool {
let trimmed = raw_response.trim();
if trimmed.is_empty() {
return false;
}
let lower = trimmed.to_ascii_lowercase();
if lower.starts_with("error:") || lower.starts_with("connection error") || lower.starts_with("failed:") {
return true;
}
let Ok(value) = serde_json::from_str::<Value>(trimmed) else {
return false;
};
let Some(object) = value.as_object() else {
return false;
};
if object.get("error").is_some() {
return true;
}
if object.get("code").and_then(Value::as_i64).is_some_and(|code| code >= 400) {
return true;
}
object
.get("status")
.and_then(Value::as_str)
.is_some_and(|status| matches!(status.to_ascii_lowercase().as_str(), "error" | "failed" | "failure"))
}
/// 从服务原始响应中提取描述文本，兼容多种 JSON 结构，解析失败时返回原字符串
pub fn extract_description(raw_response: &str) -> String {
let trimmed = raw_response.trim();
if trimmed.is_empty() {
return String::new();
}
let value: Value = match serde_json::from_str(trimmed) {
Ok(v) => v,
Err(_) => return trimmed.to_string(),
};
if let Some(desc) = find_description_value(&value) {
return desc.trim().to_string();
}
trimmed.to_string()
}
fn extract_string_field(raw_response: &str, field: &str) -> Option<String> {
let value = serde_json::from_str::<Value>(raw_response.trim()).ok()?;
find_string_field(&value, field)
}
fn find_string_field(value: &Value, field: &str) -> Option<String> {
if let Some(result) = value.get(field).and_then(Value::as_str) {
return Some(result.trim().to_string());
}
if let Some(object) = value.as_object() {
for nested in object.values() {
if let Some(result) = find_string_field(nested, field) {
return Some(result);
}
}
}
if let Some(items) = value.as_array() {
for item in items {
if let Some(result) = find_string_field(item, field) {
return Some(result);
}
}
}
None
}
fn find_description_value(value: &Value) -> Option<String> {
// 优先顶层常见字段
for key in &["image_content", "description", "text", "content", "result"] {
if let Some(v) = value.get(key).and_then(|v| v.as_str()) {
return Some(v.to_string());
}
}
// 尝试 data / result 嵌套
for nested_key in &["data", "result"] {
if let Some(nested) = value.get(nested_key) {
if let Some(v) = nested.as_str() {
return Some(v.to_string());
}
for key in &["image_content", "description", "text", "content", "result"] {
if let Some(v) = nested.get(key).and_then(|v| v.as_str()) {
return Some(v.to_string());
}
}
}
}
// 本身就是字符串
value.as_str().map(|s| s.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_description_variants() {
assert_eq!(extract_description(r#"{"description": " a picture "}"#), "a picture");
assert_eq!(extract_description(r#"{"data": {"text": "hello"}}"#), "hello");
assert_eq!(extract_description(r#"{"result": {"description": "world"}}"#), "world");
assert_eq!(
extract_description(r#"{"code":200,"message":"ok","data":{"image_content":"返回内容","filename":""}}"#),
"返回内容"
);
assert_eq!(extract_description("plain text"), "plain text");
assert_eq!(extract_description(""), "");
}
#[test]
fn test_error_responses_are_not_descriptions() {
assert!(is_error_response("Error: Connection error."));
assert!(is_error_response(r#"{"error":"Connection error"}"#));
assert!(is_error_response(r#"{"code":500,"message":"Connection error"}"#));
assert!(!is_error_response(r#"{"description":"A cat on a sofa"}"#));
assert!(!is_error_response("A picture of a connection cable"));
}
#[test]
fn test_normalize_base64_payload() {
let raw = "data:image/png;base64, iVBORw0KGgo=\n";
let normalized: String = raw
.find("base64,")
.map(|idx| &raw[idx + "base64,".len()..])
.unwrap_or(raw)
.chars()
.filter(|c| !c.is_whitespace())
.collect();
assert_eq!(normalized, "iVBORw0KGgo=");
}
}

256
src/init.sql Normal file
View File

@ -0,0 +1,256 @@
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT, -- 归属用户ID
user_name TEXT, -- 归属用户名
hash TEXT NOT NULL, -- 文件HASH
filename TEXT NOT NULL, -- 文件名称
path TEXT NOT NULL, -- 文件路径
size INTEGER NOT NULL DEFAULT 0, -- 文件大小(字节)
tags TEXT NOT NULL DEFAULT '', -- 标签
status INTEGER NOT NULL DEFAULT 0, -- 状态: 0-未处理, 1-已处理, 2-处理中, 3-不解析, -1-处理失败
parse_run_id TEXT DEFAULT NULL, -- 当前解析批次，防止并发解析和旧任务提交结果
artifact_id INTEGER DEFAULT NULL, -- 共享解析产物
log TEXT DEFAULT '', -- 日志
slice_type TEXT DEFAULT '', -- 切片类型
kb_id INTEGER DEFAULT NULL, -- 知识库ID
parse_priority INTEGER NOT NULL DEFAULT 50, -- 解析队列优先级快照，避免领取待解析文件时跨表排序
is_public INTEGER NOT NULL DEFAULT 0, -- 是否公开: 0-私有, 1-公开
meta TEXT DEFAULT NULL, -- 元数据，存储任意JSON数据
summary TEXT DEFAULT NULL, -- 文件摘要
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now'))
);
CREATE TABLE IF NOT EXISTS knowledge_bases (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT, -- 归属用户ID
user_name TEXT, -- 归属用户名
name TEXT NOT NULL, -- 知识库名称
description TEXT DEFAULT '', -- 知识库描述
kb_type TEXT NOT NULL DEFAULT 'analysis', -- 知识库类型: analysis-分析型, storage-存储型
parent_id INTEGER, -- 父级知识库ID
is_public INTEGER NOT NULL DEFAULT 0, -- 是否公开: 0-私有, 1-公开
parse_priority INTEGER NOT NULL DEFAULT 50, -- 解析优先级: 0-100，越大越优先
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY(parent_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS slices (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_id INTEGER NOT NULL, -- 文件ID
parse_run_id TEXT DEFAULT NULL, -- 生成该切片的解析批次；历史数据为空
ordinal INTEGER DEFAULT NULL, -- 切片在本次解析中的稳定顺序
is_image INTEGER NOT NULL DEFAULT 0, -- 是否为图片切片: 0-否, 1-是
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY(file_id) REFERENCES files(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
name TEXT NOT NULL,
applied_at INTEGER NOT NULL DEFAULT (strftime('%s','now'))
);
CREATE TABLE IF NOT EXISTS image_descriptions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_id INTEGER NOT NULL,
image_filename TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
raw_response TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT '',
jpg_filename TEXT DEFAULT NULL,
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now')),
UNIQUE(file_id, image_filename)
);
CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_id ON image_descriptions(file_id);
CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_filename ON image_descriptions(file_id, image_filename);
CREATE TABLE IF NOT EXISTS parse_artifacts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
artifact_key TEXT NOT NULL UNIQUE,
content_hash TEXT NOT NULL,
slice_type TEXT NOT NULL,
parser_version TEXT NOT NULL,
config_hash TEXT NOT NULL,
source_file_id INTEGER NOT NULL,
full_content TEXT DEFAULT NULL,
summary TEXT DEFAULT NULL,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now'))
);
CREATE INDEX IF NOT EXISTS idx_files_artifact_id ON files(artifact_id);
CREATE TABLE IF NOT EXISTS slice_positions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
slice_id INTEGER NOT NULL, -- 切片ID
page_idx INTEGER NOT NULL, -- 所在页码（Excel 中表示 sheet 索引）
x1 INTEGER NOT NULL,
y1 INTEGER NOT NULL,
x2 INTEGER NOT NULL,
y2 INTEGER NOT NULL,
sheet_name TEXT DEFAULT NULL, -- Excel 中所在 sheet 名称
row_num INTEGER DEFAULT NULL, -- Excel 中所在行号（1-based）
created_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY (slice_id) REFERENCES slices(id) ON DELETE CASCADE
);
-- 知识图谱相关表
-- 图节点表（实体）
CREATE TABLE IF NOT EXISTS graph_nodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL, -- 实体名称
entity_type TEXT NOT NULL, -- 实体类型
properties TEXT, -- JSON格式的属性
embedding BLOB, -- 实体embedding向量
file_id INTEGER, -- 来源文件ID
kb_id INTEGER, -- 所属知识库ID
is_public INTEGER NOT NULL DEFAULT 0, -- 是否公开: 0-私有, 1-公开
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now')),
UNIQUE(name, entity_type, kb_id) -- 同一知识库内，同类型实体名称唯一
);
-- 图边表（关系）
CREATE TABLE IF NOT EXISTS graph_edges (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_node_id INTEGER NOT NULL, -- 源节点ID
target_node_id INTEGER NOT NULL, -- 目标节点ID
relation_type TEXT NOT NULL, -- 关系类型
properties TEXT, -- JSON格式的属性
weight REAL DEFAULT 1.0, -- 关系权重
file_id INTEGER, -- 来源文件ID
created_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY (source_node_id) REFERENCES graph_nodes(id) ON DELETE CASCADE,
FOREIGN KEY (target_node_id) REFERENCES graph_nodes(id) ON DELETE CASCADE
);
-- 实体提及表（实体在文档中的出现位置）
CREATE TABLE IF NOT EXISTS entity_mentions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id INTEGER NOT NULL, -- 节点ID
slice_id INTEGER NOT NULL, -- 切片ID
start_offset INTEGER, -- 起始位置
end_offset INTEGER, -- 结束位置
context TEXT, -- 上下文片段
created_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY (node_id) REFERENCES graph_nodes(id) ON DELETE CASCADE,
FOREIGN KEY (slice_id) REFERENCES slices(id) ON DELETE CASCADE
);
-- 图快照表（存储序列化的图结构）
CREATE TABLE IF NOT EXISTS graph_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kb_id INTEGER, -- 知识库ID（NULL表示全局）
graph_data BLOB NOT NULL, -- 序列化的petgraph图
node_count INTEGER, -- 节点数量
edge_count INTEGER, -- 边数量
version INTEGER DEFAULT 1, -- 版本号
created_at INTEGER DEFAULT (strftime('%s','now')),
UNIQUE(kb_id, version)
);
-- 索引
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_parent_id ON knowledge_bases(parent_id);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_user_public_parent ON knowledge_bases(user_id, is_public, parent_id);
CREATE INDEX IF NOT EXISTS idx_files_kb_id ON files(kb_id);
CREATE INDEX IF NOT EXISTS idx_files_kb_id_created_at ON files(kb_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_files_kb_id_filename ON files(kb_id, filename);
CREATE INDEX IF NOT EXISTS idx_files_kb_id_updated_at ON files(kb_id, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_files_kb_id_status_updated_at ON files(kb_id, status, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_files_status_updated_at ON files(status, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_files_status_id ON files(status, id);
CREATE INDEX IF NOT EXISTS idx_files_id_kb_id ON files(id, kb_id);
CREATE INDEX IF NOT EXISTS idx_files_pending_created_at ON files(created_at) WHERE status = 0;
CREATE INDEX IF NOT EXISTS idx_files_user_id_created_at ON files(user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_files_hash_slice_type_status_updated_at ON files(hash, slice_type, status, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_slices_file_id_id ON slices(file_id, id);
CREATE UNIQUE INDEX IF NOT EXISTS idx_slices_file_run_ordinal
ON slices(file_id, parse_run_id, ordinal)
WHERE parse_run_id IS NOT NULL AND ordinal IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_nodes_type ON graph_nodes(entity_type);
CREATE INDEX IF NOT EXISTS idx_nodes_kb ON graph_nodes(kb_id);
CREATE INDEX IF NOT EXISTS idx_nodes_file ON graph_nodes(file_id);
CREATE INDEX IF NOT EXISTS idx_nodes_name ON graph_nodes(name);
CREATE INDEX IF NOT EXISTS idx_edges_source ON graph_edges(source_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_target ON graph_edges(target_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_relation ON graph_edges(relation_type);
CREATE INDEX IF NOT EXISTS idx_mentions_node ON entity_mentions(node_id);
CREATE INDEX IF NOT EXISTS idx_mentions_slice ON entity_mentions(slice_id);
CREATE INDEX IF NOT EXISTS idx_slice_positions_slice ON slice_positions(slice_id);
CREATE INDEX IF NOT EXISTS idx_files_meta ON files(meta);
-- 搜索词表（分词增强）
CREATE TABLE IF NOT EXISTS search_lexicon (
id INTEGER PRIMARY KEY AUTOINCREMENT,
term TEXT NOT NULL UNIQUE, -- 词条
freq INTEGER DEFAULT NULL, -- 词频
tag TEXT DEFAULT NULL, -- 词性
enabled INTEGER NOT NULL DEFAULT 1, -- 是否启用
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now'))
);
CREATE INDEX IF NOT EXISTS idx_search_lexicon_enabled ON search_lexicon(enabled);
-- 搜索同义词词典（查询扩展）
CREATE TABLE IF NOT EXISTS search_synonyms (
id INTEGER PRIMARY KEY AUTOINCREMENT,
term TEXT NOT NULL, -- 原词
synonym TEXT NOT NULL, -- 同义词
weight REAL NOT NULL DEFAULT 1.0, -- 行级权重
bidirectional INTEGER NOT NULL DEFAULT 1, -- 是否双向
enabled INTEGER NOT NULL DEFAULT 1, -- 是否启用
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now')),
UNIQUE(term, synonym)
);
CREATE INDEX IF NOT EXISTS idx_search_synonyms_term ON search_synonyms(term);
CREATE INDEX IF NOT EXISTS idx_search_synonyms_synonym ON search_synonyms(synonym);
-- 压缩文件条目表
CREATE TABLE IF NOT EXISTS archive_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_id INTEGER NOT NULL,
entry_path TEXT NOT NULL,
size INTEGER,
is_directory INTEGER DEFAULT 0,
created_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_archive_entries_file_id ON archive_entries(file_id);
-- 索引重建任务状态
CREATE TABLE IF NOT EXISTS index_rebuild_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
status TEXT NOT NULL DEFAULT 'idle', -- idle/running/completed/failed
phase TEXT NOT NULL DEFAULT '', -- 当前阶段
total_docs INTEGER NOT NULL DEFAULT 0, -- 总文档数（切片 + 全文等）
processed_docs INTEGER NOT NULL DEFAULT 0,-- 已处理文档数
started_at INTEGER DEFAULT NULL, -- 开始时间（秒级时间戳）
updated_at INTEGER DEFAULT (strftime('%s','now')),
finished_at INTEGER DEFAULT NULL, -- 完成时间
error TEXT DEFAULT NULL, -- 错误信息
meta TEXT DEFAULT NULL -- 扩展信息（JSON）
);
CREATE INDEX IF NOT EXISTS idx_index_rebuild_jobs_status ON index_rebuild_jobs(status);
CREATE INDEX IF NOT EXISTS idx_index_rebuild_jobs_updated_at ON index_rebuild_jobs(updated_at);
-- 知识库成员权限表
CREATE TABLE IF NOT EXISTS kb_permissions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kb_id INTEGER NOT NULL,
user_id TEXT NOT NULL,
permission TEXT NOT NULL DEFAULT 'viewer', -- 'viewer' | 'editor' | 'admin'
created_at INTEGER DEFAULT (strftime('%s','now')),
updated_at INTEGER DEFAULT (strftime('%s','now')),
FOREIGN KEY(kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE,
UNIQUE(kb_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_kb_permissions_kb_id ON kb_permissions(kb_id);
CREATE INDEX IF NOT EXISTS idx_kb_permissions_user_id ON kb_permissions(user_id);

62
src/lib.rs Normal file
View File

@ -0,0 +1,62 @@
use axum::{
body::Body, http::{Request, StatusCode}, middleware::Next, response::{IntoResponse, Response}
};
use base64::{Engine, engine::general_purpose::STANDARD};
pub mod api;
pub mod archive;
pub mod config;
pub mod db;
pub mod export;
pub mod file_content;
pub mod frontend;
pub mod graph;
pub mod image_description;
pub mod image_parse;
pub mod log4rs;
pub mod pdf_content;
pub mod pdf_highlight;
pub mod processor;
pub mod search;
pub mod slice_content;
/// User authentication info extracted from request headers
#[derive(Clone, Debug)]
pub struct AuthUser {
pub user_id: String,
pub user_name: String,
pub role: String,
}
impl AuthUser {
pub fn is_admin(&self) -> bool {
self.role == "admin"
}
}
fn decode_base64_if_utf8(value: &str) -> String {
match STANDARD.decode(value) {
Ok(bytes) => String::from_utf8(bytes).unwrap_or_else(|_| value.to_owned()),
Err(_) => value.to_owned(),
}
}
/// Auth middleware: extract `x-user-id` and `x-role` from headers and put them
/// into request extensions as `AuthUser` so handlers can extract `Extension<AuthUser>`.
/// Returns 401 if required headers are missing or invalid.
pub async fn auth(mut req: Request<Body>, next: Next) -> Response {
let user_id_opt = req.headers().get("x-user-id").and_then(|v| v.to_str().ok()).map(|s| s.to_owned());
let user_name_opt = req.headers().get("x-user-name").and_then(|v| v.to_str().ok()).map(decode_base64_if_utf8);
let role_opt = req.headers().get("x-role").and_then(|v| v.to_str().ok()).map(|s| s.to_owned());
match (user_id_opt, role_opt) {
(Some(user_id), Some(role)) => {
let user_name = user_name_opt.unwrap_or_default();
let auth_user = AuthUser { user_id, user_name, role };
req.extensions_mut().insert(auth_user);
next.run(req).await
}
(None, _) => (StatusCode::UNAUTHORIZED, "Missing x-user-id header").into_response(),
(_, None) => (StatusCode::UNAUTHORIZED, "Missing x-role header").into_response(),
}
}

74
src/log4rs.rs Normal file
View File

@ -0,0 +1,74 @@
use std::{env, str::FromStr};
use log4rs::{
append::console::ConsoleAppender, config::{Appender, Config, Logger, Root}, encode::pattern::PatternEncoder
};
fn parse_rust_log(rust_log: &str) -> (Option<log::LevelFilter>, Vec<(String, log::LevelFilter)>) {
let mut default_level = None;
let mut loggers = Vec::new();
for directive in rust_log.split(',') {
let directive = directive.trim();
if directive.is_empty() {
continue;
}
let (target, level_str) = match directive.split_once('=') {
Some((target, level)) => (Some(target.trim()), level.trim()),
None => (None, directive),
};
let Ok(level) = log::LevelFilter::from_str(level_str) else {
continue;
};
match target {
Some(target) if !target.is_empty() => {
loggers.push((target.to_string(), level));
}
_ => {
default_level = Some(level);
}
}
}
(default_level, loggers)
}
pub fn init() {
// 创建一个控制台 Appender
let stdout = ConsoleAppender::builder()
.encoder(Box::new(PatternEncoder::new("{d(%Y-%m-%d %H:%M:%S)} [{t}] {h({l})} - {m}{n}")))
.build();
let mut root_level = log::LevelFilter::Info;
let mut logger_levels = Vec::new();
if let Ok(rust_log) = env::var("RUST_LOG") {
let (default_level, parsed_loggers) = parse_rust_log(&rust_log);
logger_levels = parsed_loggers;
if let Some(level) = default_level {
root_level = level;
} else if !logger_levels.is_empty() {
root_level = log::LevelFilter::Off;
}
}
// Lance 的逐次 I/O span 在 INFO 级别会产生大量日志。显式配置相同 target 时仍允许覆盖。
for (target, level) in [("lance", log::LevelFilter::Warn), ("tracing::span", log::LevelFilter::Warn)] {
if !logger_levels.iter().any(|(configured_target, _)| configured_target == target) {
logger_levels.push((target.to_string(), level));
}
}
// 构建 log4rs 配置，仅输出到控制台
let mut config_builder = Config::builder().appender(Appender::builder().build("stdout", Box::new(stdout)));
for (target, level) in logger_levels {
config_builder = config_builder.logger(Logger::builder().build(target, level));
}
let config =
config_builder.build(Root::builder().appender("stdout").build(root_level)).expect("构建 log4rs 配置失败");
// 初始化日志器
log4rs::init_config(config).expect("初始化 log4rs 失败");
}

191
src/main.rs Normal file
View File

@ -0,0 +1,191 @@
#[cfg(debug_assertions)]
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
#[cfg(all(debug_assertions, feature = "profiling"))]
#[unsafe(export_name = "_rjem_malloc_conf")]
// lg_prof_sample:0 表示每次分配都采样(最详细,但性能开销大)
// lg_prof_sample:10 表示每 1KB 采样一次(推荐)
// lg_prof_sample:19 表示每 512KB 采样一次(默认值)
pub static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:10\0";
use std::{net::SocketAddr, sync::Arc};
use axum::{Router, extract::DefaultBodyLimit, middleware, response::Html, routing::get};
use chrono::Local;
use htknow::{api, auth, config, db, frontend, log4rs, processor, search};
use tokio::net::TcpListener;
use tokio_cron_scheduler::{Job, JobScheduler};
use utoipa_swagger_ui::SwaggerUi;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
log4rs::init();
// 加载配置
let cfg = config::get();
log::info!("Configuration loaded: server={}:{}", cfg.server.host, cfg.server.port);
let pool = db::init().await?;
log::info!("Initializing search engine...");
let search_init_started_at = std::time::Instant::now();
let search_engine = search::SearchEngine::init().await.with_pool(pool.clone());
log::info!("Search engine initialized in {}ms", search_init_started_at.elapsed().as_millis());
if let Err(err) = search_engine.maybe_rebuild_lancedb_from_db().await {
log::warn!("Failed to reconcile LanceDB from SQLite at startup; continuing with current index: {}", err);
}
if let Err(err) = search_engine.maybe_rebuild_tantivy_from_db().await {
log::warn!("Failed to reconcile Tantivy from SQLite at startup; continuing with current index: {}", err);
}
search::schedule_startup_index_maintenance();
match search_engine.reload_lexicon().await {
Ok(loaded) => log::info!("Search lexicon loaded: {} words", loaded),
Err(e) => log::warn!("Failed to load search lexicon at startup: {}", e),
}
let processor =
processor::FileProcessor::new(pool.clone(), search_engine.clone(), cfg.server.process_interval_secs);
if cfg.server.parse_enabled {
processor.start();
} else {
log::warn!("HTKNOW_PARSE_ENABLED=false，后台文件解析已禁用，仅支持即时解析");
}
let cron = cfg.server.lancedb_compact_cron.trim();
let _lancedb_compact_scheduler = if !cron.is_empty()
&& !cron.eq_ignore_ascii_case("off")
&& !cron.eq_ignore_ascii_case("disabled")
&& cron != "0"
{
let compact_lock = Arc::new(tokio::sync::Mutex::new(()));
let search_engine = search_engine.clone();
let sched = JobScheduler::new().await?;
let job_lock = compact_lock.clone();
let job = Job::new_async_tz(cron, Local, move |_uuid, _lock| {
let search_engine = search_engine.clone();
let job_lock = job_lock.clone();
Box::pin(async move {
let _guard = job_lock.lock().await;
match search_engine.compact_lancedb().await {
Ok(stats) => {
log::info!(
"LanceDB auto-compact done: deleted_rows={}, total_rows_before={}, total_rows_after={}, size_before_bytes={}, size_after_bytes={}",
stats.deleted_rows,
stats.total_rows_before,
stats.total_rows_after,
stats.size_before_bytes,
stats.size_after_bytes,
);
match search_engine.force_merge_tantivy_indexes().await {
Ok((index_stats, full_index_stats)) => {
log::info!(
"Tantivy auto force-merge done: index_segments={}->{}, full_index_segments={}->{}",
index_stats.before_segments,
index_stats.after_segments,
full_index_stats.before_segments,
full_index_stats.after_segments,
);
}
Err(e) => {
log::error!("Tantivy auto force-merge failed: {}", e);
}
}
}
Err(e) => {
log::error!("LanceDB auto-compact failed: {}", e);
}
}
})
})?;
sched.add(job).await?;
sched.start().await?;
log::warn!("LanceDB auto-compact enabled; avoid concurrent writes during compaction");
log::info!("LanceDB auto-compact cron: {}", cron);
Some(sched)
} else {
None
};
// API 路由需要认证
let upload_limit = cfg.server.upload_limit_mb * 1024 * 1024;
let api_router = Router::new()
.nest("/api/v1/knowledge/", api::app(pool, search_engine))
.layer(middleware::from_fn(auth))
.layer(DefaultBodyLimit::max(upload_limit));
// Swagger 路由（不需要认证）
let swagger = Router::new()
.route("/docs", get(swagger_ui_handler))
.merge(SwaggerUi::new("/swagger-ui-assets").url("/api-docs/openapi.json", api::openapi()));
// 合并路由：前端和 Swagger 不需要认证，API 需要认证
let app = Router::new().merge(swagger).merge(api_router).merge(frontend::router());
let addr = format!("{}:{}", cfg.server.host, cfg.server.port);
let listener = TcpListener::bind(&addr).await?;
log::info!("Server listening on {}", addr);
log::info!("Swagger UI available at http://{}/docs", addr);
axum::serve(listener, app.into_make_service_with_connect_info::<SocketAddr>()).await?;
Ok(())
}
/// Swagger UI HTML handler
async fn swagger_ui_handler() -> Html<&'static str> {
Html(
r#"
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>HTKnow API Documentation</title>
<link rel="stylesheet" href="/swagger-ui-assets/swagger-ui.css">
</head>
<body>
<div id="swagger-ui"></div>
<script src="/swagger-ui-assets/swagger-ui-bundle.js"></script>
<script src="/swagger-ui-assets/swagger-ui-standalone-preset.js"></script>
<script>
window.onload = function() {
SwaggerUIBundle({
url: '/api-docs/openapi.json',
dom_id: '#swagger-ui',
presets: [
SwaggerUIBundle.presets.apis,
SwaggerUIStandalonePreset
],
layout: "StandaloneLayout",
requestInterceptor: (req) => {
req.headers['x-user-id'] = localStorage.getItem('swagger-x-user-id') || '1';
req.headers['x-user-name'] = localStorage.getItem('swagger-x-user-name') || 'testuser';
req.headers['x-role'] = localStorage.getItem('swagger-x-role') || 'admin';
return req;
}
});
};
</script>
<div id="swagger-auth-config" style="position:fixed;top:10px;right:80px;z-index:1000;background:#fff;padding:10px;border:1px solid #ddd;border-radius:4px;box-shadow:0 2px 5px rgba(0,0,0,0.1);">
<input type="text" id="auth-user-id" placeholder="x-user-id" style="width:80px;padding:4px;margin-right:5px;border:1px solid #ccc;border-radius:3px;">
<input type="text" id="auth-user-name" placeholder="x-user-name" style="width:100px;padding:4px;margin-right:5px;border:1px solid #ccc;border-radius:3px;">
<input type="text" id="auth-role" placeholder="x-role" style="width:60px;padding:4px;margin-right:5px;border:1px solid #ccc;border-radius:3px;">
<button onclick="saveAuthConfig()" style="padding:4px 10px;background:#007bff;color:#fff;border:none;border-radius:3px;cursor:pointer;">保存认证</button>
</div>
<script>
document.getElementById('auth-user-id').value = localStorage.getItem('swagger-x-user-id') || '1';
document.getElementById('auth-user-name').value = localStorage.getItem('swagger-x-user-name') || 'testuser';
document.getElementById('auth-role').value = localStorage.getItem('swagger-x-role') || 'admin';
function saveAuthConfig() {
localStorage.setItem('swagger-x-user-id', document.getElementById('auth-user-id').value);
localStorage.setItem('swagger-x-user-name', document.getElementById('auth-user-name').value);
localStorage.setItem('swagger-x-role', document.getElementById('auth-role').value);
location.reload();
}
</script>
</body>
</html>
"#,
)
}

85
src/pdf_content.rs Normal file
View File

@ -0,0 +1,85 @@
use std::path::PathBuf;
use anyhow::Context;
use serde::{Deserialize, Serialize};
use crate::config;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PdfContent {
pub page_idx: i32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bbox: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_level: Option<i32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub img_path: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub table_body: Option<String>,
}
pub fn path(file_id: i64) -> PathBuf {
PathBuf::from(&config::get().storage.contents_path).join(format!("{}.json", file_id))
}
pub async fn read(file_id: i64) -> anyhow::Result<Vec<PdfContent>> {
let path = path(file_id);
let bytes = match tokio::fs::read(&path).await {
Ok(bytes) => bytes,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e).with_context(|| format!("failed to read PDF content file {}", path.display())),
};
serde_json::from_slice(&bytes).with_context(|| format!("failed to parse PDF content file {}", path.display()))
}
pub async fn write(file_id: i64, contents: &[PdfContent]) -> anyhow::Result<()> {
if contents.is_empty() {
return delete(file_id).await;
}
let path = path(file_id);
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("failed to create contents directory {}", parent.display()))?;
}
let bytes = serde_json::to_vec(contents)?;
let tmp_path = path.with_extension("json.tmp");
tokio::fs::write(&tmp_path, bytes)
.await
.with_context(|| format!("failed to write tmp PDF content file {}", tmp_path.display()))?;
tokio::fs::rename(&tmp_path, &path)
.await
.with_context(|| format!("failed to rename PDF content file to {}", path.display()))?;
Ok(())
}
pub async fn delete(file_id: i64) -> anyhow::Result<()> {
let path = path(file_id);
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).with_context(|| format!("failed to delete PDF content file {}", path.display())),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn json_round_trip() {
let rows = vec![PdfContent {
page_idx: 1,
bbox: Some("[1,2,3,4]".to_string()),
text: Some("text".to_string()),
text_level: Some(2),
img_path: None,
table_body: None,
}];
let json = serde_json::to_string(&rows).unwrap();
assert_eq!(serde_json::from_str::<Vec<PdfContent>>(&json).unwrap(), rows);
}
}

411
src/pdf_highlight.rs Normal file
View File

@ -0,0 +1,411 @@
//! PDF 高亮标注模块
//!
//! 用于在 PDF 文件中添加高亮标注
use anyhow::Result;
use log::info;
use lopdf::{
Document, Object, ObjectId, Stream, content::{Content, Operation}, dictionary
};
use serde::{Deserialize, Serialize};
/// 高亮位置信息
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HighlightPosition {
/// 页码（0-based）
pub page_idx: i32,
/// 边界框 [x1, y1, x2, y2]
pub bbox: [i32; 4],
}
/// 页面坐标系边界（用于缩放转换）
#[derive(Debug, Clone, Copy)]
pub struct PageCoordBounds {
pub min_x: f32,
pub min_y: f32,
pub max_x: f32,
pub max_y: f32,
}
/// 高亮颜色配置（RGB，范围 0.0-1.0）
const HIGHLIGHT_COLOR: [f32; 3] = [1.0, 0.8, 0.0]; // 橙黄色
/// 高亮透明度
const HIGHLIGHT_OPACITY: f32 = 0.3;
/// 为 PDF 添加高亮标注
///
/// # Arguments
/// * `pdf_bytes` - 原始 PDF 字节数据
/// * `positions` - 高亮位置列表
///
/// # Returns
/// 带高亮标注的 PDF 字节数据
pub fn add_highlights_to_pdf(pdf_bytes: &[u8], positions: &[HighlightPosition]) -> Result<Vec<u8>> {
add_highlights_to_pdf_with_bounds(pdf_bytes, positions, None)
}
/// 为 PDF 添加高亮标注（可选页面坐标边界用于缩放）
pub fn add_highlights_to_pdf_with_bounds(
pdf_bytes: &[u8], positions: &[HighlightPosition],
coord_bounds_by_page: Option<&std::collections::HashMap<i32, PageCoordBounds>>,
) -> Result<Vec<u8>> {
if positions.is_empty() {
return Ok(pdf_bytes.to_vec());
}
info!("Adding {} highlights to PDF", positions.len());
let mut doc = Document::load_mem(pdf_bytes)?;
// 获取所有页面
let pages = doc.get_pages();
info!("PDF has {} pages", pages.len());
// 按页码分组高亮位置
let mut positions_by_page: std::collections::HashMap<i32, Vec<&HighlightPosition>> =
std::collections::HashMap::new();
for pos in positions {
positions_by_page.entry(pos.page_idx).or_default().push(pos);
}
// 为每个页面添加高亮
for (page_idx, page_positions) in positions_by_page {
// PDF 页码从 1 开始，positions 的 page_idx 从 0 开始
let page_num = (page_idx + 1) as u32;
if let Some(&page_id) = pages.get(&page_num) {
// 获取页面的 MediaBox/CropBox 来确定页面尺寸与偏移（用于坐标转换）
let page_box =
get_page_box(&doc, page_id).unwrap_or(PageBox { x0: 0.0, y0: 0.0, width: 595.0, height: 842.0 });
info!(
"Page {} (idx {}): size = {}x{}, origin = ({}, {}), {} highlights",
page_num,
page_idx,
page_box.width,
page_box.height,
page_box.x0,
page_box.y0,
page_positions.len()
);
let coord_bounds = coord_bounds_by_page
.and_then(|m| m.get(&page_idx))
.copied()
.or_else(|| estimate_bounds_from_positions(&page_positions));
if let Some(bounds) = coord_bounds {
info!(
"Page {} (idx {}): coord bounds = ({:.2}, {:.2}) -> ({:.2}, {:.2})",
page_num, page_idx, bounds.min_x, bounds.min_y, bounds.max_x, bounds.max_y
);
} else {
info!("Page {} (idx {}): coord bounds missing, no scaling/offset", page_num, page_idx);
}
let bbox_transform = calc_bbox_transform_from_bounds(&page_box, coord_bounds);
info!(
"Page {} (idx {}): bbox transform scale x={:.4} y={:.4}",
page_num, page_idx, bbox_transform.scale_x, bbox_transform.scale_y
);
// 方法：添加 Highlight 注释（类似 PyMuPDF 的 add_highlight_annot）
add_highlight_annotations_to_page(&mut doc, page_id, &page_positions, &page_box, bbox_transform)?;
} else {
info!("Page {} not found in PDF", page_num);
}
}
// 保存修改后的 PDF 到内存
let mut output = Vec::new();
doc.save_to(&mut output)?;
info!("Generated highlighted PDF: {} bytes", output.len());
Ok(output)
}
/// 页面尺寸与原点
#[derive(Debug, Clone, Copy)]
struct PageBox {
x0: f32,
y0: f32,
width: f32,
height: f32,
}
/// 获取页面 MediaBox/CropBox 的尺寸与原点
fn get_page_box(doc: &Document, page_id: ObjectId) -> Option<PageBox> {
let page_dict = doc.get_dictionary(page_id).ok()?;
if let Some(page_box) =
get_box_from_dict(page_dict, b"MediaBox").or_else(|| get_box_from_dict(page_dict, b"CropBox"))
{
return Some(page_box);
}
// 如果没有，尝试从父页面获取（可继承）
if let Ok(parent_ref) = page_dict.get(b"Parent")
&& let Ok(parent_id) = parent_ref.as_reference()
&& let Ok(parent_dict) = doc.get_dictionary(parent_id)
&& let Some(page_box) =
get_box_from_dict(parent_dict, b"MediaBox").or_else(|| get_box_from_dict(parent_dict, b"CropBox"))
{
return Some(page_box);
}
None
}
fn get_box_from_dict(dict: &lopdf::Dictionary, name: &[u8]) -> Option<PageBox> {
let obj = dict.get(name).ok()?;
let arr = obj.as_array().ok()?;
if arr.len() < 4 {
return None;
}
let x1 = get_number(&arr[0]).unwrap_or(0.0);
let y1 = get_number(&arr[1]).unwrap_or(0.0);
let x2 = get_number(&arr[2]).unwrap_or(0.0);
let y2 = get_number(&arr[3]).unwrap_or(0.0);
let x_min = x1.min(x2);
let x_max = x1.max(x2);
let y_min = y1.min(y2);
let y_max = y1.max(y2);
Some(PageBox { x0: x_min, y0: y_min, width: x_max - x_min, height: y_max - y_min })
}
fn estimate_bounds_from_positions(positions: &[&HighlightPosition]) -> Option<PageCoordBounds> {
if positions.is_empty() {
return None;
}
let first = positions[0];
let mut min_x = first.bbox[0].min(first.bbox[2]) as f32;
let mut min_y = first.bbox[1].min(first.bbox[3]) as f32;
let mut max_x = first.bbox[0].max(first.bbox[2]) as f32;
let mut max_y = first.bbox[1].max(first.bbox[3]) as f32;
for pos in positions.iter().skip(1) {
let x1 = pos.bbox[0] as f32;
let y1 = pos.bbox[1] as f32;
let x2 = pos.bbox[2] as f32;
let y2 = pos.bbox[3] as f32;
min_x = min_x.min(x1.min(x2));
min_y = min_y.min(y1.min(y2));
max_x = max_x.max(x1.max(x2));
max_y = max_y.max(y1.max(y2));
}
Some(PageCoordBounds { min_x, min_y, max_x, max_y })
}
#[derive(Debug, Clone, Copy)]
struct BboxTransform {
scale_x: f32,
scale_y: f32,
offset_x: f32,
offset_y: f32,
}
fn calc_bbox_transform_from_bounds(page_box: &PageBox, bounds: Option<PageCoordBounds>) -> BboxTransform {
let Some(bounds) = bounds else {
return BboxTransform { scale_x: 1.0, scale_y: 1.0, offset_x: 0.0, offset_y: 0.0 };
};
if !bounds.min_x.is_finite() || !bounds.min_y.is_finite() || !bounds.max_x.is_finite() || !bounds.max_y.is_finite()
{
return BboxTransform { scale_x: 1.0, scale_y: 1.0, offset_x: 0.0, offset_y: 0.0 };
}
// MinerU docs show two possible normalized coordinate ranges:
// - [0, 1] (percent)
// - [0, 1000] (normalized to 1000)
let max_x = bounds.max_x.max(bounds.min_x);
let max_y = bounds.max_y.max(bounds.min_y);
if max_x <= 1.5 && max_y <= 1.5 {
return BboxTransform { scale_x: page_box.width, scale_y: page_box.height, offset_x: 0.0, offset_y: 0.0 };
}
if max_x <= 1000.0 && max_y <= 1000.0 {
return BboxTransform {
scale_x: page_box.width / 1000.0,
scale_y: page_box.height / 1000.0,
offset_x: 0.0,
offset_y: 0.0,
};
}
// Fallback: treat as absolute coords or pixels; shrink only if larger than page.
let mut scale_x = 1.0;
let mut scale_y = 1.0;
if max_x > page_box.width * 1.02 {
scale_x = page_box.width / max_x;
}
if max_y > page_box.height * 1.02 {
scale_y = page_box.height / max_y;
}
BboxTransform { scale_x, scale_y, offset_x: 0.0, offset_y: 0.0 }
}
/// 从 Object 获取数字值
fn get_number(obj: &Object) -> Option<f32> {
match obj {
Object::Integer(i) => Some(*i as f32),
Object::Real(r) => Some(*r),
_ => None,
}
}
/// 在页面上添加 Highlight 注释
fn add_highlight_annotations_to_page(
doc: &mut Document, page_id: ObjectId, positions: &[&HighlightPosition], page_box: &PageBox,
bbox_transform: BboxTransform,
) -> Result<()> {
// 为每个位置添加 Highlight 注释
for pos in positions {
// 先减去偏移量，将坐标归一化到从 0 开始，然后缩放，最后加上页面原点
let x1 = (pos.bbox[0] as f32 - bbox_transform.offset_x) * bbox_transform.scale_x + page_box.x0;
let y1_mineru = (pos.bbox[1] as f32 - bbox_transform.offset_y) * bbox_transform.scale_y;
let x2 = (pos.bbox[2] as f32 - bbox_transform.offset_x) * bbox_transform.scale_x + page_box.x0;
let y2_mineru = (pos.bbox[3] as f32 - bbox_transform.offset_y) * bbox_transform.scale_y;
// 转换 Y 坐标（MinerU 是从顶部向下，PDF 是从底部向上）
let y1 = page_box.y0 + (page_box.height - y2_mineru);
let y2 = page_box.y0 + (page_box.height - y1_mineru);
let (x1, x2) = if x1 <= x2 { (x1, x2) } else { (x2, x1) };
let (y1, y2) = if y1 <= y2 { (y1, y2) } else { (y2, y1) };
let width = x2 - x1;
let height = y2 - y1;
info!(
"Bbox raw: [{}, {}, {}, {}] -> PDF coords: ({:.2}, {:.2}) size {:.2}x{:.2}",
pos.bbox[0], pos.bbox[1], pos.bbox[2], pos.bbox[3], x1, y1, width, height
);
// 检查坐标是否在合理范围内
if width <= 0.0 || height <= 0.0 {
info!("Skipping invalid highlight: width={}, height={}", width, height);
continue;
}
if x1 < page_box.x0 - 100.0
|| x2 > page_box.x0 + page_box.width + 100.0
|| y1 < page_box.y0 - 100.0
|| y2 > page_box.y0 + page_box.height + 100.0
{
info!("Warning: highlight outside page bounds: ({:.2}, {:.2}) -> ({:.2}, {:.2})", x1, y1, x2, y2);
}
// Highlight 注释需要 Rect + QuadPoints（PDF 坐标系，原点在左下）
let rect = Object::Array(vec![x1.into(), y1.into(), x2.into(), y2.into()]);
let quad_points =
Object::Array(vec![x1.into(), y2.into(), x2.into(), y2.into(), x2.into(), y1.into(), x1.into(), y1.into()]);
let appearance_id = create_highlight_appearance(doc, width, height)?;
let annot = dictionary! {
"Type" => Object::Name(b"Annot".to_vec()),
"Subtype" => Object::Name(b"Highlight".to_vec()),
"Rect" => rect,
"QuadPoints" => quad_points,
"C" => Object::Array(vec![HIGHLIGHT_COLOR[0].into(), HIGHLIGHT_COLOR[1].into(), HIGHLIGHT_COLOR[2].into()]),
"CA" => Object::Real(HIGHLIGHT_OPACITY),
"F" => Object::Integer(4), // Print
"P" => Object::Reference(page_id),
"AP" => dictionary! { "N" => Object::Reference(appearance_id) },
};
let annot_id = doc.add_object(annot);
add_annot_to_page(doc, page_id, annot_id)?;
}
Ok(())
}
/// 将注释添加到页面的 Annots 列表
fn add_annot_to_page(doc: &mut Document, page_id: ObjectId, annot_id: ObjectId) -> Result<()> {
let page_obj = doc.get_object(page_id)?.clone();
if let Object::Dictionary(page_dict) = &page_obj {
match page_dict.get(b"Annots") {
Ok(Object::Reference(annots_ref)) => {
let annots_obj = doc.get_object(*annots_ref)?.clone();
let mut new_arr = match annots_obj {
Object::Array(arr) => arr,
_ => Vec::new(),
};
new_arr.push(Object::Reference(annot_id));
let annots_obj_mut = doc.get_object_mut(*annots_ref)?;
*annots_obj_mut = Object::Array(new_arr);
}
Ok(Object::Array(arr)) => {
let mut new_arr = arr.clone();
new_arr.push(Object::Reference(annot_id));
let page_obj_mut = doc.get_object_mut(page_id)?;
if let Object::Dictionary(page_dict_mut) = page_obj_mut {
page_dict_mut.set("Annots", Object::Array(new_arr));
}
}
_ => {
let page_obj_mut = doc.get_object_mut(page_id)?;
if let Object::Dictionary(page_dict_mut) = page_obj_mut {
page_dict_mut.set("Annots", Object::Array(vec![Object::Reference(annot_id)]));
}
}
}
}
Ok(())
}
fn create_highlight_appearance(doc: &mut Document, width: f32, height: f32) -> Result<ObjectId> {
let gs_dict = dictionary! {
"Type" => Object::Name(b"ExtGState".to_vec()),
"ca" => Object::Real(HIGHLIGHT_OPACITY),
"CA" => Object::Real(HIGHLIGHT_OPACITY),
};
let gs_id = doc.add_object(gs_dict);
let resources = dictionary! {
"ExtGState" => dictionary! {
"GS1" => Object::Reference(gs_id),
},
};
let operations = vec![
Operation::new("q", vec![]),
Operation::new("gs", vec![Object::Name(b"GS1".to_vec())]),
Operation::new("rg", vec![HIGHLIGHT_COLOR[0].into(), HIGHLIGHT_COLOR[1].into(), HIGHLIGHT_COLOR[2].into()]),
Operation::new("re", vec![0.into(), 0.into(), width.into(), height.into()]),
Operation::new("f", vec![]),
Operation::new("Q", vec![]),
];
let content = Content { operations };
let stream = Stream::new(
dictionary! {
"Type" => Object::Name(b"XObject".to_vec()),
"Subtype" => Object::Name(b"Form".to_vec()),
"BBox" => Object::Array(vec![0.into(), 0.into(), width.into(), height.into()]),
"Resources" => Object::Dictionary(resources),
},
content.encode()?,
);
Ok(doc.add_object(stream))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_highlight_position_serialization() {
let pos = HighlightPosition { page_idx: 0, bbox: [100, 200, 300, 250] };
let json = serde_json::to_string(&pos).unwrap();
assert!(json.contains("page_idx"));
assert!(json.contains("bbox"));
}
}

4165
src/processor.rs Normal file
View File

File diff suppressed because it is too large Load Diff

427
src/search/advanced.rs Normal file
View File

@ -0,0 +1,427 @@
use std::{collections::HashSet, time::Duration};
use anyhow::{Context, Result, anyhow};
use log::{info, warn};
use once_cell::sync::Lazy;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use sqlx::SqlitePool;
use super::SearchResultItem;
use crate::config;
const MAX_NEIGHBOR_SLICES: i64 = 30;
/// 复用的 LLM HTTP client（连接池）。LLM 调用超时较长，固定 600s。
static LLM_HTTP_CLIENT: Lazy<Client> =
Lazy::new(|| Client::builder().timeout(Duration::from_secs(600)).build().expect("build reqwest client"));
#[derive(Clone)]
pub struct LlmClient {
client: Client,
api_url: Option<String>,
api_key: Option<String>,
model: String,
}
impl Default for LlmClient {
fn default() -> Self {
Self::new()
}
}
impl LlmClient {
pub fn new() -> Self {
let cfg = config::get();
let llm = cfg.llm.clone();
// 复用全局 client，避免每次构造都新建连接池
Self { client: LLM_HTTP_CLIENT.clone(), api_url: llm.api_url, api_key: llm.api_key, model: llm.model }
}
pub fn is_enabled(&self) -> bool {
self.api_url.is_some()
}
pub async fn chat_json<T: serde::de::DeserializeOwned>(
&self, prompt: &str, max_tokens: usize, temperature: f32,
) -> Result<T> {
let content = self.chat(prompt, max_tokens, temperature).await?;
let clean = clean_json_like(&content);
let parsed = serde_json::from_str::<T>(&clean)
.map_err(|e| anyhow!("Failed to parse LLM JSON response: {}. content={}", e, clean))?;
Ok(parsed)
}
pub async fn chat(&self, prompt: &str, max_tokens: usize, temperature: f32) -> Result<String> {
let url = self.api_url.as_ref().ok_or_else(|| anyhow!("LLM not configured"))?;
let request = ChatRequest {
model: self.model.clone(),
messages: vec![
Message {
role: "system".to_string(),
content: "Do not output chain-of-thought. Only output final answer.".to_string(),
},
Message { role: "user".to_string(), content: prompt.to_string() },
],
max_tokens: Some(max_tokens),
temperature: Some(temperature),
};
let mut builder = self.client.post(url).json(&request);
if let Some(api_key) = &self.api_key {
builder = builder.header("Authorization", format!("Bearer {}", api_key));
}
let response = builder.send().await.context("LLM HTTP request failed")?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(anyhow!("LLM HTTP error {}: {}", status, body));
}
let body = response.text().await.context("Failed to read LLM body")?;
let parsed: ChatResponse =
serde_json::from_str(&body).map_err(|e| anyhow!("Failed to decode LLM response: {} body={}", e, body))?;
let choice =
parsed.choices.first().ok_or_else(|| anyhow!("LLM response missing choices"))?.message.content.clone();
Ok(choice)
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "snake_case")]
pub enum PlanAction {
RecentDocuments,
DocumentStructure,
PageContent,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct PlanStep {
pub action: PlanAction,
#[serde(default)]
pub comment: String,
}
pub struct QueryPlanner {
_llm: LlmClient,
}
impl QueryPlanner {
pub fn new(llm: LlmClient) -> Self {
Self { _llm: llm }
}
pub async fn plan(&self, _query: &str, _max_steps: usize) -> Vec<PlanStep> {
default_plan(2)
}
}
fn default_plan(max_steps: usize) -> Vec<PlanStep> {
let mut steps = vec![
PlanStep {
action: PlanAction::RecentDocuments, comment: "搜索相关文档并选出候选切片".to_string()
},
PlanStep {
action: PlanAction::PageContent,
comment: "按候选顺序补充上下文并由 LLM 判断可回答性，命中即停止".to_string(),
},
];
steps.truncate(max_steps.min(steps.len()));
steps
}
pub struct RelevanceJudge {
llm: LlmClient,
}
#[derive(Debug, Clone, Serialize)]
pub struct JudgeOutcome {
pub is_relevant: bool,
pub score: f32,
pub reason: String,
}
impl RelevanceJudge {
pub fn new(llm: LlmClient) -> Self {
Self { llm }
}
pub async fn judge(&self, question: &str, context: &str) -> JudgeOutcome {
if question.trim().is_empty() || context.trim().is_empty() {
return JudgeOutcome { is_relevant: false, score: 0.0, reason: "输入为空".to_string() };
}
if !self.llm.is_enabled() {
return JudgeOutcome { is_relevant: true, score: 1.0, reason: "LLM 未启用，默认通过".to_string() };
}
let prompt = format!(
r#"
你是一名检索质量评估助手。判断候选内容是否可以帮助回答问题，只输出 JSON：
{{"relevant":true/false, "score":0-1之间的小数, "reason":"简要说明"}}
问题：
{}
候选内容：
{}
"#,
question, context
);
match self.llm.chat_json::<JudgeResponse>(&prompt, 50000, 0.2).await {
Ok(resp) => {
let mut score = resp.score.unwrap_or(0.5);
if !(0.0..=1.0).contains(&score) {
score = score.clamp(0.0, 1.0);
}
info!("result: {}\nscore: {}", context, score);
JudgeOutcome {
is_relevant: resp.relevant,
score,
reason: resp.reason.unwrap_or_else(|| "LLM 无理由".to_string()),
}
}
Err(err) => {
warn!("LLM judge failed: {}", err);
JudgeOutcome { is_relevant: true, score: 0.6, reason: "LLM 判定失败，默认通过".to_string() }
}
}
}
}
#[derive(Debug, Clone, Serialize)]
pub struct ChunkSegment {
pub slice_id: i64,
pub text: String,
}
#[derive(Debug, Serialize)]
pub struct ContextChunk {
pub slice_ids: Vec<i64>,
pub content: String,
pub segments: Vec<ChunkSegment>,
}
#[derive(Clone)]
pub struct ChunkRefiner {
llm: LlmClient,
}
impl ChunkRefiner {
pub fn new(llm: LlmClient) -> Self {
Self { llm }
}
pub async fn refine(&self, question: &str, segments: &[ChunkSegment]) -> Result<RefineOutcome> {
if segments.is_empty() {
return Ok(RefineOutcome { segments: Vec::new(), reason: None });
}
if !self.llm.is_enabled() {
return Ok(RefineOutcome { segments: segments.to_vec(), reason: None });
}
let limited_segments: Vec<RefineSegmentInput> = segments
.iter()
.map(|seg| RefineSegmentInput { slice_id: seg.slice_id, text: truncate_text(&seg.text, 400) })
.collect();
let prompt = format!(
r#"
你会得到一个问题和若干切片。请保留对问题最有帮助的切片，其余丢弃。
输出 JSON：{{"keep_slice_ids":[1,2], "reason":"简短说明"}}。至少保留一个切片。
问题：
{}
切片：
{}
"#,
question,
serde_json::to_string(&limited_segments)?
);
match self.llm.chat_json::<RefineResponse>(&prompt, 50000, 0.2).await {
Ok(resp) => {
let keep_ids: HashSet<i64> = if resp.keep_slice_ids.is_empty() {
segments.iter().map(|seg| seg.slice_id).collect()
} else {
resp.keep_slice_ids.into_iter().collect()
};
let filtered: Vec<ChunkSegment> =
segments.iter().filter(|seg| keep_ids.contains(&seg.slice_id)).cloned().collect();
let final_segments = if filtered.is_empty() { segments.to_vec() } else { filtered };
Ok(RefineOutcome { segments: final_segments, reason: resp.reason })
}
Err(err) => {
warn!("LLM chunk refine failed: {}", err);
Ok(RefineOutcome { segments: segments.to_vec(), reason: None })
}
}
}
}
pub struct RefineOutcome {
pub segments: Vec<ChunkSegment>,
pub reason: Option<String>,
}
pub async fn assemble_context_chunk(
pool: &SqlitePool, center: &SearchResultItem, per_side_chars: usize,
) -> Result<ContextChunk> {
// 前后切片两条查询相互独立，并发执行
let (before_rows, after_rows) = tokio::try_join!(
fetch_before_slices(pool, center.file_id, center.id),
fetch_after_slices(pool, center.file_id, center.id),
)?;
let before = take_with_limit(before_rows, per_side_chars, true);
let after = take_with_limit(after_rows, per_side_chars, false);
let mut segments: Vec<ChunkSegment> = Vec::new();
for row in &before {
segments.push(ChunkSegment { slice_id: row.id, text: row.content.clone() });
}
segments.push(ChunkSegment { slice_id: center.id, text: center.content.clone() });
for row in &after {
segments.push(ChunkSegment { slice_id: row.id, text: row.content.clone() });
}
let slice_ids = segments.iter().map(|seg| seg.slice_id).collect();
let combined = segments.iter().map(|seg| seg.text.as_str()).collect::<Vec<_>>().join("\n");
Ok(ContextChunk { slice_ids, content: combined, segments })
}
#[derive(sqlx::FromRow, Clone)]
struct SliceRow {
id: i64,
content: String,
}
#[derive(Debug, Serialize)]
struct RefineSegmentInput {
slice_id: i64,
text: String,
}
fn truncate_text(text: &str, max_chars: usize) -> String {
if text.chars().count() <= max_chars {
return text.to_string();
}
let mut result = String::new();
for (idx, ch) in text.chars().enumerate() {
if idx >= max_chars {
result.push_str("...");
break;
}
result.push(ch);
}
result
}
fn take_with_limit(items: Vec<SliceRow>, char_limit: usize, reverse_after: bool) -> Vec<SliceRow> {
if reverse_after {
// 当前 items 为倒序
let mut selected = Vec::new();
let mut total = 0;
for row in items.into_iter() {
total += row.content.chars().count();
selected.push(row);
if total >= char_limit {
break;
}
}
selected.reverse();
selected
} else {
let mut selected = Vec::new();
let mut total = 0;
for row in items.into_iter() {
total += row.content.chars().count();
selected.push(row);
if total >= char_limit {
break;
}
}
selected
}
}
async fn fetch_before_slices(pool: &SqlitePool, file_id: i64, center_id: i64) -> Result<Vec<SliceRow>> {
let sql =
format!("SELECT id FROM slices WHERE file_id = ? AND id < ? ORDER BY id DESC LIMIT {}", MAX_NEIGHBOR_SLICES);
let ids: Vec<i64> = sqlx::query_scalar(&sql).bind(file_id).bind(center_id).fetch_all(pool).await?;
let contents = crate::slice_content::read_all(file_id).await?;
Ok(ids.into_iter().map(|id| SliceRow { id, content: contents.get(&id).cloned().unwrap_or_default() }).collect())
}
async fn fetch_after_slices(pool: &SqlitePool, file_id: i64, center_id: i64) -> Result<Vec<SliceRow>> {
let sql =
format!("SELECT id FROM slices WHERE file_id = ? AND id > ? ORDER BY id ASC LIMIT {}", MAX_NEIGHBOR_SLICES);
let ids: Vec<i64> = sqlx::query_scalar(&sql).bind(file_id).bind(center_id).fetch_all(pool).await?;
let contents = crate::slice_content::read_all(file_id).await?;
Ok(ids.into_iter().map(|id| SliceRow { id, content: contents.get(&id).cloned().unwrap_or_default() }).collect())
}
#[derive(Debug, Deserialize)]
struct RefineResponse {
#[serde(default)]
keep_slice_ids: Vec<i64>,
#[serde(default)]
reason: Option<String>,
}
#[derive(Debug, Deserialize)]
struct JudgeResponse {
relevant: bool,
#[serde(default)]
score: Option<f32>,
#[serde(default)]
reason: Option<String>,
}
#[derive(Debug, Serialize)]
struct ChatRequest {
model: String,
messages: Vec<Message>,
#[serde(skip_serializing_if = "Option::is_none")]
max_tokens: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
temperature: Option<f32>,
}
#[derive(Debug, Serialize)]
struct Message {
role: String,
content: String,
}
#[derive(Debug, Deserialize)]
struct ChatResponse {
choices: Vec<ChatChoice>,
}
#[derive(Debug, Deserialize)]
struct ChatChoice {
message: ChatMessage,
}
#[derive(Debug, Deserialize)]
struct ChatMessage {
content: String,
}
fn clean_json_like(input: &str) -> String {
let mut text = input.trim();
if let Some(rest) = text.split_once("</think>") {
text = rest.1;
}
if let Some(stripped) = text.strip_prefix("```json") {
text = stripped.trim_start();
} else if let Some(stripped) = text.strip_prefix("```") {
text = stripped.trim_start();
}
if let Some(stripped) = text.strip_suffix("```") {
text = stripped.trim_end();
}
text.trim().to_string()
}

206
src/search/chinese_tokenizer.rs Normal file
View File

@ -0,0 +1,206 @@
//! 中文分词器模块 - 提供高性能的中英文混合分词功能
use std::{collections::HashSet, sync::RwLock};
use anyhow::anyhow;
use jieba_rs::{Jieba, TokenizeMode};
use lazy_static::lazy_static;
use log::info;
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
#[derive(Debug, Clone)]
pub struct LexiconEntry {
pub term: String,
pub freq: Option<usize>,
pub tag: Option<String>,
}
// 使用全局静态 Jieba 实例，避免重复初始化
lazy_static! {
static ref JIEBA: RwLock<Jieba> = RwLock::new(Jieba::new());
// 基础停用词集合，可按需扩展
static ref STOP_WORDS: HashSet<&'static str> = {
let mut set = HashSet::new();
let content = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/stopwords.txt"));
for line in content.lines() {
let word = line.trim();
if word.is_empty() || word.starts_with('#') {
continue;
}
set.insert(word);
}
set
};
}
pub fn reload_custom_words(entries: &[LexiconEntry]) -> anyhow::Result<usize> {
let mut jieba = Jieba::new();
let mut loaded = 0usize;
for entry in entries {
let term = entry.term.trim();
if term.is_empty() {
continue;
}
let tag = entry.tag.as_deref().map(str::trim).filter(|s| !s.is_empty());
let freq = entry.freq.filter(|f| *f > 0);
jieba.add_word(term, freq, tag);
loaded += 1;
}
let mut guard = JIEBA.write().map_err(|_| anyhow!("failed to lock jieba for writing"))?;
*guard = jieba;
info!("Reloaded Jieba custom lexicon with {} words", loaded);
Ok(loaded)
}
/// 分词模式
#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
pub enum SegmentationMode {
/// 搜索模式：适合搜索引擎（召回率高但较慢）
#[default]
Search,
/// 全模式：速度较慢，但能识别更多的词汇
All,
}
/// 快速中文分词器 - 针对性能优化
#[derive(Clone)]
pub struct FastChineseTokenizer {
mode: SegmentationMode,
}
impl FastChineseTokenizer {
/// 创建新的分词器
pub fn new(mode: SegmentationMode) -> Self {
FastChineseTokenizer { mode }
}
/// 创建全模式分词器
pub fn all() -> Self {
Self::new(SegmentationMode::All)
}
/// 执行分词
pub fn segment(&self, text: &str) -> Vec<String> {
let jieba = JIEBA.read().unwrap_or_else(|e| e.into_inner());
let words = match self.mode {
SegmentationMode::Search => {
// 搜索模式：使用 cut_for_search，召回率高
jieba
.cut_for_search(text, false)
.into_iter()
.filter(|s| {
let word = s.trim();
!word.is_empty() && !STOP_WORDS.contains(word)
})
.map(|s| s.to_string())
.collect()
}
SegmentationMode::All => jieba
.cut_all(text)
.into_iter()
.filter(|s| {
let word = s.trim();
!word.is_empty() && !STOP_WORDS.contains(word)
})
.map(|s| s.to_string())
.collect(),
};
info!("Segmented mode: {:?}, text: {}, words: {:?}", self.mode, text, words);
words
}
fn tokenize_with_offsets(&self, text: &str) -> Vec<TokenInfo> {
let mode = match self.mode {
SegmentationMode::Search => TokenizeMode::Search,
SegmentationMode::All => TokenizeMode::Default,
};
let jieba = JIEBA.read().unwrap_or_else(|e| e.into_inner());
jieba
.tokenize(text, mode, false)
.into_iter()
.filter(|t| {
let word = t.word.trim();
!word.is_empty() && !STOP_WORDS.contains(word)
})
.map(|t| TokenInfo { text: t.word.to_string(), start: t.start, end: t.end })
.collect()
}
}
impl Tokenizer for FastChineseTokenizer {
type TokenStream<'a> = ChineseTokenStream<'a>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
let tokens = self.tokenize_with_offsets(text);
ChineseTokenStream::new(text, tokens)
}
}
/// Token stream for Chinese text
pub struct ChineseTokenStream<'a> {
tokens: Vec<TokenInfo>,
current_index: usize,
token: Token,
char_to_byte: Vec<usize>,
_phantom: std::marker::PhantomData<&'a ()>,
}
struct TokenInfo {
text: String,
start: usize,
end: usize,
}
impl<'a> ChineseTokenStream<'a> {
fn new(text: &'a str, tokens: Vec<TokenInfo>) -> Self {
let mut char_to_byte = Vec::with_capacity(text.chars().count() + 1);
for (byte_idx, _) in text.char_indices() {
char_to_byte.push(byte_idx);
}
char_to_byte.push(text.len());
ChineseTokenStream {
tokens,
current_index: 0,
token: Token::default(),
char_to_byte,
_phantom: std::marker::PhantomData,
}
}
}
impl<'a> TokenStream for ChineseTokenStream<'a> {
fn advance(&mut self) -> bool {
while self.current_index < self.tokens.len() {
let token_info = &self.tokens[self.current_index];
self.current_index += 1;
// 跳过空的token
if token_info.text.trim().is_empty() {
continue;
}
let char_len = self.char_to_byte.len().saturating_sub(1);
if token_info.start >= char_len || token_info.end > char_len || token_info.start >= token_info.end {
continue;
}
let start = self.char_to_byte[token_info.start];
let end = self.char_to_byte[token_info.end];
self.token.text.clear();
self.token.text.push_str(&token_info.text);
self.token.offset_from = start;
self.token.offset_to = end;
self.token.position = self.token.position.wrapping_add(1);
return true;
}
false
}
fn token(&self) -> &Token {
&self.token
}
fn token_mut(&mut self) -> &mut Token {
&mut self.token
}
}

209
src/search/embedding.rs Normal file
View File

@ -0,0 +1,209 @@
use std::time::Duration;
use anyhow::{Context, Result};
use once_cell::sync::Lazy;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use crate::config;
/// 判断图片 embedding 服务是否已配置。
pub fn image_embedding_enabled() -> bool {
config::get().services.image_embedding_url.is_some()
}
static HTTP_CLIENT: Lazy<Client> = Lazy::new(Client::new);
#[derive(Debug, Serialize)]
struct EmbeddingRequest {
model: String,
input: Vec<String>,
}
#[derive(Debug, Deserialize)]
struct EmbeddingResponse {
data: Vec<EmbeddingData>,
}
#[derive(Debug, Deserialize)]
struct EmbeddingData {
embedding: Vec<f32>,
}
/// 获取图片的 embedding 向量（从文件路径）
pub async fn get_image_embedding_from_path(path: &str, text: Option<&str>) -> Result<Vec<f32>> {
let file_name = std::path::Path::new(path).file_name().and_then(|name| name.to_str()).unwrap_or("image");
let mime = mime_guess::from_path(path).first_or_octet_stream();
let cfg = config::get();
let url = cfg
.services
.image_embedding_url
.as_deref()
.ok_or_else(|| anyhow::anyhow!("image embedding URL is not configured"))?;
let part = reqwest::multipart::Part::file(path)
.await
.with_context(|| format!("failed to open image file for embedding: {}", path))?
.file_name(file_name.to_string())
.mime_str(mime.essence_str())?;
let form = reqwest::multipart::Form::new().part("file", part).text("text", text.unwrap_or(file_name).to_string());
let response = HTTP_CLIENT
.post(url)
.timeout(Duration::from_secs(cfg.search.embedding_timeout_secs))
.multipart(form)
.send()
.await
.with_context(|| {
format!(
"image embedding request failed: url={}, file={}, timeout={}s",
url, file_name, cfg.search.embedding_timeout_secs
)
})?;
handle_image_embedding_response(response).await
}
/// 获取图片的 embedding 向量（从文件内容）
pub async fn get_image_embedding_from_bytes(
file_name: &str, content_type: Option<&str>, bytes: Vec<u8>, text: Option<&str>,
) -> Result<Vec<f32>> {
let cfg = config::get();
let url = cfg
.services
.image_embedding_url
.as_deref()
.ok_or_else(|| anyhow::anyhow!("image embedding URL is not configured"))?;
let mut part = reqwest::multipart::Part::bytes(bytes).file_name(file_name.to_string());
if let Some(content_type) = content_type {
part = part.mime_str(content_type)?;
}
let form = reqwest::multipart::Form::new().part("file", part).text("text", text.unwrap_or(file_name).to_string());
let response = HTTP_CLIENT
.post(url)
.timeout(Duration::from_secs(cfg.search.embedding_timeout_secs))
.multipart(form)
.send()
.await
.with_context(|| {
format!(
"image embedding request failed: url={}, file={}, timeout={}s",
url, file_name, cfg.search.embedding_timeout_secs
)
})?;
handle_image_embedding_response(response).await
}
async fn handle_image_embedding_response(response: reqwest::Response) -> Result<Vec<f32>> {
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
anyhow::bail!("Image embedding API error: {} - {}", status, error_text);
}
let embedding_response: EmbeddingResponse =
response.json().await.context("image embedding response decode failed")?;
embedding_response
.data
.into_iter()
.next()
.map(|data| data.embedding)
.ok_or_else(|| anyhow::anyhow!("No image embedding returned"))
}
/// 获取文本的 embedding 向量
pub async fn get_embedding(text: &str) -> Result<Vec<f32>> {
let cfg = config::get();
let query = text.trim();
if query.is_empty() {
anyhow::bail!("Embedding query cannot be empty");
}
let request = EmbeddingRequest { model: cfg.ai.embedding_model.clone(), input: vec![query.to_string()] };
let response = HTTP_CLIENT
.post(&cfg.services.embedding_url)
.timeout(Duration::from_secs(cfg.search.embedding_timeout_secs))
.json(&request)
.send()
.await
.with_context(|| {
format!(
"embedding request failed: url={}, input_chars={}, timeout={}s",
cfg.services.embedding_url,
text.chars().count(),
cfg.search.embedding_timeout_secs
)
})?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
anyhow::bail!("Embedding API error: {} - {}, input_chars={}", status, error_text, text.chars().count());
}
let embedding_response: EmbeddingResponse = response.json().await.context("embedding response decode failed")?;
let embedding = embedding_response
.data
.into_iter()
.next()
.map(|data| data.embedding)
.ok_or_else(|| anyhow::anyhow!("No embedding returned"))?;
Ok(embedding)
}
/// 批量获取文本的 embedding 向量（自动分批）
pub async fn get_embeddings(texts: &[String]) -> Result<Vec<Vec<f32>>> {
if texts.is_empty() {
return Ok(Vec::new());
}
let batch_size = config::get().ai.embedding_batch_size;
let mut all_embeddings = Vec::with_capacity(texts.len());
for chunk in texts.chunks(batch_size) {
let batch = get_embeddings_single_batch(chunk).await?;
all_embeddings.extend(batch);
}
Ok(all_embeddings)
}
async fn get_embeddings_single_batch(texts: &[String]) -> Result<Vec<Vec<f32>>> {
let cfg = config::get();
let request = EmbeddingRequest { model: cfg.ai.embedding_model.clone(), input: texts.to_vec() };
let response = HTTP_CLIENT
.post(&cfg.services.embedding_url)
.timeout(Duration::from_secs(cfg.search.embedding_timeout_secs))
.json(&request)
.send()
.await
.with_context(|| {
format!(
"batch embedding request failed: url={}, batch_size={}, total_chars={}, timeout={}s",
cfg.services.embedding_url,
texts.len(),
texts.iter().map(|t| t.chars().count()).sum::<usize>(),
cfg.search.embedding_timeout_secs
)
})?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
anyhow::bail!(
"Embedding API error: {} - {}, batch_size={}, total_chars={}",
status,
error_text,
texts.len(),
texts.iter().map(|text| text.chars().count()).sum::<usize>()
);
}
let embedding_response: EmbeddingResponse =
response.json().await.context("batch embedding response decode failed")?;
Ok(embedding_response.data.into_iter().map(|data| data.embedding).collect())
}

1411
src/search/lancedb.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1969
src/search/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

999
src/search/tantivy_engine.rs Normal file
View File

@ -0,0 +1,999 @@
use std::{
collections::{HashMap, HashSet}, path::Path, sync::{
Arc, mpsc::{self, Sender}
}, thread, time::{Duration, Instant, SystemTime, UNIX_EPOCH}
};
use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};
use log::{debug, info, warn};
use serde::Serialize;
use tantivy::{
Index, IndexReader, Result, TantivyDocument, TantivyError, Term, collector::TopDocs, directory::error::LockError, doc, merge_policy::LogMergePolicy, query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery}, schema::{FAST, Field, INDEXED, IndexRecordOption, STORED, Schema, TextFieldIndexing, TextOptions, Value as _}
};
use super::chinese_tokenizer;
use crate::config;
const ALL_TOKENIZER: &str = "all";
const INDEX_WRITER_LOCK_RETRY_MAX_ATTEMPTS: usize = 8;
const INDEX_WRITER_LOCK_RETRY_BASE_MS: u64 = 40;
#[derive(Debug, Clone)]
pub struct SynonymTerm {
pub term: String,
pub boost: f32,
}
pub type SynonymMap = HashMap<String, Vec<SynonymTerm>>;
#[derive(Clone)]
pub struct Document {
pub id: i64, // 切片 ID
pub file_id: i64, // 文件 ID
pub kb_id: Option<i64>, // 知识库 ID
pub content: String, // 内容
pub is_image: bool, // 是否为图片切片
}
/// 搜索结果项
#[derive(Debug, Clone, Serialize)]
pub struct SearchResultItem {
pub id: i64, // 切片 ID
pub file_id: i64, // 文件 ID
pub kb_id: Option<i64>, // 知识库 ID
pub content: String, // 内容
pub score: f32, // 搜索得分
}
/// 全文索引搜索结果项
#[derive(Debug, Clone, Serialize)]
pub struct FullSearchResultItem {
pub file_id: i64, // 文件 ID
pub kb_id: Option<i64>, // 知识库 ID
pub snippet: String, // 高亮片段
pub score: f32, // 搜索得分
}
#[derive(Debug, Clone, Serialize)]
pub struct ForceMergeStats {
pub before_segments: usize,
pub after_segments: usize,
pub before_docs: u64,
pub after_docs: u64,
pub before_deleted_docs: u64,
pub after_deleted_docs: u64,
pub gc_deleted_files: usize,
pub gc_failed_files: usize,
pub skipped: bool,
pub duration_ms: u128,
}
impl Document {
pub fn new(id: i64, file_id: i64, kb_id: Option<i64>, content: String) -> Self {
Document { id, file_id, kb_id, content, is_image: false }
}
pub fn with_is_image(mut self, is_image: bool) -> Self {
self.is_image = is_image;
self
}
}
/// create new index
///
/// delete if it exists
pub fn init() -> Result<(Schema, Index)> {
let cfg = config::get();
init_with_path(&cfg.search.tantivy_index_path)
}
pub fn init_full() -> Result<(Schema, Index)> {
let cfg = config::get();
init_with_path(&cfg.search.tantivy_full_index_path)
}
pub fn init_with_path(path: &str) -> Result<(Schema, Index)> {
let t0 = Instant::now();
let schema = build_schema();
info!("Tantivy init substep: build_schema() took {}ms", t0.elapsed().as_millis());
let path = Path::new(path);
let t1 = Instant::now();
let index = if path.exists() {
match Index::open_in_dir(path) {
Ok(idx) => {
// 如果 schema 缺少 is_image 字段，按时间戳备份旧索引并重建
if !schema_has_is_image(idx.schema()) {
warn!(
"Tantivy index at '{}' is missing is_image field; backing up and creating fresh index.",
path.display()
);
backup_index_dir(path)?;
std::fs::create_dir_all(path)?;
Index::create_in_dir(path, schema.clone())?
} else {
idx
}
}
Err(e) => {
warn!(
"Tantivy index at '{}' is corrupted or unreadable: {}. Removing and creating fresh index.",
path.display(),
e
);
std::fs::remove_dir_all(path)?;
std::fs::create_dir_all(path)?;
Index::create_in_dir(path, schema.clone())?
}
}
} else {
std::fs::create_dir_all(path)?;
Index::create_in_dir(path, schema.clone())?
};
info!("Tantivy init substep: open/create index at '{}' took {}ms", path.display(), t1.elapsed().as_millis());
let t2 = Instant::now();
register_tokenizers(&index);
info!("Tantivy init substep: register_tokenizers() took {}ms", t2.elapsed().as_millis());
Ok((schema, index))
}
fn schema_has_is_image(schema: Schema) -> bool {
schema.get_field("is_image").is_ok()
}
fn backup_index_dir(path: &Path) -> std::io::Result<()> {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
let backup_path = path.with_file_name(format!(
"{}.backup.{}",
path.file_name().and_then(|n| n.to_str()).unwrap_or("tantivy_index"),
timestamp
));
warn!("Backing up old Tantivy index from {} to {}", path.display(), backup_path.display());
std::fs::rename(path, &backup_path)?;
Ok(())
}
fn is_lock_busy_error(err: &TantivyError) -> bool {
matches!(err, TantivyError::LockFailure(LockError::LockBusy, _))
}
/// Retry opening a Tantivy index writer with exponential backoff.
///
/// `$sleep` is a closure that performs the actual sleep for the given number of
/// milliseconds.
macro_rules! retry_open_writer {
($index:expr, $sleep:expr) => {{
let mut delay_ms = INDEX_WRITER_LOCK_RETRY_BASE_MS;
let mut attempt = 1;
loop {
match open_writer($index) {
Ok(writer) => break Ok(writer),
Err(err) if is_lock_busy_error(&err) && attempt < INDEX_WRITER_LOCK_RETRY_MAX_ATTEMPTS => {
warn!(
"Tantivy index writer lock busy (attempt {}/{}), retry in {}ms",
attempt, INDEX_WRITER_LOCK_RETRY_MAX_ATTEMPTS, delay_ms
);
$sleep(delay_ms);
delay_ms = (delay_ms * 2).min(800);
attempt += 1;
}
Err(err) => break Err(err),
}
}
}};
}
async fn create_writer(index: &Index) -> tantivy::Result<tantivy::IndexWriter> {
retry_open_writer!(index, |ms| std::thread::sleep(Duration::from_millis(ms)))
}
fn create_writer_blocking(index: &Index) -> tantivy::Result<tantivy::IndexWriter> {
retry_open_writer!(index, |ms| std::thread::sleep(Duration::from_millis(ms)))
}
fn open_writer(index: &Index) -> tantivy::Result<tantivy::IndexWriter> {
let writer_memory = config::get().search.tantivy_memory_mb * 1_000_000;
let writer = index.writer(writer_memory)?;
// 设置 merge policy，减少小 segment 数量
writer.set_merge_policy(Box::new(LogMergePolicy::default()));
Ok(writer)
}
pub async fn create_writer_with_timing(index: &Index, label: &str) -> tantivy::Result<tantivy::IndexWriter> {
let start = Instant::now();
let writer = create_writer(index).await?;
debug!("tantivy_writer_create label={} duration_ms={}", label, start.elapsed().as_millis());
Ok(writer)
}
fn create_writer_with_timing_blocking(index: &Index, label: &str) -> tantivy::Result<tantivy::IndexWriter> {
let start = Instant::now();
let writer = create_writer_blocking(index)?;
debug!("tantivy_writer_create label={} duration_ms={}", label, start.elapsed().as_millis());
Ok(writer)
}
/// 每个 Tantivy 索引对应一个长期存活的 writer 线程。
///
/// `IndexWriter` 不是 `Send`/`Sync`，不能放在 `Arc<Mutex<...>>` 里跨越 await，因此在线程内部持有 writer，
/// 外部通过 channel 发送写/删除/merge 任务，并通过 oneshot 等待结果。每次操作后仍执行 commit，保留原
/// 有的数据安全语义。
pub struct IndexWriterHandle {
tx: Sender<WriterJob>,
thread: Option<thread::JoinHandle<()>>,
}
struct WriterJob {
op: WriterOp,
reply: Option<tokio::sync::oneshot::Sender<WriterResult>>,
}
enum WriterOp {
WriteBatch(Vec<Document>),
DeleteTerms { field_name: &'static str, ids: Vec<i64> },
ForceMerge,
Shutdown,
}
enum WriterResult {
Ok,
WriteBatch,
ForceMerge(ForceMergeStats),
Err(String),
}
impl IndexWriterHandle {
pub async fn open(index: Index, schema: Schema, label: String) -> anyhow::Result<Arc<Self>> {
let (init_tx, init_rx) = tokio::sync::oneshot::channel::<anyhow::Result<()>>();
let (tx, rx) = mpsc::channel::<WriterJob>();
let thread_name = format!("tantivy-writer-{}", label);
let thread = thread::Builder::new().name(thread_name).spawn(move || {
let mut writer = match create_writer_with_timing_blocking(&index, &label) {
Ok(w) => {
let _ = init_tx.send(Ok(()));
Some(w)
}
Err(e) => {
let _ = init_tx.send(Err(anyhow::anyhow!(e)));
return;
}
};
while let Ok(job) = rx.recv() {
let current_writer = match writer.take() {
Some(w) => w,
None => {
let result = WriterResult::Err("tantivy writer is not available".to_string());
if let Some(reply) = job.reply {
let _ = reply.send(result);
}
continue;
}
};
let (result, maybe_writer) = match job.op {
WriterOp::Shutdown => {
drop(current_writer);
break;
}
WriterOp::WriteBatch(docs) => {
if docs.is_empty() {
(WriterResult::WriteBatch, Some(current_writer))
} else {
let mut writer = current_writer;
let result = match add_documents(&mut writer, &schema, docs) {
Ok(count) => match commit_writer(&mut writer, &label, count) {
Ok(()) => WriterResult::WriteBatch,
Err(e) => WriterResult::Err(e.to_string()),
},
Err(e) => WriterResult::Err(e.to_string()),
};
(result, Some(writer))
}
}
WriterOp::DeleteTerms { field_name, ids } => {
if ids.is_empty() {
(WriterResult::Ok, Some(current_writer))
} else {
let mut writer = current_writer;
let field = get_field(&schema, field_name);
let delete_start = Instant::now();
for id in &ids {
let term = Term::from_field_i64(field, *id);
writer.delete_term(term);
}
debug!(
"tantivy_delete_terms target={} count={} duration_ms={}",
field_name,
ids.len(),
delete_start.elapsed().as_millis()
);
let result = match commit_writer(&mut writer, &label, ids.len()) {
Ok(()) => WriterResult::Ok,
Err(e) => WriterResult::Err(e.to_string()),
};
(result, Some(writer))
}
}
WriterOp::ForceMerge => match force_merge_with_writer(&index, current_writer, &label) {
Ok((stats, new_writer)) => (WriterResult::ForceMerge(stats), Some(new_writer)),
Err(e) => (WriterResult::Err(e.to_string()), None),
},
};
writer = maybe_writer;
if let Some(reply) = job.reply {
let _ = reply.send(result);
}
// writer 为 None 时说明 force_merge 失败且未能重建，退出循环
if writer.is_none() {
warn!("tantivy writer thread {} exiting after force_merge failure", label);
break;
}
}
// writer 在这里 drop，释放目录锁
})?;
init_rx.await??;
Ok(Arc::new(Self { tx, thread: Some(thread) }))
}
pub async fn write_batch(&self, docs: Vec<Document>) -> anyhow::Result<()> {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
self.tx
.send(WriterJob { op: WriterOp::WriteBatch(docs), reply: Some(reply_tx) })
.map_err(|e| anyhow::anyhow!("tantivy writer channel closed: {e}"))?;
match reply_rx.await? {
WriterResult::Ok | WriterResult::WriteBatch => Ok(()),
WriterResult::ForceMerge(_) => Err(anyhow::anyhow!("unexpected force_merge result from write_batch")),
WriterResult::Err(e) => Err(anyhow::anyhow!("tantivy write_batch failed: {e}")),
}
}
pub async fn delete_by_field(&self, field_name: &'static str, ids: &[i64]) -> anyhow::Result<()> {
if ids.is_empty() {
return Ok(());
}
let ids = ids.to_vec();
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
self.tx
.send(WriterJob { op: WriterOp::DeleteTerms { field_name, ids }, reply: Some(reply_tx) })
.map_err(|e| anyhow::anyhow!("tantivy writer channel closed: {e}"))?;
match reply_rx.await? {
WriterResult::Ok => Ok(()),
WriterResult::WriteBatch => Err(anyhow::anyhow!("unexpected write result from delete")),
WriterResult::ForceMerge(_) => Err(anyhow::anyhow!("unexpected force_merge result from delete")),
WriterResult::Err(e) => Err(anyhow::anyhow!("tantivy delete failed: {e}")),
}
}
pub async fn force_merge(&self) -> anyhow::Result<ForceMergeStats> {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
self.tx
.send(WriterJob { op: WriterOp::ForceMerge, reply: Some(reply_tx) })
.map_err(|e| anyhow::anyhow!("tantivy writer channel closed: {e}"))?;
match reply_rx.await? {
WriterResult::ForceMerge(stats) => Ok(stats),
WriterResult::Ok => Err(anyhow::anyhow!("unexpected ok result from force_merge")),
WriterResult::WriteBatch => Err(anyhow::anyhow!("unexpected write result from force_merge")),
WriterResult::Err(e) => Err(anyhow::anyhow!("tantivy force_merge failed: {e}")),
}
}
}
impl Drop for IndexWriterHandle {
fn drop(&mut self) {
if let Some(thread) = self.thread.take() {
let _ = self.tx.send(WriterJob { op: WriterOp::Shutdown, reply: None });
let _ = thread.join();
}
}
}
fn force_merge_with_writer(
index: &Index, mut writer: tantivy::IndexWriter, label: &str,
) -> anyhow::Result<(ForceMergeStats, tantivy::IndexWriter)> {
let start = Instant::now();
let before_metas = index.searchable_segment_metas()?;
let before_segments = before_metas.len();
let before_docs = segment_docs(&before_metas);
let before_deleted_docs = segment_deleted_docs(&before_metas);
if before_segments < 2 {
let (gc_deleted_files, gc_failed_files) = garbage_collect_index_files(&writer, "force_merge_skipped")?;
writer.wait_merging_threads()?;
let new_writer = create_writer_with_timing_blocking(index, &format!("{}_gc", label))?;
return Ok((
ForceMergeStats {
before_segments,
after_segments: before_segments,
before_docs,
after_docs: before_docs,
before_deleted_docs,
after_deleted_docs: before_deleted_docs,
gc_deleted_files,
gc_failed_files,
skipped: true,
duration_ms: start.elapsed().as_millis(),
},
new_writer,
));
}
let segment_ids = before_metas.iter().map(|meta| meta.id()).collect::<Vec<_>>();
writer.merge(&segment_ids).wait()?;
writer.wait_merging_threads()?;
let new_writer = create_writer_with_timing_blocking(index, &format!("{}_gc", label))?;
let (gc_deleted_files, gc_failed_files) = garbage_collect_index_files(&new_writer, "force_merge")?;
let after_metas = index.searchable_segment_metas()?;
Ok((
ForceMergeStats {
before_segments,
after_segments: after_metas.len(),
before_docs,
after_docs: segment_docs(&after_metas),
before_deleted_docs,
after_deleted_docs: segment_deleted_docs(&after_metas),
gc_deleted_files,
gc_failed_files,
skipped: false,
duration_ms: start.elapsed().as_millis(),
},
new_writer,
))
}
pub async fn create_rebuild_writer(index: &Index, label: &str) -> tantivy::Result<tantivy::IndexWriter> {
create_writer_with_timing(index, label).await
}
pub fn add_documents(
index_writer: &mut tantivy::IndexWriter, schema: &Schema, docs: impl IntoIterator<Item=Document>,
) -> tantivy::Result<usize> {
let mut count = 0_usize;
for doc in docs {
index_writer.add_document(create_document(doc, schema))?;
count += 1;
}
Ok(count)
}
pub fn commit_writer(index_writer: &mut tantivy::IndexWriter, label: &str, doc_count: usize) -> tantivy::Result<()> {
let commit_start = Instant::now();
index_writer.commit()?;
debug!("tantivy_commit label={} docs={} duration_ms={}", label, doc_count, commit_start.elapsed().as_millis());
Ok(())
}
pub fn search_sync(
reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec<i64>>, kb_ids: Option<&Vec<i64>>,
filter_is_image: Option<bool>, synonym_map: Option<&SynonymMap>,
) -> anyhow::Result<Vec<SearchResultItem>> {
let cfg = config::get();
let searcher = reader.searcher();
let tantivy_query = build_query(query, file_ids, kb_ids, filter_is_image, schema, synonym_map)?;
let search_start = Instant::now();
let top_docs = searcher.search(&tantivy_query, &TopDocs::with_limit(cfg.search.limit))?;
debug!("Tantivy searcher.search {}ms", search_start.elapsed().as_millis());
let mut results = vec![];
let mut doc_total_ms: u128 = 0;
let mut doc_max_ms: u128 = 0;
let mut doc_count: usize = 0;
for (score, doc_address) in top_docs {
let doc_start = Instant::now();
let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
let doc_elapsed = doc_start.elapsed().as_millis();
doc_total_ms += doc_elapsed;
if doc_elapsed > doc_max_ms {
doc_max_ms = doc_elapsed;
}
doc_count += 1;
let id = retrieved_doc.get_first(get_field(schema, "id")).and_then(|v| v.as_i64()).unwrap_or(0);
let file_id = retrieved_doc.get_first(get_field(schema, "file_id")).and_then(|v| v.as_i64()).unwrap_or(0);
let kb_id = retrieved_doc.get_first(get_field(schema, "kb_id")).and_then(|v| v.as_i64());
let content = retrieved_doc
.get_first(get_field(schema, "content"))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.unwrap_or_default();
results.push(SearchResultItem { id, file_id, kb_id, content, score });
}
debug!("Tantivy searcher.doc total={}ms max={}ms count={}", doc_total_ms, doc_max_ms, doc_count);
Ok(results)
}
pub async fn search(
reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec<i64>>, kb_ids: Option<&Vec<i64>>,
filter_is_image: Option<bool>, synonym_map: Option<&SynonymMap>,
) -> anyhow::Result<Vec<SearchResultItem>> {
search_sync(reader, schema, query, file_ids, kb_ids, filter_is_image, synonym_map)
}
#[allow(clippy::too_many_arguments)]
pub fn search_with_snippet_sync(
reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec<i64>>, kb_ids: Option<&Vec<i64>>,
filter_is_image: Option<bool>, max_chars: usize, synonym_map: Option<&SynonymMap>,
) -> anyhow::Result<Vec<FullSearchResultItem>> {
let cfg = config::get();
let searcher = reader.searcher();
let tantivy_query = build_query(query, file_ids, kb_ids, filter_is_image, schema, synonym_map)?;
let search_start = Instant::now();
let top_docs = searcher.search(&tantivy_query, &TopDocs::with_limit(cfg.search.limit))?;
debug!("Tantivy full searcher.search {}ms", search_start.elapsed().as_millis());
let terms = build_snippet_terms(query, synonym_map);
let matcher = build_matcher(&terms);
let mut results = vec![];
let mut doc_total_ms: u128 = 0;
let mut doc_max_ms: u128 = 0;
let mut doc_count: usize = 0;
for (score, doc_address) in top_docs {
let doc_start = Instant::now();
let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
let doc_elapsed = doc_start.elapsed().as_millis();
doc_total_ms += doc_elapsed;
if doc_elapsed > doc_max_ms {
doc_max_ms = doc_elapsed;
}
doc_count += 1;
let file_id = retrieved_doc.get_first(get_field(schema, "file_id")).and_then(|v| v.as_i64()).unwrap_or(0);
let kb_id = retrieved_doc.get_first(get_field(schema, "kb_id")).and_then(|v| v.as_i64());
let content = retrieved_doc.get_first(get_field(schema, "content")).and_then(|v| v.as_str()).unwrap_or("");
let snippet = build_best_snippet(content, matcher.as_ref(), terms.len(), max_chars);
results.push(FullSearchResultItem { file_id, kb_id, snippet, score });
}
debug!("Tantivy full searcher.doc total={}ms max={}ms count={}", doc_total_ms, doc_max_ms, doc_count);
Ok(results)
}
#[allow(clippy::too_many_arguments)]
pub async fn search_with_snippet(
reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec<i64>>, kb_ids: Option<&Vec<i64>>,
filter_is_image: Option<bool>, max_chars: usize, synonym_map: Option<&SynonymMap>,
) -> anyhow::Result<Vec<FullSearchResultItem>> {
// 搜索 + 取文档 + 生成 snippet 均为 CPU 密集的同步操作，放到阻塞线程池执行，避免阻塞异步运行时。
let reader = reader.clone();
let schema = schema.clone();
let query = query.to_string();
let file_ids = file_ids.cloned();
let kb_ids = kb_ids.cloned();
let synonym_map = synonym_map.cloned();
tokio::task::spawn_blocking(move || {
search_with_snippet_sync(
&reader,
&schema,
&query,
file_ids.as_ref(),
kb_ids.as_ref(),
filter_is_image,
max_chars,
synonym_map.as_ref(),
)
})
.await
.map_err(|e| anyhow::anyhow!("search_with_snippet task panicked: {e}"))?
}
fn garbage_collect_index_files(writer: &tantivy::IndexWriter, label: &str) -> anyhow::Result<(usize, usize)> {
let start = Instant::now();
let gc_result = writer.garbage_collect_files().wait()?;
let deleted_files = gc_result.deleted_files.len();
let failed_files = gc_result.failed_to_delete_files.len();
if failed_files > 0 {
warn!(
"tantivy_gc label={} deleted_files={} failed_files={} duration_ms={}",
label,
deleted_files,
failed_files,
start.elapsed().as_millis()
);
} else {
debug!(
"tantivy_gc label={} deleted_files={} failed_files={} duration_ms={}",
label,
deleted_files,
failed_files,
start.elapsed().as_millis()
);
}
Ok((deleted_files, failed_files))
}
fn segment_docs(metas: &[tantivy::SegmentMeta]) -> u64 {
metas.iter().map(|meta| u64::from(meta.num_docs())).sum()
}
fn segment_deleted_docs(metas: &[tantivy::SegmentMeta]) -> u64 {
metas.iter().map(|meta| u64::from(meta.num_deleted_docs())).sum()
}
fn build_query(
input: &str, file_ids: Option<&Vec<i64>>, kb_ids: Option<&Vec<i64>>, filter_is_image: Option<bool>,
schema: &Schema, synonym_map: Option<&SynonymMap>,
) -> tantivy::Result<Box<dyn Query>> {
let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
let query_terms = build_query_terms(input, synonym_map);
// content 至少命中一个
let mut content_queries = Vec::new();
for query_term in query_terms {
let tq = TermQuery::new(
Term::from_field_text(get_field(schema, "content"), query_term.term.as_str()),
IndexRecordOption::Basic,
);
if (query_term.boost - 1.0).abs() > f32::EPSILON {
content_queries
.push((Occur::Should, Box::new(BoostQuery::new(Box::new(tq), query_term.boost)) as Box<dyn Query>));
} else {
content_queries.push((Occur::Should, Box::new(tq) as Box<dyn Query>));
}
}
let content_bool = BooleanQuery::new(content_queries);
subqueries.push((Occur::Must, Box::new(content_bool)));
if let Some(ids) = file_ids
&& !ids.is_empty()
{
let mut file_id_queries = Vec::new();
let file_id_field = get_field(schema, "file_id");
for file_id in ids {
let file_id_query = TermQuery::new(Term::from_field_i64(file_id_field, *file_id), IndexRecordOption::Basic);
file_id_queries.push((Occur::Should, Box::new(file_id_query) as Box<dyn Query>));
}
if !file_id_queries.is_empty() {
let file_ids_bool_query = BooleanQuery::new(file_id_queries);
subqueries.push((Occur::Must, Box::new(file_ids_bool_query)));
}
}
if let Some(ids) = kb_ids
&& !ids.is_empty()
{
let mut kb_id_queries = Vec::new();
let kb_id_field = get_field(schema, "kb_id");
for kb_id in ids {
let kb_id_query = TermQuery::new(Term::from_field_i64(kb_id_field, *kb_id), IndexRecordOption::Basic);
kb_id_queries.push((Occur::Should, Box::new(kb_id_query) as Box<dyn Query>));
}
if !kb_id_queries.is_empty() {
let kb_ids_bool_query = BooleanQuery::new(kb_id_queries);
subqueries.push((Occur::Must, Box::new(kb_ids_bool_query)));
}
}
if let Some(is_image) = filter_is_image {
let is_image_field = get_field(schema, "is_image");
let term = Term::from_field_i64(is_image_field, i64::from(is_image));
subqueries.push((Occur::Must, Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box<dyn Query>));
}
Ok(Box::new(BooleanQuery::new(subqueries)))
}
fn create_document(doc: Document, schema: &Schema) -> TantivyDocument {
let mut document = doc! {
get_field(schema, "id") => doc.id,
get_field(schema, "file_id") => doc.file_id,
get_field(schema, "content") => doc.content,
get_field(schema, "is_image") => i64::from(doc.is_image),
};
if let Some(kb_id) = doc.kb_id {
document.add_i64(get_field(schema, "kb_id"), kb_id);
}
document
}
fn get_field(schema: &Schema, field: &str) -> Field {
schema.get_field(field).unwrap_or_else(|_| panic!("Field '{}' not found in schema", field))
}
fn perform_segmentation(text: &str, mode: chinese_tokenizer::SegmentationMode) -> Vec<String> {
chinese_tokenizer::FastChineseTokenizer::new(mode).segment(text)
}
fn register_tokenizers(index: &Index) {
let all_tokenizer = chinese_tokenizer::FastChineseTokenizer::all();
index.tokenizers().register(ALL_TOKENIZER, all_tokenizer);
}
fn build_schema() -> Schema {
let mut schema_builder = Schema::builder();
schema_builder.add_i64_field("id", INDEXED | STORED | FAST); // 切片 id
schema_builder.add_i64_field("file_id", INDEXED | STORED | FAST); // 文件 id
schema_builder.add_i64_field("kb_id", INDEXED | STORED | FAST); // 知识库 id
schema_builder.add_i64_field("is_image", INDEXED | FAST); // 是否为图片切片
let text_options = TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(ALL_TOKENIZER)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored();
schema_builder.add_text_field("content", text_options); // 内容
schema_builder.build()
}
#[derive(Debug, Clone, Copy)]
struct MatchInfo {
start_char: usize,
end_char: usize,
term_idx: usize,
}
fn build_snippet_terms(query: &str, synonym_map: Option<&SynonymMap>) -> Vec<String> {
let mut terms = Vec::new();
let mut seen = HashSet::new();
let segmented = perform_segmentation(query, chinese_tokenizer::SegmentationMode::Search);
for term in segmented.into_iter().chain(std::iter::once(query.trim().to_string())) {
let trimmed = term.trim();
if trimmed.is_empty() {
continue;
}
if seen.insert(trimmed.to_string()) {
terms.push(trimmed.to_string());
}
if let Some(map) = synonym_map
&& let Some(synonyms) = map.get(trimmed)
{
for synonym in synonyms {
let syn = synonym.term.trim();
if syn.is_empty() {
continue;
}
if seen.insert(syn.to_string()) {
terms.push(syn.to_string());
}
}
}
}
terms.sort_by_key(|b| std::cmp::Reverse(b.len()));
terms
}
#[derive(Debug, Clone)]
struct QueryTerm {
term: String,
boost: f32,
}
fn build_query_terms(input: &str, synonym_map: Option<&SynonymMap>) -> Vec<QueryTerm> {
let mut terms = Vec::new();
let mut seen = HashSet::new();
let segmented = perform_segmentation(input, chinese_tokenizer::SegmentationMode::Search);
for term in segmented {
let trimmed = term.trim();
if trimmed.is_empty() {
continue;
}
if seen.insert(trimmed.to_string()) {
terms.push(QueryTerm { term: trimmed.to_string(), boost: 1.0 });
}
if let Some(map) = synonym_map
&& let Some(synonyms) = map.get(trimmed)
{
for synonym in synonyms {
let syn = synonym.term.trim();
if syn.is_empty() || syn == trimmed {
continue;
}
if seen.insert(syn.to_string()) {
terms.push(QueryTerm { term: syn.to_string(), boost: synonym.boost.max(0.01) });
}
}
}
}
if terms.is_empty() {
let raw = input.trim();
if !raw.is_empty() {
terms.push(QueryTerm { term: raw.to_string(), boost: 1.0 });
if let Some(map) = synonym_map
&& let Some(synonyms) = map.get(raw)
{
for synonym in synonyms {
let syn = synonym.term.trim();
if syn.is_empty() || syn == raw {
continue;
}
if seen.insert(syn.to_string()) {
terms.push(QueryTerm { term: syn.to_string(), boost: synonym.boost.max(0.01) });
}
}
}
}
}
terms
}
fn build_matcher(terms: &[String]) -> Option<AhoCorasick> {
if terms.is_empty() {
return None;
}
AhoCorasickBuilder::new().ascii_case_insensitive(true).match_kind(MatchKind::LeftmostLongest).build(terms).ok()
}
fn build_best_snippet(content: &str, matcher: Option<&AhoCorasick>, term_count: usize, max_chars: usize) -> String {
if content.is_empty() {
return String::new();
}
let char_to_byte = build_char_index(content);
let char_len = char_to_byte.len().saturating_sub(1);
let max_chars = max_chars.max(1);
if char_len <= max_chars {
return highlight_range(content, matcher, &char_to_byte, 0, char_len, false, false);
}
let matches =
if let Some(matcher) = matcher { collect_matches(content, matcher, &char_to_byte) } else { Vec::new() };
if matches.is_empty() {
return highlight_range(content, matcher, &char_to_byte, 0, max_chars, false, true);
}
let (best_left, best_right) = select_best_window(&matches, term_count, max_chars);
let left_start = matches[best_left].start_char;
let right_end = matches[best_right].end_char;
let span = right_end.saturating_sub(left_start);
let extra = max_chars.saturating_sub(span);
let desired_start = left_start.saturating_sub(extra / 2);
let lower_bound = right_end.saturating_sub(max_chars);
let upper_bound = left_start.min(char_len.saturating_sub(max_chars));
let start = desired_start.clamp(lower_bound, upper_bound);
let end = start.saturating_add(max_chars).min(char_len);
highlight_range(content, matcher, &char_to_byte, start, end, start > 0, end < char_len)
}
fn collect_matches(content: &str, matcher: &AhoCorasick, char_to_byte: &[usize]) -> Vec<MatchInfo> {
let mut matches = Vec::new();
for m in matcher.find_iter(content) {
let start_char = byte_to_char_start(char_to_byte, m.start());
let end_char = byte_to_char_end(char_to_byte, m.end());
if end_char > start_char {
matches.push(MatchInfo { start_char, end_char, term_idx: m.pattern().as_usize() });
}
}
matches
}
fn select_best_window(matches: &[MatchInfo], term_count: usize, max_chars: usize) -> (usize, usize) {
let mut counts = vec![0usize; term_count];
let mut unique = 0usize;
let mut right = 0usize;
let mut best_left = 0usize;
let mut best_right = 0usize;
let mut best_unique = 0usize;
let mut best_total = 0usize;
let mut best_span = usize::MAX;
for left in 0..matches.len() {
if right < left {
right = left;
}
let window_start = matches[left].start_char;
let window_end = window_start.saturating_add(max_chars);
while right < matches.len() && matches[right].end_char <= window_end {
let term_idx = matches[right].term_idx;
if term_idx < term_count {
if counts[term_idx] == 0 {
unique += 1;
}
counts[term_idx] += 1;
}
right += 1;
}
let total = right.saturating_sub(left);
if total > 0 {
let rightmost_end = matches[right - 1].end_char;
let span = rightmost_end.saturating_sub(matches[left].start_char);
let better = unique > best_unique
|| (unique == best_unique && total > best_total)
|| (unique == best_unique && total == best_total && span < best_span)
|| (unique == best_unique
&& total == best_total
&& span == best_span
&& matches[left].start_char < matches[best_left].start_char);
if better {
best_unique = unique;
best_total = total;
best_span = span;
best_left = left;
best_right = right - 1;
}
}
let term_idx = matches[left].term_idx;
if term_idx < term_count {
counts[term_idx] = counts[term_idx].saturating_sub(1);
if counts[term_idx] == 0 {
unique = unique.saturating_sub(1);
}
}
}
(best_left, best_right)
}
fn build_char_index(text: &str) -> Vec<usize> {
let mut char_to_byte = Vec::with_capacity(text.chars().count() + 1);
for (byte_idx, _) in text.char_indices() {
char_to_byte.push(byte_idx);
}
char_to_byte.push(text.len());
char_to_byte
}
fn byte_to_char_start(char_to_byte: &[usize], byte_idx: usize) -> usize {
match char_to_byte.binary_search(&byte_idx) {
Ok(idx) => idx,
Err(idx) => idx.saturating_sub(1),
}
}
fn byte_to_char_end(char_to_byte: &[usize], byte_idx: usize) -> usize {
match char_to_byte.binary_search(&byte_idx) {
Ok(idx) => idx,
Err(idx) => idx,
}
}
fn highlight_range(
content: &str, matcher: Option<&AhoCorasick>, char_to_byte: &[usize], start_char: usize, end_char: usize,
prefix: bool, suffix: bool,
) -> String {
let start = *char_to_byte.get(start_char).unwrap_or(&0);
let end = *char_to_byte.get(end_char).unwrap_or(&content.len());
let slice = &content[start..end.min(content.len())];
let mut output = String::new();
if prefix {
output.push_str("...");
}
if let Some(matcher) = matcher {
let mut last = 0usize;
for m in matcher.find_iter(slice) {
let start_idx = m.start();
let end_idx = m.end();
if start_idx > last {
output.push_str(&escape_html(&slice[last..start_idx]));
}
output.push_str("<b>");
output.push_str(&escape_html(&slice[start_idx..end_idx]));
output.push_str("</b>");
last = end_idx;
}
if last < slice.len() {
output.push_str(&escape_html(&slice[last..]));
}
} else {
output.push_str(&escape_html(slice));
}
if suffix {
output.push_str("...");
}
output
}
fn escape_html(input: &str) -> String {
let mut escaped = String::with_capacity(input.len());
for ch in input.chars() {
match ch {
'&' => escaped.push_str("&amp;"),
'<' => escaped.push_str("&lt;"),
'>' => escaped.push_str("&gt;"),
'"' => escaped.push_str("&quot;"),
'\'' => escaped.push_str("&#39;"),
_ => escaped.push(ch),
}
}
escaped
}

54
src/slice_content.rs Normal file
View File

@ -0,0 +1,54 @@
use std::{collections::HashMap, path::PathBuf};
use anyhow::Context;
use crate::config;
fn content_path(file_id: i64) -> PathBuf {
PathBuf::from(&config::get().storage.slice_contents_path).join(format!("{}.json", file_id))
}
/// 一个源文件的全部切片正文。正文不再放入 SQLite，以减少 WAL 和主库体积。
pub async fn read_all(file_id: i64) -> anyhow::Result<HashMap<i64, String>> {
let path = content_path(file_id);
match tokio::fs::read(&path).await {
Ok(bytes) => {
serde_json::from_slice(&bytes).with_context(|| format!("invalid slice content file {}", path.display()))
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(HashMap::new()),
Err(e) => Err(e).with_context(|| format!("failed to read slice content file {}", path.display())),
}
}
pub async fn upsert_many(file_id: i64, rows: &[(i64, String)]) -> anyhow::Result<()> {
if rows.is_empty() {
return Ok(());
}
let mut contents = read_all(file_id).await?;
for (id, content) in rows {
contents.insert(*id, content.clone());
}
write_all(file_id, &contents).await
}
pub async fn write_all(file_id: i64, contents: &HashMap<i64, String>) -> anyhow::Result<()> {
let path = content_path(file_id);
if contents.is_empty() {
return delete(file_id).await;
}
let parent = path.parent().expect("slice content path has parent");
tokio::fs::create_dir_all(parent).await?;
let bytes = serde_json::to_vec(contents)?;
let tmp = path.with_extension("json.tmp");
tokio::fs::write(&tmp, bytes).await.with_context(|| format!("failed to write {}", tmp.display()))?;
tokio::fs::rename(&tmp, &path).await.with_context(|| format!("failed to rename {}", path.display()))
}
pub async fn delete(file_id: i64) -> anyhow::Result<()> {
let path = content_path(file_id);
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).with_context(|| format!("failed to delete slice content file {}", path.display())),
}
}

1395
stopwords.txt Normal file
View File

File diff suppressed because it is too large Load Diff

28
test_result.md Normal file
View File

@ -0,0 +1,28 @@
# Load Testing with 990 files test
## knowledge_base
测试时间：26-03-26
 hey -n 1000 -c 50 -H 'x-user-id: 1' -H 'x-role: admin' http://192.168.0.46:11000/api/v1/knowledge/knowledge_base/
Summary:
Total: 1.0624 secs
Slowest: 0.1404 secs
Fastest: 0.0050 secs
Average: 0.0501 secs
Requests/sec: 941.3084
Total data: 1646000 bytes
Size/request: 1646 bytes
测试时间：26-03-26
 hey -n 1000 -c 50 -m POST -H 'Content-Type: application/json' -H 'x-user-id: 1' -H 'x-role: admin' -d '{"description": "压测", "is_public": true, "kb_type": "analysis", "name": "压测知识库", "parent_id": 1}' http://192.168.0.46:11000/api/v1/knowledge/knowledge_base/
 hey -n 1000 -c 50 -H 'x-user-id: 1' -H 'x-role: admin' http://192.168.0.46:11000/api/v1/knowledge/knowledge_base/2
Summary:
Total: 1.9395 secs
Slowest: 0.3394 secs
Fastest: 0.0091 secs
Average: 0.0915 secs
Requests/sec: 515.5856

247
tests/api_concurrency.rs Normal file
View File

@ -0,0 +1,247 @@
mod common;
use std::{
fs, time::{Duration, Instant}
};
use axum::http::StatusCode;
use common::*;
use futures::future::join_all;
#[derive(Debug, Clone)]
struct RequestResult {
status: StatusCode,
duration: Duration,
}
#[derive(Debug)]
struct ConcurrencyMetrics {
total: usize,
success: usize,
failed: usize,
mean_ms: f64,
p50_ms: f64,
p95_ms: f64,
p99_ms: f64,
rps: f64,
total_duration_ms: f64,
}
impl std::fmt::Display for ConcurrencyMetrics {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"total={} success={} failed={} rps={:.1} mean={:.1}ms p50={:.1}ms p95={:.1}ms p99={:.1}ms total_time={:.1}ms",
self.total,
self.success,
self.failed,
self.rps,
self.mean_ms,
self.p50_ms,
self.p95_ms,
self.p99_ms,
self.total_duration_ms
)
}
}
fn percentile(sorted: &[f64], p: f64) -> f64 {
if sorted.is_empty() {
return 0.0;
}
let idx = ((sorted.len() - 1) as f64 * p) as usize;
sorted[idx]
}
fn compute_metrics(results: Vec<RequestResult>, total_duration: Duration) -> ConcurrencyMetrics {
let total = results.len();
let success = results.iter().filter(|r| r.status.is_success()).count();
let failed = total.saturating_sub(success);
let mut durations: Vec<f64> = results.iter().map(|r| r.duration.as_secs_f64() * 1000.0).collect();
durations.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let mean_ms = durations.iter().sum::<f64>() / total.max(1) as f64;
let p50_ms = percentile(&durations, 0.5);
let p95_ms = percentile(&durations, 0.95);
let p99_ms = percentile(&durations, 0.99);
let total_duration_ms = total_duration.as_secs_f64() * 1000.0;
let rps = total as f64 / total_duration.as_secs_f64().max(f64::EPSILON);
ConcurrencyMetrics { total, success, failed, mean_ms, p50_ms, p95_ms, p99_ms, rps, total_duration_ms }
}
fn kb_create_body(name: &str) -> Value {
serde_json::json!({
"name": name,
"description": "perf test knowledge base",
"kb_type": "analysis",
"parent_id": null,
"is_public": false
})
}
#[tokio::test]
async fn concurrent_kb_list_read_perf() {
let app = app().await;
let user = TestUser::new("perf-kb-list");
// Seed a few KBs so the list endpoint has real work to do.
for _ in 0..5 {
let req = authed_json_request(
"POST",
"/api/v1/knowledge/knowledge_base/",
&user,
kb_create_body(&format!("Seed KB {}", next_seq())),
);
let _ = app.clone().oneshot(req).await.unwrap();
}
const REQUEST_COUNT: usize = 200;
const CONCURRENCY: usize = 50;
let start = Instant::now();
let futures: Vec<_> = (0..REQUEST_COUNT)
.map(|i| {
let app = app.clone();
let user = TestUser::with_role(&format!("perf-kb-list-user-{}", i), "admin");
async move {
let req_start = Instant::now();
let req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &user);
let res = app.oneshot(req).await.unwrap();
RequestResult { status: res.status(), duration: req_start.elapsed() }
}
})
.collect();
let results = join_all(futures).await;
let metrics = compute_metrics(results, start.elapsed());
println!("concurrent_kb_list_read_perf ({} requests, concurrency={}): {}", REQUEST_COUNT, CONCURRENCY, metrics);
assert_eq!(metrics.success, metrics.total, "all KB list requests should succeed");
assert!(metrics.p95_ms < 500.0, "p95 latency should be below 500ms, got {:.1}ms", metrics.p95_ms);
}
#[tokio::test]
async fn concurrent_kb_create_write_perf() {
let app = app().await;
const REQUEST_COUNT: usize = 100;
const CONCURRENCY: usize = 20;
let start = Instant::now();
let futures: Vec<_> = (0..REQUEST_COUNT)
.map(|i| {
let app = app.clone();
let user = TestUser::with_role(&format!("perf-kb-create-user-{}", i), "admin");
async move {
let req_start = Instant::now();
let body = kb_create_body(&format!("Perf KB {}", next_seq()));
let req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, body);
let res = app.oneshot(req).await.unwrap();
RequestResult { status: res.status(), duration: req_start.elapsed() }
}
})
.collect();
let results = join_all(futures).await;
let metrics = compute_metrics(results, start.elapsed());
println!("concurrent_kb_create_write_perf ({} requests, concurrency={}): {}", REQUEST_COUNT, CONCURRENCY, metrics);
assert_eq!(metrics.success, metrics.total, "all KB create requests should succeed");
assert!(metrics.p95_ms < 1000.0, "p95 latency should be below 1000ms, got {:.1}ms", metrics.p95_ms);
}
#[tokio::test]
async fn concurrent_file_list_read_perf() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let user = TestUser::new("perf-file-list");
let kb_id = insert_kb(&pool, &user, "Perf File KB", "analysis", None, false).await;
// Seed a few files so the file list endpoint has real work to do.
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
for _ in 0..5 {
let path = file_dir.join(format!("perf-file-{}.txt", next_seq()));
fs::write(&path, b"perf test content").unwrap();
insert_file(&pool, &user, "perf.txt", &path, Some(kb_id), vec!["perf".to_string()], false).await;
}
const REQUEST_COUNT: usize = 150;
const CONCURRENCY: usize = 30;
let start = Instant::now();
let futures: Vec<_> = (0..REQUEST_COUNT)
.map(|i| {
let app = app.clone();
let user = TestUser::with_role(&format!("perf-file-list-user-{}", i), "admin");
async move {
let req_start = Instant::now();
let req = authed_empty_request("GET", "/api/v1/knowledge/files/", &user);
let res = app.oneshot(req).await.unwrap();
RequestResult { status: res.status(), duration: req_start.elapsed() }
}
})
.collect();
let results = join_all(futures).await;
let metrics = compute_metrics(results, start.elapsed());
println!("concurrent_file_list_read_perf ({} requests, concurrency={}): {}", REQUEST_COUNT, CONCURRENCY, metrics);
assert_eq!(metrics.success, metrics.total, "all file list requests should succeed");
assert!(metrics.p95_ms < 500.0, "p95 latency should be below 500ms, got {:.1}ms", metrics.p95_ms);
}
#[tokio::test]
async fn concurrent_mixed_read_write_perf() {
let app = app().await;
let user = TestUser::new("perf-mixed");
// Seed a few KBs.
for _ in 0..3 {
let req = authed_json_request(
"POST",
"/api/v1/knowledge/knowledge_base/",
&user,
kb_create_body(&format!("Seed {}", next_seq())),
);
let _ = app.clone().oneshot(req).await.unwrap();
}
const REQUEST_COUNT: usize = 200;
const CONCURRENCY: usize = 40;
// 70% read, 30% write.
const WRITE_RATIO: usize = 7;
let start = Instant::now();
let futures: Vec<_> = (0..REQUEST_COUNT)
.map(|i| {
let app = app.clone();
let user = TestUser::with_role(&format!("perf-mixed-user-{}", i), "admin");
async move {
let req_start = Instant::now();
let is_read = i % 10 < WRITE_RATIO;
let req = if is_read {
authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &user)
} else {
let body = kb_create_body(&format!("Mixed Perf KB {}", next_seq()));
authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, body)
};
let res = app.oneshot(req).await.unwrap();
RequestResult { status: res.status(), duration: req_start.elapsed() }
}
})
.collect();
let results = join_all(futures).await;
let metrics = compute_metrics(results, start.elapsed());
println!(
"concurrent_mixed_read_write_perf ({} requests, concurrency={}, 70% read / 30% write): {}",
REQUEST_COUNT, CONCURRENCY, metrics
);
assert!(metrics.success >= metrics.total * 99 / 100, "success rate should be >= 99%");
assert!(metrics.p95_ms < 1000.0, "p95 latency should be below 1000ms, got {:.1}ms", metrics.p95_ms);
}

918
tests/api_integration.rs Normal file
View File

@ -0,0 +1,918 @@
mod common;
use std::fs;
use common::*;
#[tokio::test]
async fn knowledge_base_flow() {
let app = app().await;
let user = TestUser::new("kb-flow");
let unauth_req =
Request::builder().method("GET").uri("/api/v1/knowledge/knowledge_base/").body(Body::empty()).unwrap();
let unauth_res = app.clone().oneshot(unauth_req).await.unwrap();
assert_eq!(unauth_res.status(), StatusCode::UNAUTHORIZED);
let create_body = serde_json::json!({
"name": "Integration KB",
"description": "kb for integration tests",
"kb_type": "analysis",
"parent_id": null,
"is_public": false
});
let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, create_body);
let create_res = app.clone().oneshot(create_req).await.unwrap();
assert_eq!(create_res.status(), StatusCode::OK);
let created = response_json(create_res).await;
let kb_id = created["id"].as_i64().expect("created kb id");
let list_req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &user);
let list_res = app.clone().oneshot(list_req).await.unwrap();
assert_eq!(list_res.status(), StatusCode::OK);
let list_bytes = list_res.into_body().collect().await.unwrap().to_bytes();
let list: Vec<Value> = serde_json::from_slice(&list_bytes).unwrap();
assert!(list.iter().any(|kb| kb["id"].as_i64() == Some(kb_id) && kb["name"].as_str() == Some("Integration KB")));
let update_body = serde_json::json!({
"parent_id": kb_id
});
let update_req =
authed_json_request("PUT", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &user, update_body);
let update_res = app.clone().oneshot(update_req).await.unwrap();
assert_eq!(update_res.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn knowledge_base_public_and_reparse() {
let app = app().await;
let user = TestUser::new("kb-public");
let create_body = serde_json::json!({
"name": "Public KB",
"description": "kb for public tests",
"kb_type": "analysis",
"parent_id": null,
"is_public": false
});
let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, create_body);
let create_res = app.clone().oneshot(create_req).await.unwrap();
assert_eq!(create_res.status(), StatusCode::OK);
let created = response_json(create_res).await;
let kb_id = created["id"].as_i64().expect("created kb id");
let public_req = authed_json_request(
"PUT",
format!("/api/v1/knowledge/knowledge_base/{}", kb_id),
&user,
serde_json::json!({ "is_public": true }),
);
let public_res = app.clone().oneshot(public_req).await.unwrap();
assert_eq!(public_res.status(), StatusCode::OK);
let public_json = response_json(public_res).await;
assert_eq!(public_json["is_public"].as_bool(), Some(true));
let reparse_req = authed_empty_request("POST", "/api/v1/knowledge/knowledge_base/reparse", &user);
let reparse_res = app.clone().oneshot(reparse_req).await.unwrap();
assert_eq!(reparse_res.status(), StatusCode::OK);
let reparse_json = response_json(reparse_res).await;
assert_eq!(reparse_json["kb_count"].as_i64(), Some(1));
assert_eq!(reparse_json["file_count"].as_i64(), Some(0));
}
#[tokio::test]
async fn knowledge_base_tree_and_detail_flow() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let user = TestUser::new("kb-tree");
let root_kb_id = insert_kb(&pool, &user, "Root KB", "analysis", None, false).await;
let child_kb_id = insert_kb(&pool, &user, "Child KB", "analysis", Some(root_kb_id), false).await;
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let root_file_path = file_dir.join(format!("root-file-{}.txt", next_seq()));
let child_file_path = file_dir.join(format!("child-file-{}.txt", next_seq()));
fs::write(&root_file_path, b"root file").unwrap();
fs::write(&child_file_path, b"child file").unwrap();
let root_file_id =
insert_file(&pool, &user, "root-file.txt", &root_file_path, Some(root_kb_id), vec!["root".to_string()], false)
.await;
let child_file_id = insert_file(
&pool,
&user,
"child-file.txt",
&child_file_path,
Some(child_kb_id),
vec!["child".to_string()],
false,
)
.await;
let tree_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/tree?kb_id={}", root_kb_id), &user);
let tree_res = app.clone().oneshot(tree_req).await.unwrap();
assert_eq!(tree_res.status(), StatusCode::OK);
let tree_json = response_json(tree_res).await;
let tree_nodes = tree_json.as_array().expect("tree nodes");
assert_eq!(tree_nodes.len(), 1);
let root_node = &tree_nodes[0];
assert_eq!(root_node["id"].as_i64(), Some(root_kb_id));
assert!(root_node["files"].as_array().unwrap().iter().any(|file| file["id"].as_i64() == Some(root_file_id)));
assert!(root_node["children"].as_array().unwrap().iter().any(|child| {
child["id"].as_i64() == Some(child_kb_id)
&& child["files"].as_array().unwrap().iter().any(|file| file["id"].as_i64() == Some(child_file_id))
}));
let detail_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", root_kb_id), &user);
let detail_res = app.clone().oneshot(detail_req).await.unwrap();
assert_eq!(detail_res.status(), StatusCode::OK);
let detail_json = response_json(detail_res).await;
assert_eq!(detail_json["id"].as_i64(), Some(root_kb_id));
assert!(detail_json["files"].is_null(), "detail should no longer return files");
// New paginated files endpoint
let kb_files_req = authed_empty_request(
"GET",
format!("/api/v1/knowledge/knowledge_base/{}/files?filename=root-file", root_kb_id),
&user,
);
let kb_files_res = app.clone().oneshot(kb_files_req).await.unwrap();
assert_eq!(kb_files_res.status(), StatusCode::OK);
let kb_files_json = response_json(kb_files_res).await;
let kb_files_items = kb_files_json["items"].as_array().expect("kb files items");
assert!(kb_files_items.iter().any(|file| file["id"].as_i64() == Some(root_file_id)));
assert!(kb_files_json["total"].as_i64().unwrap_or(0) >= 1);
// Pagination
let kb_files_page_req = authed_empty_request(
"GET",
format!("/api/v1/knowledge/knowledge_base/{}/files?size=1&page=1", root_kb_id),
&user,
);
let kb_files_page_res = app.clone().oneshot(kb_files_page_req).await.unwrap();
assert_eq!(kb_files_page_res.status(), StatusCode::OK);
let kb_files_page_json = response_json(kb_files_page_res).await;
assert!(kb_files_page_json["total"].as_i64().unwrap_or(0) >= 1);
assert!(kb_files_page_json["items"].as_array().expect("items").len() <= 1);
// Tag filter
let kb_files_tag_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/files?tag=root", root_kb_id), &user);
let kb_files_tag_res = app.clone().oneshot(kb_files_tag_req).await.unwrap();
assert_eq!(kb_files_tag_res.status(), StatusCode::OK);
let kb_files_tag_json = response_json(kb_files_tag_res).await;
assert!(
kb_files_tag_json["items"]
.as_array()
.expect("tag items")
.iter()
.all(|file| { file["id"].as_i64() == Some(root_file_id) })
);
}
#[tokio::test]
async fn knowledge_base_tag_stats_respect_scope_data_quality_and_permissions() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let owner = TestUser::with_role("tag-owner", "user");
let viewer = TestUser::with_role("tag-viewer", "user");
let root_kb_id = insert_kb(&pool, &owner, "Tag Root", "analysis", None, true).await;
let child_kb_id = insert_kb(&pool, &owner, "Tag Child", "analysis", Some(root_kb_id), false).await;
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let private_path = file_dir.join(format!("tag-private-{}.txt", next_seq()));
let public_path = file_dir.join(format!("tag-public-{}.txt", next_seq()));
let child_path = file_dir.join(format!("tag-child-{}.txt", next_seq()));
let invalid_path = file_dir.join(format!("tag-invalid-{}.txt", next_seq()));
fs::write(&private_path, b"private").unwrap();
fs::write(&public_path, b"public").unwrap();
fs::write(&child_path, b"child").unwrap();
fs::write(&invalid_path, b"invalid").unwrap();
insert_file(
&pool,
&owner,
"tag-private.txt",
&private_path,
Some(root_kb_id),
vec!["alpha".to_string(), "shared".to_string(), "shared".to_string()],
false,
)
.await;
insert_file(
&pool,
&owner,
"tag-public.txt",
&public_path,
Some(root_kb_id),
vec!["alpha".to_string(), "".to_string(), "beta".to_string()],
true,
)
.await;
insert_file(
&pool,
&owner,
"tag-child.txt",
&child_path,
Some(child_kb_id),
vec!["child".to_string(), "shared".to_string()],
true,
)
.await;
let invalid_file_id =
insert_file(&pool, &owner, "tag-invalid.txt", &invalid_path, Some(root_kb_id), vec![], false).await;
sqlx::query("UPDATE files SET tags = 'not-json' WHERE id = ?").bind(invalid_file_id).execute(&pool).await.unwrap();
let direct_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/tags", root_kb_id), &owner);
let direct_res = app.clone().oneshot(direct_req).await.unwrap();
assert_eq!(direct_res.status(), StatusCode::OK);
let direct_json = response_json(direct_res).await;
assert_eq!(direct_json["kb_id"].as_i64(), Some(root_kb_id));
assert_eq!(
direct_json["tags"],
serde_json::json!([
{"tag": "alpha", "file_count": 2},
{"tag": "beta", "file_count": 1},
{"tag": "shared", "file_count": 1}
])
);
let descendants_req = authed_empty_request(
"GET",
format!("/api/v1/knowledge/knowledge_base/{}/tags?include_descendants=true", root_kb_id),
&owner,
);
let descendants_res = app.clone().oneshot(descendants_req).await.unwrap();
assert_eq!(descendants_res.status(), StatusCode::OK);
let descendants_json = response_json(descendants_res).await;
assert_eq!(
descendants_json["tags"],
serde_json::json!([
{"tag": "alpha", "file_count": 2},
{"tag": "shared", "file_count": 2},
{"tag": "beta", "file_count": 1},
{"tag": "child", "file_count": 1}
])
);
let viewer_req = authed_empty_request(
"GET",
format!("/api/v1/knowledge/knowledge_base/{}/tags?include_descendants=true", root_kb_id),
&viewer,
);
let viewer_res = app.clone().oneshot(viewer_req).await.unwrap();
assert_eq!(viewer_res.status(), StatusCode::OK);
let viewer_json = response_json(viewer_res).await;
assert_eq!(
viewer_json["tags"],
serde_json::json!([
{"tag": "alpha", "file_count": 1},
{"tag": "beta", "file_count": 1}
])
);
let inaccessible_kb_id = insert_kb(&pool, &owner, "Tag Private", "analysis", None, false).await;
let inaccessible_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/tags", inaccessible_kb_id), &viewer);
let inaccessible_res = app.clone().oneshot(inaccessible_req).await.unwrap();
assert_eq!(inaccessible_res.status(), StatusCode::NOT_FOUND);
let admin = TestUser::new("tag-admin");
let missing_req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/9223372036854775807/tags", &admin);
let missing_res = app.oneshot(missing_req).await.unwrap();
assert_eq!(missing_res.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn file_endpoints_flow() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let user = TestUser::new("file");
let kb_id = insert_kb(&pool, &user, "File KB", "analysis", None, false).await;
let file_suffix = next_seq();
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let file_path = file_dir.join(format!("file-{}.txt", file_suffix));
let file_contents = b"file contents";
fs::write(&file_path, file_contents).unwrap();
let file_id = insert_file(
&pool,
&user,
"file.txt",
&file_path,
Some(kb_id),
vec!["tag1".to_string(), "tag2".to_string()],
false,
)
.await;
let slice_id = insert_slice(&pool, file_id, "slice content").await;
insert_slice_position(&pool, slice_id, 1, [1, 2, 3, 4]).await;
let list_req = authed_empty_request("GET", "/api/v1/knowledge/files/", &user);
let list_res = app.clone().oneshot(list_req).await.unwrap();
assert_eq!(list_res.status(), StatusCode::OK);
let list_json = response_json(list_res).await;
let list = list_json["items"].as_array().expect("file list items");
assert!(list_json["total"].as_i64().unwrap_or(0) >= 1);
assert!(list.iter().any(|f| f["id"].as_i64() == Some(file_id)));
let list_tag_req = authed_empty_request("GET", "/api/v1/knowledge/files/?tag=tag1", &user);
let list_tag_res = app.clone().oneshot(list_tag_req).await.unwrap();
assert_eq!(list_tag_res.status(), StatusCode::OK);
let list_tag_json = response_json(list_tag_res).await;
let list_tag = list_tag_json["items"].as_array().expect("file list items");
assert!(list_tag.iter().any(|f| f["id"].as_i64() == Some(file_id)));
let list_kb_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/?kb_id={}", kb_id), &user);
let list_kb_res = app.clone().oneshot(list_kb_req).await.unwrap();
assert_eq!(list_kb_res.status(), StatusCode::OK);
let list_kb_json = response_json(list_kb_res).await;
let list_kb = list_kb_json["items"].as_array().expect("file list items");
assert!(list_kb.iter().any(|f| f["id"].as_i64() == Some(file_id)));
let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/{}", file_id), &user);
let get_res = app.clone().oneshot(get_req).await.unwrap();
assert_eq!(get_res.status(), StatusCode::OK);
let get_json = response_json(get_res).await;
assert_eq!(get_json["id"].as_i64(), Some(file_id));
let update_body = serde_json::json!({
"filename": "renamed.txt",
"tags": ["tag3"]
});
let update_req = authed_json_request("PUT", format!("/api/v1/knowledge/files/{}", file_id), &user, update_body);
let update_res = app.clone().oneshot(update_req).await.unwrap();
assert_eq!(update_res.status(), StatusCode::OK);
let update_json = response_json(update_res).await;
assert_eq!(update_json["filename"].as_str(), Some("renamed.txt"));
let updated_tags: Vec<String> = serde_json::from_str(update_json["tags"].as_str().unwrap()).unwrap();
assert_eq!(updated_tags, vec!["tag3".to_string()]);
let download_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/{}/download", file_id), &user);
let download_res = app.clone().oneshot(download_req).await.unwrap();
assert_eq!(download_res.status(), StatusCode::OK);
let disposition =
download_res.headers().get(header::CONTENT_DISPOSITION).and_then(|v| v.to_str().ok()).unwrap_or("");
assert!(disposition.contains("renamed.txt"));
let download_bytes = download_res.into_body().collect().await.unwrap().to_bytes();
assert_eq!(download_bytes.as_ref(), file_contents);
let slices_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/{}/slices", file_id), &user);
let slices_res = app.clone().oneshot(slices_req).await.unwrap();
assert_eq!(slices_res.status(), StatusCode::OK);
let slices_json = response_json(slices_res).await;
let slices = slices_json.as_array().expect("slice list");
let slice = slices.iter().find(|s| s["id"].as_i64() == Some(slice_id)).expect("slice entry");
assert_eq!(slice["file_id"].as_i64(), Some(file_id));
assert_eq!(slice["content"].as_str(), Some("slice content"));
assert!(slice["positions"].is_null(), "slices endpoint should not return positions");
let delete_req = authed_empty_request("DELETE", format!("/api/v1/knowledge/files/{}", file_id), &user);
let delete_res = app.clone().oneshot(delete_req).await.unwrap();
assert_eq!(delete_res.status(), StatusCode::OK);
assert!(!file_path.exists());
}
#[tokio::test]
async fn slice_highlight_by_id() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let user = TestUser::new("slice_highlight");
let kb_id = insert_kb(&pool, &user, "Slice Highlight KB", "analysis", None, false).await;
let file_suffix = next_seq();
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let file_path = file_dir.join(format!("highlight-{}.txt", file_suffix));
fs::write(&file_path, b"highlight content").unwrap();
let file_id = insert_file(&pool, &user, "highlight.txt", &file_path, Some(kb_id), Vec::new(), false).await;
let slice_id = insert_slice(&pool, file_id, "slice content").await;
insert_slice_position(&pool, slice_id, 2, [10, 20, 100, 30]).await;
// 查询切片高亮信息
let highlight_req = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight", file_id, slice_id),
&user,
);
let highlight_res = app.clone().oneshot(highlight_req).await.unwrap();
assert_eq!(highlight_res.status(), StatusCode::OK);
let highlight_json = response_json(highlight_res).await;
let positions = highlight_json["positions"].as_array().expect("positions array");
assert_eq!(positions.len(), 1);
assert_eq!(positions[0]["page_idx"].as_i64(), Some(2));
let bbox = positions[0]["bbox"].as_array().expect("bbox array");
assert_eq!(bbox.len(), 4);
assert_eq!(bbox[0].as_i64(), Some(10));
assert_eq!(bbox[3].as_i64(), Some(30));
// 切片不属于该文件应返回 400
let other_file_id = insert_file(&pool, &user, "other.txt", &file_path, Some(kb_id), Vec::new(), false).await;
let bad_req = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight", other_file_id, slice_id),
&user,
);
let bad_res = app.clone().oneshot(bad_req).await.unwrap();
assert_eq!(bad_res.status(), StatusCode::BAD_REQUEST);
// 不存在的切片应返回 404
let missing_req =
authed_empty_request("GET", format!("/api/v1/knowledge/files/{}/slices/{}/highlight", file_id, 999999), &user);
let missing_res = app.clone().oneshot(missing_req).await.unwrap();
assert_eq!(missing_res.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn slice_highlight_page_by_id() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let user = TestUser::new("slice_highlight_page");
let kb_id = insert_kb(&pool, &user, "Slice Highlight Page KB", "analysis", None, false).await;
let file_suffix = next_seq();
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let file_path = file_dir.join(format!("highlight-page-{}.txt", file_suffix));
fs::write(&file_path, b"highlight content").unwrap();
let file_id = insert_file(&pool, &user, "highlight-page.txt", &file_path, Some(kb_id), Vec::new(), false).await;
// 为文件写入 pdf_content，使 highlight-page 能按内容字数判断
let pdf_rows = vec![
htknow::pdf_content::PdfContent {
page_idx: 0,
bbox: Some("[10,20,100,30]".to_string()),
text: Some("a".to_string()),
text_level: None,
img_path: None,
table_body: None,
},
htknow::pdf_content::PdfContent {
page_idx: 1,
bbox: Some("[10,20,100,30]".to_string()),
text: Some("a".to_string()),
text_level: None,
img_path: None,
table_body: None,
},
];
htknow::pdf_content::write(file_id, &pdf_rows).await.unwrap();
// 场景 1：第一页内容字数少于阈值（19 * 1 = 19 < 20）且存在第二页，应返回第二页
let slice_id_a = insert_slice(&pool, file_id, "slice a").await;
for _ in 0..19 {
insert_slice_position(&pool, slice_id_a, 0, [10, 20, 100, 30]).await;
}
insert_slice_position(&pool, slice_id_a, 1, [10, 20, 100, 30]).await;
let req_a = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_a),
&user,
);
let res_a = app.clone().oneshot(req_a).await.unwrap();
assert_eq!(res_a.status(), StatusCode::OK);
let json_a = response_json(res_a).await;
assert_eq!(json_a["page_idx"].as_i64(), Some(1));
// 场景 2：第一页内容字数达到阈值（21 * 1 = 21 >= 20），应返回第一页
let slice_id_b = insert_slice(&pool, file_id, "slice b").await;
for _ in 0..21 {
insert_slice_position(&pool, slice_id_b, 0, [10, 20, 100, 30]).await;
}
insert_slice_position(&pool, slice_id_b, 1, [10, 20, 100, 30]).await;
let req_b = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_b),
&user,
);
let res_b = app.clone().oneshot(req_b).await.unwrap();
assert_eq!(res_b.status(), StatusCode::OK);
let json_b = response_json(res_b).await;
assert_eq!(json_b["page_idx"].as_i64(), Some(0));
// 场景 3：只有一页且内容字数少于阈值（无 pdf_content 匹配，按位置数量兜底为 1），仍返回第一页
let slice_id_c = insert_slice(&pool, file_id, "slice c").await;
insert_slice_position(&pool, slice_id_c, 2, [10, 20, 100, 30]).await;
let req_c = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_c),
&user,
);
let res_c = app.clone().oneshot(req_c).await.unwrap();
assert_eq!(res_c.status(), StatusCode::OK);
let json_c = response_json(res_c).await;
assert_eq!(json_c["page_idx"].as_i64(), Some(2));
// 场景 4：切片没有高亮位置应返回 404
let slice_id_d = insert_slice(&pool, file_id, "slice d").await;
let req_d = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_d),
&user,
);
let res_d = app.clone().oneshot(req_d).await.unwrap();
assert_eq!(res_d.status(), StatusCode::NOT_FOUND);
// 场景 5：切片不属于该文件应返回 400
let other_file_id = insert_file(&pool, &user, "other.txt", &file_path, Some(kb_id), Vec::new(), false).await;
let req_bad = authed_empty_request(
"GET",
format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", other_file_id, slice_id_a),
&user,
);
let res_bad = app.clone().oneshot(req_bad).await.unwrap();
assert_eq!(res_bad.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn graph_endpoints_flow() {
let app = app().await;
let pool = get_pool().await;
let env = setup_env();
let user = TestUser::new("graph");
let kb_id = insert_kb(&pool, &user, "Graph KB", "analysis", None, false).await;
let file_suffix = next_seq();
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let file_path = file_dir.join(format!("graph-{}.txt", file_suffix));
fs::write(&file_path, b"graph file").unwrap();
let file_id = insert_file(&pool, &user, "graph.txt", &file_path, Some(kb_id), Vec::new(), false).await;
let slice_id = insert_slice(&pool, file_id, "mention context").await;
let node_a_id = insert_graph_node(
&pool,
"Alpha",
"device",
Some(serde_json::json!({ "origin": "test" })),
Some(file_id),
Some(kb_id),
)
.await;
let node_b_id = insert_graph_node(&pool, "Beta", "component", None, Some(file_id), Some(kb_id)).await;
insert_graph_edge(&pool, node_a_id, node_b_id, "related_to", Some(file_id)).await;
insert_entity_mention(&pool, node_a_id, slice_id, "Alpha mention").await;
let search_req =
authed_empty_request("GET", format!("/api/v1/knowledge/graph/entities?kb_id={}&q=Alpha", kb_id), &user);
let search_res = app.clone().oneshot(search_req).await.unwrap();
assert_eq!(search_res.status(), StatusCode::OK);
let search_json = response_json(search_res).await;
let search_list = search_json.as_array().expect("entity list");
assert!(search_list.iter().any(|entity| entity["id"].as_i64() == Some(node_a_id)));
let entity_req = authed_empty_request("GET", format!("/api/v1/knowledge/graph/entities/{}", node_a_id), &user);
let entity_res = app.clone().oneshot(entity_req).await.unwrap();
assert_eq!(entity_res.status(), StatusCode::OK);
let entity_json = response_json(entity_res).await;
assert_eq!(entity_json["entity"]["id"].as_i64(), Some(node_a_id));
let neighbors = entity_json["neighbors"].as_array().expect("neighbors");
assert_eq!(neighbors.len(), 1);
assert_eq!(neighbors[0]["entity"]["id"].as_i64(), Some(node_b_id));
assert_eq!(neighbors[0]["direction"].as_str(), Some("outgoing"));
let mentions = entity_json["mentions"].as_array().expect("mentions");
assert_eq!(mentions.len(), 1);
assert_eq!(mentions[0]["file_id"].as_i64(), Some(file_id));
let stats_req = authed_empty_request("GET", format!("/api/v1/knowledge/graph/stats?kb_id={}", kb_id), &user);
let stats_res = app.clone().oneshot(stats_req).await.unwrap();
assert_eq!(stats_res.status(), StatusCode::OK);
let stats_json = response_json(stats_res).await;
assert_eq!(stats_json["node_count"].as_i64(), Some(2));
assert_eq!(stats_json["edge_count"].as_i64(), Some(1));
let entity_types = stats_json["entity_types"].as_object().expect("entity types");
assert_eq!(entity_types.get("device").and_then(|v| v.as_i64()), Some(1));
assert_eq!(entity_types.get("component").and_then(|v| v.as_i64()), Some(1));
let relation_types = stats_json["relation_types"].as_object().expect("relation types");
assert_eq!(relation_types.get("related_to").and_then(|v| v.as_i64()), Some(1));
}
#[tokio::test]
async fn search_full_empty_and_image_requires_file() {
let app = app().await;
let user = TestUser::new("search");
let full_req = authed_empty_request("GET", "/api/v1/knowledge/search/full?query=missing", &user);
let full_res = app.clone().oneshot(full_req).await.unwrap();
assert_eq!(full_res.status(), StatusCode::OK);
let full_json = response_json(full_res).await;
assert_eq!(full_json["results"].as_array().map(|v| v.len()), Some(0));
let boundary = format!("boundary-{}", next_seq());
let body = multipart_body(&boundary, &[("text", "sample")]);
let image_req = authed_multipart_request("POST", "/api/v1/knowledge/search/image", &user, &boundary, body);
let image_res = app.clone().oneshot(image_req).await.unwrap();
assert_eq!(image_res.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn kb_permission_viewer_can_read_but_not_modify() {
let app = app().await;
let owner = TestUser::with_role("kb-perm-owner", "user");
let viewer = TestUser::with_role("kb-perm-viewer", "user");
// Owner creates a private KB
let create_body = serde_json::json!({
"name": "Permission Test KB",
"description": "kb for permission tests",
"kb_type": "analysis",
"parent_id": null,
"is_public": false
});
let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &owner, create_body);
let create_res = app.clone().oneshot(create_req).await.unwrap();
assert_eq!(create_res.status(), StatusCode::OK);
let created = response_json(create_res).await;
let kb_id = created["id"].as_i64().expect("created kb id");
assert_eq!(created["current_user_permission"].as_str(), Some("admin"));
// Owner grants viewer permission
let grant_req = authed_json_request(
"POST",
format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id),
&owner,
serde_json::json!({ "user_id": viewer.id, "permission": "viewer" }),
);
let grant_res = app.clone().oneshot(grant_req).await.unwrap();
assert_eq!(grant_res.status(), StatusCode::OK);
// Viewer can list the KB
let list_req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &viewer);
let list_res = app.clone().oneshot(list_req).await.unwrap();
assert_eq!(list_res.status(), StatusCode::OK);
let list_bytes = list_res.into_body().collect().await.unwrap().to_bytes();
let list: Vec<Value> = serde_json::from_slice(&list_bytes).unwrap();
let kb = list.iter().find(|k| k["id"].as_i64() == Some(kb_id)).expect("viewer sees the kb");
assert_eq!(kb["current_user_permission"].as_str(), Some("viewer"));
// Viewer can get detail
let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &viewer);
let get_res = app.clone().oneshot(get_req).await.unwrap();
assert_eq!(get_res.status(), StatusCode::OK);
let get_json = response_json(get_res).await;
assert_eq!(get_json["current_user_permission"].as_str(), Some("viewer"));
// Viewer cannot update the KB
let update_req = authed_json_request(
"PUT",
format!("/api/v1/knowledge/knowledge_base/{}", kb_id),
&viewer,
serde_json::json!({ "name": "Hacked" }),
);
let update_res = app.clone().oneshot(update_req).await.unwrap();
assert_eq!(update_res.status(), StatusCode::FORBIDDEN);
// Viewer cannot reparse
let reparse_req =
authed_empty_request("POST", format!("/api/v1/knowledge/knowledge_base/{}/reparse", kb_id), &viewer);
let reparse_res = app.clone().oneshot(reparse_req).await.unwrap();
assert_eq!(reparse_res.status(), StatusCode::FORBIDDEN);
// Viewer cannot delete
let delete_req = authed_empty_request("DELETE", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &viewer);
let delete_res = app.clone().oneshot(delete_req).await.unwrap();
assert_eq!(delete_res.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn kb_permission_editor_can_upload_but_not_delete() {
let app = app().await;
let env = setup_env();
let owner = TestUser::with_role("kb-editor-owner", "user");
let editor = TestUser::with_role("kb-editor-editor", "user");
// Owner creates a private storage KB (so we can upload without parse complications)
let create_body = serde_json::json!({
"name": "Editor Test KB",
"description": "kb for editor permission tests",
"kb_type": "storage",
"parent_id": null,
"is_public": false
});
let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &owner, create_body);
let create_res = app.clone().oneshot(create_req).await.unwrap();
assert_eq!(create_res.status(), StatusCode::OK);
let kb_id = response_json(create_res).await["id"].as_i64().unwrap();
// Owner grants editor permission
let grant_req = authed_json_request(
"POST",
format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id),
&owner,
serde_json::json!({ "user_id": editor.id, "permission": "editor" }),
);
let grant_res = app.clone().oneshot(grant_req).await.unwrap();
assert_eq!(grant_res.status(), StatusCode::OK);
// Editor can update the KB name/description
let update_req = authed_json_request(
"PUT",
format!("/api/v1/knowledge/knowledge_base/{}", kb_id),
&editor,
serde_json::json!({ "name": "Renamed by Editor", "description": "updated" }),
);
let update_res = app.clone().oneshot(update_req).await.unwrap();
assert_eq!(update_res.status(), StatusCode::OK);
// Editor cannot change visibility
let vis_req = authed_json_request(
"PUT",
format!("/api/v1/knowledge/knowledge_base/{}", kb_id),
&editor,
serde_json::json!({ "is_public": true }),
);
let vis_res = app.clone().oneshot(vis_req).await.unwrap();
assert_eq!(vis_res.status(), StatusCode::FORBIDDEN);
// Editor can upload a file
let file_dir = env.data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let test_file = file_dir.join(format!("editor-upload-{}.txt", next_seq()));
fs::write(&test_file, b"editor upload test").unwrap();
let boundary = format!("boundary-{}", next_seq());
let upload_req = authed_multipart_request_with_file(
"POST",
"/api/v1/knowledge/files/",
&editor,
&boundary,
&[("kb_id", &kb_id.to_string()), ("slice_type", "text")],
"file",
"test.txt",
b"editor upload test",
);
let upload_res = app.clone().oneshot(upload_req).await.unwrap();
assert_eq!(upload_res.status(), StatusCode::OK);
// Editor cannot delete the KB
let delete_req = authed_empty_request("DELETE", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &editor);
let delete_res = app.clone().oneshot(delete_req).await.unwrap();
assert_eq!(delete_res.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn kb_permission_admin_can_manage_permissions() {
let app = app().await;
let owner = TestUser::with_role("kb-admin-owner", "user");
let viewer = TestUser::with_role("kb-admin-viewer", "user");
// Owner creates KB
let create_body = serde_json::json!({
"name": "Admin Perm Test KB",
"description": "test",
"kb_type": "analysis",
"is_public": false
});
let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &owner, create_body);
let create_res = app.clone().oneshot(create_req).await.unwrap();
let kb_id = response_json(create_res).await["id"].as_i64().unwrap();
// Owner lists permissions (should be empty initially except no explicit rows)
let list_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), &owner);
let list_res = app.clone().oneshot(list_req).await.unwrap();
assert_eq!(list_res.status(), StatusCode::OK);
// Owner grants viewer permission
let grant_req = authed_json_request(
"POST",
format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id),
&owner,
serde_json::json!({ "user_id": viewer.id, "permission": "viewer" }),
);
let grant_res = app.clone().oneshot(grant_req).await.unwrap();
assert_eq!(grant_res.status(), StatusCode::OK);
let grant_json = response_json(grant_res).await;
assert_eq!(grant_json["user_id"].as_str(), Some(viewer.id.as_str()));
assert_eq!(grant_json["permission"].as_str(), Some("viewer"));
// Viewer cannot call permission APIs
let viewer_list_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), &viewer);
let viewer_list_res = app.clone().oneshot(viewer_list_req).await.unwrap();
assert_eq!(viewer_list_res.status(), StatusCode::FORBIDDEN);
// Owner upgrades viewer to admin
let upgrade_req = authed_json_request(
"POST",
format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id),
&owner,
serde_json::json!({ "user_id": viewer.id, "permission": "admin" }),
);
let upgrade_res = app.clone().oneshot(upgrade_req).await.unwrap();
assert_eq!(upgrade_res.status(), StatusCode::OK);
// Now viewer (as KB admin) can list permissions
let list2_req =
authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), &viewer);
let list2_res = app.clone().oneshot(list2_req).await.unwrap();
assert_eq!(list2_res.status(), StatusCode::OK);
// Owner removes viewer permission
let remove_req = authed_empty_request(
"DELETE",
format!("/api/v1/knowledge/knowledge_base/{}/permissions/{}", kb_id, viewer.id),
&owner,
);
let remove_res = app.clone().oneshot(remove_req).await.unwrap();
assert_eq!(remove_res.status(), StatusCode::OK);
// After removal, viewer can no longer access the KB
let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &viewer);
let get_res = app.clone().oneshot(get_req).await.unwrap();
assert_eq!(get_res.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn kb_permission_search_filters_unauthorized_kb() {
let app = app().await;
let pool = get_pool().await;
let owner = TestUser::with_role("kb-search-owner", "user");
let other = TestUser::with_role("kb-search-other", "user");
// Owner creates a private KB with a file
let kb_id = insert_kb(&pool, &owner, "Search Permission KB", "analysis", None, false).await;
let file_dir = setup_env().data_dir.join("files");
fs::create_dir_all(&file_dir).unwrap();
let test_file = file_dir.join(format!("search-perm-{}.txt", next_seq()));
fs::write(&test_file, b"secret content about dragons").unwrap();
let file_id = insert_file(&pool, &owner, "secret-perm.txt", &test_file, Some(kb_id), vec![], false).await;
// Set file as completed so it appears in full search
sqlx::query("UPDATE files SET status = 1 WHERE id = ?").bind(file_id).execute(&pool).await.unwrap();
// Owner can use full-search filename filter and find the file
let owner_search_req =
authed_empty_request("GET", "/api/v1/knowledge/search/full?filename=secret-perm.txt", &owner);
let owner_search_res = app.clone().oneshot(owner_search_req).await.unwrap();
assert_eq!(owner_search_res.status(), StatusCode::OK);
let owner_json = response_json(owner_search_res).await;
let owner_results = owner_json["results"].as_array().expect("results");
assert!(
owner_results.iter().any(|r| r["file"]["id"].as_i64() == Some(file_id)),
"owner should find their own file"
);
// Other user without permission cannot see the result
let other_search_req =
authed_empty_request("GET", "/api/v1/knowledge/search/full?filename=secret-perm.txt", &other);
let other_search_res = app.clone().oneshot(other_search_req).await.unwrap();
assert_eq!(other_search_res.status(), StatusCode::OK);
let other_json = response_json(other_search_res).await;
let other_results = other_json["results"].as_array().expect("results");
assert!(
!other_results.iter().any(|r| r["file"]["id"].as_i64() == Some(file_id)),
"other user should not see unauthorized file"
);
// Grant viewer permission and verify search now returns result
let grant_req = authed_json_request(
"POST",
format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id),
&owner,
serde_json::json!({ "user_id": other.id, "permission": "viewer" }),
);
let grant_res = app.clone().oneshot(grant_req).await.unwrap();
assert_eq!(grant_res.status(), StatusCode::OK);
// Verify other can now access the KB detail
let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &other);
let get_res = app.clone().oneshot(get_req).await.unwrap();
assert_eq!(get_res.status(), StatusCode::OK);
let get_json = response_json(get_res).await;
assert_eq!(get_json["current_user_permission"].as_str(), Some("viewer"));
let granted_search_req =
authed_empty_request("GET", "/api/v1/knowledge/search/full?filename=secret-perm.txt", &other);
let granted_search_res = app.clone().oneshot(granted_search_req).await.unwrap();
assert_eq!(granted_search_res.status(), StatusCode::OK);
let granted_json = response_json(granted_search_res).await;
let granted_results = granted_json["results"].as_array().expect("results");
assert!(
granted_results.iter().any(|r| r["file"]["id"].as_i64() == Some(file_id)),
"viewer should now find the file"
);
}

293
tests/common/mod.rs Normal file
View File

@ -0,0 +1,293 @@
#![allow(dead_code, unused_imports)]
use std::{
fs, path::PathBuf, sync::{
OnceLock, atomic::{AtomicUsize, Ordering}
}, time::{SystemTime, UNIX_EPOCH}
};
use axum::{Router, extract::DefaultBodyLimit, middleware};
pub use axum::{
body::Body, http::{Request, Response, StatusCode, header}
};
use htknow::{api, auth, db, search::SearchEngine, slice_content};
pub use http_body_util::BodyExt;
pub use serde_json::Value;
use sqlx::SqlitePool;
use tokio::sync::OnceCell;
pub use tower::ServiceExt;
pub struct TestEnv {
pub data_dir: PathBuf,
}
static TEST_ENV: OnceLock<TestEnv> = OnceLock::new();
static APP: OnceCell<Router> = OnceCell::const_new();
static TEST_SEQ: AtomicUsize = AtomicUsize::new(1);
pub fn next_seq() -> usize {
TEST_SEQ.fetch_add(1, Ordering::Relaxed)
}
pub fn setup_env() -> &'static TestEnv {
TEST_ENV.get_or_init(|| {
let nonce = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
let base_dir = std::env::temp_dir().join(format!("htknow-it-{}", nonce));
std::fs::create_dir_all(&base_dir).expect("create test data dir");
let db_path = base_dir.join("test.sqlite");
unsafe {
std::env::set_var("HTKNOW_DATA_DIR", base_dir.to_string_lossy().as_ref());
std::env::set_var("DATABASE_URL", format!("sqlite://{}", db_path.display()));
std::env::set_var("HTKNOW_DB_MAX_CONNECTIONS", "10");
std::env::set_var("HTKNOW_DB_INIT_DEFAULT_KBS", "false");
std::env::set_var("HTKNOW_SERVER_UPLOAD_LIMIT_MB", "1");
std::env::set_var("HTKNOW_SEARCH_LIMIT", "5");
}
TestEnv { data_dir: base_dir }
})
}
pub async fn build_app() -> Router {
setup_env();
let pool = db::init().await.expect("init db");
let search_engine = SearchEngine::init().await.with_pool(pool.clone());
let upload_limit = htknow::config::get().server.upload_limit_mb * 1024 * 1024;
Router::new()
.nest("/api/v1/knowledge/", api::app(pool, search_engine))
.layer(middleware::from_fn(auth))
.layer(DefaultBodyLimit::max(upload_limit))
}
pub async fn app() -> Router {
APP.get_or_init(|| async { build_app().await }).await.clone()
}
pub async fn get_pool() -> SqlitePool {
let _ = app().await;
db::init().await.expect("init db")
}
pub struct TestUser {
pub id: String,
pub name: String,
pub role: String,
}
impl TestUser {
pub fn new(prefix: &str) -> Self {
Self::with_role(prefix, "admin")
}
pub fn with_role(prefix: &str, role: &str) -> Self {
let seq = next_seq();
Self { id: format!("{}-{}", prefix, seq), name: format!("{}-name-{}", prefix, seq), role: role.to_string() }
}
}
pub fn authed_request(
method: &str, uri: String, user: &TestUser, body: Body, content_type: Option<String>,
) -> Request<Body> {
let mut builder = Request::builder().method(method).uri(uri);
builder = builder.header("x-user-id", &user.id).header("x-user-name", &user.name).header("x-role", &user.role);
if let Some(content_type) = content_type {
builder = builder.header(header::CONTENT_TYPE, content_type);
}
builder.body(body).unwrap()
}
pub fn authed_empty_request(method: &str, uri: impl Into<String>, user: &TestUser) -> Request<Body> {
authed_request(method, uri.into(), user, Body::empty(), None)
}
pub fn authed_json_request(method: &str, uri: impl Into<String>, user: &TestUser, body: Value) -> Request<Body> {
authed_request(method, uri.into(), user, Body::from(body.to_string()), Some("application/json".to_string()))
}
pub fn authed_multipart_request(
method: &str, uri: impl Into<String>, user: &TestUser, boundary: &str, body: Vec<u8>,
) -> Request<Body> {
let content_type = format!("multipart/form-data; boundary={}", boundary);
authed_request(method, uri.into(), user, Body::from(body), Some(content_type))
}
pub async fn response_json(res: Response<Body>) -> Value {
let bytes = res.into_body().collect().await.unwrap().to_bytes();
if bytes.is_empty() {
panic!("response body is empty");
}
serde_json::from_slice(&bytes).unwrap_or_else(|e| {
panic!("failed to parse JSON response: {}\nbody: {}", e, String::from_utf8_lossy(&bytes));
})
}
pub async fn insert_kb(
pool: &SqlitePool, user: &TestUser, name: &str, kb_type: &str, parent_id: Option<i64>, is_public: bool,
) -> i64 {
let is_public = if is_public { 1 } else { 0 };
sqlx::query(
"INSERT INTO knowledge_bases (user_id, user_name, name, description, kb_type, parent_id, is_public) \
VALUES (?, ?, ?, ?, ?, ?, ?)",
)
.bind(&user.id)
.bind(&user.name)
.bind(name)
.bind("test knowledge base")
.bind(kb_type)
.bind(parent_id)
.bind(is_public)
.execute(pool)
.await
.unwrap()
.last_insert_rowid()
}
pub async fn insert_file(
pool: &SqlitePool, user: &TestUser, filename: &str, path: &PathBuf, kb_id: Option<i64>, tags: Vec<String>,
is_public: bool,
) -> i64 {
let is_public = if is_public { 1 } else { 0 };
let tags_json = serde_json::to_string(&tags).unwrap();
let hash = format!("hash-{}", next_seq());
let size = fs::metadata(path).map(|meta| meta.len() as i64).unwrap_or(0);
sqlx::query(
"INSERT INTO files (user_id, user_name, hash, filename, path, size, slice_type, kb_id, is_public, tags, status, log) \
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
)
.bind(&user.id)
.bind(&user.name)
.bind(hash)
.bind(filename)
.bind(path.to_string_lossy().as_ref())
.bind(size)
.bind("text")
.bind(kb_id)
.bind(is_public)
.bind(tags_json)
.bind(0)
.bind("")
.execute(pool)
.await
.unwrap()
.last_insert_rowid()
}
pub async fn insert_slice(pool: &SqlitePool, file_id: i64, content: &str) -> i64 {
let id = sqlx::query("INSERT INTO slices (file_id) VALUES (?)")
.bind(file_id)
.execute(pool)
.await
.unwrap()
.last_insert_rowid();
slice_content::upsert_many(file_id, &[(id, content.to_string())]).await.unwrap();
id
}
pub async fn insert_slice_position(pool: &SqlitePool, slice_id: i64, page_idx: i32, bbox: [i32; 4]) {
insert_slice_position_full(pool, slice_id, page_idx, bbox, None, None).await;
}
pub async fn insert_slice_position_full(
pool: &SqlitePool, slice_id: i64, page_idx: i32, bbox: [i32; 4], sheet_name: Option<&str>, row_num: Option<i32>,
) {
sqlx::query(
"INSERT INTO slice_positions (slice_id, page_idx, x1, y1, x2, y2, sheet_name, row_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
)
.bind(slice_id)
.bind(page_idx)
.bind(bbox[0])
.bind(bbox[1])
.bind(bbox[2])
.bind(bbox[3])
.bind(sheet_name)
.bind(row_num)
.execute(pool)
.await
.unwrap();
}
pub async fn insert_graph_node(
pool: &SqlitePool, name: &str, entity_type: &str, properties: Option<Value>, file_id: Option<i64>,
kb_id: Option<i64>,
) -> i64 {
let props = properties.map(|value| value.to_string());
sqlx::query(
"INSERT INTO graph_nodes (name, entity_type, properties, file_id, kb_id, is_public) \
VALUES (?, ?, ?, ?, ?, 0)",
)
.bind(name)
.bind(entity_type)
.bind(props)
.bind(file_id)
.bind(kb_id)
.execute(pool)
.await
.unwrap()
.last_insert_rowid()
}
pub async fn insert_graph_edge(
pool: &SqlitePool, source_node_id: i64, target_node_id: i64, relation_type: &str, file_id: Option<i64>,
) -> i64 {
sqlx::query("INSERT INTO graph_edges (source_node_id, target_node_id, relation_type, file_id) VALUES (?, ?, ?, ?)")
.bind(source_node_id)
.bind(target_node_id)
.bind(relation_type)
.bind(file_id)
.execute(pool)
.await
.unwrap()
.last_insert_rowid()
}
pub async fn insert_entity_mention(pool: &SqlitePool, node_id: i64, slice_id: i64, context: &str) -> i64 {
sqlx::query("INSERT INTO entity_mentions (node_id, slice_id, context) VALUES (?, ?, ?)")
.bind(node_id)
.bind(slice_id)
.bind(context)
.execute(pool)
.await
.unwrap()
.last_insert_rowid()
}
pub fn multipart_body(boundary: &str, fields: &[(&str, &str)]) -> Vec<u8> {
let mut body = Vec::new();
for (name, value) in fields {
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
body.extend_from_slice(format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name).as_bytes());
body.extend_from_slice(value.as_bytes());
body.extend_from_slice(b"\r\n");
}
body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
body
}
#[allow(clippy::too_many_arguments)]
// Helper: build multipart request that includes an actual file field
pub fn authed_multipart_request_with_file(
method: &str, uri: impl Into<String>, user: &TestUser, boundary: &str, extra_fields: &[(&str, &str)],
field_name: &str, file_name: &str, file_content: &[u8],
) -> Request<Body> {
let mut body = Vec::new();
for (name, value) in extra_fields {
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
body.extend_from_slice(format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name).as_bytes());
body.extend_from_slice(value.as_bytes());
body.extend_from_slice(b"\r\n");
}
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
body.extend_from_slice(
format!("Content-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\n", field_name, file_name).as_bytes(),
);
body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
body.extend_from_slice(file_content);
body.extend_from_slice(b"\r\n");
body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
let content_type = format!("multipart/form-data; boundary={}", boundary);
authed_request(method, uri.into(), user, Body::from(body), Some(content_type))
}

3
todo.md Normal file
View File

@ -0,0 +1,3 @@
文档：122-06A0014-B01001_发动机-维修手册.pdf
搜索：保养主推进柴油机的步骤
搜索方式：高级搜索