fix image parsing and add docker build script

Dockerfile.rust-builder

@@ -0,0 +1,10 @@
+# Pinned to Debian Bookworm so the compiled binary is compatible with the
+# Debian Bookworm runtime image used by Dockerfile (GLIBC 2.36).
+FROM rust:1-bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# lance-* generates protobuf bindings during compilation.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends protobuf-compiler libprotobuf-dev \
+    && rm -rf /var/lib/apt/lists/*

build-docker.sh

@@ -1,62 +1,114 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# Build HTKnow on a server without installing Node.js or Rust on the host.
+#
+# Examples:
+#   ./build-docker.sh
+#   ./build-docker.sh --tag gwknow:v0.0.2
+#   ./build-docker.sh --skip-frontend --tag gwknow:v0.0.3
+#   IMAGE_TAG=v0.0.3 ./build-docker.sh
 
-set -e
+set -Eeuo pipefail
 
-# Parse command line arguments
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-BUILD_MODE="release"
+cd "$ROOT_DIR"
-PROFILING_FLAG=""
+
-CUSTOM_IMAGE_TAG="${IMAGE_TAG:-}"
+NODE_IMAGE="${NODE_IMAGE:-docker.1ms.run/node:22}"
-if [ "${1:-}" = "debug" ] || [ "${1:-}" = "--debug" ]; then
+RUST_BUILDER_IMAGE="${RUST_BUILDER_IMAGE:-htknow-rust-builder:bookworm}"
-    BUILD_MODE="debug"
+SKIP_FRONTEND=false
-    PROFILING_FLAG="--features profiling"
+REBUILD_BUILDER=false
+IMAGE_TAG="${IMAGE_TAG:-}"
+
+usage() {
+    cat <<'EOF'
+Usage: ./build-docker.sh [options]
+
+Options:
+  --tag IMAGE          Target image tag, for example gwknow:v0.0.2.
+  --skip-frontend      Do not rebuild frontend/dist (safe for Rust-only changes).
+  --rebuild-builder    Rebuild the cached Rust + protoc builder image.
+  -h, --help           Show this help.
+
+Environment overrides:
+  IMAGE_TAG, NODE_IMAGE, RUST_BUILDER_IMAGE
+
+If --tag and IMAGE_TAG are omitted, a new gwknow:YYYYmmdd-HHMMSS tag is used.
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --tag)
+            [[ $# -ge 2 ]] || { echo "--tag requires an image tag" >&2; exit 2; }
+            IMAGE_TAG="$2"
+            shift 2
+            ;;
+        --skip-frontend|--backend-only)
+            SKIP_FRONTEND=true
+            shift
+            ;;
+        --rebuild-builder)
+            REBUILD_BUILDER=true
+            shift
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1" >&2
+            usage >&2
+            exit 2
+            ;;
+    esac
+done
+
+command -v docker >/dev/null || { echo "docker is not installed or is not in PATH" >&2; exit 1; }
+docker info >/dev/null 2>&1 || { echo "Docker daemon is unavailable" >&2; exit 1; }
+
+if [[ -z "$IMAGE_TAG" ]]; then
+    IMAGE_TAG="gwknow:$(date +%Y%m%d-%H%M%S)"
+elif [[ "$IMAGE_TAG" != *:* ]]; then
+    IMAGE_TAG="gwknow:${IMAGE_TAG}"
 fi
 
-# Build frontend
+HOST_UID="$(id -u)"
-cd frontend
+HOST_GID="$(id -g)"
-npm install
+mkdir -p .cargo-cache .npm-cache
-npm run build
-cd ..
 
-if [ "$BUILD_MODE" = "debug" ]; then
+if [[ "$REBUILD_BUILDER" == true ]] || ! docker image inspect "$RUST_BUILDER_IMAGE" >/dev/null 2>&1; then
-    echo "Building htknow for debug with profiling..."
+    echo "==> Building Rust builder image: $RUST_BUILDER_IMAGE"
-    cargo build ${PROFILING_FLAG}
+    docker build -t "$RUST_BUILDER_IMAGE" -f Dockerfile.rust-builder .
-    if [ -n "$CUSTOM_IMAGE_TAG" ]; then
+fi
-        IMAGE_TAG="${CUSTOM_IMAGE_TAG}-debug"
+
+if [[ "$SKIP_FRONTEND" == false ]]; then
+    echo "==> Building frontend with $NODE_IMAGE"
+    docker run --rm \
+        --user "${HOST_UID}:${HOST_GID}" \
+        -e HOME=/tmp \
+        -e npm_config_cache=/npm-cache \
+        -v "$ROOT_DIR":/app \
+        -v "$ROOT_DIR/.npm-cache":/npm-cache \
+        -w /app/frontend \
+        "$NODE_IMAGE" \
+        sh -c 'npm ci && npm run build'
 else
-        IMAGE_TAG="$(date +%Y%m%d%H%M)-debug"
+    echo "==> Skipping frontend build (--skip-frontend)"
-    fi
-    DOCKERFILE="Dockerfile.debug"
-    echo "Using Dockerfile.debug with jemalloc profiling tools"
-else
-    echo "Building htknow for release (profiling disabled)..."
-    cargo build --release
-    if [ -n "$CUSTOM_IMAGE_TAG" ]; then
-        IMAGE_TAG="${CUSTOM_IMAGE_TAG}"
-    else
-        IMAGE_TAG="$(date +%Y%m%d%H%M)"
-    fi
-    DOCKERFILE="Dockerfile"
 fi
 
-# Build Docker image
+echo "==> Building Rust release binary with $RUST_BUILDER_IMAGE"
-echo "Building Docker image with ${DOCKERFILE}..."
+docker run --rm \
-docker build -f "${DOCKERFILE}" -t "htknow:${IMAGE_TAG}" -t htknow:latest .
+    --user "${HOST_UID}:${HOST_GID}" \
+    -e HOME=/tmp \
+    -e CARGO_HOME=/cargo-cache \
+    -v "$ROOT_DIR":/app \
+    -v "$ROOT_DIR/.cargo-cache":/cargo-cache \
+    -w /app \
+    "$RUST_BUILDER_IMAGE" \
+    bash -c 'cargo build --locked --release'
 
-echo "Build complete!"
+echo "==> Building runtime image: $IMAGE_TAG"
-echo ""
+docker build -f Dockerfile -t "$IMAGE_TAG" .
-echo "Image tag: htknow:${IMAGE_TAG}"
+
-echo "Latest tag: htknow:latest"
+echo
-echo "BUILT_IMAGE=htknow:${IMAGE_TAG}"
+echo "Build complete: $IMAGE_TAG"
-echo "BUILT_LATEST_IMAGE=htknow:latest"
+echo "To deploy, update the compose image tag and recreate only the intended service."
-echo "Build mode: ${BUILD_MODE}"
-echo ""
-echo "To run the container:"
-echo "  docker-compose up -d"
-echo ""
-echo "Or manually:"
-echo "  docker run -d -p 8080:8080 -v \$(pwd)/data:/app/data --name htknow htknow:${IMAGE_TAG}"
-echo ""
-echo "Usage:"
-echo "  ./build-docker.sh          # Build in release mode (default)"
-echo "  ./build-docker.sh debug    # Build in debug mode"
-echo "  IMAGE_TAG=v1.2.3 ./build-docker.sh  # Use custom image tag"

src/processor.rs

@@ -196,7 +196,12 @@ fn enrich_content_list_with_image_descriptions(content_list: &mut [ContentItem],
     for item in content_list.iter_mut() {
         if item.typ == "image"
             && let Some(img_name) = item.img_path.as_deref()
-            && let Some(description) = desc_map.get(img_name).filter(|s| !s.trim().is_empty())
+            // MinerU 的 content_list 常用 `images/foo.jpg`，而 images 映射和缓存表
+            // 使用 `foo.jpg`。两种写法都要能找到同一条图片描述。
+            && let Some(description) = desc_map
+                .get(img_name)
+                .or_else(|| img_name.strip_prefix("images/").and_then(|name| desc_map.get(name)))
+                .filter(|s| !s.trim().is_empty())
         {
             let caption = description.clone();
             let captions = item.image_caption.get_or_insert_with(Vec::new);
@@ -207,6 +212,29 @@ fn enrich_content_list_with_image_descriptions(content_list: &mut [ContentItem],
     }
 }
 
+/// 单个上传图片经 MinerU 解析时，只保留第一个图片块。
+///
+/// MinerU 有时会为同一张栅格图返回页面渲染图和图片块两个 JPG；两者写入同一
+/// 切片只会产生重复的图片引用。该规则仅用于上传图片（而不是 PDF），避免影响
+/// PDF 内多张真实图片的解析结果。
+fn retain_one_image_block_for_uploaded_image(content_list: &mut Vec<ContentItem>) -> usize {
+    let mut found_image = false;
+    let mut removed = 0;
+    content_list.retain(|item| {
+        if item.typ != "image" {
+            return true;
+        }
+        if found_image {
+            removed += 1;
+            false
+        } else {
+            found_image = true;
+            true
+        }
+    });
+    removed
+}
+
 async fn claim_pending_file_by_id(pool: &SqlitePool, file_id: i64) -> anyhow::Result<Option<File>> {
     let sql = format!(
         "UPDATE files
@@ -1933,7 +1961,10 @@ impl FileProcessor {
 
     /// MinerU 不支持的图片格式需要先转为 PNG；仅在转换失败时使用轻量级入库兜底。
     fn requires_mineru_image_conversion(filename_lower: &str) -> bool {
-        filename_lower.ends_with(".svg") || filename_lower.ends_with(".ico") || filename_lower.ends_with(".avif")
+        filename_lower.ends_with(".svg")
+            || filename_lower.ends_with(".ico")
+            || filename_lower.ends_with(".avif")
+            || filename_lower.ends_with(".bmp")
     }
 
     /// 将 MinerU 不支持的图片临时规范化为 PNG。
@@ -1964,7 +1995,14 @@ impl FileProcessor {
             )
             .await
         } else {
-            let input = format!("{}[0]", file.path);
+            // 上传文件落盘时使用 UUID 作为路径，没有原始扩展名。ICO 的魔数识别在
+            // Debian ImageMagick 6 中不可靠，因此必须显式指定解码格式；否则会报
+            // `no decode delegate for this image format ''` 并走保底路径。
+            let input = if filename_lower.ends_with(".ico") {
+                format!("ICO:{}[0]", file.path)
+            } else {
+                format!("{}[0]", file.path)
+            };
             let args = vec![
                 input,
                 "-background".to_string(),
@@ -2124,6 +2162,12 @@ impl FileProcessor {
                 Ok(content_list)
             })
             .await?;
+            if is_image {
+                let removed = retain_one_image_block_for_uploaded_image(&mut content_list);
+                if removed > 0 {
+                    warn!("Keeping one MinerU image block for uploaded image {}, dropped {} duplicate blocks", file.filename, removed);
+                }
+            }
         } else {
             // 没有数据，调用 MinerU API
             let mineru_result = self.call_mineru_api(file, is_image, timing.as_deref_mut()).await?;
@@ -2132,6 +2176,12 @@ impl FileProcessor {
                 Ok(serde_json::from_str(&mineru_result.content_list)?)
             })
             .await?;
+            if is_image {
+                let removed = retain_one_image_block_for_uploaded_image(&mut content_list);
+                if removed > 0 {
+                    warn!("Keeping one MinerU image block for uploaded image {}, dropped {} duplicate blocks", file.filename, removed);
+                }
+            }
             if !self.ensure_file_exists(file.id, "before writing pdf contents").await? {
                 return Ok(());
             }
@@ -2233,8 +2283,9 @@ impl FileProcessor {
                     Err(err) => warn!("Failed to parse upload image {}: {}", file.filename, err),
                 }
             }
-            if !content_list.iter().any(|item| item.typ == "image" && item.img_path.as_deref() == Some(&file.filename))
+            // MinerU 已经提取到图片时，不能再把上传原图追加一次；否则一个 BMP 会同时
-            {
+            // 产生 MinerU 的派生 JPG 引用和原始 BMP 引用，造成重复切片。
+            if !content_list.iter().any(|item| item.typ == "image") {
                 let caption = desc_map.get(&file.filename).cloned().unwrap_or_default();
                 content_list.push(ContentItem {
                     typ: "image".to_string(),
@@ -4143,7 +4194,7 @@ mod tests {
 
     #[test]
     fn unsupported_mineru_image_formats_require_conversion() {
-        for filename in ["icon.ico", "diagram.svg", "photo.avif"] {
+        for filename in ["icon.ico", "diagram.svg", "photo.avif", "photo.bmp"] {
             assert!(crate::search::is_image_file(filename));
             assert!(FileProcessor::requires_mineru_image_conversion(filename));
         }
@@ -4162,4 +4213,40 @@ mod tests {
         assert_eq!(captions.len(), 1);
         assert_eq!(captions[0], "animated image");
     }
+
+    #[test]
+    fn image_description_matches_mineru_images_prefix() {
+        let mut items = vec![image_content_item("images/mineru-output.jpg")];
+        let descriptions = HashMap::from([("mineru-output.jpg".to_string(), "MinerU description".to_string())]);
+
+        enrich_content_list_with_image_descriptions(&mut items, &descriptions);
+
+        let captions = items[0].image_caption.as_ref().expect("image caption should be present");
+        assert_eq!(captions, &vec!["MinerU description".to_string()]);
+    }
+
+    #[test]
+    fn uploaded_image_keeps_only_one_mineru_image_block() {
+        let mut items = vec![
+            image_content_item("images/page-render.jpg"),
+            ContentItem {
+                typ: "text".to_string(),
+                bbox: vec![],
+                page_idx: 0,
+                text: Some("OCR text".to_string()),
+                text_level: None,
+                text_format: None,
+                img_path: None,
+                image_caption: None,
+                table_body: None,
+                table_caption: None,
+            },
+            image_content_item("images/image-block.jpg"),
+        ];
+
+        assert_eq!(retain_one_image_block_for_uploaded_image(&mut items), 1);
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[0].img_path.as_deref(), Some("images/page-render.jpg"));
+        assert_eq!(items[1].text.as_deref(), Some("OCR text"));
+    }
 }