เตรียมความพร้อมสำหรับการ Deploy บน HF Spaces: เพิ่ม ONNX Reranker, แก้ไข Static Path และปรับปรุง Asset Downloader

webapp/tipitaka-api/app/main.py

@@ -83,7 +83,7 @@ async def health():
 # ── Serve built frontend in production (Docker/HF Space) ──
 if settings.SERVE_STATIC:
     static_dir = settings.STATIC_DIR or os.path.join(
-        os.path.dirname(__file__), "..", "..", "web", "dist"
+        os.path.dirname(__file__), "..", "..", "tipitaka-web", "dist"
     )
     if os.path.isdir(static_dir):
         from fastapi.responses import FileResponse, HTMLResponse

webapp/tipitaka-api/download_assets.py

@@ -19,84 +19,100 @@ settings = get_settings()
 DB_PATH = Path(settings.DATABASE_PATH)
 QDRANT_DIR = Path(settings.QDRANT_PATH)
 SNAPSHOT_DIR = Path(settings.SNAPSHOT_DIR)
+MODELS_DIR = Path(__file__).resolve().parent / "models"
 
 # User's local snapshot path fallback (for verification in verify_assets)
 USER_LOCAL_SNAPSHOT_DIR = Path(r"F:\_Ai\Tipitaka-AI-Expert\Tipitaka-Data\data\snapshots")
 
 BUCKET_ID = "dhammawatthumpra/tipitaka-storage"
+RERANK_REPO_ID = "jinaai/jina-reranker-v2-base-multilingual"
 
 # Files to check/download
 BUCKET_FILES = [
     ("tipitaka_mcu.db", str(DB_PATH)),
-    ("qdrant/tipitaka_chunks.snapshot", "tipitaka_chunks.snapshot"),
-    ("qdrant/tipitaka_scripture.snapshot", "tipitaka_scripture.snapshot"),
+    ("qdrant/tipitaka_chunks.snapshot", str(SNAPSHOT_DIR / "tipitaka_chunks.snapshot")),
+    ("qdrant/tipitaka_scripture.snapshot", str(SNAPSHOT_DIR / "tipitaka_scripture.snapshot")),
 ]
 
+# Essential Reranker files (ONNX)
+RERANK_FILES = [
+    "config.json",
+    "tokenizer.json",
+    "tokenizer_config.json",
+    "onnx/model_int8.onnx",
+]
 
-def check_file_exists(local_path_name: str) -> bool:
-    """Check if a file exists either in project SNAPSHOT_DIR or USER_LOCAL_SNAPSHOT_DIR."""
-    # If it's the DB, check DB_PATH
-    if local_path_name == str(DB_PATH):
-        return DB_PATH.exists()
-    
-    # If it's a snapshot, check both locations
-    in_project = SNAPSHOT_DIR / local_path_name
-    in_user_local = USER_LOCAL_SNAPSHOT_DIR / local_path_name
+
+def check_file_exists(path: Path) -> bool:
+    """Check if a file exists locally or in user fallback."""
+    if path.exists():
+        return True
     
-    return in_project.exists() or in_user_local.exists()
+    # Check fallback for snapshots
+    if "snapshot" in path.name:
+        fallback = USER_LOCAL_SNAPSHOT_DIR / path.name
+        if fallback.exists():
+            return True
+            
+    return False
 
 
 def download_files() -> None:
     """Download missing files from HF bucket."""
     hf_token = os.getenv("HF_TOKEN", "")
+    from huggingface_hub import hf_hub_download
     
-    # Filter to files that don't exist yet in any location
-    pending = []
-    for remote, local_name in BUCKET_FILES:
-        if not check_file_exists(local_name if "snapshot" in local_name else str(DB_PATH)):
-            # Determine destination: if it's the DB, use DB_PATH, else use project SNAPSHOT_DIR
-            dest = DB_PATH if "db" in local_name else (SNAPSHOT_DIR / local_name)
-            pending.append((remote, str(dest)))
-
-    if not pending:
-        print("Success: All assets already exist in project or local storage - skipping download.")
-        return
-
-    if not hf_token:
-        print("WARNING: HF_TOKEN not set — skipping asset download.")
-        return
-
-    print(f"Downloading {len(pending)} files from bucket '{BUCKET_ID}'...")
-
-    try:
-        from huggingface_hub import hf_hub_download
-        
-        for remote, dest_str in pending:
-            dest_path = Path(dest_str)
-            print(f"Downloading {remote} -> {dest_path}")
-            
-            dest_path.parent.mkdir(parents=True, exist_ok=True)
-            
-            # Note: hf_hub_download with local_dir will create subfolders if remote has slashes
-            # So if remote is "qdrant/foo.snapshot" and local_dir is DATA_DIR, 
-            # it becomes DATA_DIR/qdrant/foo.snapshot.
-            # But we want it in DATA_DIR/snapshots/foo.snapshot.
-            # So we use local_dir_use_symlinks=False and manually move if needed, 
-            # or just download to a temp and move.
+    # 1. Download Core Assets (DB & Snapshots)
+    pending_core = []
+    for remote, local_path_str in BUCKET_FILES:
+        local_path = Path(local_path_str)
+        if not check_file_exists(local_path):
+            pending_core.append((remote, local_path))
+
+    if pending_core:
+        if not hf_token:
+            print("WARNING: HF_TOKEN not set — cannot download core assets.")
+        else:
+            print(f"Downloading {len(pending_core)} core assets from '{BUCKET_ID}'...")
+            for remote, dest_path in pending_core:
+                try:
+                    print(f"Downloading {remote} -> {dest_path}")
+                    dest_path.parent.mkdir(parents=True, exist_ok=True)
+                    downloaded = hf_hub_download(
+                        repo_id=BUCKET_ID,
+                        filename=remote,
+                        repo_type="dataset",
+                        token=hf_token
+                    )
+                    import shutil
+                    shutil.copy(downloaded, dest_path)
+                except Exception as e:
+                    print(f"FAILED to download {remote}: {e}")
+
+    # 2. Download Reranker Assets (if missing)
+    rerank_dir = MODELS_DIR / "jina-v2-onnx"
+    pending_rerank = []
+    for filename in RERANK_FILES:
+        local_path = rerank_dir / filename
+        if not local_path.exists():
+            pending_rerank.append(filename)
             
-            downloaded_path = hf_hub_download(
-                repo_id=BUCKET_ID,
-                filename=remote,
-                repo_type="dataset",
-                token=hf_token
-            )
-            
-            import shutil
-            shutil.copy(downloaded_path, dest_path)
-            
-        print("All files processed successfully.")
-    except Exception as e:
-        print(f"ERROR downloading files: {e}")
+    if pending_rerank:
+        print(f"Downloading {len(pending_rerank)} reranker assets from '{RERANK_REPO_ID}'...")
+        for filename in pending_rerank:
+            try:
+                dest_path = rerank_dir / filename
+                print(f"Downloading {filename} -> {dest_path}")
+                dest_path.parent.mkdir(parents=True, exist_ok=True)
+                downloaded = hf_hub_download(
+                    repo_id=RERANK_REPO_ID,
+                    filename=filename,
+                    token=hf_token if hf_token else None
+                )
+                import shutil
+                shutil.copy(downloaded, dest_path)
+            except Exception as e:
+                print(f"FAILED to download reranker file {filename}: {e}")
 
 
 def verify_assets() -> bool:
@@ -108,27 +124,27 @@ def verify_assets() -> bool:
     chunks_exists = (SNAPSHOT_DIR / "tipitaka_chunks.snapshot").exists() or \
                     (USER_LOCAL_SNAPSHOT_DIR / "tipitaka_chunks.snapshot").exists()
     
-    scripture_exists = (SNAPSHOT_DIR / "tipitaka_scripture.snapshot").exists() or \
-                       (USER_LOCAL_SNAPSHOT_DIR / "tipitaka_scripture.snapshot").exists()
-
     if not chunks_exists:
         missing.append("tipitaka_chunks.snapshot")
-    if not scripture_exists:
-        missing.append("tipitaka_scripture.snapshot")
+
+    rerank_model = MODELS_DIR / "jina-v2-onnx" / "onnx" / "model_int8.onnx"
+    if not rerank_model.exists():
+        missing.append("reranker_model (model_int8.onnx)")
 
     if missing:
-        print(f"WARNING: Missing critical assets in any location: {missing}")
+        print(f"WARNING: Missing assets: {missing}")
         return False
     
-    print("Success: All critical assets verified (either in project or local storage).")
+    print("Success: All critical assets verified.")
     return True
 
 
 if __name__ == "__main__":
-    # Ensure directories exist
     QDRANT_DIR.mkdir(parents=True, exist_ok=True)
     SNAPSHOT_DIR.mkdir(parents=True, exist_ok=True)
+    MODELS_DIR.mkdir(parents=True, exist_ok=True)
     
     download_files()
     verify_assets()
     print("Asset management complete.")
+

webapp/tipitaka-api/requirements.txt

@@ -9,3 +9,8 @@ sentence-transformers>=3.0
 huggingface_hub>=0.20
 torch>=2.0
 pythainlp>=5.0
+onnxruntime>=1.17.0
+transformers>=4.40.0
+numpy>=1.24.0
+httpx>=0.27.0
+anyio>=4.0.0