Download api/routers/classification.py from Basel0/qshub: direct link, hf CLI and curl.
- Browser
- Download file 6.68 kB
-
https://huggingface.co/spaces/Basel0/qshub/resolve/main/api/routers/classification.py
- Command line
-
hf download hf://spaces/Basel0/qshub/api/routers/classification.py
-
curl -L -o classification.py https://huggingface.co/spaces/Basel0/qshub/resolve/main/api/routers/classification.py
6.68 kB
| from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks | |
| from pydantic import BaseModel | |
| from typing import List, Dict, Any | |
| import os | |
| import json | |
| import numpy as np | |
| from PIL import Image | |
| from api.auth import get_current_user | |
| from utils.db import safe_query, safe_execute | |
| from pdf_engine.smart_classifier import PAGE_ITEMS_MAP | |
| from workflow.step2_classify import _classify_single, _ai_classify | |
| router = APIRouter() | |
| CACHE_ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "_qto_cache") | |
| os.makedirs(CACHE_ROOT, exist_ok=True) | |
| class RunExtractionReq(BaseModel): | |
| project_id: int | |
| class SaveClassificationReq(BaseModel): | |
| project_id: int | |
| classified_pages: List[Dict[str, Any]] | |
| async def auto_classify(req: RunExtractionReq, current_user: dict = Depends(get_current_user)): | |
| df_state = safe_query("SELECT state_data FROM qto_active_projects WHERE user_id=%s AND project_id=%s", (current_user["id"], req.project_id)) | |
| if df_state.empty or not df_state.iloc[0]["state_data"]: | |
| raise HTTPException(status_code=400, detail="No active project uploaded.") | |
| state_data = json.loads(df_state.iloc[0]["state_data"]) | |
| project_cache = os.path.join(CACHE_ROOT, str(req.project_id)) | |
| # Re-extract FULL text from cached PDFs on disk for accurate keyword classification. | |
| # The DB only stores truncated 250-char previews to stay under packet limits, | |
| # but keyword matching needs the full page text. Reading from disk via mmap is | |
| # instant and uses virtually no RAM. | |
| from pdf_engine.pdf_loader import extract_page_text | |
| str_boundaries = state_data.get("str_boundaries") or [] | |
| str_texts = [] | |
| for b in str_boundaries: | |
| pdf_p = os.path.join(project_cache, b.get("pdf_path", "")) | |
| exists = os.path.exists(pdf_p) | |
| print(f"[classify/auto] STR PDF path={pdf_p} exists={exists}") | |
| if exists: | |
| extracted = extract_page_text(pdf_p) | |
| print(f"[classify/auto] STR extracted {len(extracted)} pages, first 200 chars: {extracted[0][:200] if extracted else '(none)'}") | |
| str_texts.extend(extracted) | |
| # Fallback to DB-stored (truncated) texts if PDFs are missing from disk | |
| if not str_texts: | |
| str_texts = state_data.get("str_texts") or [] | |
| arch_boundaries = state_data.get("arch_boundaries") or [] | |
| arch_texts = [] | |
| for b in arch_boundaries: | |
| pdf_p = os.path.join(project_cache, b.get("pdf_path", "")) | |
| exists = os.path.exists(pdf_p) | |
| print(f"[classify/auto] ARCH PDF path={pdf_p} exists={exists}") | |
| if exists: | |
| extracted = extract_page_text(pdf_p) | |
| print(f"[classify/auto] ARCH extracted {len(extracted)} pages, first 200 chars: {extracted[0][:200] if extracted else '(none)'}") | |
| arch_texts.extend(extracted) | |
| if not arch_texts: | |
| arch_texts = state_data.get("arch_texts") or [] | |
| classified = [] | |
| # Keyword classification using full page text | |
| for i, txt in enumerate(str_texts): | |
| classified.append({ | |
| "pdf": "structural", | |
| "page_index": i, | |
| "page_num": i + 1, | |
| "text_preview": txt[:100].replace("\n", " "), | |
| "image_url": f"/cache/{req.project_id}/str_page_{i}.jpg", | |
| **_classify_single(txt, "structural"), | |
| }) | |
| for i, txt in enumerate(arch_texts): | |
| classified.append({ | |
| "pdf": "architectural", | |
| "page_index": i, | |
| "page_num": i + 1, | |
| "text_preview": txt[:100].replace("\n", " "), | |
| "image_url": f"/cache/{req.project_id}/arch_page_{i}.jpg", | |
| **_classify_single(txt, "architectural"), | |
| }) | |
| state_data["classified_pages"] = classified | |
| state_data["current_step"] = 2 | |
| from utils.db import upsert_active_state | |
| upsert_active_state(current_user["id"], req.project_id, 2, state_data) | |
| return { | |
| "classified_pages": classified, | |
| "next_step": 3, | |
| "_debug": { | |
| "cache_dir": project_cache, | |
| "cache_exists": os.path.isdir(project_cache), | |
| "cache_files": os.listdir(project_cache) if os.path.isdir(project_cache) else [], | |
| "str_boundaries": str_boundaries, | |
| "arch_boundaries": arch_boundaries, | |
| "str_texts_count": len(str_texts), | |
| "arch_texts_count": len(arch_texts), | |
| "str_text_sample": str_texts[0][:300] if str_texts else "(empty)", | |
| "arch_text_sample": arch_texts[0][:300] if arch_texts else "(empty)", | |
| "unknown_count": sum(1 for p in classified if p.get("detected_type") == "unknown"), | |
| "total_classified": len(classified), | |
| } | |
| } | |
| def _background_record_memory(user_id: int, pages: list): | |
| try: | |
| from engine.qto_memory import record_mapping | |
| for p in pages: | |
| dtype = (p.get("detected_type") or "").strip() | |
| if dtype and dtype != "unknown": | |
| record_mapping(user_id, dtype[:120], dtype) | |
| except Exception as e: | |
| print(f"qto_memory record error: {e}") | |
| async def save_classification(req: SaveClassificationReq, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user)): | |
| try: | |
| df_state = safe_query("SELECT state_data FROM qto_active_projects WHERE user_id=%s AND project_id=%s", (current_user["id"], req.project_id)) | |
| state_data = {} | |
| if not df_state.empty and df_state.iloc[0]["state_data"]: | |
| try: | |
| state_data = json.loads(df_state.iloc[0]["state_data"]) | |
| except Exception: | |
| state_data = {} | |
| state_data["classified_pages"] = req.classified_pages | |
| state_data["current_step"] = 3 | |
| from utils.db import upsert_active_state | |
| success, msg = upsert_active_state(current_user["id"], req.project_id, 3, state_data) | |
| if not success: | |
| print(f"[classify/save] DB ERROR: {msg}") | |
| raise HTTPException(status_code=500, detail=f"DB error: {msg}") | |
| # Learning loop pushed to background to prevent 502 Proxy Timeout | |
| background_tasks.add_task(_background_record_memory, current_user["id"], req.classified_pages or []) | |
| return {"message": "Classifications saved successfully."} | |
| except HTTPException: | |
| raise | |
| except Exception as e: | |
| print(f"[classify/save] UNHANDLED ERROR: {e}") | |
| import traceback; traceback.print_exc() | |
| raise HTTPException(status_code=500, detail=f"Server error: {str(e)}") | |