qshub / api /routers /classification.py
basel00omar's picture
Add ARCH debug logging to auto_classify
d0d7322
Raw History Blame Contribute Delete
6.68 kB
from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks
from pydantic import BaseModel
from typing import List, Dict, Any
import os
import json
import numpy as np
from PIL import Image
from api.auth import get_current_user
from utils.db import safe_query, safe_execute
from pdf_engine.smart_classifier import PAGE_ITEMS_MAP
from workflow.step2_classify import _classify_single, _ai_classify
router = APIRouter()
CACHE_ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "_qto_cache")
os.makedirs(CACHE_ROOT, exist_ok=True)
class RunExtractionReq(BaseModel):
project_id: int
class SaveClassificationReq(BaseModel):
project_id: int
classified_pages: List[Dict[str, Any]]
@router.post("/classify/auto")
async def auto_classify(req: RunExtractionReq, current_user: dict = Depends(get_current_user)):
df_state = safe_query("SELECT state_data FROM qto_active_projects WHERE user_id=%s AND project_id=%s", (current_user["id"], req.project_id))
if df_state.empty or not df_state.iloc[0]["state_data"]:
raise HTTPException(status_code=400, detail="No active project uploaded.")
state_data = json.loads(df_state.iloc[0]["state_data"])
project_cache = os.path.join(CACHE_ROOT, str(req.project_id))
# Re-extract FULL text from cached PDFs on disk for accurate keyword classification.
# The DB only stores truncated 250-char previews to stay under packet limits,
# but keyword matching needs the full page text. Reading from disk via mmap is
# instant and uses virtually no RAM.
from pdf_engine.pdf_loader import extract_page_text
str_boundaries = state_data.get("str_boundaries") or []
str_texts = []
for b in str_boundaries:
pdf_p = os.path.join(project_cache, b.get("pdf_path", ""))
exists = os.path.exists(pdf_p)
print(f"[classify/auto] STR PDF path={pdf_p} exists={exists}")
if exists:
extracted = extract_page_text(pdf_p)
print(f"[classify/auto] STR extracted {len(extracted)} pages, first 200 chars: {extracted[0][:200] if extracted else '(none)'}")
str_texts.extend(extracted)
# Fallback to DB-stored (truncated) texts if PDFs are missing from disk
if not str_texts:
str_texts = state_data.get("str_texts") or []
arch_boundaries = state_data.get("arch_boundaries") or []
arch_texts = []
for b in arch_boundaries:
pdf_p = os.path.join(project_cache, b.get("pdf_path", ""))
exists = os.path.exists(pdf_p)
print(f"[classify/auto] ARCH PDF path={pdf_p} exists={exists}")
if exists:
extracted = extract_page_text(pdf_p)
print(f"[classify/auto] ARCH extracted {len(extracted)} pages, first 200 chars: {extracted[0][:200] if extracted else '(none)'}")
arch_texts.extend(extracted)
if not arch_texts:
arch_texts = state_data.get("arch_texts") or []
classified = []
# Keyword classification using full page text
for i, txt in enumerate(str_texts):
classified.append({
"pdf": "structural",
"page_index": i,
"page_num": i + 1,
"text_preview": txt[:100].replace("\n", " "),
"image_url": f"/cache/{req.project_id}/str_page_{i}.jpg",
**_classify_single(txt, "structural"),
})
for i, txt in enumerate(arch_texts):
classified.append({
"pdf": "architectural",
"page_index": i,
"page_num": i + 1,
"text_preview": txt[:100].replace("\n", " "),
"image_url": f"/cache/{req.project_id}/arch_page_{i}.jpg",
**_classify_single(txt, "architectural"),
})
state_data["classified_pages"] = classified
state_data["current_step"] = 2
from utils.db import upsert_active_state
upsert_active_state(current_user["id"], req.project_id, 2, state_data)
return {
"classified_pages": classified,
"next_step": 3,
"_debug": {
"cache_dir": project_cache,
"cache_exists": os.path.isdir(project_cache),
"cache_files": os.listdir(project_cache) if os.path.isdir(project_cache) else [],
"str_boundaries": str_boundaries,
"arch_boundaries": arch_boundaries,
"str_texts_count": len(str_texts),
"arch_texts_count": len(arch_texts),
"str_text_sample": str_texts[0][:300] if str_texts else "(empty)",
"arch_text_sample": arch_texts[0][:300] if arch_texts else "(empty)",
"unknown_count": sum(1 for p in classified if p.get("detected_type") == "unknown"),
"total_classified": len(classified),
}
}
def _background_record_memory(user_id: int, pages: list):
try:
from engine.qto_memory import record_mapping
for p in pages:
dtype = (p.get("detected_type") or "").strip()
if dtype and dtype != "unknown":
record_mapping(user_id, dtype[:120], dtype)
except Exception as e:
print(f"qto_memory record error: {e}")
@router.post("/classify/save")
async def save_classification(req: SaveClassificationReq, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user)):
try:
df_state = safe_query("SELECT state_data FROM qto_active_projects WHERE user_id=%s AND project_id=%s", (current_user["id"], req.project_id))
state_data = {}
if not df_state.empty and df_state.iloc[0]["state_data"]:
try:
state_data = json.loads(df_state.iloc[0]["state_data"])
except Exception:
state_data = {}
state_data["classified_pages"] = req.classified_pages
state_data["current_step"] = 3
from utils.db import upsert_active_state
success, msg = upsert_active_state(current_user["id"], req.project_id, 3, state_data)
if not success:
print(f"[classify/save] DB ERROR: {msg}")
raise HTTPException(status_code=500, detail=f"DB error: {msg}")
# Learning loop pushed to background to prevent 502 Proxy Timeout
background_tasks.add_task(_background_record_memory, current_user["id"], req.classified_pages or [])
return {"message": "Classifications saved successfully."}
except HTTPException:
raise
except Exception as e:
print(f"[classify/save] UNHANDLED ERROR: {e}")
import traceback; traceback.print_exc()
raise HTTPException(status_code=500, detail=f"Server error: {str(e)}")