File size: 6,677 Bytes
16455b0
1b4b6d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0f3ab13
1b4b6d2
0102eee
 
 
 
 
 
0f3ab13
0102eee
 
 
d0d7322
 
 
 
 
 
0102eee
 
 
 
 
0f3ab13
0102eee
 
 
d0d7322
 
 
 
 
 
0102eee
 
 
1b4b6d2
 
 
0102eee
1b4b6d2
 
 
 
 
 
0102eee
1b4b6d2
 
 
 
 
 
 
 
 
0102eee
1b4b6d2
 
 
 
 
 
cc1ec7a
 
1b4b6d2
 
 
d0d7322
 
 
 
 
 
 
 
 
 
 
 
 
 
1b4b6d2
 
16455b0
 
 
 
 
 
 
 
 
 
1b4b6d2
16455b0
1b4b6d2
8e18bfe
 
 
 
 
 
 
 
 
cc1ec7a
8e18bfe
 
cc1ec7a
 
8e18bfe
 
 
 
1b4b6d2
16455b0
 
8e18bfe
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks
from pydantic import BaseModel
from typing import List, Dict, Any
import os
import json
import numpy as np
from PIL import Image

from api.auth import get_current_user
from utils.db import safe_query, safe_execute
from pdf_engine.smart_classifier import PAGE_ITEMS_MAP
from workflow.step2_classify import _classify_single, _ai_classify

router = APIRouter()

CACHE_ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "_qto_cache")
os.makedirs(CACHE_ROOT, exist_ok=True)

class RunExtractionReq(BaseModel):
    project_id: int

class SaveClassificationReq(BaseModel):
    project_id: int
    classified_pages: List[Dict[str, Any]]

@router.post("/classify/auto")
async def auto_classify(req: RunExtractionReq, current_user: dict = Depends(get_current_user)):
    df_state = safe_query("SELECT state_data FROM qto_active_projects WHERE user_id=%s AND project_id=%s", (current_user["id"], req.project_id))
    if df_state.empty or not df_state.iloc[0]["state_data"]:
        raise HTTPException(status_code=400, detail="No active project uploaded.")
        
    state_data = json.loads(df_state.iloc[0]["state_data"])
    project_cache = os.path.join(CACHE_ROOT, str(req.project_id))
    
    # Re-extract FULL text from cached PDFs on disk for accurate keyword classification.
    # The DB only stores truncated 250-char previews to stay under packet limits,
    # but keyword matching needs the full page text. Reading from disk via mmap is
    # instant and uses virtually no RAM.
    from pdf_engine.pdf_loader import extract_page_text
    
    str_boundaries = state_data.get("str_boundaries") or []
    str_texts = []
    for b in str_boundaries:
        pdf_p = os.path.join(project_cache, b.get("pdf_path", ""))
        exists = os.path.exists(pdf_p)
        print(f"[classify/auto] STR PDF path={pdf_p} exists={exists}")
        if exists:
            extracted = extract_page_text(pdf_p)
            print(f"[classify/auto] STR extracted {len(extracted)} pages, first 200 chars: {extracted[0][:200] if extracted else '(none)'}")
            str_texts.extend(extracted)
    
    # Fallback to DB-stored (truncated) texts if PDFs are missing from disk
    if not str_texts:
        str_texts = state_data.get("str_texts") or []
    
    arch_boundaries = state_data.get("arch_boundaries") or []
    arch_texts = []
    for b in arch_boundaries:
        pdf_p = os.path.join(project_cache, b.get("pdf_path", ""))
        exists = os.path.exists(pdf_p)
        print(f"[classify/auto] ARCH PDF path={pdf_p} exists={exists}")
        if exists:
            extracted = extract_page_text(pdf_p)
            print(f"[classify/auto] ARCH extracted {len(extracted)} pages, first 200 chars: {extracted[0][:200] if extracted else '(none)'}")
            arch_texts.extend(extracted)
    
    if not arch_texts:
        arch_texts = state_data.get("arch_texts") or []
    
    classified = []
    
    # Keyword classification using full page text
    for i, txt in enumerate(str_texts):
        classified.append({
            "pdf": "structural",
            "page_index": i,
            "page_num": i + 1,
            "text_preview": txt[:100].replace("\n", " "),
            "image_url": f"/cache/{req.project_id}/str_page_{i}.jpg",
            **_classify_single(txt, "structural"),
        })
        
    for i, txt in enumerate(arch_texts):
        classified.append({
            "pdf": "architectural",
            "page_index": i,
            "page_num": i + 1,
            "text_preview": txt[:100].replace("\n", " "),
            "image_url": f"/cache/{req.project_id}/arch_page_{i}.jpg",
            **_classify_single(txt, "architectural"),
        })

    state_data["classified_pages"] = classified
    state_data["current_step"] = 2
    
    from utils.db import upsert_active_state
    upsert_active_state(current_user["id"], req.project_id, 2, state_data)
    
    return {
        "classified_pages": classified,
        "next_step": 3,
        "_debug": {
            "cache_dir": project_cache,
            "cache_exists": os.path.isdir(project_cache),
            "cache_files": os.listdir(project_cache) if os.path.isdir(project_cache) else [],
            "str_boundaries": str_boundaries,
            "arch_boundaries": arch_boundaries,
            "str_texts_count": len(str_texts),
            "arch_texts_count": len(arch_texts),
            "str_text_sample": str_texts[0][:300] if str_texts else "(empty)",
            "arch_text_sample": arch_texts[0][:300] if arch_texts else "(empty)",
            "unknown_count": sum(1 for p in classified if p.get("detected_type") == "unknown"),
            "total_classified": len(classified),
        }
    }

def _background_record_memory(user_id: int, pages: list):
    try:
        from engine.qto_memory import record_mapping
        for p in pages:
            dtype = (p.get("detected_type") or "").strip()
            if dtype and dtype != "unknown":
                record_mapping(user_id, dtype[:120], dtype)
    except Exception as e:
        print(f"qto_memory record error: {e}")

@router.post("/classify/save")
async def save_classification(req: SaveClassificationReq, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user)):
    try:
        df_state = safe_query("SELECT state_data FROM qto_active_projects WHERE user_id=%s AND project_id=%s", (current_user["id"], req.project_id))
        
        state_data = {}
        if not df_state.empty and df_state.iloc[0]["state_data"]:
            try:
                state_data = json.loads(df_state.iloc[0]["state_data"])
            except Exception:
                state_data = {}
        
        state_data["classified_pages"] = req.classified_pages
        state_data["current_step"] = 3
        
        from utils.db import upsert_active_state
        success, msg = upsert_active_state(current_user["id"], req.project_id, 3, state_data)

        if not success:
            print(f"[classify/save] DB ERROR: {msg}")
            raise HTTPException(status_code=500, detail=f"DB error: {msg}")

        # Learning loop pushed to background to prevent 502 Proxy Timeout
        background_tasks.add_task(_background_record_memory, current_user["id"], req.classified_pages or [])

        return {"message": "Classifications saved successfully."}
    except HTTPException:
        raise
    except Exception as e:
        print(f"[classify/save] UNHANDLED ERROR: {e}")
        import traceback; traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Server error: {str(e)}")