Spaces:
Runtime error
Runtime error
Download app/main.py from progressive-intelligence-lab/docuflow: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/app/main.py
- Command line
-
hf download hf://spaces/progressive-intelligence-lab/docuflow/app/main.py
-
curl -L -o main.py https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/app/main.py
15.5 kB
| from fastapi import FastAPI, UploadFile, File, HTTPException, Body | |
| from fastapi.templating import Jinja2Templates | |
| from fastapi.responses import HTMLResponse | |
| from fastapi.staticfiles import StaticFiles | |
| from fastapi import Request | |
| import os | |
| from uuid import uuid4 | |
| from typing import Dict, List | |
| # ํ์ํ ๋ชจ๋ ์ํฌํธ (์ฌ๋ฌ๋ถ์ด ๊ตฌํํ ๋ชจ๋) | |
| from .embedding_utils import split_text, get_embedding, rerank | |
| from .supabase_utils import ( | |
| save_embedding, | |
| save_embeddings_batch, | |
| search_similar_embeddings, | |
| search_by_keyword, | |
| get_document_id_by_hash, | |
| ) | |
| from .config import ( | |
| UPLOAD_DIR, | |
| ALLOWED_EXTENSIONS, | |
| OPENAI_API_KEY, | |
| OPENAI_CHAT_MODEL, | |
| EMBED_WORKERS, | |
| ) | |
| import pdfplumber | |
| from openai import ( | |
| OpenAI, | |
| RateLimitError, | |
| AuthenticationError, | |
| APITimeoutError, | |
| APIConnectionError, | |
| ) | |
| from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception | |
| import hashlib | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| app = FastAPI(title="DocuFlow AI Agent") | |
| client = OpenAI(api_key=OPENAI_API_KEY) | |
| async def health_check(): | |
| """ํฌ์ค์ฒดํฌ ์๋ํฌ์ธํธ - ๋ฐฐํฌ ์ํ ํ์ธ์ฉ""" | |
| return {"status": "healthy"} | |
| def _is_transient_error(e: Exception) -> bool: | |
| """ํ์์์ยท์ฐ๊ฒฐ ์ค๋ฅยทrate limit(quota ๋ถ์กฑ ์ ์ธ)์ ์ฌ์๋ ๋์์ผ๋ก ํ๋จ""" | |
| if isinstance(e, (APITimeoutError, APIConnectionError)): | |
| return True | |
| if isinstance(e, RateLimitError): | |
| return e.code != "insufficient_quota" | |
| return False | |
| def _call_llm(messages: list, model: str) -> str: | |
| response = client.chat.completions.create(model=model, messages=messages) | |
| return response.choices[0].message.content | |
| # ํ ํ๋ฆฟ ๋ฐ ์ ์ ํ์ผ ์ค์ | |
| templates = Jinja2Templates(directory="app/templates") | |
| app.mount("/static", StaticFiles(directory="app/static"), name="static") | |
| def allowed_file(filename: str) -> bool: | |
| return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS | |
| def _table_to_markdown(table: list[list]) -> str: | |
| """pdfplumber ํ ์ด๋ธ(๋ฆฌ์คํธ์ ๋ฆฌ์คํธ)์ ๋งํฌ๋ค์ด ํ ์ด๋ธ ๋ฌธ์์ด๋ก ๋ณํํ๋ค.""" | |
| if not table: | |
| return "" | |
| # None โ ๋น ๋ฌธ์์ด, ์ ๋ด ์ค๋ฐ๊ฟ โ ๊ณต๋ฐฑ | |
| cleaned = [] | |
| for row in table: | |
| cleaned.append([(cell or "").replace("\n", " ").strip() for cell in row]) | |
| # ๋ชจ๋ ์ ์ด ๋น์ด์์ผ๋ฉด ๋น ๋ฌธ์์ด ๋ฐํ | |
| if all(cell == "" for row in cleaned for cell in row): | |
| return "" | |
| header = cleaned[0] | |
| col_count = len(header) | |
| lines = [ | |
| "| " + " | ".join(header) + " |", | |
| "| " + " | ".join(["---"] * col_count) + " |", | |
| ] | |
| for row in cleaned[1:]: | |
| # ์ด ์๊ฐ ๋ง์ง ์์ผ๋ฉด ํจ๋ฉ | |
| padded = ( | |
| row + [""] * (col_count - len(row)) | |
| if len(row) < col_count | |
| else row[:col_count] | |
| ) | |
| lines.append("| " + " | ".join(padded) + " |") | |
| return "\n".join(lines) | |
| def extract_text_from_pdf(file_path: str) -> str: | |
| """ | |
| PDF ํ์ผ์์ ํ ์คํธ๋ฅผ ์ถ์ถํ๋ ํจ์. | |
| ํ(table)๊ฐ ์๋ ํ์ด์ง๋ ๋งํฌ๋ค์ด ํ ์ด๋ธ๋ก ๋ณํํ์ฌ ๊ตฌ์กฐ๋ฅผ ๋ณด์กดํ๋ค. | |
| """ | |
| all_pages_text = [] | |
| with pdfplumber.open(file_path) as pdf: | |
| for page in pdf.pages: | |
| tables = page.find_tables() | |
| print(f"[DEBUG] page {page.page_number}: {len(tables)} tables found") | |
| # ํ ์ด๋ธ์ด ์์ผ๋ฉด ๊ธฐ์กด ๋น ๋ฅธ ๊ฒฝ๋ก | |
| if not tables: | |
| page_text = page.extract_text() | |
| if page_text: | |
| all_pages_text.append(f"[p.{page.page_number}] {page_text}") | |
| continue | |
| # ํ ์ด๋ธ์ด ์๋ ํ์ด์ง: ํ ์ด๋ธ๊ณผ ๋น-ํ ์ด๋ธ ์์ญ์ ๋ถ๋ฆฌํ์ฌ ์ถ์ถ | |
| segments = [] # (y์ขํ, ํ ์คํธ) ํํ ๋ฆฌ์คํธ | |
| # ํ ์ด๋ธ bbox ์์ง | |
| table_bboxes = [] | |
| for table in tables: | |
| bbox = table.bbox # (x0, top, x1, bottom) | |
| table_bboxes.append(bbox) | |
| md = _table_to_markdown(table.extract()) | |
| if md: | |
| segments.append((bbox[1], md)) # top y์ขํ ๊ธฐ์ค | |
| # ๋น-ํ ์ด๋ธ ์์ญ ํ ์คํธ ์ถ์ถ | |
| # ํ์ด์ง๋ฅผ ํ ์ด๋ธ ์์ญ ๊ธฐ์ค์ผ๋ก cropํ์ฌ ๋๋จธ์ง ํ ์คํธ ์ถ์ถ | |
| page_height = page.height | |
| page_width = page.width | |
| # ํ ์ด๋ธ ์์ญ์ y์ขํ ์์ผ๋ก ์ ๋ ฌ | |
| sorted_bboxes = sorted(table_bboxes, key=lambda b: b[1]) | |
| # ๋น-ํ ์ด๋ธ ์์ญ ๊ตฌ๊ฐ ๊ณ์ฐ | |
| non_table_regions = [] | |
| current_top = 0 | |
| for bbox in sorted_bboxes: | |
| if bbox[1] > current_top: | |
| non_table_regions.append((0, current_top, page_width, bbox[1])) | |
| current_top = max(current_top, bbox[3]) | |
| if current_top < page_height: | |
| non_table_regions.append((0, current_top, page_width, page_height)) | |
| for region in non_table_regions: | |
| try: | |
| cropped = page.crop(region) | |
| region_text = cropped.extract_text() | |
| if region_text: | |
| segments.append((region[1], region_text)) | |
| except Exception: | |
| pass | |
| # y์ขํ ์์ผ๋ก ์ ๋ ฌํ์ฌ ์ฝ๊ธฐ ์์ ์ ์ง | |
| segments.sort(key=lambda s: s[0]) | |
| page_text = "\n".join(seg[1] for seg in segments) | |
| if page_text: | |
| all_pages_text.append(f"[p.{page.page_number}] {page_text}") | |
| return "\n".join(all_pages_text) | |
| async def root_page(request: Request): | |
| return templates.TemplateResponse("index.html", {"request": request}) | |
| async def upload_document(file: UploadFile = File(...)): | |
| """ | |
| ๋ฌธ์ ์ ๋ก๋ ๋ฐ ์๋ฒ ๋ฉ ์ฒ๋ฆฌ ์๋ํฌ์ธํธ | |
| ํ๋ก์ธ์ค: | |
| 1. ์ ๋ก๋๋ ํ์ผ์ ๋ก์ปฌ(UPLOAD_DIR)์ ์ ์ฅ | |
| 2. PDF์์ ํ ์คํธ ์ถ์ถ | |
| 3. ํ ์คํธ๋ฅผ ์ฒญํฌ๋ก ๋ถํ | |
| 4. ๊ฐ ์ฒญํฌ๋ฅผ ์๋ฒ ๋ฉํ๊ณ DB์ ์ ์ฅ | |
| """ | |
| document_id = str(uuid4()) | |
| if not allowed_file(file.filename): | |
| raise HTTPException(status_code=400, detail="ํ์ฉ๋์ง ์๋ ํ์ผ ํ์์ ๋๋ค.") | |
| try: | |
| content = await file.read() | |
| file_hash = hashlib.sha256(content).hexdigest() | |
| existing_id = get_document_id_by_hash(file_hash) | |
| if existing_id: | |
| return { | |
| "message": "์ด๋ฏธ ๋์ผํ ๋ฌธ์๊ฐ ์ ๋ก๋๋์ด ์์ต๋๋ค.", | |
| "document_id": existing_id, | |
| } | |
| # 1. ํ์ผ ์ ์ฅ | |
| file_path = os.path.join(UPLOAD_DIR, f"{document_id}_{file.filename}") | |
| with open(file_path, "wb") as f: | |
| f.write(content) | |
| # 2. ํ ์คํธ ์ถ์ถ | |
| t0 = time.perf_counter() | |
| text = extract_text_from_pdf(file_path) | |
| t1 = time.perf_counter() | |
| # 3. ์ฒญํฌ ๋ถํ | |
| chunks = split_text(text) | |
| t2 = time.perf_counter() | |
| # 4. ์๋ฒ ๋ฉ ๋ฐ ์ ์ฅ | |
| def _embed_chunk(chunk: str): | |
| return get_embedding(chunk) | |
| with ThreadPoolExecutor(max_workers=EMBED_WORKERS) as executor: | |
| embeddings = list(executor.map(_embed_chunk, chunks)) | |
| save_embeddings_batch(document_id, chunks, embeddings, file_hash=file_hash) | |
| t3 = time.perf_counter() | |
| return { | |
| "message": "๋ฌธ์ ์ฒ๋ฆฌ๊ฐ ์๋ฃ๋์์ต๋๋ค.", | |
| "document_id": document_id, | |
| "chunks_count": len(chunks), | |
| "timing": { | |
| "extract_text_sec": round(t1 - t0, 2), | |
| "split_text_sec": round(t2 - t1, 2), | |
| "embed_and_save_sec": round(t3 - t2, 2), | |
| "total_sec": round(t3 - t0, 2), | |
| }, | |
| } | |
| except AuthenticationError: | |
| print("์ค๋ฅ ๋ฐ์: OpenAI API ํค๊ฐ ์ ํจํ์ง ์์ต๋๋ค.") | |
| raise HTTPException( | |
| status_code=500, | |
| detail="์๋ฒ ์ค์ ์ค๋ฅ๊ฐ ๋ฐ์ํ์ต๋๋ค. ๊ด๋ฆฌ์์๊ฒ ๋ฌธ์ํด์ฃผ์ธ์.", | |
| ) | |
| except APITimeoutError: | |
| print("์ค๋ฅ ๋ฐ์: OpenAI API ์์ฒญ ์๊ฐ ์ด๊ณผ") | |
| raise HTTPException( | |
| status_code=504, | |
| detail="์์ฒญ ์ฒ๋ฆฌ ์๊ฐ์ด ์ด๊ณผ๋์์ต๋๋ค. ์ ์ ํ ๋ค์ ์๋ํด์ฃผ์ธ์.", | |
| ) | |
| except Exception as e: | |
| print(f"์ค๋ฅ ๋ฐ์: {str(e)}") | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| finally: | |
| pass | |
| async def question_answering(question: Dict[str, str] = Body(...)): | |
| """ | |
| RAG ๊ธฐ๋ฐ ์ง์์๋ต ์๋ํฌ์ธํธ | |
| ํ๋ก์ธ์ค: | |
| 1. ์ฟผ๋ฆฌ ๋ฆฌ๋ผ์ดํ | |
| 2. ์ง๋ฌธ ์๋ฒ ๋ฉ | |
| 3. ํ์ด๋ธ๋ฆฌ๋ ๊ฒ์ (๋ฒกํฐ + ํค์๋) | |
| 4. ๋ฆฌ๋ญํน | |
| 5. LLM ์๋ต ์์ฑ | |
| """ | |
| try: | |
| if "document_id" not in question: | |
| raise HTTPException(status_code=400, detail="document_id is required") | |
| user_question = question["query"] | |
| document_id = question["document_id"] | |
| # 1. ์ฟผ๋ฆฌ ๋ฆฌ๋ผ์ดํ : LLM์ผ๋ก ๊ฒ์ ์ต์ ํ ์ฟผ๋ฆฌ ์์ฑ | |
| rewrite_prompt = ( | |
| "๋น์ ์ ๋ฌธ์ ๊ฒ์ ์ฟผ๋ฆฌ ์ต์ ํ ์ ๋ฌธ๊ฐ์ ๋๋ค.\n" | |
| "๋ค์ ์ฌ์ฉ์ ์ง๋ฌธ์ ๋ฌธ์ ๊ฒ์์ ์ ํฉํ ํํ๋ก ๋ณํํ์ธ์.\n\n" | |
| "๊ท์น:\n" | |
| "1. ํต์ฌ ๋ช ์ฌ์ ๊ณ ์ ๋ช ์ฌ๋ฅผ ๋ณด์กดํ์ธ์\n" | |
| "2. ํ๊ตญ์ด ์กฐ์ฌ(์/๋/์ด/๊ฐ/์/๋ฅผ/์)๋ฅผ ์ ๊ฑฐํ์ธ์\n" | |
| "3. ๊ฒ์์ ๋ถํ์ํ ํํ(์๋ ค์ฃผ์ธ์, ๋ฌด์์ธ๊ฐ์ ๋ฑ)์ ์ ๊ฑฐํ์ธ์\n" | |
| "4. ๋์์ด๋ ๊ด๋ จ ์ฉ์ด๊ฐ ์๋ค๋ฉด ์ถ๊ฐํ์ธ์\n" | |
| "5. ๋ณํ๋ ํค์๋/๊ตฌ๋ฌธ๋ง ์ถ๋ ฅํ์ธ์ (์ค๋ช ์์ด)\n\n" | |
| f"์ง๋ฌธ: {user_question}" | |
| ) | |
| rewritten_query = _call_llm( | |
| messages=[{"role": "user", "content": rewrite_prompt}], | |
| model=OPENAI_CHAT_MODEL, | |
| ).strip() | |
| print(f"\n{'='*60}") | |
| print(f" [QA] ์๋ณธ ์ง๋ฌธ: {user_question}") | |
| print(f" [QA] ๋ฆฌ๋ผ์ดํ : {rewritten_query}") | |
| print(f"{'='*60}") | |
| # 2. ์ง๋ฌธ ์๋ฒ ๋ฉ (๋ฆฌ๋ผ์ดํ ๋ ์ฟผ๋ฆฌ ์ฌ์ฉ) | |
| question_embedding = get_embedding(rewritten_query) | |
| # 3. ํ์ด๋ธ๋ฆฌ๋ ๊ฒ์: ๋ฒกํฐ ๊ฒ์ + ํค์๋ ๊ฒ์ | |
| vector_chunks = search_similar_embeddings(question_embedding, document_id) | |
| keyword_chunks = search_by_keyword(user_question, document_id) | |
| # ์ค๋ณต ์ ๊ฑฐ ํ ํฉ์นจ | |
| seen_contents = set() | |
| all_chunks_content = [] | |
| for chunk in vector_chunks: | |
| content = chunk["content"] | |
| if content not in seen_contents: | |
| seen_contents.add(content) | |
| all_chunks_content.append(content) | |
| for chunk in keyword_chunks: | |
| content = chunk["content"] | |
| if content not in seen_contents: | |
| seen_contents.add(content) | |
| all_chunks_content.append(content) | |
| print(f"\n ํ์ด๋ธ๋ฆฌ๋ ๊ฒ์ ๊ฒฐ๊ณผ:") | |
| print(f" ๋ฒกํฐ ๊ฒ์: {len(vector_chunks)}๊ฑด") | |
| print(f" ํค์๋ ๊ฒ์: {len(keyword_chunks)}๊ฑด") | |
| print(f" ์ค๋ณต ์ ๊ฑฐ ํ: {len(all_chunks_content)}๊ฑด") | |
| # 4. ๋น ๊ฒฐ๊ณผ ๋ฐฉ์ด: ๊ฒ์ ๊ฒฐ๊ณผ๊ฐ ์์ผ๋ฉด ์ฆ์ ๋ฐํ | |
| if not all_chunks_content: | |
| print(f" โ ๊ฒ์ ๊ฒฐ๊ณผ ์์ โ ๋น ์๋ต ๋ฐํ") | |
| return {"answer": "๋ฌธ์์์ ํด๋น ์ ๋ณด๋ฅผ ์ฐพ์ ์ ์์ต๋๋ค.", "sources": []} | |
| # 5. ๋ฆฌ๋ญํน: Cross-encoder๋ก ์์ 5๊ฐ ์ ๋ณ | |
| reranked_chunks = rerank(user_question, all_chunks_content, top_k=5) | |
| print(f"\n ๋ฆฌ๋ญํน ํ ์ต์ข {len(reranked_chunks)}๊ฑด ์ ๋ณ:") | |
| for i, chunk in enumerate(reranked_chunks, 1): | |
| preview = chunk[:60].replace("\n", " ") | |
| print(f" [{i}] {preview}...") | |
| print(f"{'='*60}\n") | |
| # 6. ํ๋กฌํํธ ๊ตฌ์ฑ | |
| context = "\n\n".join( | |
| f"[์ฐธ๊ณ ์๋ฃ {i}]\n{chunk}" for i, chunk in enumerate(reranked_chunks, 1) | |
| ) | |
| system_prompt = ( | |
| "[์ญํ ]\n" | |
| "- ๋๋ ์ ๋ก๋๋ ๋ฌธ์๋ฅผ ๊ทผ๊ฑฐ๋ก๋ง ๋ต๋ณํ๋ RAG ์ด์์คํดํธ๋ค.\n" | |
| "[ํ๋ ์์น]\n" | |
| "- ์ ๊ณต๋ ์ฐธ๊ณ ๋ฌธ์์ ๊ทผ๊ฑฐ๊ฐ ์๋ ๋ด์ฉ๋ง ๋ต๋ณํ๋ค.\n" | |
| "- ์ฐธ๊ณ ๋ฌธ์์ ์๋ ์ฌ์ค์ ์ถ์ ํ์ง ์๋๋ค.\n" | |
| "- ์ง๋ฌธ์ด ๋ชจํธํ๋ฉด ํ์ํ ์ ๋ณด๋ฅผ ๋จผ์ ๋ฌผ์ด๋ณธ๋ค.\n" | |
| "- ๋ฌธ์ ์ธ ์ง๋ฌธ์ด๋ฉด '๋ฌธ์์์ ํ์ธํ ์ ์๋ค'๋ผ๊ณ ๋ช ์ํ๋ค.\n" | |
| "[๊ธ์ง ์ฌํญ]\n" | |
| "- ์ฐธ๊ณ ๋ฌธ์์ ์๋ ์ ๋ณด๋ฅผ ์ถ์ , ์ถ๋ก , ์ง์ด๋ด์ง ์๋๋ค.\n" | |
| "- ํ์ ์ด ์์ผ๋ฉด 'ํ์ธํ ์ ์๋ค'๋ผ๊ณ ๋ง ๋ตํ๋ค.\n" | |
| "[๋ต๋ณ ์ ๊ฒ์ฆ]\n" | |
| "- ๋ต๋ณ์ ๊ฐ ๋ฌธ์ฅ์ด ์ฐธ๊ณ ๋ฌธ์์ ๊ทผ๊ฑฐ๊ฐ ์๋์ง ํ์ธํ๋ค.\n" | |
| "- ๊ทผ๊ฑฐ๊ฐ ์๋ ๋ฌธ์ฅ์ ์ญ์ ํ๊ฑฐ๋ '๋ฌธ์์ ๋ช ์๋์ง ์์'์ผ๋ก ํ์ํ๋ค.\n" | |
| "[๋ถํ์ค์ฑ ์ฒ๋ฆฌ]\n" | |
| "- ์ฐธ๊ณ ๋ฌธ์์ ๋ถ๋ถ์ ์ผ๋ก๋ง ์ธ๊ธ๋ ๋ด์ฉ์ '์ ํ์ ์ผ๋ก ํ์ธ๋จ' ๋ฑ์ผ๋ก ํํํ๋ค.\n" | |
| "- ๊ด๋ จ ๋ด์ฉ์ด ์ ํ ์์ผ๋ฉด ์ถ์ธกํ์ง ์๊ณ 'ํด๋น ์ ๋ณด๋ ๋ฌธ์์ ์์ต๋๋ค'๋ผ๊ณ ๋ง ๋ตํ๋ค.\n" | |
| "[๊ทผ๊ฑฐ ์ ์]\n" | |
| "- ๊ทผ๊ฑฐ ์ ์ ์ ํ์ด์ง ๋ฒํธ([p.N])๊ฐ ์์ผ๋ฉด ํฌํจํ์ธ์.\n" | |
| "[์๋ต ํ์]\n" | |
| "- ๋ต๋ณ:\n" | |
| "- ๊ทผ๊ฑฐ:\n" | |
| "- ํ๊ณ/์ถ๊ฐ ํ์ธ ํ์์ฌํญ:" | |
| ) | |
| # 7. LLM ์๋ต ์์ฑ | |
| messages = [ | |
| {"role": "system", "content": f"{system_prompt}\n\n[์ฐธ๊ณ ๋ฌธ์]\n{context}"}, | |
| {"role": "user", "content": user_question}, | |
| ] | |
| answer = _call_llm(messages=messages, model=OPENAI_CHAT_MODEL) | |
| return { | |
| "answer": answer, | |
| "sources": reranked_chunks, | |
| } | |
| except AuthenticationError: | |
| print("์ค๋ฅ ๋ฐ์: OpenAI API ํค๊ฐ ์ ํจํ์ง ์์ต๋๋ค.") | |
| raise HTTPException( | |
| status_code=500, | |
| detail="์๋ฒ ์ค์ ์ค๋ฅ๊ฐ ๋ฐ์ํ์ต๋๋ค. ๊ด๋ฆฌ์์๊ฒ ๋ฌธ์ํด์ฃผ์ธ์.", | |
| ) | |
| except APITimeoutError: | |
| print("์ค๋ฅ ๋ฐ์: OpenAI API ์์ฒญ ์๊ฐ ์ด๊ณผ") | |
| raise HTTPException( | |
| status_code=504, | |
| detail="์์ฒญ ์ฒ๋ฆฌ ์๊ฐ์ด ์ด๊ณผ๋์์ต๋๋ค. ์ ์ ํ ๋ค์ ์๋ํด์ฃผ์ธ์.", | |
| ) | |
| except RateLimitError as e: | |
| if e.code == "insufficient_quota": | |
| raise HTTPException( | |
| status_code=402, | |
| detail="OpenAI API ํฌ๋ ๋ง์ด ๋ถ์กฑํฉ๋๋ค. ํ๋์ ํ์ธํด์ฃผ์ธ์.", | |
| ) | |
| raise HTTPException( | |
| status_code=429, | |
| detail="OpenAI API ์์ฒญ ํ๋๋ฅผ ์ด๊ณผํ์ต๋๋ค. ์ ์ ํ ๋ค์ ์๋ํด์ฃผ์ธ์.", | |
| ) | |
| except Exception as e: | |
| print(f"์ค๋ฅ ๋ฐ์: {str(e)}") | |
| raise HTTPException(status_code=500, detail=str(e)) | |