Spaces:
Runtime error
Runtime error
jch1223 commited on
Commit ยท
74b444e
1
Parent(s): 9228365
[delta-02] draft: split_text
Browse files- app/config.py +7 -3
- app/embedding_utils.py +23 -1
- app/main.py +25 -2
app/config.py
CHANGED
|
@@ -8,7 +8,7 @@ PROVIDER = os.getenv("PROVIDER", "ollama").lower()
|
|
| 8 |
|
| 9 |
# Ollama ์ค์ (๊ธฐ๋ณธ๊ฐ - ๋ก์ปฌ ๋ชจ๋ธ ์ฌ์ฉ)
|
| 10 |
OLLAMA_BASE = os.getenv("OLLAMA_BASE", "http://localhost:11434")
|
| 11 |
-
OLLAMA_EMBED_MODEL = os.getenv("OLLAMA_EMBED_MODEL", "
|
| 12 |
OLLAMA_CHAT_MODEL = os.getenv("OLLAMA_CHAT_MODEL", "llama3.1")
|
| 13 |
|
| 14 |
# OpenAI ์ค์ (์ ํ์ฌํญ - PROVIDER=openai ์ผ ๋๋ง ์ฌ์ฉ)
|
|
@@ -20,10 +20,14 @@ OLLAMA_CHAT_MODEL = os.getenv("OLLAMA_CHAT_MODEL", "llama3.1")
|
|
| 20 |
# OpenAI ์ฌ์ฉ ์ ํ์ ํค ๊ฒ์ฆ
|
| 21 |
if PROVIDER == "openai":
|
| 22 |
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
|
| 23 |
-
OPENAI_EMBEDDING_MODEL = os.getenv(
|
|
|
|
|
|
|
| 24 |
OPENAI_CHAT_MODEL = os.getenv("OPENAI_CHAT_MODEL", "gpt-3.5-turbo")
|
| 25 |
if not OPENAI_API_KEY:
|
| 26 |
-
raise ValueError(
|
|
|
|
|
|
|
| 27 |
|
| 28 |
UPLOAD_DIR = "app/uploads"
|
| 29 |
|
|
|
|
| 8 |
|
| 9 |
# Ollama ์ค์ (๊ธฐ๋ณธ๊ฐ - ๋ก์ปฌ ๋ชจ๋ธ ์ฌ์ฉ)
|
| 10 |
OLLAMA_BASE = os.getenv("OLLAMA_BASE", "http://localhost:11434")
|
| 11 |
+
OLLAMA_EMBED_MODEL = os.getenv("OLLAMA_EMBED_MODEL", "bge-m3")
|
| 12 |
OLLAMA_CHAT_MODEL = os.getenv("OLLAMA_CHAT_MODEL", "llama3.1")
|
| 13 |
|
| 14 |
# OpenAI ์ค์ (์ ํ์ฌํญ - PROVIDER=openai ์ผ ๋๋ง ์ฌ์ฉ)
|
|
|
|
| 20 |
# OpenAI ์ฌ์ฉ ์ ํ์ ํค ๊ฒ์ฆ
|
| 21 |
if PROVIDER == "openai":
|
| 22 |
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
|
| 23 |
+
OPENAI_EMBEDDING_MODEL = os.getenv(
|
| 24 |
+
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
|
| 25 |
+
)
|
| 26 |
OPENAI_CHAT_MODEL = os.getenv("OPENAI_CHAT_MODEL", "gpt-3.5-turbo")
|
| 27 |
if not OPENAI_API_KEY:
|
| 28 |
+
raise ValueError(
|
| 29 |
+
"PROVIDER=openai๋ก ์ค์ ํ์ง๋ง OPENAI_API_KEY๊ฐ ์ค์ ๋์ง ์์์ต๋๋ค."
|
| 30 |
+
)
|
| 31 |
|
| 32 |
UPLOAD_DIR = "app/uploads"
|
| 33 |
|
app/embedding_utils.py
CHANGED
|
@@ -1,4 +1,8 @@
|
|
| 1 |
from typing import List
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
|
| 4 |
def split_text(
|
|
@@ -26,7 +30,25 @@ def split_text(
|
|
| 26 |
# ---------------------------------------------------------
|
| 27 |
# [Day 1] ๊ณผ์ : ์ฌ๊ธฐ์ ํ
์คํธ ๋ถํ ๋ก์ง์ ์์ฑํ์ธ์.
|
| 28 |
# ---------------------------------------------------------
|
| 29 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
def get_embedding(text: str) -> List[float]:
|
|
|
|
| 1 |
from typing import List
|
| 2 |
+
import nltk
|
| 3 |
+
from sentence_transformers import SentenceTransformer
|
| 4 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
| 5 |
+
import numpy as np
|
| 6 |
|
| 7 |
|
| 8 |
def split_text(
|
|
|
|
| 30 |
# ---------------------------------------------------------
|
| 31 |
# [Day 1] ๊ณผ์ : ์ฌ๊ธฐ์ ํ
์คํธ ๋ถํ ๋ก์ง์ ์์ฑํ์ธ์.
|
| 32 |
# ---------------------------------------------------------
|
| 33 |
+
model = SentenceTransformer("BAAI/bge-m3")
|
| 34 |
+
sentences = nltk.sent_tokenize(text)
|
| 35 |
+
embeddings = model.encode(sentences)
|
| 36 |
+
|
| 37 |
+
# ์ธ์ ๋ฌธ์ฅ ์ ์ฌ๋ ๊ณ์ฐ
|
| 38 |
+
similarities = [
|
| 39 |
+
cosine_similarity([embeddings[i]], [embeddings[i + 1]])[0][0]
|
| 40 |
+
for i in range(len(embeddings) - 1)
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
# ์ ์ฌ๋๊ฐ ์๊ณ๊ฐ ์ดํ์ธ ์ง์ ์ ๊ฒฝ๊ณ๋ก ์ฒญํฌ ๊ตฌ์ฑ
|
| 44 |
+
threshold = 0.5
|
| 45 |
+
chunks, current = [], [sentences[0]]
|
| 46 |
+
for i, sim in enumerate(similarities):
|
| 47 |
+
if sim < threshold:
|
| 48 |
+
chunks.append(" ".join(current))
|
| 49 |
+
current = []
|
| 50 |
+
current.append(sentences[i + 1])
|
| 51 |
+
chunks.append(" ".join(current))
|
| 52 |
|
| 53 |
|
| 54 |
def get_embedding(text: str) -> List[float]:
|
app/main.py
CHANGED
|
@@ -12,6 +12,8 @@ from .embedding_utils import split_text, get_embedding
|
|
| 12 |
from .supabase_utils import save_embedding, search_similar_embeddings
|
| 13 |
from .config import UPLOAD_DIR, ALLOWED_EXTENSIONS
|
| 14 |
|
|
|
|
|
|
|
| 15 |
app = FastAPI(title="DocuFlow AI Agent")
|
| 16 |
|
| 17 |
# ํ
ํ๋ฆฟ ๋ฐ ์ ์ ํ์ผ ์ค์
|
|
@@ -35,7 +37,13 @@ def extract_text_from_pdf(file_path: str) -> str:
|
|
| 35 |
# ---------------------------------------------------------
|
| 36 |
# [Day 1] ๊ณผ์ : PDF ํ
์คํธ ์ถ์ถ ๋ก์ง ์์ฑ
|
| 37 |
# ---------------------------------------------------------
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
|
| 41 |
@app.get("/", response_class=HTMLResponse)
|
|
@@ -58,6 +66,9 @@ async def upload_document(file: UploadFile = File(...)):
|
|
| 58 |
"""
|
| 59 |
document_id = str(uuid4())
|
| 60 |
|
|
|
|
|
|
|
|
|
|
| 61 |
try:
|
| 62 |
# ---------------------------------------------------------
|
| 63 |
# [Day 1 & 2] ๊ณผ์ : ํ์ดํ๋ผ์ธ ๊ตฌํ
|
|
@@ -67,6 +78,15 @@ async def upload_document(file: UploadFile = File(...)):
|
|
| 67 |
# 4. ์๋ฒ ๋ฉ ๋ฐ ์ ์ฅ
|
| 68 |
# ---------------------------------------------------------
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
return {
|
| 71 |
"message": "๋ฌธ์ ์ฒ๋ฆฌ๊ฐ ์๋ฃ๋์์ต๋๋ค.",
|
| 72 |
"document_id": document_id,
|
|
@@ -121,7 +141,10 @@ async def question_answering(question: Dict[str, str] = Body(...)):
|
|
| 121 |
# answer = ...
|
| 122 |
# similar_chunks = ...
|
| 123 |
|
| 124 |
-
return {
|
|
|
|
|
|
|
|
|
|
| 125 |
|
| 126 |
except Exception as e:
|
| 127 |
print(f"์ค๋ฅ ๋ฐ์: {str(e)}")
|
|
|
|
| 12 |
from .supabase_utils import save_embedding, search_similar_embeddings
|
| 13 |
from .config import UPLOAD_DIR, ALLOWED_EXTENSIONS
|
| 14 |
|
| 15 |
+
import pdfplumber
|
| 16 |
+
|
| 17 |
app = FastAPI(title="DocuFlow AI Agent")
|
| 18 |
|
| 19 |
# ํ
ํ๋ฆฟ ๋ฐ ์ ์ ํ์ผ ์ค์
|
|
|
|
| 37 |
# ---------------------------------------------------------
|
| 38 |
# [Day 1] ๊ณผ์ : PDF ํ
์คํธ ์ถ์ถ ๋ก์ง ์์ฑ
|
| 39 |
# ---------------------------------------------------------
|
| 40 |
+
text = ""
|
| 41 |
+
with pdfplumber.open(file_path) as pdf:
|
| 42 |
+
for page in pdf.pages:
|
| 43 |
+
page_text = page.extract_text()
|
| 44 |
+
if page_text:
|
| 45 |
+
text += page_text + "\n"
|
| 46 |
+
return text
|
| 47 |
|
| 48 |
|
| 49 |
@app.get("/", response_class=HTMLResponse)
|
|
|
|
| 66 |
"""
|
| 67 |
document_id = str(uuid4())
|
| 68 |
|
| 69 |
+
if not allowed_file(file.filename):
|
| 70 |
+
raise HTTPException(status_code=400, detail="ํ์ฉ๋์ง ์๋ ํ์ผ ํ์์
๋๋ค.")
|
| 71 |
+
|
| 72 |
try:
|
| 73 |
# ---------------------------------------------------------
|
| 74 |
# [Day 1 & 2] ๊ณผ์ : ํ์ดํ๋ผ์ธ ๊ตฌํ
|
|
|
|
| 78 |
# 4. ์๋ฒ ๋ฉ ๋ฐ ์ ์ฅ
|
| 79 |
# ---------------------------------------------------------
|
| 80 |
|
| 81 |
+
# 1. ํ์ผ ์ ์ฅ
|
| 82 |
+
file_path = os.path.join(UPLOAD_DIR, f"{document_id}_{file.filename}")
|
| 83 |
+
with open(file_path, "wb") as f:
|
| 84 |
+
content = await file.read()
|
| 85 |
+
f.write(content)
|
| 86 |
+
|
| 87 |
+
# 2. ํ
์คํธ ์ถ์ถ
|
| 88 |
+
text = extract_text_from_pdf(file_path)
|
| 89 |
+
|
| 90 |
return {
|
| 91 |
"message": "๋ฌธ์ ์ฒ๋ฆฌ๊ฐ ์๋ฃ๋์์ต๋๋ค.",
|
| 92 |
"document_id": document_id,
|
|
|
|
| 141 |
# answer = ...
|
| 142 |
# similar_chunks = ...
|
| 143 |
|
| 144 |
+
return {
|
| 145 |
+
"answer": "์์ง AI๊ฐ ์ฐ๊ฒฐ๋์ง ์์์ต๋๋ค. ์ฝ๋๋ฅผ ์์ฑํด์ฃผ์ธ์!",
|
| 146 |
+
"sources": [],
|
| 147 |
+
}
|
| 148 |
|
| 149 |
except Exception as e:
|
| 150 |
print(f"์ค๋ฅ ๋ฐ์: {str(e)}")
|