jch1223 commited on
Commit
74b444e
ยท
1 Parent(s): 9228365

[delta-02] draft: split_text

Browse files
Files changed (3) hide show
  1. app/config.py +7 -3
  2. app/embedding_utils.py +23 -1
  3. app/main.py +25 -2
app/config.py CHANGED
@@ -8,7 +8,7 @@ PROVIDER = os.getenv("PROVIDER", "ollama").lower()
8
 
9
  # Ollama ์„ค์ • (๊ธฐ๋ณธ๊ฐ’ - ๋กœ์ปฌ ๋ชจ๋ธ ์‚ฌ์šฉ)
10
  OLLAMA_BASE = os.getenv("OLLAMA_BASE", "http://localhost:11434")
11
- OLLAMA_EMBED_MODEL = os.getenv("OLLAMA_EMBED_MODEL", "nomic-embed-text")
12
  OLLAMA_CHAT_MODEL = os.getenv("OLLAMA_CHAT_MODEL", "llama3.1")
13
 
14
  # OpenAI ์„ค์ • (์„ ํƒ์‚ฌํ•ญ - PROVIDER=openai ์ผ ๋•Œ๋งŒ ์‚ฌ์šฉ)
@@ -20,10 +20,14 @@ OLLAMA_CHAT_MODEL = os.getenv("OLLAMA_CHAT_MODEL", "llama3.1")
20
  # OpenAI ์‚ฌ์šฉ ์‹œ ํ•„์ˆ˜ ํ‚ค ๊ฒ€์ฆ
21
  if PROVIDER == "openai":
22
  OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
23
- OPENAI_EMBEDDING_MODEL = os.getenv("OPENAI_EMBEDDING_MODEL", "text-embedding-3-small")
 
 
24
  OPENAI_CHAT_MODEL = os.getenv("OPENAI_CHAT_MODEL", "gpt-3.5-turbo")
25
  if not OPENAI_API_KEY:
26
- raise ValueError("PROVIDER=openai๋กœ ์„ค์ •ํ–ˆ์ง€๋งŒ OPENAI_API_KEY๊ฐ€ ์„ค์ •๋˜์ง€ ์•Š์•˜์Šต๋‹ˆ๋‹ค.")
 
 
27
 
28
  UPLOAD_DIR = "app/uploads"
29
 
 
8
 
9
  # Ollama ์„ค์ • (๊ธฐ๋ณธ๊ฐ’ - ๋กœ์ปฌ ๋ชจ๋ธ ์‚ฌ์šฉ)
10
  OLLAMA_BASE = os.getenv("OLLAMA_BASE", "http://localhost:11434")
11
+ OLLAMA_EMBED_MODEL = os.getenv("OLLAMA_EMBED_MODEL", "bge-m3")
12
  OLLAMA_CHAT_MODEL = os.getenv("OLLAMA_CHAT_MODEL", "llama3.1")
13
 
14
  # OpenAI ์„ค์ • (์„ ํƒ์‚ฌํ•ญ - PROVIDER=openai ์ผ ๋•Œ๋งŒ ์‚ฌ์šฉ)
 
20
  # OpenAI ์‚ฌ์šฉ ์‹œ ํ•„์ˆ˜ ํ‚ค ๊ฒ€์ฆ
21
  if PROVIDER == "openai":
22
  OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
23
+ OPENAI_EMBEDDING_MODEL = os.getenv(
24
+ "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
25
+ )
26
  OPENAI_CHAT_MODEL = os.getenv("OPENAI_CHAT_MODEL", "gpt-3.5-turbo")
27
  if not OPENAI_API_KEY:
28
+ raise ValueError(
29
+ "PROVIDER=openai๋กœ ์„ค์ •ํ–ˆ์ง€๋งŒ OPENAI_API_KEY๊ฐ€ ์„ค์ •๋˜์ง€ ์•Š์•˜์Šต๋‹ˆ๋‹ค."
30
+ )
31
 
32
  UPLOAD_DIR = "app/uploads"
33
 
app/embedding_utils.py CHANGED
@@ -1,4 +1,8 @@
1
  from typing import List
 
 
 
 
2
 
3
 
4
  def split_text(
@@ -26,7 +30,25 @@ def split_text(
26
  # ---------------------------------------------------------
27
  # [Day 1] ๊ณผ์ œ: ์—ฌ๊ธฐ์— ํ…์ŠคํŠธ ๋ถ„ํ•  ๋กœ์ง์„ ์ž‘์„ฑํ•˜์„ธ์š”.
28
  # ---------------------------------------------------------
29
- pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
 
31
 
32
  def get_embedding(text: str) -> List[float]:
 
1
  from typing import List
2
+ import nltk
3
+ from sentence_transformers import SentenceTransformer
4
+ from sklearn.metrics.pairwise import cosine_similarity
5
+ import numpy as np
6
 
7
 
8
  def split_text(
 
30
  # ---------------------------------------------------------
31
  # [Day 1] ๊ณผ์ œ: ์—ฌ๊ธฐ์— ํ…์ŠคํŠธ ๋ถ„ํ•  ๋กœ์ง์„ ์ž‘์„ฑํ•˜์„ธ์š”.
32
  # ---------------------------------------------------------
33
+ model = SentenceTransformer("BAAI/bge-m3")
34
+ sentences = nltk.sent_tokenize(text)
35
+ embeddings = model.encode(sentences)
36
+
37
+ # ์ธ์ ‘ ๋ฌธ์žฅ ์œ ์‚ฌ๋„ ๊ณ„์‚ฐ
38
+ similarities = [
39
+ cosine_similarity([embeddings[i]], [embeddings[i + 1]])[0][0]
40
+ for i in range(len(embeddings) - 1)
41
+ ]
42
+
43
+ # ์œ ์‚ฌ๋„๊ฐ€ ์ž„๊ณ„๊ฐ’ ์ดํ•˜์ธ ์ง€์ ์„ ๊ฒฝ๊ณ„๋กœ ์ฒญํฌ ๊ตฌ์„ฑ
44
+ threshold = 0.5
45
+ chunks, current = [], [sentences[0]]
46
+ for i, sim in enumerate(similarities):
47
+ if sim < threshold:
48
+ chunks.append(" ".join(current))
49
+ current = []
50
+ current.append(sentences[i + 1])
51
+ chunks.append(" ".join(current))
52
 
53
 
54
  def get_embedding(text: str) -> List[float]:
app/main.py CHANGED
@@ -12,6 +12,8 @@ from .embedding_utils import split_text, get_embedding
12
  from .supabase_utils import save_embedding, search_similar_embeddings
13
  from .config import UPLOAD_DIR, ALLOWED_EXTENSIONS
14
 
 
 
15
  app = FastAPI(title="DocuFlow AI Agent")
16
 
17
  # ํ…œํ”Œ๋ฆฟ ๋ฐ ์ •์  ํŒŒ์ผ ์„ค์ •
@@ -35,7 +37,13 @@ def extract_text_from_pdf(file_path: str) -> str:
35
  # ---------------------------------------------------------
36
  # [Day 1] ๊ณผ์ œ: PDF ํ…์ŠคํŠธ ์ถ”์ถœ ๋กœ์ง ์ž‘์„ฑ
37
  # ---------------------------------------------------------
38
- pass
 
 
 
 
 
 
39
 
40
 
41
  @app.get("/", response_class=HTMLResponse)
@@ -58,6 +66,9 @@ async def upload_document(file: UploadFile = File(...)):
58
  """
59
  document_id = str(uuid4())
60
 
 
 
 
61
  try:
62
  # ---------------------------------------------------------
63
  # [Day 1 & 2] ๊ณผ์ œ: ํŒŒ์ดํ”„๋ผ์ธ ๊ตฌํ˜„
@@ -67,6 +78,15 @@ async def upload_document(file: UploadFile = File(...)):
67
  # 4. ์ž„๋ฒ ๋”ฉ ๋ฐ ์ €์žฅ
68
  # ---------------------------------------------------------
69
 
 
 
 
 
 
 
 
 
 
70
  return {
71
  "message": "๋ฌธ์„œ ์ฒ˜๋ฆฌ๊ฐ€ ์™„๋ฃŒ๋˜์—ˆ์Šต๋‹ˆ๋‹ค.",
72
  "document_id": document_id,
@@ -121,7 +141,10 @@ async def question_answering(question: Dict[str, str] = Body(...)):
121
  # answer = ...
122
  # similar_chunks = ...
123
 
124
- return {"answer": "์•„์ง AI๊ฐ€ ์—ฐ๊ฒฐ๋˜์ง€ ์•Š์•˜์Šต๋‹ˆ๋‹ค. ์ฝ”๋“œ๋ฅผ ์™„์„ฑํ•ด์ฃผ์„ธ์š”!", "sources": []}
 
 
 
125
 
126
  except Exception as e:
127
  print(f"์˜ค๋ฅ˜ ๋ฐœ์ƒ: {str(e)}")
 
12
  from .supabase_utils import save_embedding, search_similar_embeddings
13
  from .config import UPLOAD_DIR, ALLOWED_EXTENSIONS
14
 
15
+ import pdfplumber
16
+
17
  app = FastAPI(title="DocuFlow AI Agent")
18
 
19
  # ํ…œํ”Œ๋ฆฟ ๋ฐ ์ •์  ํŒŒ์ผ ์„ค์ •
 
37
  # ---------------------------------------------------------
38
  # [Day 1] ๊ณผ์ œ: PDF ํ…์ŠคํŠธ ์ถ”์ถœ ๋กœ์ง ์ž‘์„ฑ
39
  # ---------------------------------------------------------
40
+ text = ""
41
+ with pdfplumber.open(file_path) as pdf:
42
+ for page in pdf.pages:
43
+ page_text = page.extract_text()
44
+ if page_text:
45
+ text += page_text + "\n"
46
+ return text
47
 
48
 
49
  @app.get("/", response_class=HTMLResponse)
 
66
  """
67
  document_id = str(uuid4())
68
 
69
+ if not allowed_file(file.filename):
70
+ raise HTTPException(status_code=400, detail="ํ—ˆ์šฉ๋˜์ง€ ์•Š๋Š” ํŒŒ์ผ ํ˜•์‹์ž…๋‹ˆ๋‹ค.")
71
+
72
  try:
73
  # ---------------------------------------------------------
74
  # [Day 1 & 2] ๊ณผ์ œ: ํŒŒ์ดํ”„๋ผ์ธ ๊ตฌํ˜„
 
78
  # 4. ์ž„๋ฒ ๋”ฉ ๋ฐ ์ €์žฅ
79
  # ---------------------------------------------------------
80
 
81
+ # 1. ํŒŒ์ผ ์ €์žฅ
82
+ file_path = os.path.join(UPLOAD_DIR, f"{document_id}_{file.filename}")
83
+ with open(file_path, "wb") as f:
84
+ content = await file.read()
85
+ f.write(content)
86
+
87
+ # 2. ํ…์ŠคํŠธ ์ถ”์ถœ
88
+ text = extract_text_from_pdf(file_path)
89
+
90
  return {
91
  "message": "๋ฌธ์„œ ์ฒ˜๋ฆฌ๊ฐ€ ์™„๋ฃŒ๋˜์—ˆ์Šต๋‹ˆ๋‹ค.",
92
  "document_id": document_id,
 
141
  # answer = ...
142
  # similar_chunks = ...
143
 
144
+ return {
145
+ "answer": "์•„์ง AI๊ฐ€ ์—ฐ๊ฒฐ๋˜์ง€ ์•Š์•˜์Šต๋‹ˆ๋‹ค. ์ฝ”๋“œ๋ฅผ ์™„์„ฑํ•ด์ฃผ์„ธ์š”!",
146
+ "sources": [],
147
+ }
148
 
149
  except Exception as e:
150
  print(f"์˜ค๋ฅ˜ ๋ฐœ์ƒ: {str(e)}")