docuflow / app /embedding_utils.py
woongmin's picture
[Delta-02] Drill C retry/backoff ๋˜๋Š” graceful fallback ๋™์ž‘ ํ™•์ธ ํ…Œ์ŠคํŠธ ์ฝ”๋“œ ์ œ๊ฑฐ ver.2
7ed0ad9
Raw History Blame Contribute Delete
6.76 kB
from typing import List
import nltk
import numpy as np
from openai import OpenAI, RateLimitError, APITimeoutError, APIConnectionError
from sentence_transformers import CrossEncoder
from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception
from app.config import OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL
client = OpenAI(api_key=OPENAI_API_KEY)
# Cross-encoder ๋ฆฌ๋žญ์ปค (์ฒซ ํ˜ธ์ถœ ์‹œ ๋ชจ๋ธ ๋‹ค์šด๋กœ๋“œ)
reranker = CrossEncoder("BAAI/bge-reranker-v2-m3")
# ๋ฌธ์žฅ ์ˆ˜๊ฐ€ ์ด ๊ฐ’์„ ์ดˆ๊ณผํ•˜๋ฉด ์˜๋ฏธ ๊ธฐ๋ฐ˜ ๋ถ„ํ•  ๋Œ€์‹  ๊ธ€์ž ์ˆ˜ ๊ธฐ๋ฐ˜ ๋ถ„ํ•  ์‚ฌ์šฉ
SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING = 500
def _simple_split_by_chars(
text: str, chunk_size: int = 1000, chunk_overlap: int = 200
) -> List[str]:
"""์ž„๋ฒ ๋”ฉ ์—†์ด ๊ธ€์ž ์ˆ˜ ๊ธฐ์ค€์œผ๋กœ๋งŒ ์ฒญํฌ ๋ถ„ํ• . ๋Œ€์šฉ๋Ÿ‰ ๋ฌธ์„œ์šฉ fallback."""
text = text.strip()
if not text:
return []
chunks: List[str] = []
start = 0
while start < len(text):
end = min(start + chunk_size, len(text))
if end < len(text):
split_pos = text.rfind(" ", start, end + 1)
if split_pos != -1:
end = split_pos + 1
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
start = (
max(0, end - chunk_overlap)
if chunk_overlap > 0 and end < len(text)
else end
)
return chunks
def _is_transient_error(e: Exception) -> bool:
"""ํƒ€์ž„์•„์›ƒยท์—ฐ๊ฒฐ ์˜ค๋ฅ˜ยทrate limit(quota ๋ถ€์กฑ ์ œ์™ธ)์„ ์žฌ์‹œ๋„ ๋Œ€์ƒ์œผ๋กœ ํŒ๋‹จ"""
if isinstance(e, (APITimeoutError, APIConnectionError)):
return True
if isinstance(e, RateLimitError):
return e.code != "insufficient_quota"
return False
@retry(
retry=retry_if_exception(_is_transient_error),
wait=wait_exponential(multiplier=1, min=2, max=30),
stop=stop_after_attempt(3),
)
def _openai_embed(text: str) -> List[float]:
response = client.embeddings.create(
model=OPENAI_EMBEDDING_MODEL,
input=text,
)
return response.data[0].embedding
def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
norm = np.linalg.norm(a) * np.linalg.norm(b)
if norm == 0:
return 0.0
return float(np.dot(a, b) / norm)
def _semantic_chunk(
sentences: List[str], chunk_size: int = 1000, chunk_overlap: int = 200
) -> List[str]:
"""๋ฌธ์žฅ ๋ฆฌ์ŠคํŠธ๋ฅผ ์ž„๋ฒ ๋”ฉ ์œ ์‚ฌ๋„ ๊ธฐ๋ฐ˜์œผ๋กœ ์˜๋ฏธ ๋‹จ์œ„ ์ฒญํฌ๋กœ ๋ถ„ํ• ํ•œ๋‹ค."""
if len(sentences) <= 1:
text = " ".join(sentences).strip()
return [text] if text else []
embeddings = [np.array(_openai_embed(s)) for s in sentences]
similarities = [
_cosine_similarity(embeddings[i], embeddings[i + 1])
for i in range(len(embeddings) - 1)
]
mean_sim = np.mean(similarities)
std_sim = np.std(similarities)
threshold = mean_sim - std_sim
# ์˜๋ฏธ ๊ฒฝ๊ณ„์—์„œ ๋ฌธ์žฅ ๊ทธ๋ฃน ๋ถ„ํ• 
groups: List[List[str]] = [[sentences[0]]]
for i, sim in enumerate(similarities):
if sim < threshold:
groups.append([])
groups[-1].append(sentences[i + 1])
# chunk_size ์ ์šฉ: ์ž‘์€ ๊ทธ๋ฃน ๋ณ‘ํ•ฉ, ํฐ ๊ทธ๋ฃน ๊ฐ•์ œ ๋ถ„ํ• 
merged: List[str] = []
buffer = ""
for group in groups:
group_text = " ".join(group)
if buffer and len(buffer) + len(group_text) + 1 > chunk_size:
merged.append(buffer.strip())
buffer = ""
buffer = (buffer + " " + group_text).strip() if buffer else group_text
if buffer:
merged.append(buffer.strip())
# ํฐ ์ฒญํฌ ๊ฐ•์ œ ๋ถ„ํ• 
chunks: List[str] = []
for chunk in merged:
while len(chunk) > chunk_size:
split_pos = chunk[:chunk_size].rfind(" ")
if split_pos == -1:
split_pos = chunk_size
chunks.append(chunk[:split_pos].strip())
chunk = chunk[split_pos:].strip()
if chunk:
chunks.append(chunk)
# chunk_overlap ์ ์šฉ
if chunk_overlap > 0 and len(chunks) > 1:
overlapped = [chunks[0]]
for i in range(1, len(chunks)):
prev = chunks[i - 1]
overlap_text = prev[-chunk_overlap:] if len(prev) > chunk_overlap else prev
space_pos = overlap_text.find(" ")
if space_pos != -1:
overlap_text = overlap_text[space_pos + 1 :]
overlapped.append((overlap_text + " " + chunks[i]).strip())
chunks = overlapped
return chunks
def split_text(
text: str, chunk_size: int = 1000, chunk_overlap: int = 200
) -> List[str]:
"""
๊ธด ํ…์ŠคํŠธ๋ฅผ ์˜๋ฏธ ๊ธฐ๋ฐ˜์œผ๋กœ ์ฒญํฌ๋กœ ๋ถ„ํ• ํ•˜๋Š” ํ•จ์ˆ˜.
1. nltk ๋ฌธ์žฅ ๋ถ„๋ฆฌ
2. OpenAI ์ž„๋ฒ ๋”ฉ์œผ๋กœ ์ธ์ ‘ ๋ฌธ์žฅ ์œ ์‚ฌ๋„ ๊ณ„์‚ฐ
3. ์ ์‘์  threshold(ํ‰๊ท  - 1ฯƒ)๋กœ ์˜๋ฏธ ๊ฒฝ๊ณ„ ๋ถ„ํ• 
4. chunk_size ์ ์šฉ (์ž‘์€ ๊ทธ๋ฃน ๋ณ‘ํ•ฉ, ํฐ ๊ทธ๋ฃน ๊ฐ•์ œ ๋ถ„ํ• )
5. chunk_overlap ์ ์šฉ
"""
sentences = nltk.sent_tokenize(text)
if len(sentences) <= 1:
return [text.strip()] if text.strip() else []
# ๋Œ€์šฉ๋Ÿ‰ ๋ฌธ์„œ: ๋ฌธ๋‹จ ๋‹จ์œ„๋กœ ๋‚˜๋ˆˆ ๋’ค ๊ฐ ๋ฌธ๋‹จ์— ์‹œ๋งจํ‹ฑ ์ฒญํ‚น ์ ์šฉ
if len(sentences) > SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING:
paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
all_chunks: List[str] = []
for para in paragraphs:
para_sentences = nltk.sent_tokenize(para)
if len(para_sentences) <= 1:
if para:
all_chunks.append(para)
elif len(para_sentences) <= SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING:
all_chunks.extend(
_semantic_chunk(para_sentences, chunk_size, chunk_overlap)
)
else:
all_chunks.extend(
_simple_split_by_chars(para, chunk_size, chunk_overlap)
)
return all_chunks if all_chunks else [text.strip()]
return _semantic_chunk(sentences, chunk_size, chunk_overlap)
def get_embedding(text: str) -> List[float]:
"""
ํ…์ŠคํŠธ์˜ ๋ฒกํ„ฐ ์ž„๋ฒ ๋”ฉ์„ ์ƒ์„ฑํ•˜๋Š” ํ•จ์ˆ˜.
OpenAI Embeddings API๋ฅผ ์‚ฌ์šฉํ•œ๋‹ค.
Args:
text (str): ์ž„๋ฒ ๋”ฉํ•  ํ…์ŠคํŠธ
Returns:
List[float]: ์ž„๋ฒ ๋”ฉ ๋ฒกํ„ฐ
"""
return _openai_embed(text)
def rerank(query: str, chunks: List[str], top_k: int = 5) -> List[str]:
"""Cross-encoder๋กœ query-chunk ์Œ์„ ์žฌํ‰๊ฐ€ํ•˜์—ฌ ๊ด€๋ จ๋„ ๋†’์€ ์ฒญํฌ๋ฅผ ์ƒ์œ„๋กœ ์ •๋ ฌ."""
if not chunks:
return []
pairs = [[query, chunk] for chunk in chunks]
scores = reranker.predict(pairs)
ranked = sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True)
return [chunk for chunk, _ in ranked[:top_k]]