Spaces:
Runtime error
Runtime error
File size: 6,759 Bytes
1e570c8 74b444e 37c1245 afc8097 7ed0ad9 0c6fd3a 7ed0ad9 0a7fe27 afc8097 1990560 afc8097 1990560 0a7fe27 37c1245 2c18c5e 37c1245 2c18c5e 37c1245 2c18c5e 0c6fd3a 0a7fe27 0c6fd3a 0a7fe27 1e570c8 f537bfd 1e570c8 f537bfd 0a7fe27 f537bfd 1990560 0c6fd3a 74b444e 0a7fe27 74b444e 0a7fe27 74b444e 0a7fe27 1e570c8 f537bfd 0c6fd3a f537bfd 1e570c8 0c6fd3a 1e570c8 0c6fd3a afc8097 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 | from typing import List
import nltk
import numpy as np
from openai import OpenAI, RateLimitError, APITimeoutError, APIConnectionError
from sentence_transformers import CrossEncoder
from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception
from app.config import OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL
client = OpenAI(api_key=OPENAI_API_KEY)
# Cross-encoder ๋ฆฌ๋ญ์ปค (์ฒซ ํธ์ถ ์ ๋ชจ๋ธ ๋ค์ด๋ก๋)
reranker = CrossEncoder("BAAI/bge-reranker-v2-m3")
# ๋ฌธ์ฅ ์๊ฐ ์ด ๊ฐ์ ์ด๊ณผํ๋ฉด ์๋ฏธ ๊ธฐ๋ฐ ๋ถํ ๋์ ๊ธ์ ์ ๊ธฐ๋ฐ ๋ถํ ์ฌ์ฉ
SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING = 500
def _simple_split_by_chars(
text: str, chunk_size: int = 1000, chunk_overlap: int = 200
) -> List[str]:
"""์๋ฒ ๋ฉ ์์ด ๊ธ์ ์ ๊ธฐ์ค์ผ๋ก๋ง ์ฒญํฌ ๋ถํ . ๋์ฉ๋ ๋ฌธ์์ฉ fallback."""
text = text.strip()
if not text:
return []
chunks: List[str] = []
start = 0
while start < len(text):
end = min(start + chunk_size, len(text))
if end < len(text):
split_pos = text.rfind(" ", start, end + 1)
if split_pos != -1:
end = split_pos + 1
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
start = (
max(0, end - chunk_overlap)
if chunk_overlap > 0 and end < len(text)
else end
)
return chunks
def _is_transient_error(e: Exception) -> bool:
"""ํ์์์ยท์ฐ๊ฒฐ ์ค๋ฅยทrate limit(quota ๋ถ์กฑ ์ ์ธ)์ ์ฌ์๋ ๋์์ผ๋ก ํ๋จ"""
if isinstance(e, (APITimeoutError, APIConnectionError)):
return True
if isinstance(e, RateLimitError):
return e.code != "insufficient_quota"
return False
@retry(
retry=retry_if_exception(_is_transient_error),
wait=wait_exponential(multiplier=1, min=2, max=30),
stop=stop_after_attempt(3),
)
def _openai_embed(text: str) -> List[float]:
response = client.embeddings.create(
model=OPENAI_EMBEDDING_MODEL,
input=text,
)
return response.data[0].embedding
def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
norm = np.linalg.norm(a) * np.linalg.norm(b)
if norm == 0:
return 0.0
return float(np.dot(a, b) / norm)
def _semantic_chunk(
sentences: List[str], chunk_size: int = 1000, chunk_overlap: int = 200
) -> List[str]:
"""๋ฌธ์ฅ ๋ฆฌ์คํธ๋ฅผ ์๋ฒ ๋ฉ ์ ์ฌ๋ ๊ธฐ๋ฐ์ผ๋ก ์๋ฏธ ๋จ์ ์ฒญํฌ๋ก ๋ถํ ํ๋ค."""
if len(sentences) <= 1:
text = " ".join(sentences).strip()
return [text] if text else []
embeddings = [np.array(_openai_embed(s)) for s in sentences]
similarities = [
_cosine_similarity(embeddings[i], embeddings[i + 1])
for i in range(len(embeddings) - 1)
]
mean_sim = np.mean(similarities)
std_sim = np.std(similarities)
threshold = mean_sim - std_sim
# ์๋ฏธ ๊ฒฝ๊ณ์์ ๋ฌธ์ฅ ๊ทธ๋ฃน ๋ถํ
groups: List[List[str]] = [[sentences[0]]]
for i, sim in enumerate(similarities):
if sim < threshold:
groups.append([])
groups[-1].append(sentences[i + 1])
# chunk_size ์ ์ฉ: ์์ ๊ทธ๋ฃน ๋ณํฉ, ํฐ ๊ทธ๋ฃน ๊ฐ์ ๋ถํ
merged: List[str] = []
buffer = ""
for group in groups:
group_text = " ".join(group)
if buffer and len(buffer) + len(group_text) + 1 > chunk_size:
merged.append(buffer.strip())
buffer = ""
buffer = (buffer + " " + group_text).strip() if buffer else group_text
if buffer:
merged.append(buffer.strip())
# ํฐ ์ฒญํฌ ๊ฐ์ ๋ถํ
chunks: List[str] = []
for chunk in merged:
while len(chunk) > chunk_size:
split_pos = chunk[:chunk_size].rfind(" ")
if split_pos == -1:
split_pos = chunk_size
chunks.append(chunk[:split_pos].strip())
chunk = chunk[split_pos:].strip()
if chunk:
chunks.append(chunk)
# chunk_overlap ์ ์ฉ
if chunk_overlap > 0 and len(chunks) > 1:
overlapped = [chunks[0]]
for i in range(1, len(chunks)):
prev = chunks[i - 1]
overlap_text = prev[-chunk_overlap:] if len(prev) > chunk_overlap else prev
space_pos = overlap_text.find(" ")
if space_pos != -1:
overlap_text = overlap_text[space_pos + 1 :]
overlapped.append((overlap_text + " " + chunks[i]).strip())
chunks = overlapped
return chunks
def split_text(
text: str, chunk_size: int = 1000, chunk_overlap: int = 200
) -> List[str]:
"""
๊ธด ํ
์คํธ๋ฅผ ์๋ฏธ ๊ธฐ๋ฐ์ผ๋ก ์ฒญํฌ๋ก ๋ถํ ํ๋ ํจ์.
1. nltk ๋ฌธ์ฅ ๋ถ๋ฆฌ
2. OpenAI ์๋ฒ ๋ฉ์ผ๋ก ์ธ์ ๋ฌธ์ฅ ์ ์ฌ๋ ๊ณ์ฐ
3. ์ ์์ threshold(ํ๊ท - 1ฯ)๋ก ์๋ฏธ ๊ฒฝ๊ณ ๋ถํ
4. chunk_size ์ ์ฉ (์์ ๊ทธ๋ฃน ๋ณํฉ, ํฐ ๊ทธ๋ฃน ๊ฐ์ ๋ถํ )
5. chunk_overlap ์ ์ฉ
"""
sentences = nltk.sent_tokenize(text)
if len(sentences) <= 1:
return [text.strip()] if text.strip() else []
# ๋์ฉ๋ ๋ฌธ์: ๋ฌธ๋จ ๋จ์๋ก ๋๋ ๋ค ๊ฐ ๋ฌธ๋จ์ ์๋งจํฑ ์ฒญํน ์ ์ฉ
if len(sentences) > SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING:
paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
all_chunks: List[str] = []
for para in paragraphs:
para_sentences = nltk.sent_tokenize(para)
if len(para_sentences) <= 1:
if para:
all_chunks.append(para)
elif len(para_sentences) <= SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING:
all_chunks.extend(
_semantic_chunk(para_sentences, chunk_size, chunk_overlap)
)
else:
all_chunks.extend(
_simple_split_by_chars(para, chunk_size, chunk_overlap)
)
return all_chunks if all_chunks else [text.strip()]
return _semantic_chunk(sentences, chunk_size, chunk_overlap)
def get_embedding(text: str) -> List[float]:
"""
ํ
์คํธ์ ๋ฒกํฐ ์๋ฒ ๋ฉ์ ์์ฑํ๋ ํจ์.
OpenAI Embeddings API๋ฅผ ์ฌ์ฉํ๋ค.
Args:
text (str): ์๋ฒ ๋ฉํ ํ
์คํธ
Returns:
List[float]: ์๋ฒ ๋ฉ ๋ฒกํฐ
"""
return _openai_embed(text)
def rerank(query: str, chunks: List[str], top_k: int = 5) -> List[str]:
"""Cross-encoder๋ก query-chunk ์์ ์ฌํ๊ฐํ์ฌ ๊ด๋ จ๋ ๋์ ์ฒญํฌ๋ฅผ ์์๋ก ์ ๋ ฌ."""
if not chunks:
return []
pairs = [[query, chunk] for chunk in chunks]
scores = reranker.predict(pairs)
ranked = sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True)
return [chunk for chunk, _ in ranked[:top_k]]
|