from typing import List import nltk import numpy as np from openai import OpenAI, RateLimitError, APITimeoutError, APIConnectionError from sentence_transformers import CrossEncoder from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception from app.config import OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL client = OpenAI(api_key=OPENAI_API_KEY) # Cross-encoder 리랭커 (첫 호출 시 모델 다운로드) reranker = CrossEncoder("BAAI/bge-reranker-v2-m3") # 문장 수가 이 값을 초과하면 의미 기반 분할 대신 글자 수 기반 분할 사용 SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING = 500 def _simple_split_by_chars( text: str, chunk_size: int = 1000, chunk_overlap: int = 200 ) -> List[str]: """임베딩 없이 글자 수 기준으로만 청크 분할. 대용량 문서용 fallback.""" text = text.strip() if not text: return [] chunks: List[str] = [] start = 0 while start < len(text): end = min(start + chunk_size, len(text)) if end < len(text): split_pos = text.rfind(" ", start, end + 1) if split_pos != -1: end = split_pos + 1 chunk = text[start:end].strip() if chunk: chunks.append(chunk) start = ( max(0, end - chunk_overlap) if chunk_overlap > 0 and end < len(text) else end ) return chunks def _is_transient_error(e: Exception) -> bool: """타임아웃·연결 오류·rate limit(quota 부족 제외)을 재시도 대상으로 판단""" if isinstance(e, (APITimeoutError, APIConnectionError)): return True if isinstance(e, RateLimitError): return e.code != "insufficient_quota" return False @retry( retry=retry_if_exception(_is_transient_error), wait=wait_exponential(multiplier=1, min=2, max=30), stop=stop_after_attempt(3), ) def _openai_embed(text: str) -> List[float]: response = client.embeddings.create( model=OPENAI_EMBEDDING_MODEL, input=text, ) return response.data[0].embedding def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: norm = np.linalg.norm(a) * np.linalg.norm(b) if norm == 0: return 0.0 return float(np.dot(a, b) / norm) def _semantic_chunk( sentences: List[str], chunk_size: int = 1000, chunk_overlap: int = 200 ) -> List[str]: """문장 리스트를 임베딩 유사도 기반으로 의미 단위 청크로 분할한다.""" if len(sentences) <= 1: text = " ".join(sentences).strip() return [text] if text else [] embeddings = [np.array(_openai_embed(s)) for s in sentences] similarities = [ _cosine_similarity(embeddings[i], embeddings[i + 1]) for i in range(len(embeddings) - 1) ] mean_sim = np.mean(similarities) std_sim = np.std(similarities) threshold = mean_sim - std_sim # 의미 경계에서 문장 그룹 분할 groups: List[List[str]] = [[sentences[0]]] for i, sim in enumerate(similarities): if sim < threshold: groups.append([]) groups[-1].append(sentences[i + 1]) # chunk_size 적용: 작은 그룹 병합, 큰 그룹 강제 분할 merged: List[str] = [] buffer = "" for group in groups: group_text = " ".join(group) if buffer and len(buffer) + len(group_text) + 1 > chunk_size: merged.append(buffer.strip()) buffer = "" buffer = (buffer + " " + group_text).strip() if buffer else group_text if buffer: merged.append(buffer.strip()) # 큰 청크 강제 분할 chunks: List[str] = [] for chunk in merged: while len(chunk) > chunk_size: split_pos = chunk[:chunk_size].rfind(" ") if split_pos == -1: split_pos = chunk_size chunks.append(chunk[:split_pos].strip()) chunk = chunk[split_pos:].strip() if chunk: chunks.append(chunk) # chunk_overlap 적용 if chunk_overlap > 0 and len(chunks) > 1: overlapped = [chunks[0]] for i in range(1, len(chunks)): prev = chunks[i - 1] overlap_text = prev[-chunk_overlap:] if len(prev) > chunk_overlap else prev space_pos = overlap_text.find(" ") if space_pos != -1: overlap_text = overlap_text[space_pos + 1 :] overlapped.append((overlap_text + " " + chunks[i]).strip()) chunks = overlapped return chunks def split_text( text: str, chunk_size: int = 1000, chunk_overlap: int = 200 ) -> List[str]: """ 긴 텍스트를 의미 기반으로 청크로 분할하는 함수. 1. nltk 문장 분리 2. OpenAI 임베딩으로 인접 문장 유사도 계산 3. 적응적 threshold(평균 - 1σ)로 의미 경계 분할 4. chunk_size 적용 (작은 그룹 병합, 큰 그룹 강제 분할) 5. chunk_overlap 적용 """ sentences = nltk.sent_tokenize(text) if len(sentences) <= 1: return [text.strip()] if text.strip() else [] # 대용량 문서: 문단 단위로 나눈 뒤 각 문단에 시맨틱 청킹 적용 if len(sentences) > SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING: paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()] all_chunks: List[str] = [] for para in paragraphs: para_sentences = nltk.sent_tokenize(para) if len(para_sentences) <= 1: if para: all_chunks.append(para) elif len(para_sentences) <= SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING: all_chunks.extend( _semantic_chunk(para_sentences, chunk_size, chunk_overlap) ) else: all_chunks.extend( _simple_split_by_chars(para, chunk_size, chunk_overlap) ) return all_chunks if all_chunks else [text.strip()] return _semantic_chunk(sentences, chunk_size, chunk_overlap) def get_embedding(text: str) -> List[float]: """ 텍스트의 벡터 임베딩을 생성하는 함수. OpenAI Embeddings API를 사용한다. Args: text (str): 임베딩할 텍스트 Returns: List[float]: 임베딩 벡터 """ return _openai_embed(text) def rerank(query: str, chunks: List[str], top_k: int = 5) -> List[str]: """Cross-encoder로 query-chunk 쌍을 재평가하여 관련도 높은 청크를 상위로 정렬.""" if not chunks: return [] pairs = [[query, chunk] for chunk in chunks] scores = reranker.predict(pairs) ranked = sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True) return [chunk for chunk, _ in ranked[:top_k]]