Spaces:
Runtime error
Runtime error
Download app/embedding_utils.py from progressive-intelligence-lab/docuflow: direct link, hf CLI and curl.
- Browser
- Download file 6.76 kB
-
https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/app/embedding_utils.py
- Command line
-
hf download hf://spaces/progressive-intelligence-lab/docuflow/app/embedding_utils.py
-
curl -L -o embedding_utils.py https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/app/embedding_utils.py
6.76 kB
| from typing import List | |
| import nltk | |
| import numpy as np | |
| from openai import OpenAI, RateLimitError, APITimeoutError, APIConnectionError | |
| from sentence_transformers import CrossEncoder | |
| from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception | |
| from app.config import OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL | |
| client = OpenAI(api_key=OPENAI_API_KEY) | |
| # Cross-encoder ๋ฆฌ๋ญ์ปค (์ฒซ ํธ์ถ ์ ๋ชจ๋ธ ๋ค์ด๋ก๋) | |
| reranker = CrossEncoder("BAAI/bge-reranker-v2-m3") | |
| # ๋ฌธ์ฅ ์๊ฐ ์ด ๊ฐ์ ์ด๊ณผํ๋ฉด ์๋ฏธ ๊ธฐ๋ฐ ๋ถํ ๋์ ๊ธ์ ์ ๊ธฐ๋ฐ ๋ถํ ์ฌ์ฉ | |
| SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING = 500 | |
| def _simple_split_by_chars( | |
| text: str, chunk_size: int = 1000, chunk_overlap: int = 200 | |
| ) -> List[str]: | |
| """์๋ฒ ๋ฉ ์์ด ๊ธ์ ์ ๊ธฐ์ค์ผ๋ก๋ง ์ฒญํฌ ๋ถํ . ๋์ฉ๋ ๋ฌธ์์ฉ fallback.""" | |
| text = text.strip() | |
| if not text: | |
| return [] | |
| chunks: List[str] = [] | |
| start = 0 | |
| while start < len(text): | |
| end = min(start + chunk_size, len(text)) | |
| if end < len(text): | |
| split_pos = text.rfind(" ", start, end + 1) | |
| if split_pos != -1: | |
| end = split_pos + 1 | |
| chunk = text[start:end].strip() | |
| if chunk: | |
| chunks.append(chunk) | |
| start = ( | |
| max(0, end - chunk_overlap) | |
| if chunk_overlap > 0 and end < len(text) | |
| else end | |
| ) | |
| return chunks | |
| def _is_transient_error(e: Exception) -> bool: | |
| """ํ์์์ยท์ฐ๊ฒฐ ์ค๋ฅยทrate limit(quota ๋ถ์กฑ ์ ์ธ)์ ์ฌ์๋ ๋์์ผ๋ก ํ๋จ""" | |
| if isinstance(e, (APITimeoutError, APIConnectionError)): | |
| return True | |
| if isinstance(e, RateLimitError): | |
| return e.code != "insufficient_quota" | |
| return False | |
| def _openai_embed(text: str) -> List[float]: | |
| response = client.embeddings.create( | |
| model=OPENAI_EMBEDDING_MODEL, | |
| input=text, | |
| ) | |
| return response.data[0].embedding | |
| def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: | |
| norm = np.linalg.norm(a) * np.linalg.norm(b) | |
| if norm == 0: | |
| return 0.0 | |
| return float(np.dot(a, b) / norm) | |
| def _semantic_chunk( | |
| sentences: List[str], chunk_size: int = 1000, chunk_overlap: int = 200 | |
| ) -> List[str]: | |
| """๋ฌธ์ฅ ๋ฆฌ์คํธ๋ฅผ ์๋ฒ ๋ฉ ์ ์ฌ๋ ๊ธฐ๋ฐ์ผ๋ก ์๋ฏธ ๋จ์ ์ฒญํฌ๋ก ๋ถํ ํ๋ค.""" | |
| if len(sentences) <= 1: | |
| text = " ".join(sentences).strip() | |
| return [text] if text else [] | |
| embeddings = [np.array(_openai_embed(s)) for s in sentences] | |
| similarities = [ | |
| _cosine_similarity(embeddings[i], embeddings[i + 1]) | |
| for i in range(len(embeddings) - 1) | |
| ] | |
| mean_sim = np.mean(similarities) | |
| std_sim = np.std(similarities) | |
| threshold = mean_sim - std_sim | |
| # ์๋ฏธ ๊ฒฝ๊ณ์์ ๋ฌธ์ฅ ๊ทธ๋ฃน ๋ถํ | |
| groups: List[List[str]] = [[sentences[0]]] | |
| for i, sim in enumerate(similarities): | |
| if sim < threshold: | |
| groups.append([]) | |
| groups[-1].append(sentences[i + 1]) | |
| # chunk_size ์ ์ฉ: ์์ ๊ทธ๋ฃน ๋ณํฉ, ํฐ ๊ทธ๋ฃน ๊ฐ์ ๋ถํ | |
| merged: List[str] = [] | |
| buffer = "" | |
| for group in groups: | |
| group_text = " ".join(group) | |
| if buffer and len(buffer) + len(group_text) + 1 > chunk_size: | |
| merged.append(buffer.strip()) | |
| buffer = "" | |
| buffer = (buffer + " " + group_text).strip() if buffer else group_text | |
| if buffer: | |
| merged.append(buffer.strip()) | |
| # ํฐ ์ฒญํฌ ๊ฐ์ ๋ถํ | |
| chunks: List[str] = [] | |
| for chunk in merged: | |
| while len(chunk) > chunk_size: | |
| split_pos = chunk[:chunk_size].rfind(" ") | |
| if split_pos == -1: | |
| split_pos = chunk_size | |
| chunks.append(chunk[:split_pos].strip()) | |
| chunk = chunk[split_pos:].strip() | |
| if chunk: | |
| chunks.append(chunk) | |
| # chunk_overlap ์ ์ฉ | |
| if chunk_overlap > 0 and len(chunks) > 1: | |
| overlapped = [chunks[0]] | |
| for i in range(1, len(chunks)): | |
| prev = chunks[i - 1] | |
| overlap_text = prev[-chunk_overlap:] if len(prev) > chunk_overlap else prev | |
| space_pos = overlap_text.find(" ") | |
| if space_pos != -1: | |
| overlap_text = overlap_text[space_pos + 1 :] | |
| overlapped.append((overlap_text + " " + chunks[i]).strip()) | |
| chunks = overlapped | |
| return chunks | |
| def split_text( | |
| text: str, chunk_size: int = 1000, chunk_overlap: int = 200 | |
| ) -> List[str]: | |
| """ | |
| ๊ธด ํ ์คํธ๋ฅผ ์๋ฏธ ๊ธฐ๋ฐ์ผ๋ก ์ฒญํฌ๋ก ๋ถํ ํ๋ ํจ์. | |
| 1. nltk ๋ฌธ์ฅ ๋ถ๋ฆฌ | |
| 2. OpenAI ์๋ฒ ๋ฉ์ผ๋ก ์ธ์ ๋ฌธ์ฅ ์ ์ฌ๋ ๊ณ์ฐ | |
| 3. ์ ์์ threshold(ํ๊ท - 1ฯ)๋ก ์๋ฏธ ๊ฒฝ๊ณ ๋ถํ | |
| 4. chunk_size ์ ์ฉ (์์ ๊ทธ๋ฃน ๋ณํฉ, ํฐ ๊ทธ๋ฃน ๊ฐ์ ๋ถํ ) | |
| 5. chunk_overlap ์ ์ฉ | |
| """ | |
| sentences = nltk.sent_tokenize(text) | |
| if len(sentences) <= 1: | |
| return [text.strip()] if text.strip() else [] | |
| # ๋์ฉ๋ ๋ฌธ์: ๋ฌธ๋จ ๋จ์๋ก ๋๋ ๋ค ๊ฐ ๋ฌธ๋จ์ ์๋งจํฑ ์ฒญํน ์ ์ฉ | |
| if len(sentences) > SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING: | |
| paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()] | |
| all_chunks: List[str] = [] | |
| for para in paragraphs: | |
| para_sentences = nltk.sent_tokenize(para) | |
| if len(para_sentences) <= 1: | |
| if para: | |
| all_chunks.append(para) | |
| elif len(para_sentences) <= SENTENCE_LIMIT_FOR_SEMANTIC_CHUNKING: | |
| all_chunks.extend( | |
| _semantic_chunk(para_sentences, chunk_size, chunk_overlap) | |
| ) | |
| else: | |
| all_chunks.extend( | |
| _simple_split_by_chars(para, chunk_size, chunk_overlap) | |
| ) | |
| return all_chunks if all_chunks else [text.strip()] | |
| return _semantic_chunk(sentences, chunk_size, chunk_overlap) | |
| def get_embedding(text: str) -> List[float]: | |
| """ | |
| ํ ์คํธ์ ๋ฒกํฐ ์๋ฒ ๋ฉ์ ์์ฑํ๋ ํจ์. | |
| OpenAI Embeddings API๋ฅผ ์ฌ์ฉํ๋ค. | |
| Args: | |
| text (str): ์๋ฒ ๋ฉํ ํ ์คํธ | |
| Returns: | |
| List[float]: ์๋ฒ ๋ฉ ๋ฒกํฐ | |
| """ | |
| return _openai_embed(text) | |
| def rerank(query: str, chunks: List[str], top_k: int = 5) -> List[str]: | |
| """Cross-encoder๋ก query-chunk ์์ ์ฌํ๊ฐํ์ฌ ๊ด๋ จ๋ ๋์ ์ฒญํฌ๋ฅผ ์์๋ก ์ ๋ ฌ.""" | |
| if not chunks: | |
| return [] | |
| pairs = [[query, chunk] for chunk in chunks] | |
| scores = reranker.predict(pairs) | |
| ranked = sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True) | |
| return [chunk for chunk, _ in ranked[:top_k]] | |