Spaces:
Runtime error
Runtime error
Download app/supabase_utils.py from progressive-intelligence-lab/docuflow: direct link, hf CLI and curl.
- Browser
- Download file 6.24 kB
-
https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/app/supabase_utils.py
- Command line
-
hf download hf://spaces/progressive-intelligence-lab/docuflow/app/supabase_utils.py
-
curl -L -o supabase_utils.py https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/app/supabase_utils.py
6.24 kB
| from supabase import create_client, Client | |
| from typing import List, Dict, Any | |
| import os | |
| # ํ๊ฒฝ ๋ณ์ ๋ก๋ ๋ฐ Supabase ํด๋ผ์ด์ธํธ ์ด๊ธฐํ | |
| BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100")) | |
| SUPABASE_URL = os.getenv("SUPABASE_URL") | |
| SUPABASE_KEY = os.getenv("SUPABASE_KEY") | |
| if not SUPABASE_URL or not SUPABASE_KEY: | |
| raise ValueError("SUPABASE_URL๊ณผ SUPABASE_KEY๊ฐ ์ค์ ๋์ด ์์ด์ผ ํฉ๋๋ค.") | |
| supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY) | |
| def save_embedding( | |
| document_id: str, | |
| chunk: str, | |
| embedding: List[float], | |
| chunk_index: int, | |
| file_hash: str | None = None, | |
| ) -> Dict: | |
| """ | |
| [TODO Day 2] ๋ฌธ์ ์ฒญํฌ์ ์๋ฒ ๋ฉ ๋ฒกํฐ๋ฅผ Supabase DB์ ์ ์ฅํ๋ ํจ์์ ๋๋ค. | |
| ์๊ตฌ์ฌํญ: | |
| 1. README.md์ ๋ฐ์ดํฐ๋ฒ ์ด์ค ๊ตฌ์ฑ ์น์ ์์ ์์ฑํ ํ ์ด๋ธ์ ๋ฐ์ดํฐ๋ฅผ ์ ์ฅํ์ธ์. | |
| - ์ด๋ค ์ปฌ๋ผ์ ์ด๋ค ๊ฐ์ ์ ์ฅํด์ผ ํ๋์ง SQL ์คํค๋ง๋ฅผ ์ง์ ํ์ธํ์ธ์. | |
| 2. Supabase Python ํด๋ผ์ด์ธํธ ์ฌ์ฉ ๋ฐฉ๋ฒ์ ์ง์ ์กฐ์ฌํ์ธ์. | |
| - ํํธ: ์์์ ์ด๋ฏธ ์ด๊ธฐํ๋ `supabase` ํด๋ผ์ด์ธํธ๋ฅผ ํ์ฉํ์ธ์. | |
| Args: | |
| document_id (str): ๋ฌธ์ ID | |
| chunk (str): ํ ์คํธ ์ฒญํฌ ๋ด์ฉ | |
| embedding (List[float]): ์๋ฒ ๋ฉ ๋ฒกํฐ | |
| chunk_index (int): ์ฒญํฌ ์์ ์ธ๋ฑ์ค | |
| Returns: | |
| Dict: ์ ์ฅ๋ ๋ฐ์ดํฐ ์ ๋ณด | |
| """ | |
| # --------------------------------------------------------- | |
| # [Day 2] ๊ณผ์ : ์ฌ๊ธฐ์ DB ์ ์ฅ ๋ก์ง์ ์์ฑํ์ธ์. | |
| # --------------------------------------------------------- | |
| if file_hash: | |
| metadata = {"file_hash": file_hash} | |
| else: | |
| metadata = {} | |
| data = { | |
| "document_id": document_id, | |
| "content": chunk, | |
| "embedding": embedding, | |
| "chunk_index": chunk_index, | |
| "metadata": metadata, | |
| } | |
| result = supabase.table("document_chunks").insert(data).execute() | |
| return result.data | |
| def save_embeddings_batch( | |
| document_id: str, | |
| chunks: List[str], | |
| embeddings: List[List[float]], | |
| file_hash: str | None = None, | |
| ) -> None: | |
| """ | |
| ์ฌ๋ฌ ์ฒญํฌ๋ฅผ ํ ๋ฒ์ DB์ ์ ์ฅํฉ๋๋ค. | |
| """ | |
| metadata = {"file_hash": file_hash} if file_hash else {} | |
| rows = [ | |
| { | |
| "document_id": document_id, | |
| "content": chunk, | |
| "embedding": embedding, | |
| "chunk_index": i, | |
| "metadata": metadata, | |
| } | |
| for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)) | |
| ] | |
| # Supabase๋ ํ ๋ฒ์ ์ฌ๋ฌ ํ insert ์ง์ | |
| for i in range(0, len(rows), BATCH_SIZE): | |
| batch = rows[i : i + BATCH_SIZE] | |
| supabase.table("document_chunks").insert(batch).execute() | |
| def search_similar_embeddings( | |
| query_embedding: List[float], | |
| document_id: str, | |
| limit: int = 10, | |
| similarity_threshold: float = 0.6, | |
| ) -> List[Dict[str, Any]]: | |
| """ | |
| [TODO Day 3] ์ฌ์ฉ์์ ์ง๋ฌธ ๋ฒกํฐ์ ์ ์ฌํ ๋ฌธ์ ์ฒญํฌ๋ฅผ ๊ฒ์ํ๋ ํจ์์ ๋๋ค. | |
| ์๊ตฌ์ฌํญ: | |
| 1. README.md์ ๋ฐ์ดํฐ๋ฒ ์ด์ค ๊ตฌ์ฑ ์น์ ์์ ์์ฑํ RPC ํจ์๋ฅผ ํ์ฉํ์ฌ ์ ์ฌ๋ ๊ฒ์์ ์ํํ์ธ์. | |
| - ์ด๋ค ์ธ์๋ฅผ ์ ๋ฌํด์ผ ํ๋์ง SQL ์คํค๋ง์ RPC ํจ์ ์ ์๋ฅผ ์ง์ ํ์ธํ์ธ์. | |
| 2. ์ ์ฌ๋(similarity)๊ฐ similarity_threshold ์ด์์ธ ๊ฒฐ๊ณผ๋ง ํํฐ๋งํ์ธ์. (์ ํ์ฌํญ) | |
| Args: | |
| query_embedding (List[float]): ์ง๋ฌธ์ ์๋ฒ ๋ฉ ๋ฒกํฐ | |
| document_id (str): ๊ฒ์ํ ๋ฌธ์ ID | |
| limit (int): ๋ฐํํ ์ต๋ ๊ฒฐ๊ณผ ์ | |
| similarity_threshold (float): ์ ์ฌ๋ ์๊ณ๊ฐ | |
| Returns: | |
| List[Dict[str, Any]]: ์ ์ฌํ ๋ฌธ์ ์ฒญํฌ ๋ฆฌ์คํธ | |
| """ | |
| result = supabase.rpc( | |
| "match_documents", | |
| { | |
| "query_embedding": query_embedding, | |
| "match_count": limit, | |
| "p_document_id": document_id, | |
| }, | |
| ).execute() | |
| # ๋๋ฒ๊น : ๋ฒกํฐ ๊ฒ์ ๊ฒฐ๊ณผ ์์ฝ | |
| print(f"\n{'='*60}") | |
| print( | |
| f" ๋ฒกํฐ ๊ฒ์ ๊ฒฐ๊ณผ (์ด {len(result.data)}๊ฑด, ์๊ณ๊ฐ >= {similarity_threshold})" | |
| ) | |
| print(f"{'='*60}") | |
| for i, row in enumerate(result.data, 1): | |
| sim = row.get("similarity", 0) | |
| preview = row.get("content", "")[:60].replace("\n", " ") | |
| marker = "โ" if sim >= similarity_threshold else "โ" | |
| print(f" {marker} [{i:2d}] ์ ์ฌ๋ {sim:.4f} | {preview}...") | |
| print(f"{'='*60}") | |
| filtered = [ | |
| row for row in result.data if row.get("similarity", 0) >= similarity_threshold | |
| ] | |
| print(f" โ ํต๊ณผ: {len(filtered)}๊ฑด / ์ ์ธ: {len(result.data) - len(filtered)}๊ฑด") | |
| return filtered | |
| def search_by_keyword( | |
| query: str, document_id: str, limit: int = 5 | |
| ) -> List[Dict[str, Any]]: | |
| """ํค์๋ ๊ธฐ๋ฐ ๊ฒ์ (๋ฒกํฐ ๊ฒ์ ๋ณด์์ฉ). ๊ณ ์ ๋ช ์ฌ ๋ฑ ์ ํํ ๋งค์นญ์ด ํ์ํ ๋ ์ฌ์ฉ.""" | |
| STOP_WORDS = { | |
| "์", | |
| "๋", | |
| "์ด", | |
| "๊ฐ", | |
| "์", | |
| "๋ฅผ", | |
| "์", | |
| "์", | |
| "์์", | |
| "๋ก", | |
| "์ผ๋ก", | |
| "์", | |
| "๊ณผ", | |
| "๋", | |
| "๋ง", | |
| "์๊ฒ", | |
| "ํ", | |
| "ํ๋", | |
| "์๋", | |
| "์๋", | |
| "์๋ ค์ฃผ์ธ์", | |
| "์๋ ค์ค", | |
| "๋ฌด์์ธ๊ฐ์", | |
| "๋ญ์ผ", | |
| "์ด๋ค", | |
| } | |
| words = query.replace("?", "").replace(".", "").replace(",", "").split() | |
| keywords = [w for w in words if len(w) > 1 and w not in STOP_WORDS] | |
| if not keywords: | |
| return [] | |
| seen = set() | |
| results = [] | |
| for kw in keywords: | |
| rows = ( | |
| supabase.table("document_chunks") | |
| .select("content") | |
| .eq("document_id", document_id) | |
| .ilike("content", f"%{kw}%") | |
| .limit(limit) | |
| .execute() | |
| ).data | |
| for row in rows: | |
| if row["content"] not in seen: | |
| seen.add(row["content"]) | |
| results.append(row) | |
| return results[:limit] | |
| def get_document_id_by_hash(file_hash: str) -> str: | |
| result = supabase.rpc("get_document_by_hash", {"p_hash": file_hash}).execute() | |
| if result.data: | |
| return result.data[0]["document_id"] | |
| return None | |