docuflow / app /supabase_utils.py
woongmin's picture
[Delta-02] Drill B ํ…Œ์ŠคํŠธ ํ™•์ธ ํ›„ ์ฝ”๋“œ ์›์ƒ ๋ณต๊ตฌ
31e7ade
Raw History Blame Contribute Delete
6.24 kB
from supabase import create_client, Client
from typing import List, Dict, Any
import os
# ํ™˜๊ฒฝ ๋ณ€์ˆ˜ ๋กœ๋“œ ๋ฐ Supabase ํด๋ผ์ด์–ธํŠธ ์ดˆ๊ธฐํ™”
BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100"))
SUPABASE_URL = os.getenv("SUPABASE_URL")
SUPABASE_KEY = os.getenv("SUPABASE_KEY")
if not SUPABASE_URL or not SUPABASE_KEY:
raise ValueError("SUPABASE_URL๊ณผ SUPABASE_KEY๊ฐ€ ์„ค์ •๋˜์–ด ์žˆ์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค.")
supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
def save_embedding(
document_id: str,
chunk: str,
embedding: List[float],
chunk_index: int,
file_hash: str | None = None,
) -> Dict:
"""
[TODO Day 2] ๋ฌธ์„œ ์ฒญํฌ์™€ ์ž„๋ฒ ๋”ฉ ๋ฒกํ„ฐ๋ฅผ Supabase DB์— ์ €์žฅํ•˜๋Š” ํ•จ์ˆ˜์ž…๋‹ˆ๋‹ค.
์š”๊ตฌ์‚ฌํ•ญ:
1. README.md์˜ ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค ๊ตฌ์„ฑ ์„น์…˜์—์„œ ์ƒ์„ฑํ•œ ํ…Œ์ด๋ธ”์— ๋ฐ์ดํ„ฐ๋ฅผ ์ €์žฅํ•˜์„ธ์š”.
- ์–ด๋–ค ์ปฌ๋Ÿผ์— ์–ด๋–ค ๊ฐ’์„ ์ €์žฅํ•ด์•ผ ํ•˜๋Š”์ง€ SQL ์Šคํ‚ค๋งˆ๋ฅผ ์ง์ ‘ ํ™•์ธํ•˜์„ธ์š”.
2. Supabase Python ํด๋ผ์ด์–ธํŠธ ์‚ฌ์šฉ ๋ฐฉ๋ฒ•์€ ์ง์ ‘ ์กฐ์‚ฌํ•˜์„ธ์š”.
- ํžŒํŠธ: ์œ„์—์„œ ์ด๋ฏธ ์ดˆ๊ธฐํ™”๋œ `supabase` ํด๋ผ์ด์–ธํŠธ๋ฅผ ํ™œ์šฉํ•˜์„ธ์š”.
Args:
document_id (str): ๋ฌธ์„œ ID
chunk (str): ํ…์ŠคํŠธ ์ฒญํฌ ๋‚ด์šฉ
embedding (List[float]): ์ž„๋ฒ ๋”ฉ ๋ฒกํ„ฐ
chunk_index (int): ์ฒญํฌ ์ˆœ์„œ ์ธ๋ฑ์Šค
Returns:
Dict: ์ €์žฅ๋œ ๋ฐ์ดํ„ฐ ์ •๋ณด
"""
# ---------------------------------------------------------
# [Day 2] ๊ณผ์ œ: ์—ฌ๊ธฐ์— DB ์ €์žฅ ๋กœ์ง์„ ์ž‘์„ฑํ•˜์„ธ์š”.
# ---------------------------------------------------------
if file_hash:
metadata = {"file_hash": file_hash}
else:
metadata = {}
data = {
"document_id": document_id,
"content": chunk,
"embedding": embedding,
"chunk_index": chunk_index,
"metadata": metadata,
}
result = supabase.table("document_chunks").insert(data).execute()
return result.data
def save_embeddings_batch(
document_id: str,
chunks: List[str],
embeddings: List[List[float]],
file_hash: str | None = None,
) -> None:
"""
์—ฌ๋Ÿฌ ์ฒญํฌ๋ฅผ ํ•œ ๋ฒˆ์— DB์— ์ €์žฅํ•ฉ๋‹ˆ๋‹ค.
"""
metadata = {"file_hash": file_hash} if file_hash else {}
rows = [
{
"document_id": document_id,
"content": chunk,
"embedding": embedding,
"chunk_index": i,
"metadata": metadata,
}
for i, (chunk, embedding) in enumerate(zip(chunks, embeddings))
]
# Supabase๋Š” ํ•œ ๋ฒˆ์— ์—ฌ๋Ÿฌ ํ–‰ insert ์ง€์›
for i in range(0, len(rows), BATCH_SIZE):
batch = rows[i : i + BATCH_SIZE]
supabase.table("document_chunks").insert(batch).execute()
def search_similar_embeddings(
query_embedding: List[float],
document_id: str,
limit: int = 10,
similarity_threshold: float = 0.6,
) -> List[Dict[str, Any]]:
"""
[TODO Day 3] ์‚ฌ์šฉ์ž์˜ ์งˆ๋ฌธ ๋ฒกํ„ฐ์™€ ์œ ์‚ฌํ•œ ๋ฌธ์„œ ์ฒญํฌ๋ฅผ ๊ฒ€์ƒ‰ํ•˜๋Š” ํ•จ์ˆ˜์ž…๋‹ˆ๋‹ค.
์š”๊ตฌ์‚ฌํ•ญ:
1. README.md์˜ ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค ๊ตฌ์„ฑ ์„น์…˜์—์„œ ์ƒ์„ฑํ•œ RPC ํ•จ์ˆ˜๋ฅผ ํ™œ์šฉํ•˜์—ฌ ์œ ์‚ฌ๋„ ๊ฒ€์ƒ‰์„ ์ˆ˜ํ–‰ํ•˜์„ธ์š”.
- ์–ด๋–ค ์ธ์ž๋ฅผ ์ „๋‹ฌํ•ด์•ผ ํ•˜๋Š”์ง€ SQL ์Šคํ‚ค๋งˆ์˜ RPC ํ•จ์ˆ˜ ์ •์˜๋ฅผ ์ง์ ‘ ํ™•์ธํ•˜์„ธ์š”.
2. ์œ ์‚ฌ๋„(similarity)๊ฐ€ similarity_threshold ์ด์ƒ์ธ ๊ฒฐ๊ณผ๋งŒ ํ•„ํ„ฐ๋งํ•˜์„ธ์š”. (์„ ํƒ์‚ฌํ•ญ)
Args:
query_embedding (List[float]): ์งˆ๋ฌธ์˜ ์ž„๋ฒ ๋”ฉ ๋ฒกํ„ฐ
document_id (str): ๊ฒ€์ƒ‰ํ•  ๋ฌธ์„œ ID
limit (int): ๋ฐ˜ํ™˜ํ•  ์ตœ๋Œ€ ๊ฒฐ๊ณผ ์ˆ˜
similarity_threshold (float): ์œ ์‚ฌ๋„ ์ž„๊ณ„๊ฐ’
Returns:
List[Dict[str, Any]]: ์œ ์‚ฌํ•œ ๋ฌธ์„œ ์ฒญํฌ ๋ฆฌ์ŠคํŠธ
"""
result = supabase.rpc(
"match_documents",
{
"query_embedding": query_embedding,
"match_count": limit,
"p_document_id": document_id,
},
).execute()
# ๋””๋ฒ„๊น…: ๋ฒกํ„ฐ ๊ฒ€์ƒ‰ ๊ฒฐ๊ณผ ์š”์•ฝ
print(f"\n{'='*60}")
print(
f" ๋ฒกํ„ฐ ๊ฒ€์ƒ‰ ๊ฒฐ๊ณผ (์ด {len(result.data)}๊ฑด, ์ž„๊ณ„๊ฐ’ >= {similarity_threshold})"
)
print(f"{'='*60}")
for i, row in enumerate(result.data, 1):
sim = row.get("similarity", 0)
preview = row.get("content", "")[:60].replace("\n", " ")
marker = "โœ“" if sim >= similarity_threshold else "โœ—"
print(f" {marker} [{i:2d}] ์œ ์‚ฌ๋„ {sim:.4f} | {preview}...")
print(f"{'='*60}")
filtered = [
row for row in result.data if row.get("similarity", 0) >= similarity_threshold
]
print(f" โ†’ ํ†ต๊ณผ: {len(filtered)}๊ฑด / ์ œ์™ธ: {len(result.data) - len(filtered)}๊ฑด")
return filtered
def search_by_keyword(
query: str, document_id: str, limit: int = 5
) -> List[Dict[str, Any]]:
"""ํ‚ค์›Œ๋“œ ๊ธฐ๋ฐ˜ ๊ฒ€์ƒ‰ (๋ฒกํ„ฐ ๊ฒ€์ƒ‰ ๋ณด์™„์šฉ). ๊ณ ์œ ๋ช…์‚ฌ ๋“ฑ ์ •ํ™•ํ•œ ๋งค์นญ์ด ํ•„์š”ํ•  ๋•Œ ์‚ฌ์šฉ."""
STOP_WORDS = {
"์€",
"๋Š”",
"์ด",
"๊ฐ€",
"์„",
"๋ฅผ",
"์˜",
"์—",
"์—์„œ",
"๋กœ",
"์œผ๋กœ",
"์™€",
"๊ณผ",
"๋„",
"๋งŒ",
"์—๊ฒŒ",
"ํ•œ",
"ํ•˜๋Š”",
"์žˆ๋Š”",
"์—†๋Š”",
"์•Œ๋ ค์ฃผ์„ธ์š”",
"์•Œ๋ ค์ค˜",
"๋ฌด์—‡์ธ๊ฐ€์š”",
"๋ญ์•ผ",
"์–ด๋–ค",
}
words = query.replace("?", "").replace(".", "").replace(",", "").split()
keywords = [w for w in words if len(w) > 1 and w not in STOP_WORDS]
if not keywords:
return []
seen = set()
results = []
for kw in keywords:
rows = (
supabase.table("document_chunks")
.select("content")
.eq("document_id", document_id)
.ilike("content", f"%{kw}%")
.limit(limit)
.execute()
).data
for row in rows:
if row["content"] not in seen:
seen.add(row["content"])
results.append(row)
return results[:limit]
def get_document_id_by_hash(file_hash: str) -> str:
result = supabase.rpc("get_document_by_hash", {"p_hash": file_hash}).execute()
if result.data:
return result.data[0]["document_id"]
return None