Spaces:
Sleeping
Sleeping
File size: 3,552 Bytes
8bc6b50 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | """Script to ingest knowledge (templates) into Supabase."""
import os
import json
import asyncio
import httpx
from dotenv import load_dotenv
from pathlib import Path
# Load env variables
env_path = Path(__file__).resolve().parent.parent / ".env"
load_dotenv(env_path)
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from app.services.embedding_service import EmbeddingService
async def ingest_knowledge():
supabase_url = os.environ.get("SUPABASE_URL", "").rstrip("/")
secret_key = os.environ.get("SUPABASE_SECRET_KEY", "")
if not supabase_url or not secret_key:
print("Missing Supabase credentials in .env")
return
# Initialize Embedding Service
print("Loading Gemini embedding model...")
embedding_service = EmbeddingService()
# Load templates
templates_path = Path(__file__).resolve().parent.parent / "app" / "data" / "roadmap_templates.json"
with open(templates_path, "r", encoding="utf-8") as f:
templates = json.load(f)
print(f"Found {len(templates)} templates. Starting ingestion...")
endpoint = f"{supabase_url}/rest/v1/roadmap_knowledge"
headers = {
"apikey": secret_key,
"Authorization": f"Bearer {secret_key}",
"Content-Type": "application/json",
"Prefer": "return=minimal",
}
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
for template in templates:
title = template["title"]
print(f"Ingesting: {title}")
# Process the template into a rich Markdown representation
md_lines = []
md_lines.append(f"# Roadmap: {title}")
description = template.get("description", "")
if description:
md_lines.append(f"**Description:** {description}\n")
md_lines.append("## Phases & Topics")
for idx, phase in enumerate(template.get("phases", [])):
phase_title = phase.get("title", f"Phase {idx+1}")
md_lines.append(f"### {idx+1}. {phase_title}")
phase_desc = phase.get("description", "")
if phase_desc:
md_lines.append(f"_{phase_desc}_")
topics = phase.get("topics", [])
if topics:
md_lines.append("Topics covered:")
for t in topics:
if isinstance(t, dict):
t_title = t.get("title", "")
t_desc = t.get("description", "")
md_lines.append(f"- **{t_title}**: {t_desc}")
else:
md_lines.append(f"- {t}")
md_lines.append("")
content_str = "\n".join(md_lines)
# Embed the entire structured markdown to capture deep semantic meaning
embedding = embedding_service.embed_text(content_str.lower())
payload = {
"topic_title": title,
"content": content_str,
"embedding": embedding
}
try:
response = await client.post(endpoint, headers=headers, json=payload)
response.raise_for_status()
except Exception as e:
print(f"Failed to insert {title}: {e}")
await asyncio.sleep(0.5)
print("Ingestion complete.")
if __name__ == "__main__":
asyncio.run(ingest_knowledge())
|