"""Script to ingest knowledge (templates) into Supabase.""" import os import json import asyncio import httpx from dotenv import load_dotenv from pathlib import Path # Load env variables env_path = Path(__file__).resolve().parent.parent / ".env" load_dotenv(env_path) import sys sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from app.services.embedding_service import EmbeddingService async def ingest_knowledge(): supabase_url = os.environ.get("SUPABASE_URL", "").rstrip("/") secret_key = os.environ.get("SUPABASE_SECRET_KEY", "") if not supabase_url or not secret_key: print("Missing Supabase credentials in .env") return # Initialize Embedding Service print("Loading Gemini embedding model...") embedding_service = EmbeddingService() # Load templates templates_path = Path(__file__).resolve().parent.parent / "app" / "data" / "roadmap_templates.json" with open(templates_path, "r", encoding="utf-8") as f: templates = json.load(f) print(f"Found {len(templates)} templates. Starting ingestion...") endpoint = f"{supabase_url}/rest/v1/roadmap_knowledge" headers = { "apikey": secret_key, "Authorization": f"Bearer {secret_key}", "Content-Type": "application/json", "Prefer": "return=minimal", } async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client: for template in templates: title = template["title"] print(f"Ingesting: {title}") # Process the template into a rich Markdown representation md_lines = [] md_lines.append(f"# Roadmap: {title}") description = template.get("description", "") if description: md_lines.append(f"**Description:** {description}\n") md_lines.append("## Phases & Topics") for idx, phase in enumerate(template.get("phases", [])): phase_title = phase.get("title", f"Phase {idx+1}") md_lines.append(f"### {idx+1}. {phase_title}") phase_desc = phase.get("description", "") if phase_desc: md_lines.append(f"_{phase_desc}_") topics = phase.get("topics", []) if topics: md_lines.append("Topics covered:") for t in topics: if isinstance(t, dict): t_title = t.get("title", "") t_desc = t.get("description", "") md_lines.append(f"- **{t_title}**: {t_desc}") else: md_lines.append(f"- {t}") md_lines.append("") content_str = "\n".join(md_lines) # Embed the entire structured markdown to capture deep semantic meaning embedding = embedding_service.embed_text(content_str.lower()) payload = { "topic_title": title, "content": content_str, "embedding": embedding } try: response = await client.post(endpoint, headers=headers, json=payload) response.raise_for_status() except Exception as e: print(f"Failed to insert {title}: {e}") await asyncio.sleep(0.5) print("Ingestion complete.") if __name__ == "__main__": asyncio.run(ingest_knowledge())