scripts/knowledge_ingester.py

"""Script to ingest knowledge (templates) into Supabase."""

import os
import json
import asyncio
import httpx
from dotenv import load_dotenv
from pathlib import Path

# Load env variables
env_path = Path(__file__).resolve().parent.parent / ".env"
load_dotenv(env_path)

import sys
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from app.services.embedding_service import EmbeddingService

async def ingest_knowledge():
    supabase_url = os.environ.get("SUPABASE_URL", "").rstrip("/")
    secret_key = os.environ.get("SUPABASE_SECRET_KEY", "")
    
    if not supabase_url or not secret_key:
        print("Missing Supabase credentials in .env")
        return

    # Initialize Embedding Service
    print("Loading Gemini embedding model...")
    embedding_service = EmbeddingService()

    # Load templates
    templates_path = Path(__file__).resolve().parent.parent / "app" / "data" / "roadmap_templates.json"
    with open(templates_path, "r", encoding="utf-8") as f:
        templates = json.load(f)

    print(f"Found {len(templates)} templates. Starting ingestion...")

    endpoint = f"{supabase_url}/rest/v1/roadmap_knowledge"
    headers = {
        "apikey": secret_key,
        "Authorization": f"Bearer {secret_key}",
        "Content-Type": "application/json",
        "Prefer": "return=minimal",
    }

    async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
        for template in templates:
            title = template["title"]
            print(f"Ingesting: {title}")
            
            # Process the template into a rich Markdown representation
            md_lines = []
            md_lines.append(f"# Roadmap: {title}")
            description = template.get("description", "")
            if description:
                md_lines.append(f"**Description:** {description}\n")
            
            md_lines.append("## Phases & Topics")
            for idx, phase in enumerate(template.get("phases", [])):
                phase_title = phase.get("title", f"Phase {idx+1}")
                md_lines.append(f"### {idx+1}. {phase_title}")
                phase_desc = phase.get("description", "")
                if phase_desc:
                    md_lines.append(f"_{phase_desc}_")
                
                topics = phase.get("topics", [])
                if topics:
                    md_lines.append("Topics covered:")
                    for t in topics:
                        if isinstance(t, dict):
                            t_title = t.get("title", "")
                            t_desc = t.get("description", "")
                            md_lines.append(f"- **{t_title}**: {t_desc}")
                        else:
                            md_lines.append(f"- {t}")
                md_lines.append("")
            
            content_str = "\n".join(md_lines)
            
            # Embed the entire structured markdown to capture deep semantic meaning
            embedding = embedding_service.embed_text(content_str.lower())

            payload = {
                "topic_title": title,
                "content": content_str,
                "embedding": embedding
            }

            try:
                response = await client.post(endpoint, headers=headers, json=payload)
                response.raise_for_status()
            except Exception as e:
                print(f"Failed to insert {title}: {e}")
            await asyncio.sleep(0.5)

    print("Ingestion complete.")

if __name__ == "__main__":
    asyncio.run(ingest_knowledge())