""" GitHub service — Fetch repos + READMEs via Clerk OAuth, summarize with Haiku, index. """ import os import logging from datetime import datetime import httpx logger = logging.getLogger("cereal.services.github") CLERK_API_URL = os.getenv("CLERK_API_URL", "https://api.clerk.com/v1") CLERK_SECRET_KEY = os.getenv("CLERK_SECRET_KEY", "") async def _get_all_github_tokens(user_id: str) -> list[dict]: """ Return all GitHub tokens for this user from Clerk. Each dict: { "token": str, "login": str } """ async with httpx.AsyncClient() as client: resp = await client.get( f"{CLERK_API_URL}/users/{user_id}/oauth_access_tokens/oauth_github", headers={"Authorization": f"Bearer {CLERK_SECRET_KEY}"}, ) resp.raise_for_status() tokens = resp.json() if not tokens: raise ValueError("No GitHub OAuth tokens found for user") results = [] for t in tokens: token = t["token"] try: me = await client.get( "https://api.github.com/user", headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github.v3+json"}, ) login = me.json().get("login", "unknown") except Exception: login = "unknown" results.append({"token": token, "login": login}) return results async def fetch_and_index_github(user_id: str) -> dict: """Index repos from ALL connected GitHub accounts.""" from services.elasticsearch import bulk_index, delete_user_docs from services.llm import summarize_readme accounts = await _get_all_github_tokens(user_id) logger.info(f"{len(accounts)} GitHub account(s) for {user_id}") await delete_user_docs("github_projects", user_id) all_docs = [] now = datetime.utcnow().isoformat() async with httpx.AsyncClient() as client: for acct in accounts: token = acct["token"] login = acct["login"] repos = [] page = 1 while True: r = await client.get( "https://api.github.com/user/repos", headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github.v3+json"}, params={"sort":"updated","per_page":30,"page":page,"affiliation":"owner"}, ) r.raise_for_status() batch = r.json() if not batch: break repos.extend(batch) page += 1 if page > 5: break # Cap at 150 repos per account for repo in repos: if repo.get("fork"): continue readme = "" try: rr = await client.get( f"https://api.github.com/repos/{repo['full_name']}/readme", headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github.v3.raw"}, ) if rr.status_code == 200: readme = rr.text except Exception: pass summary = "" if readme: try: summary = await summarize_readme(readme, repo["name"]) except Exception: summary = readme[:500] all_docs.append({ "user_id": user_id, "github_account": login, # track which account "repo_name": repo["name"], "repo_url": repo["html_url"], "stars": repo.get("stargazers_count", 0), "language": repo.get("language", ""), "topics": repo.get("topics", []), "readme_summary": summary or f"{repo['name']}: {repo.get('description','')}", "indexed_at": now, }) if all_docs: await bulk_index("github_projects", all_docs) logger.info(f"Indexed {len(all_docs)} repos across {len(accounts)} account(s)") return {"repo_count": len(all_docs), "account_count": len(accounts)}