Spaces:
Sleeping
Sleeping
| """ | |
| GitHub service — Fetch repos + READMEs via Clerk OAuth, summarize with Haiku, index. | |
| """ | |
| import os | |
| import logging | |
| from datetime import datetime | |
| import httpx | |
| logger = logging.getLogger("cereal.services.github") | |
| CLERK_API_URL = os.getenv("CLERK_API_URL", "https://api.clerk.com/v1") | |
| CLERK_SECRET_KEY = os.getenv("CLERK_SECRET_KEY", "") | |
| async def _get_all_github_tokens(user_id: str) -> list[dict]: | |
| """ | |
| Return all GitHub tokens for this user from Clerk. | |
| Each dict: { "token": str, "login": str } | |
| """ | |
| async with httpx.AsyncClient() as client: | |
| resp = await client.get( | |
| f"{CLERK_API_URL}/users/{user_id}/oauth_access_tokens/oauth_github", | |
| headers={"Authorization": f"Bearer {CLERK_SECRET_KEY}"}, | |
| ) | |
| resp.raise_for_status() | |
| tokens = resp.json() | |
| if not tokens: | |
| raise ValueError("No GitHub OAuth tokens found for user") | |
| results = [] | |
| for t in tokens: | |
| token = t["token"] | |
| try: | |
| me = await client.get( | |
| "https://api.github.com/user", | |
| headers={"Authorization": f"Bearer {token}", | |
| "Accept": "application/vnd.github.v3+json"}, | |
| ) | |
| login = me.json().get("login", "unknown") | |
| except Exception: | |
| login = "unknown" | |
| results.append({"token": token, "login": login}) | |
| return results | |
| async def fetch_and_index_github(user_id: str) -> dict: | |
| """Index repos from ALL connected GitHub accounts.""" | |
| from services.elasticsearch import bulk_index, delete_user_docs | |
| from services.llm import summarize_readme | |
| accounts = await _get_all_github_tokens(user_id) | |
| logger.info(f"{len(accounts)} GitHub account(s) for {user_id}") | |
| await delete_user_docs("github_projects", user_id) | |
| all_docs = [] | |
| now = datetime.utcnow().isoformat() | |
| async with httpx.AsyncClient() as client: | |
| for acct in accounts: | |
| token = acct["token"] | |
| login = acct["login"] | |
| repos = [] | |
| page = 1 | |
| while True: | |
| r = await client.get( | |
| "https://api.github.com/user/repos", | |
| headers={"Authorization": f"Bearer {token}", | |
| "Accept": "application/vnd.github.v3+json"}, | |
| params={"sort":"updated","per_page":30,"page":page,"affiliation":"owner"}, | |
| ) | |
| r.raise_for_status() | |
| batch = r.json() | |
| if not batch: break | |
| repos.extend(batch) | |
| page += 1 | |
| if page > 5: break # Cap at 150 repos per account | |
| for repo in repos: | |
| if repo.get("fork"): continue | |
| readme = "" | |
| try: | |
| rr = await client.get( | |
| f"https://api.github.com/repos/{repo['full_name']}/readme", | |
| headers={"Authorization": f"Bearer {token}", | |
| "Accept": "application/vnd.github.v3.raw"}, | |
| ) | |
| if rr.status_code == 200: readme = rr.text | |
| except Exception: pass | |
| summary = "" | |
| if readme: | |
| try: summary = await summarize_readme(readme, repo["name"]) | |
| except Exception: summary = readme[:500] | |
| all_docs.append({ | |
| "user_id": user_id, | |
| "github_account": login, # track which account | |
| "repo_name": repo["name"], | |
| "repo_url": repo["html_url"], | |
| "stars": repo.get("stargazers_count", 0), | |
| "language": repo.get("language", ""), | |
| "topics": repo.get("topics", []), | |
| "readme_summary": summary or f"{repo['name']}: {repo.get('description','')}", | |
| "indexed_at": now, | |
| }) | |
| if all_docs: | |
| await bulk_index("github_projects", all_docs) | |
| logger.info(f"Indexed {len(all_docs)} repos across {len(accounts)} account(s)") | |
| return {"repo_count": len(all_docs), "account_count": len(accounts)} | |