Shivoo29 commited on
Commit
6ca8e48
Β·
0 Parent(s):

fix: set PLAYWRIGHT_BROWSERS_PATH so chromium is found at runtime by user

Browse files
Dockerfile ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.12-slim
2
+
3
+ RUN useradd -m -u 1000 user
4
+ ENV PATH="/home/user/.local/bin:$PATH"
5
+ ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
6
+
7
+ WORKDIR /app
8
+
9
+ RUN apt-get update && apt-get install -y \
10
+ wget curl gnupg \
11
+ libnss3 libatk1.0-0 libatk-bridge2.0-0 \
12
+ libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 \
13
+ libxfixes3 libxrandr2 libgbm1 libasound2 \
14
+ && rm -rf /var/lib/apt/lists/*
15
+
16
+ COPY backend/requirements.txt .
17
+ RUN pip install --no-cache-dir -r requirements.txt
18
+
19
+ RUN mkdir /ms-playwright && chmod 777 /ms-playwright
20
+ RUN playwright install chromium --with-deps
21
+
22
+ COPY --chown=user backend/ ./backend/
23
+
24
+ USER user
25
+
26
+ ENV DEV_MODE=false
27
+ ENV PORT=7860
28
+
29
+ EXPOSE 7860
30
+
31
+ CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "7860"]
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: DocForge API
3
+ emoji: πŸ”¨
4
+ colorFrom: blue
5
+ colorTo: indigo
6
+ sdk: docker
7
+ app_port: 7860
8
+ pinned: false
9
+ ---
10
+
11
+ # DocForge API
12
+
13
+ Backend API for DocForge β€” AI-powered documentation context file generator.
backend/.env.example ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DocForge Backend Environment Variables
2
+ # Copy this to .env and fill in your values
3
+
4
+ # ── Auth ──────────────────────────────────────────────────────────────────────
5
+ # Secret key for signing JWTs β€” change this in production!
6
+ SECRET_KEY=change_me_to_a_long_random_string_in_production
7
+
8
+ # Fernet key for encrypting stored API keys (32-byte URL-safe base64)
9
+ # Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
10
+ # If omitted, derived from SECRET_KEY (fine for dev, use a real key in prod)
11
+ ENCRYPTION_KEY=
12
+
13
+ # URL of your frontend (used in device-flow verification_uri + OAuth redirects)
14
+ FRONTEND_URL=http://localhost:3000
15
+
16
+ # URL of this backend (used to build OAuth callback URIs)
17
+ BACKEND_URL=http://localhost:8000
18
+
19
+ # ── OAuth (Google + GitHub sign-in) ──────────────────────────────────────────
20
+ # Google: https://console.cloud.google.com/apis/credentials
21
+ # Authorised redirect URI to add: BACKEND_URL/api/auth/oauth/google/callback
22
+ GOOGLE_CLIENT_ID=
23
+ GOOGLE_CLIENT_SECRET=
24
+
25
+ # GitHub: https://github.com/settings/developers β†’ OAuth Apps
26
+ # Callback URL to set: BACKEND_URL/api/auth/oauth/github/callback
27
+ GITHUB_CLIENT_ID=
28
+ GITHUB_CLIENT_SECRET=
29
+
30
+ # ── Fallback AI keys (used when user has no keys stored) ─────────────────────
31
+ # Optional: lets unauthenticated / keyless users still get a result.
32
+ # Leave blank to require every user to supply their own key.
33
+ GEMINI_API_KEY=
34
+ GROQ_API_KEY=
35
+
36
+ # ── Storage ───────────────────────────────────────────────────────────────────
37
+ # Redis URL (optional β€” uses in-memory cache if not set)
38
+ REDIS_URL=redis://localhost:6379
39
+
40
+ # Supabase (recommended for production β€” stores users, API keys, cache, device codes)
41
+ # Required tables: users, user_api_keys, device_codes, cache_entries, corrections
42
+ SUPABASE_URL=your_supabase_url
43
+ SUPABASE_KEY=your_supabase_key
44
+
45
+ # ── Misc ──────────────────────────────────────────────────────────────────────
46
+ # Playwright timeout in milliseconds
47
+ PLAYWRIGHT_TIMEOUT=60000
48
+
49
+ # Set to true to use in-memory fallbacks (no Redis/Supabase required)
50
+ # Perfect for local development
51
+ DEV_MODE=true
backend/Dockerfile ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.12-slim
2
+
3
+ WORKDIR /app
4
+
5
+ # Install system dependencies for Playwright
6
+ RUN apt-get update && apt-get install -y \
7
+ wget \
8
+ gnupg \
9
+ && rm -rf /var/lib/apt/lists/*
10
+
11
+ COPY requirements.txt .
12
+ RUN pip install --no-cache-dir -r requirements.txt
13
+
14
+ # Install Playwright browser binaries
15
+ RUN playwright install chromium --with-deps
16
+
17
+ COPY . .
18
+
19
+ ENV DEV_MODE=false
20
+ ENV PORT=8000
21
+
22
+ EXPOSE 8000
23
+
24
+ CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
backend/__init__.py ADDED
File without changes
backend/main.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ DocForge FastAPI Backend
3
+
4
+ Thin client / fat backend architecture:
5
+ - All heavy lifting (crawling, structuring, caching) happens here
6
+ - VS Code extension and MCP server are thin HTTP clients to this API
7
+
8
+ Run locally:
9
+ uvicorn backend.main:app --reload --port 8000
10
+
11
+ Or via Makefile:
12
+ make dev-backend
13
+ """
14
+
15
+ import logging
16
+ import os
17
+ import time
18
+
19
+ from dotenv import load_dotenv
20
+ from fastapi import FastAPI, Request
21
+
22
+ load_dotenv()
23
+ from fastapi.middleware.cors import CORSMiddleware
24
+ from fastapi.responses import JSONResponse
25
+
26
+ from .routers import auth, context, corrections, search, versions, oauth
27
+
28
+ # ── Logging setup ─────────────────────────────────────────────────────────────
29
+ logging.basicConfig(
30
+ level=logging.INFO,
31
+ format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
32
+ )
33
+ logger = logging.getLogger(__name__)
34
+
35
+ # ── App initialization ────────────────────────────────────────────────────────
36
+ app = FastAPI(
37
+ title="DocForge API",
38
+ description=(
39
+ "Turn any library's documentation into a prompt-ready .context.md file, "
40
+ "pinned to the exact version you're using."
41
+ ),
42
+ version="1.0.0",
43
+ docs_url="/docs",
44
+ redoc_url="/redoc",
45
+ )
46
+
47
+ # Allow VS Code extension and MCP server to call from any origin during dev
48
+ app.add_middleware(
49
+ CORSMiddleware,
50
+ allow_origins=["*"],
51
+ allow_methods=["*"],
52
+ allow_headers=["*"],
53
+ )
54
+
55
+ # ── Request timing middleware ─────────────────────────────────────────────────
56
+ @app.middleware("http")
57
+ async def log_request_time(request: Request, call_next):
58
+ t0 = time.time()
59
+ response = await call_next(request)
60
+ elapsed_ms = (time.time() - t0) * 1000
61
+ logger.info("%s %s β†’ %d (%.0fms)", request.method, request.url.path, response.status_code, elapsed_ms)
62
+ return response
63
+
64
+
65
+ # ── Routes ────────────────────────────────────────────────────────────────────
66
+ app.include_router(auth.router)
67
+ app.include_router(oauth.router)
68
+ app.include_router(context.router)
69
+ app.include_router(corrections.router)
70
+ app.include_router(search.router)
71
+ app.include_router(versions.router)
72
+
73
+
74
+ @app.get("/api/health", tags=["health"])
75
+ async def health_check() -> dict:
76
+ """Health check endpoint β€” used by VS Code extension to verify backend connectivity."""
77
+ return {
78
+ "status": "ok",
79
+ "service": "docforge-api",
80
+ "version": "1.0.0",
81
+ "dev_mode": os.getenv("DEV_MODE", "true").lower() == "true",
82
+ }
83
+
84
+
85
+ @app.get("/", tags=["root"])
86
+ async def root() -> dict:
87
+ return {
88
+ "message": "DocForge API is running",
89
+ "docs": "/docs",
90
+ "health": "/api/health",
91
+ }
92
+
93
+
94
+ # ── Global error handler ──────────────────────────────────────────────────────
95
+ @app.exception_handler(Exception)
96
+ async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse:
97
+ logger.exception("Unhandled exception on %s: %s", request.url, exc)
98
+ return JSONResponse(
99
+ status_code=500,
100
+ content={"detail": f"Internal server error: {str(exc)}"},
101
+ )
backend/models/__init__.py ADDED
File without changes
backend/models/schemas.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pydantic models for DocForge API request/response schemas."""
2
+
3
+ from datetime import date
4
+ from enum import Enum
5
+ from typing import Any
6
+ from uuid import UUID
7
+
8
+ from pydantic import BaseModel, Field
9
+
10
+
11
+ class InputType(str, Enum):
12
+ npm = "npm"
13
+ pypi = "pypi"
14
+ url = "url"
15
+ github = "github"
16
+ paste = "paste"
17
+ crates = "crates"
18
+ rubygems = "rubygems"
19
+ pubdev = "pubdev"
20
+ nuget = "nuget"
21
+ maven = "maven"
22
+ hex = "hex"
23
+ cran = "cran"
24
+
25
+
26
+ class OutputFormat(str, Enum):
27
+ context_md = "context_md"
28
+ json = "json"
29
+ mcp = "mcp"
30
+
31
+
32
+ class JobStatus(str, Enum):
33
+ queued = "queued"
34
+ processing = "processing"
35
+ complete = "complete"
36
+ failed = "failed"
37
+
38
+
39
+ class ContextRequest(BaseModel):
40
+ input: str = Field(..., description="Package name, URL, or GitHub repo")
41
+ input_type: InputType = Field(InputType.npm, description="Type of input")
42
+ components: list[str] | None = Field(None, description="Specific components to extract")
43
+ output_format: OutputFormat = Field(OutputFormat.context_md, description="Output format")
44
+ content: str | None = Field(None, description="Raw content for paste input type")
45
+ # Client-supplied API keys (used when no Bearer token is present)
46
+ gemini_key: str | None = Field(None, description="Gemini API key (CLI local-key mode)")
47
+ groq_key: str | None = Field(None, description="Groq API key (CLI local-key mode)")
48
+
49
+
50
+ class JobCreatedResponse(BaseModel):
51
+ job_id: str
52
+ status: JobStatus = JobStatus.queued
53
+
54
+
55
+ class PropSchema(BaseModel):
56
+ name: str
57
+ type: str
58
+ required: bool = False
59
+ default: str | None = None
60
+
61
+
62
+ class ComponentSchema(BaseModel):
63
+ name: str
64
+ import_path: str | None = None
65
+ install_command: str | None = None
66
+ props: list[PropSchema] = Field(default_factory=list)
67
+ gotchas: list[str] = Field(default_factory=list)
68
+ peer_dependencies: list[str] = Field(default_factory=list)
69
+ usage_example: str | None = None
70
+ last_verified: str = Field(default_factory=lambda: date.today().isoformat())
71
+
72
+
73
+ class StructuredLibrary(BaseModel):
74
+ library: str
75
+ version: str
76
+ description: str | None = None
77
+ components: list[ComponentSchema] = Field(default_factory=list)
78
+
79
+
80
+ class ContextJobResult(BaseModel):
81
+ status: JobStatus
82
+ job_id: str
83
+ library: str | None = None
84
+ version: str | None = None
85
+ output: str | None = None
86
+ components: list[dict[str, Any]] = Field(default_factory=list)
87
+ error: str | None = None
88
+
89
+
90
+ class SearchResult(BaseModel):
91
+ name: str
92
+ version: str
93
+ cached_at: str
94
+
95
+
96
+ class SearchResponse(BaseModel):
97
+ results: list[SearchResult]
98
+
99
+
100
+ class VersionsResponse(BaseModel):
101
+ package: str
102
+ versions: list[str]
backend/models/user_models.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel, EmailStr
2
+ from typing import Optional
3
+
4
+
5
+ # ── Request / Response models ──────────────────────────────────────────────────
6
+
7
+ class RegisterRequest(BaseModel):
8
+ email: str
9
+ password: str
10
+ gemini_key: Optional[str] = None
11
+ groq_key: Optional[str] = None
12
+
13
+
14
+ class LoginRequest(BaseModel):
15
+ email: str
16
+ password: str
17
+
18
+
19
+ class UpdateKeysRequest(BaseModel):
20
+ gemini_key: Optional[str] = None
21
+ groq_key: Optional[str] = None
22
+
23
+
24
+ class AuthResponse(BaseModel):
25
+ access_token: str
26
+ token_type: str = "bearer"
27
+ user_id: str
28
+ email: str
29
+
30
+
31
+ class UserProfile(BaseModel):
32
+ user_id: str
33
+ email: str
34
+ has_gemini_key: bool
35
+ has_groq_key: bool
36
+
37
+
38
+ # ── Device flow models (for CLI / extension auth) ─────────────────────────────
39
+
40
+ class DeviceInitResponse(BaseModel):
41
+ device_code: str # opaque, sent by CLI to poll
42
+ user_code: str # human-readable code shown in terminal
43
+ verification_uri: str # URL user opens in browser
44
+ expires_in: int = 900 # seconds
45
+ interval: int = 5 # polling interval seconds
46
+
47
+
48
+ class DeviceVerifyRequest(BaseModel):
49
+ user_code: str # code the user types on the website
50
+
51
+
52
+ class DevicePollResponse(BaseModel):
53
+ status: str # "pending" | "complete" | "expired"
54
+ access_token: Optional[str] = None
55
+ email: Optional[str] = None
backend/requirements.txt ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi==0.115.0
2
+ uvicorn[standard]==0.30.6
3
+ httpx==0.27.2
4
+ pydantic>=2.11.0
5
+ python-dotenv==1.0.1
6
+ # Playwright for JS-rendered docs sites
7
+ playwright>=1.50.0
8
+ # HTML to Markdown conversion
9
+ html2text==2024.2.26
10
+ # Redis (production cache)
11
+ redis==5.1.0
12
+ # Supabase (production storage)
13
+ supabase==2.9.0
14
+ # PDF/document conversion via IBM Docling (optional β€” install separately)
15
+ # docling==2.0.0
16
+ # Google Gemini
17
+ google-generativeai==0.8.3
18
+ # Auth
19
+ python-jose[cryptography]==3.5.0
20
+ passlib[bcrypt]==1.7.4
21
+ cryptography==46.0.5
backend/routers/__init__.py ADDED
File without changes
backend/routers/auth.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Auth router β€” all endpoints under /api/auth
3
+
4
+ Public:
5
+ POST /api/auth/register create account (+ optional API keys)
6
+ POST /api/auth/login email + password β†’ JWT
7
+ POST /api/auth/device/init CLI / extension requests a device code
8
+ GET /api/auth/device/poll CLI polls until user verifies on web
9
+ POST /api/auth/device/verify web UI posts user_code to mark verified
10
+
11
+ Protected (requires Bearer token):
12
+ GET /api/auth/me current user profile
13
+ PUT /api/auth/keys save / update API keys
14
+ POST /api/auth/logout (client-side: just discard the token)
15
+ """
16
+
17
+ import os
18
+ from fastapi import APIRouter, HTTPException, Depends
19
+ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
20
+
21
+ from backend.models.user_models import (
22
+ RegisterRequest, LoginRequest, UpdateKeysRequest,
23
+ AuthResponse, UserProfile,
24
+ DeviceInitResponse, DeviceVerifyRequest, DevicePollResponse,
25
+ )
26
+ from backend.services import auth_service
27
+
28
+ router = APIRouter(prefix="/api/auth", tags=["auth"])
29
+ _bearer = HTTPBearer(auto_error=False)
30
+
31
+ FRONTEND_URL = os.getenv("FRONTEND_URL", "http://localhost:3000")
32
+
33
+
34
+ # ── Auth dependency ───────────────────────────────────────────────────────────
35
+
36
+ async def current_user(
37
+ creds: HTTPAuthorizationCredentials = Depends(_bearer),
38
+ ) -> dict:
39
+ if not creds:
40
+ raise HTTPException(status_code=401, detail="Not authenticated")
41
+ payload = auth_service.decode_token(creds.credentials)
42
+ if not payload:
43
+ raise HTTPException(status_code=401, detail="Invalid or expired token")
44
+ user = await auth_service.get_user_by_id(payload["sub"])
45
+ if not user:
46
+ raise HTTPException(status_code=401, detail="User not found")
47
+ return user
48
+
49
+
50
+ # ── Public endpoints ──────────────────────────────────────────────────────────
51
+
52
+ @router.post("/register", response_model=AuthResponse)
53
+ async def register(body: RegisterRequest):
54
+ hashed = auth_service.hash_password(body.password)
55
+ user_id = await auth_service.create_user(body.email, hashed)
56
+ if not user_id:
57
+ raise HTTPException(status_code=409, detail="Email already registered")
58
+
59
+ # Optionally save API keys during registration
60
+ if body.gemini_key or body.groq_key:
61
+ await auth_service.save_api_keys(user_id, body.gemini_key, body.groq_key)
62
+
63
+ token = auth_service.create_access_token(user_id, body.email)
64
+ return AuthResponse(access_token=token, user_id=user_id, email=body.email)
65
+
66
+
67
+ @router.post("/login", response_model=AuthResponse)
68
+ async def login(body: LoginRequest):
69
+ user = await auth_service.get_user_by_email(body.email)
70
+ if not user or not auth_service.verify_password(body.password, user["password_hash"]):
71
+ raise HTTPException(status_code=401, detail="Invalid email or password")
72
+
73
+ token = auth_service.create_access_token(user["id"], user["email"])
74
+ return AuthResponse(access_token=token, user_id=user["id"], email=user["email"])
75
+
76
+
77
+ # ── Device flow ───────────────────────────────────────────────────────────────
78
+
79
+ @router.post("/device/init", response_model=DeviceInitResponse)
80
+ async def device_init():
81
+ """CLI / extension calls this to get a device code."""
82
+ result = await auth_service.create_device_code()
83
+ return DeviceInitResponse(
84
+ device_code=result["device_code"],
85
+ user_code=result["user_code"],
86
+ verification_uri=f"{FRONTEND_URL}/auth/device",
87
+ )
88
+
89
+
90
+ @router.get("/device/poll", response_model=DevicePollResponse)
91
+ async def device_poll(device_code: str):
92
+ """CLI polls this every 5 s until status == 'complete'."""
93
+ result = await auth_service.poll_device_code(device_code)
94
+ return DevicePollResponse(**result)
95
+
96
+
97
+ @router.post("/device/verify")
98
+ async def device_verify(
99
+ body: DeviceVerifyRequest,
100
+ user: dict = Depends(current_user),
101
+ ):
102
+ """Web UI posts this after the logged-in user enters their user_code."""
103
+ ok = await auth_service.verify_device_code(body.user_code, user["id"])
104
+ if not ok:
105
+ raise HTTPException(status_code=400, detail="Invalid or expired code")
106
+ return {"ok": True}
107
+
108
+
109
+ # ── Protected endpoints ───────────────────────────────────────────────────────
110
+
111
+ @router.get("/me", response_model=UserProfile)
112
+ async def me(user: dict = Depends(current_user)):
113
+ keys = await auth_service.get_api_keys(user["id"])
114
+ return UserProfile(
115
+ user_id=user["id"],
116
+ email=user["email"],
117
+ has_gemini_key=bool(keys.get("gemini_key")),
118
+ has_groq_key=bool(keys.get("groq_key")),
119
+ )
120
+
121
+
122
+ @router.put("/keys")
123
+ async def update_keys(
124
+ body: UpdateKeysRequest,
125
+ user: dict = Depends(current_user),
126
+ ):
127
+ if not body.gemini_key and not body.groq_key:
128
+ raise HTTPException(status_code=400, detail="Provide at least one key")
129
+ await auth_service.save_api_keys(user["id"], body.gemini_key, body.groq_key)
130
+ return {"ok": True, "message": "API keys saved"}
backend/routers/context.py ADDED
@@ -0,0 +1,468 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ /api/context routes β€” the core DocForge pipeline endpoint.
3
+
4
+ POST /api/context β†’ queues a job, returns job_id immediately
5
+ GET /api/context/{job_id} β†’ returns job status and result when done
6
+ """
7
+
8
+ import logging
9
+ import time
10
+ import uuid
11
+ from datetime import date
12
+ from urllib.parse import urlparse
13
+
14
+ from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
15
+
16
+ from ..models.schemas import (
17
+ ContextJobResult,
18
+ ContextRequest,
19
+ InputType,
20
+ JobCreatedResponse,
21
+ JobStatus,
22
+ OutputFormat,
23
+ )
24
+ from ..services import cache
25
+ from ..services.formatter import format_context_md
26
+ from ..services.ingestion.npm_ingester import (
27
+ resolve_npm_package,
28
+ resolve_pypi_package,
29
+ )
30
+ from ..services.ingestion.paste_ingester import ingest_paste
31
+ from ..services.ingestion.registries_ingester import (
32
+ resolve_crates,
33
+ resolve_cran,
34
+ resolve_hex,
35
+ resolve_maven,
36
+ resolve_nuget,
37
+ resolve_pubdev,
38
+ resolve_rubygems,
39
+ )
40
+ from ..services.ingestion.search_ingester import find_docs_url
41
+ from ..services.ingestion.url_ingester import crawl_url
42
+ from ..services.structurer import structure_with_gemini
43
+
44
+ logger = logging.getLogger(__name__)
45
+ router = APIRouter(prefix="/api/context", tags=["context"])
46
+
47
+
48
+ @router.post("", response_model=JobCreatedResponse)
49
+ async def create_context_job(
50
+ request: ContextRequest,
51
+ background_tasks: BackgroundTasks,
52
+ http_request: Request,
53
+ ) -> JobCreatedResponse:
54
+ """
55
+ Queue a documentation context generation job.
56
+ Returns immediately with a job_id for polling.
57
+
58
+ If a valid Bearer token is provided the user's stored API keys are used for
59
+ the AI step; otherwise falls back to env-var keys (dev / open access).
60
+ """
61
+ # ── Auth: prefer stored keys from Bearer token, fall back to request body ─
62
+ gemini_key: str | None = None
63
+ groq_key: str | None = None
64
+
65
+ auth_header = http_request.headers.get("authorization", "")
66
+ if auth_header.startswith("Bearer "):
67
+ from ..services.auth_service import decode_token, get_api_keys
68
+ payload = decode_token(auth_header.split(" ", 1)[1])
69
+ if payload:
70
+ keys = await get_api_keys(payload["sub"])
71
+ gemini_key = keys.get("gemini_key")
72
+ groq_key = keys.get("groq_key")
73
+
74
+ # CLI local-key mode: keys sent directly in the request body
75
+ if not gemini_key and not groq_key:
76
+ gemini_key = request.gemini_key or None
77
+ groq_key = request.groq_key or None
78
+
79
+ # Enforce user-supplied key β€” DocForge does not maintain a server-level key.
80
+ # DEV_MODE bypasses this so local dev works without a key configured.
81
+ import os as _os
82
+ if not gemini_key and not _os.getenv("DEV_MODE", "false").lower() == "true":
83
+ raise HTTPException(
84
+ status_code=400,
85
+ detail=(
86
+ "Gemini API key required. "
87
+ "Get a free key at https://aistudio.google.com/app/apikey "
88
+ "then add it in DocForge Settings."
89
+ ),
90
+ )
91
+
92
+ job_id = str(uuid.uuid4())
93
+
94
+ # Store initial job state
95
+ await cache.set_job(job_id, {
96
+ "status": JobStatus.queued,
97
+ "job_id": job_id,
98
+ "input": request.input,
99
+ "input_type": request.input_type,
100
+ })
101
+
102
+ # Run the heavy pipeline work in the background so the HTTP response
103
+ # returns instantly (crawling takes 30–90s)
104
+ background_tasks.add_task(
105
+ _run_pipeline,
106
+ job_id=job_id,
107
+ request=request,
108
+ gemini_key=gemini_key,
109
+ groq_key=groq_key,
110
+ )
111
+
112
+ logger.info("Queued job %s for input=%s type=%s", job_id, request.input, request.input_type)
113
+ return JobCreatedResponse(job_id=job_id, status=JobStatus.queued)
114
+
115
+
116
+ @router.get("/{job_id}", response_model=ContextJobResult)
117
+ async def get_context_job(job_id: str) -> ContextJobResult:
118
+ """Poll job status. Returns output when status == 'complete'."""
119
+ job = await cache.get_job(job_id)
120
+ if job is None:
121
+ raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
122
+ return ContextJobResult(**job)
123
+
124
+
125
+ # ---------------------------------------------------------------------------
126
+ # Background pipeline
127
+ # ---------------------------------------------------------------------------
128
+
129
+ async def _run_pipeline(
130
+ job_id: str,
131
+ request: ContextRequest,
132
+ gemini_key: str | None = None,
133
+ groq_key: str | None = None,
134
+ ) -> None:
135
+ """
136
+ Full DocForge pipeline:
137
+ 1. Resolve package / fetch URL / clone repo
138
+ 2. Crawl documentation
139
+ 3. Structure with Gemini (using user's stored key if authenticated)
140
+ 4. Format as .context.md
141
+ 5. Cache result
142
+ """
143
+ t_start = time.time()
144
+
145
+ async def update_status(status: str, **extra: object) -> None:
146
+ job = await cache.get_job(job_id) or {}
147
+ job.update({"status": status, "job_id": job_id, **extra})
148
+ await cache.set_job(job_id, job)
149
+
150
+ try:
151
+ await update_status(JobStatus.processing)
152
+
153
+ # ── Step 1: Resolve metadata ──────────────────────────────────────
154
+ library = request.input
155
+ version = "latest"
156
+ docs_url: str | None = None
157
+ version_tag: str | None = None
158
+ copypaste_docs: str | None = None # pre-fetched docs for copy-paste libs
159
+ pre_structured: dict | None = None # pre-built structured dict (skips LLM)
160
+
161
+ if request.input_type == InputType.npm:
162
+ # If the user somehow passed a URL as an npm package, treat it as a URL crawl
163
+ if request.input.startswith(("http://", "https://")):
164
+ logger.warning("npm input looks like a URL β€” switching to URL crawl: %s", request.input)
165
+ raw_docs = await crawl_url(request.input)
166
+ structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key)
167
+ output = format_context_md(structured) if request.output_format == OutputFormat.context_md else __import__("json").dumps(structured, indent=2)
168
+ structured["cached_at"] = date.today().isoformat()
169
+ await cache.set_cached_context(library, version, structured)
170
+ await update_status(JobStatus.complete, library=library, version=version, output=output, components=structured.get("components", []))
171
+ return
172
+ meta = await resolve_npm_package(request.input)
173
+ library = meta["name"]
174
+ version = meta["version"]
175
+ docs_url = meta.get("homepage")
176
+ version_tag = meta.get("_version_tag") # set when version not on npm but GitHub exists
177
+ # Copy-paste library: docs already fetched during resolution
178
+ if meta.get("_is_copypaste"):
179
+ copypaste_docs = meta.get("_raw_docs", "")
180
+ pre_structured = meta.get("_structured") or None
181
+ logger.info(
182
+ "Copy-paste library '%s' detected via %s β€” skipping crawl (%d chars pre-fetched)",
183
+ library, meta.get("_copypaste_method"), len(copypaste_docs),
184
+ )
185
+ else:
186
+ logger.info("Resolved npm: %s@%s docs=%s tag=%s", library, version, docs_url, version_tag)
187
+
188
+ elif request.input_type == InputType.pypi:
189
+ meta = await resolve_pypi_package(request.input)
190
+ library = meta["name"]
191
+ version = meta["version"]
192
+ docs_url = meta.get("homepage")
193
+ logger.info("Resolved PyPI: %s@%s docs=%s", library, version, docs_url)
194
+
195
+ elif request.input_type == InputType.crates:
196
+ meta = await resolve_crates(request.input)
197
+ library = meta["name"]
198
+ version = meta["version"]
199
+ docs_url = meta.get("homepage")
200
+ logger.info("Resolved crates.io: %s@%s docs=%s", library, version, docs_url)
201
+
202
+ elif request.input_type == InputType.rubygems:
203
+ meta = await resolve_rubygems(request.input)
204
+ library = meta["name"]
205
+ version = meta["version"]
206
+ docs_url = meta.get("homepage")
207
+ logger.info("Resolved RubyGems: %s@%s docs=%s", library, version, docs_url)
208
+
209
+ elif request.input_type == InputType.pubdev:
210
+ meta = await resolve_pubdev(request.input)
211
+ library = meta["name"]
212
+ version = meta["version"]
213
+ docs_url = meta.get("homepage")
214
+ logger.info("Resolved pub.dev: %s@%s docs=%s", library, version, docs_url)
215
+
216
+ elif request.input_type == InputType.nuget:
217
+ meta = await resolve_nuget(request.input)
218
+ library = meta["name"]
219
+ version = meta["version"]
220
+ docs_url = meta.get("homepage")
221
+ logger.info("Resolved NuGet: %s@%s docs=%s", library, version, docs_url)
222
+
223
+ elif request.input_type == InputType.maven:
224
+ meta = await resolve_maven(request.input)
225
+ library = meta["name"]
226
+ version = meta["version"]
227
+ docs_url = meta.get("homepage")
228
+ logger.info("Resolved Maven: %s@%s docs=%s", library, version, docs_url)
229
+
230
+ elif request.input_type == InputType.hex:
231
+ meta = await resolve_hex(request.input)
232
+ library = meta["name"]
233
+ version = meta["version"]
234
+ docs_url = meta.get("homepage")
235
+ logger.info("Resolved Hex: %s@%s docs=%s", library, version, docs_url)
236
+
237
+ elif request.input_type == InputType.cran:
238
+ meta = await resolve_cran(request.input)
239
+ library = meta["name"]
240
+ version = meta["version"]
241
+ docs_url = meta.get("homepage")
242
+ logger.info("Resolved CRAN: %s@%s docs=%s", library, version, docs_url)
243
+
244
+ elif request.input_type == InputType.url:
245
+ # Derive a clean library name from the domain so the cache key is
246
+ # human-readable ("reactbits") rather than the raw URL.
247
+ # GitHub URLs keep their repo name; plain URLs use the domain stem.
248
+ if "github.com" in request.input:
249
+ # e.g. https://github.com/DavidHDev/react-bits β†’ react-bits
250
+ parts = [p for p in urlparse(request.input).path.split("/") if p]
251
+ library = parts[1] if len(parts) >= 2 else request.input
252
+ else:
253
+ domain = urlparse(request.input).netloc.removeprefix("www.")
254
+ library = domain.split(".")[0] # "reactbits" from "reactbits.dev"
255
+ logger.info("URL input '%s' β†’ library name '%s' for cache/structuring", request.input, library)
256
+
257
+ # ── Step 2: Check cache (skip expensive crawl if already done) ────
258
+ cached = await cache.get_cached_context(library, version)
259
+ if cached:
260
+ output = format_context_md(cached) if request.output_format == OutputFormat.context_md else str(cached)
261
+ await update_status(
262
+ JobStatus.complete,
263
+ library=library,
264
+ version=version,
265
+ output=output,
266
+ components=cached.get("components", []),
267
+ )
268
+ logger.info("Cache hit for %s@%s, job %s done in %.1fs", library, version, job_id, time.time() - t_start)
269
+ return
270
+
271
+ # ── Step 3: Crawl documentation ───────────────────────────────────
272
+ raw_docs = ""
273
+
274
+ # Copy-paste library β€” docs were fetched during resolution, skip crawl
275
+ if copypaste_docs is not None:
276
+ raw_docs = copypaste_docs
277
+
278
+ elif request.input_type == InputType.paste:
279
+ raw_docs = await ingest_paste(request.content or request.input)
280
+
281
+ elif request.input_type == InputType.github:
282
+ from ..services.ingestion.github_ingester import ingest_github_repo
283
+ raw_docs = await ingest_github_repo(request.input)
284
+
285
+ elif request.input_type == InputType.url:
286
+ if "github.com" in request.input:
287
+ from ..services.ingestion.github_ingester import ingest_github_repo
288
+ raw_docs = await ingest_github_repo(request.input)
289
+ else:
290
+ import asyncio as _asyncio
291
+ from ..services.ingestion.url_ingester import extract_github_url
292
+ from ..services.ingestion.copypaste_ingester import (
293
+ check_jsrepo, check_shadcn_registry, detect_and_ingest,
294
+ )
295
+
296
+ # library is already set to the clean name_hint in Step 1
297
+ # ── Parallel: all three lightweight checks at once ─────────────
298
+ # check_shadcn_registry hits /r/registry.json etc. β€” it does NOT
299
+ # need a GitHub URL, so it runs alongside the other two.
300
+ github_url, jsrepo_name_docs, shadcn_result = await _asyncio.gather(
301
+ extract_github_url(request.input),
302
+ check_jsrepo(library, None),
303
+ check_shadcn_registry(request.input, library),
304
+ )
305
+
306
+ # shadcn registry returns a pre-built structured dict β€” skip LLM entirely
307
+ if shadcn_result is not None:
308
+ pre_structured = shadcn_result
309
+ logger.info(
310
+ "URL '%s' β†’ shadcn registry match for '%s' (%d components, skipping LLM)",
311
+ request.input, library, len(pre_structured.get("components", [])),
312
+ )
313
+
314
+ else:
315
+ # Prefer jsrepo registry hits for the raw-docs path
316
+ quick_docs = jsrepo_name_docs
317
+
318
+ # If we also have a GitHub URL, do a targeted jsrepo check
319
+ # (raw GitHub paths + jsrepo/{owner}/{repo}).
320
+ jsrepo_targeted: str | None = None
321
+ if github_url and not quick_docs:
322
+ jsrepo_targeted = await check_jsrepo(library, github_url)
323
+
324
+ cp_docs = jsrepo_targeted or quick_docs
325
+
326
+ if cp_docs:
327
+ raw_docs = cp_docs
328
+ logger.info("URL '%s' β†’ registry match for '%s' (%d chars)", request.input, library, len(raw_docs))
329
+
330
+ elif github_url:
331
+ logger.info("URL '%s' β†’ GitHub repo found: %s", request.input, github_url)
332
+ # copy-paste detection with full github_url context
333
+ cp_result = await detect_and_ingest(
334
+ name=library,
335
+ github_url=github_url,
336
+ homepage=request.input,
337
+ )
338
+ if cp_result:
339
+ raw_docs = cp_result.get("_raw_docs", "")
340
+ logger.info(
341
+ "URL '%s' β†’ copy-paste lib (method=%s) %d chars",
342
+ request.input, cp_result.get("_copypaste_method"), len(raw_docs),
343
+ )
344
+ else:
345
+ # Not copy-paste β€” GitHub ingester still beats Playwright
346
+ from ..services.ingestion.github_ingester import ingest_github_repo
347
+ raw_docs = await ingest_github_repo(github_url)
348
+ logger.info("URL '%s' β†’ GitHub ingestion (%d chars)", request.input, len(raw_docs))
349
+
350
+ else:
351
+ # No GitHub found β€” fall back to Playwright
352
+ raw_docs = await crawl_url(request.input)
353
+ logger.info("URL '%s' β†’ Playwright crawl (%d chars)", request.input, len(raw_docs))
354
+
355
+ # Sparse content β€” try GitHub search by domain name.
356
+ # Only accept a fallback repo whose homepage points back to the
357
+ # same domain we started from (prevents picking up unrelated
358
+ # packages like MCP wrappers that happen to mention the name).
359
+ if len(raw_docs) < 20_000:
360
+ import httpx
361
+ input_domain = urlparse(request.input).netloc.removeprefix("www.")
362
+ async with httpx.AsyncClient(timeout=10) as _client:
363
+ from ..services.ingestion.npm_ingester import _github_search
364
+ fallback = await _github_search(_client, library)
365
+ if fallback:
366
+ # Quick check: does the GitHub repo's homepage match?
367
+ try:
368
+ meta_resp = await _client.get(
369
+ f"https://api.github.com/repos/{fallback.removeprefix('https://github.com/')}",
370
+ headers={"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"},
371
+ timeout=5,
372
+ )
373
+ if meta_resp.is_success:
374
+ repo_homepage = meta_resp.json().get("homepage") or ""
375
+ if input_domain not in repo_homepage:
376
+ logger.info(
377
+ "GitHub fallback '%s' homepage '%s' doesn't match domain '%s' β€” skipping",
378
+ fallback, repo_homepage, input_domain,
379
+ )
380
+ fallback = None
381
+ except Exception:
382
+ fallback = None # if we can't verify, skip it
383
+
384
+ if fallback:
385
+ logger.info(
386
+ "Sparse Playwright content (%d chars) for '%s' β€” trying GitHub: %s",
387
+ len(raw_docs), request.input, fallback,
388
+ )
389
+ from ..services.ingestion.github_ingester import ingest_github_repo
390
+ github_docs = await ingest_github_repo(fallback)
391
+ if len(github_docs) > len(raw_docs):
392
+ raw_docs = github_docs
393
+
394
+ else:
395
+ # npm or pypi: crawl the resolved docs URL
396
+ if version_tag and not docs_url:
397
+ # Version not on npm but GitHub repo known β€” use git tag directly
398
+ from ..services.ingestion.github_ingester import ingest_github_repo
399
+ repo_url = meta.get("repository", "")
400
+ import re as _re
401
+ repo_url = _re.sub(r"^git\+", "", repo_url)
402
+ logger.info("Using GitHub tag %s for %s", version_tag, repo_url)
403
+ raw_docs = await ingest_github_repo(repo_url, version_tag=version_tag)
404
+ elif docs_url:
405
+ if "github.com" in docs_url:
406
+ from ..services.ingestion.github_ingester import ingest_github_repo
407
+ raw_docs = await ingest_github_repo(docs_url, version_tag=version_tag)
408
+ else:
409
+ raw_docs = await crawl_url(docs_url)
410
+ else:
411
+ # Web search fallback: query DuckDuckGo for docs
412
+ found_url = await find_docs_url(f"{library} {version}")
413
+ if found_url:
414
+ logger.info("Web search found docs for %s: %s", library, found_url)
415
+ if "github.com" in found_url:
416
+ from ..services.ingestion.github_ingester import ingest_github_repo
417
+ raw_docs = await ingest_github_repo(found_url)
418
+ else:
419
+ raw_docs = await crawl_url(found_url)
420
+ else:
421
+ logger.warning("No docs URL found for %s, sending minimal stub", library)
422
+ raw_docs = f"Library: {library}\nVersion: {version}\nNo documentation URL found."
423
+
424
+ logger.info("Crawl done for %s@%s: %d chars in %.1fs", library, version, len(raw_docs), time.time() - t_start)
425
+
426
+ # ── Step 4: Structure with Gemini ─────────────────────────────────
427
+ # Skip LLM for copy-paste libraries with pre-built structured data
428
+ if pre_structured is not None:
429
+ structured = pre_structured
430
+ structured.setdefault("library", library)
431
+ structured.setdefault("version", version)
432
+ logger.info("Using pre-built structure for %s@%s (%d components)", library, version, len(structured.get("components", [])))
433
+ else:
434
+ structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key)
435
+
436
+ # Filter to requested components if specified
437
+ if request.components:
438
+ requested = {c.lower() for c in request.components}
439
+ structured["components"] = [
440
+ c for c in structured.get("components", [])
441
+ if c.get("name", "").lower() in requested
442
+ ]
443
+
444
+ # ── Step 5: Format output ─────────────────────────────────────────
445
+ if request.output_format == OutputFormat.context_md:
446
+ output = format_context_md(structured)
447
+ else:
448
+ import json
449
+ output = json.dumps(structured, indent=2)
450
+
451
+ # ── Step 6: Cache and complete ────────────────────────────────────
452
+ structured["cached_at"] = date.today().isoformat()
453
+ await cache.set_cached_context(library, version, structured)
454
+
455
+ total = time.time() - t_start
456
+ logger.info("Pipeline complete for %s@%s in %.1fs", library, version, total)
457
+
458
+ await update_status(
459
+ JobStatus.complete,
460
+ library=library,
461
+ version=version,
462
+ output=output,
463
+ components=structured.get("components", []),
464
+ )
465
+
466
+ except Exception as exc:
467
+ logger.exception("Pipeline failed for job %s: %s", job_id, exc)
468
+ await update_status(JobStatus.failed, error=str(exc))
backend/routers/corrections.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """POST /api/corrections β€” submit a correction for a library's extracted docs."""
2
+
3
+ import os
4
+ import uuid
5
+ from typing import Optional
6
+
7
+ from fastapi import APIRouter
8
+ from pydantic import BaseModel
9
+
10
+ router = APIRouter(prefix="/api/corrections", tags=["corrections"])
11
+
12
+ DEV_MODE = os.getenv("DEV_MODE", "true").lower() == "true"
13
+ SUPABASE_URL = os.getenv("SUPABASE_URL", "")
14
+ SUPABASE_KEY = os.getenv("SUPABASE_KEY", "")
15
+
16
+ # In-memory store for DEV_MODE
17
+ _corrections: dict[str, dict] = {}
18
+
19
+
20
+ class CorrectionRequest(BaseModel):
21
+ library: str
22
+ version: str
23
+ component_name: str
24
+ field: str # "gotcha" | "prop" | "usage_example" | "import_path"
25
+ correction: str
26
+ reporter_email: Optional[str] = None
27
+
28
+
29
+ class CorrectionResponse(BaseModel):
30
+ correction_id: str
31
+ status: str
32
+
33
+
34
+ @router.post("", response_model=CorrectionResponse)
35
+ async def submit_correction(req: CorrectionRequest) -> CorrectionResponse:
36
+ """Submit a correction for a library's extracted documentation."""
37
+ correction_id = str(uuid.uuid4())
38
+
39
+ if DEV_MODE or not (SUPABASE_URL and SUPABASE_KEY):
40
+ _corrections[correction_id] = {
41
+ "id": correction_id,
42
+ "library": req.library,
43
+ "version": req.version,
44
+ "component_name": req.component_name,
45
+ "field": req.field,
46
+ "correction": req.correction,
47
+ "reporter_email": req.reporter_email,
48
+ "status": "pending",
49
+ }
50
+ else:
51
+ import httpx
52
+ async with httpx.AsyncClient(timeout=10) as client:
53
+ await client.post(
54
+ f"{SUPABASE_URL}/rest/v1/corrections",
55
+ headers={
56
+ "apikey": SUPABASE_KEY,
57
+ "Authorization": f"Bearer {SUPABASE_KEY}",
58
+ "Content-Type": "application/json",
59
+ "Prefer": "return=minimal",
60
+ },
61
+ json={
62
+ "id": correction_id,
63
+ "library": req.library,
64
+ "version": req.version,
65
+ "component_name": req.component_name,
66
+ "field": req.field,
67
+ "correction": req.correction,
68
+ "reporter_email": req.reporter_email,
69
+ "status": "pending",
70
+ },
71
+ )
72
+
73
+ return CorrectionResponse(correction_id=correction_id, status="pending")
74
+
75
+
76
+ @router.get("/{library}")
77
+ async def get_corrections(library: str, version: Optional[str] = None) -> dict:
78
+ """Get approved corrections for a library."""
79
+ if DEV_MODE or not (SUPABASE_URL and SUPABASE_KEY):
80
+ results = [
81
+ c for c in _corrections.values()
82
+ if c["library"] == library and c["status"] == "approved"
83
+ and (version is None or c["version"] == version)
84
+ ]
85
+ return {"corrections": results}
86
+
87
+ import httpx
88
+ params = f"library=eq.{library}&status=eq.approved"
89
+ if version:
90
+ params += f"&version=eq.{version}"
91
+ async with httpx.AsyncClient(timeout=10) as client:
92
+ resp = await client.get(
93
+ f"{SUPABASE_URL}/rest/v1/corrections?{params}",
94
+ headers={"apikey": SUPABASE_KEY, "Authorization": f"Bearer {SUPABASE_KEY}"},
95
+ )
96
+ return {"corrections": resp.json() if resp.status_code == 200 else []}
backend/routers/oauth.py ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ OAuth router β€” Google and GitHub sign-in / sign-up
3
+
4
+ GET /api/auth/oauth/google β†’ redirect to Google consent page
5
+ GET /api/auth/oauth/google/callback β†’ exchange code, create/find user, redirect to frontend
6
+ GET /api/auth/oauth/github β†’ redirect to GitHub consent page
7
+ GET /api/auth/oauth/github/callback β†’ exchange code, create/find user, redirect to frontend
8
+
9
+ After successful auth the user is sent to:
10
+ FRONTEND_URL/auth/callback?token=<jwt>&email=<email>
11
+ On failure they land at:
12
+ FRONTEND_URL/auth/login?error=<message>
13
+ """
14
+
15
+ import os
16
+ import httpx
17
+ from urllib.parse import urlencode
18
+
19
+ from fastapi import APIRouter
20
+ from fastapi.responses import RedirectResponse
21
+
22
+ from backend.services import auth_service
23
+
24
+ router = APIRouter(prefix="/api/auth/oauth", tags=["oauth"])
25
+
26
+ FRONTEND_URL = os.getenv("FRONTEND_URL", "http://localhost:3000")
27
+ BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8000")
28
+ GOOGLE_CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID", "")
29
+ GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET", "")
30
+ GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "")
31
+ GITHUB_CLIENT_SECRET = os.getenv("GITHUB_CLIENT_SECRET", "")
32
+
33
+ GOOGLE_REDIRECT_URI = f"{BACKEND_URL}/api/auth/oauth/google/callback"
34
+ GITHUB_REDIRECT_URI = f"{BACKEND_URL}/api/auth/oauth/github/callback"
35
+
36
+
37
+ def _error_redirect(msg: str) -> RedirectResponse:
38
+ return RedirectResponse(f"{FRONTEND_URL}/auth/login?error={msg.replace(' ', '+')}")
39
+
40
+
41
+ # ── Google ────────────────────────────────────────────────────────────────────
42
+
43
+ @router.get("/google")
44
+ async def google_login(source: str = "web", callback_port: str = ""):
45
+ if not GOOGLE_CLIENT_ID:
46
+ return _error_redirect("Google OAuth not configured")
47
+ state = f"vscode_local:{callback_port}" if source == "vscode_local" and callback_port else source
48
+ params = urlencode({
49
+ "client_id": GOOGLE_CLIENT_ID,
50
+ "redirect_uri": GOOGLE_REDIRECT_URI,
51
+ "response_type": "code",
52
+ "scope": "openid email profile",
53
+ "access_type": "offline",
54
+ "state": state,
55
+ })
56
+ return RedirectResponse(f"https://accounts.google.com/o/oauth2/v2/auth?{params}")
57
+
58
+
59
+ @router.get("/google/callback")
60
+ async def google_callback(code: str = "", error: str = "", state: str = "web"):
61
+ if error or not code:
62
+ return _error_redirect("Google sign-in cancelled")
63
+
64
+ async with httpx.AsyncClient() as client:
65
+ token_res = await client.post(
66
+ "https://oauth2.googleapis.com/token",
67
+ data={
68
+ "code": code,
69
+ "client_id": GOOGLE_CLIENT_ID,
70
+ "client_secret": GOOGLE_CLIENT_SECRET,
71
+ "redirect_uri": GOOGLE_REDIRECT_URI,
72
+ "grant_type": "authorization_code",
73
+ },
74
+ )
75
+ if not token_res.is_success:
76
+ return _error_redirect("Google token exchange failed")
77
+
78
+ access_token = token_res.json().get("access_token")
79
+ if not access_token:
80
+ return _error_redirect("No access token from Google")
81
+
82
+ info_res = await client.get(
83
+ "https://www.googleapis.com/oauth2/v2/userinfo",
84
+ headers={"Authorization": f"Bearer {access_token}"},
85
+ )
86
+ if not info_res.is_success:
87
+ return _error_redirect("Could not fetch Google profile")
88
+
89
+ email = info_res.json().get("email")
90
+
91
+ if not email:
92
+ return _error_redirect("No email in Google profile")
93
+
94
+ user = await auth_service.get_or_create_oauth_user(email)
95
+ token = auth_service.create_access_token(user["id"], user["email"])
96
+ if state.startswith("vscode_local:"):
97
+ port = state.split(":", 1)[1]
98
+ return RedirectResponse(f"http://localhost:{port}/?token={token}&email={email}")
99
+ if state == "vscode":
100
+ return RedirectResponse(
101
+ f"vscode://docforge.docforge/callback?token={token}&email={email}"
102
+ )
103
+ return RedirectResponse(
104
+ f"{FRONTEND_URL}/auth/callback?token={token}&email={email}"
105
+ )
106
+
107
+
108
+ # ── GitHub ────────────────────────────────────────────────────────────────────
109
+
110
+ @router.get("/github")
111
+ async def github_login(source: str = "web", callback_port: str = ""):
112
+ if not GITHUB_CLIENT_ID:
113
+ return _error_redirect("GitHub OAuth not configured")
114
+ state = f"vscode_local:{callback_port}" if source == "vscode_local" and callback_port else source
115
+ params = urlencode({
116
+ "client_id": GITHUB_CLIENT_ID,
117
+ "redirect_uri": GITHUB_REDIRECT_URI,
118
+ "scope": "user:email",
119
+ "state": state,
120
+ })
121
+ return RedirectResponse(f"https://github.com/login/oauth/authorize?{params}")
122
+
123
+
124
+ @router.get("/github/callback")
125
+ async def github_callback(code: str = "", error: str = "", state: str = "web"):
126
+ if error or not code:
127
+ return _error_redirect("GitHub sign-in cancelled")
128
+
129
+ async with httpx.AsyncClient() as client:
130
+ token_res = await client.post(
131
+ "https://github.com/login/oauth/access_token",
132
+ headers={"Accept": "application/json"},
133
+ data={
134
+ "client_id": GITHUB_CLIENT_ID,
135
+ "client_secret": GITHUB_CLIENT_SECRET,
136
+ "code": code,
137
+ "redirect_uri": GITHUB_REDIRECT_URI,
138
+ },
139
+ )
140
+ if not token_res.is_success:
141
+ return _error_redirect("GitHub token exchange failed")
142
+
143
+ gh_token = token_res.json().get("access_token")
144
+ if not gh_token:
145
+ return _error_redirect("No access token from GitHub")
146
+
147
+ # Prefer verified primary email from /user/emails
148
+ emails_res = await client.get(
149
+ "https://api.github.com/user/emails",
150
+ headers={
151
+ "Authorization": f"Bearer {gh_token}",
152
+ "Accept": "application/vnd.github+json",
153
+ },
154
+ )
155
+ email = None
156
+ if emails_res.is_success:
157
+ for entry in emails_res.json():
158
+ if entry.get("primary") and entry.get("verified"):
159
+ email = entry["email"]
160
+ break
161
+
162
+ # Fallback to public profile email
163
+ if not email:
164
+ user_res = await client.get(
165
+ "https://api.github.com/user",
166
+ headers={"Authorization": f"Bearer {gh_token}"},
167
+ )
168
+ if user_res.is_success:
169
+ email = user_res.json().get("email")
170
+
171
+ if not email:
172
+ return _error_redirect("No verified email on GitHub account")
173
+
174
+ user = await auth_service.get_or_create_oauth_user(email)
175
+ token = auth_service.create_access_token(user["id"], user["email"])
176
+ if state.startswith("vscode_local:"):
177
+ port = state.split(":", 1)[1]
178
+ return RedirectResponse(f"http://localhost:{port}/?token={token}&email={email}")
179
+ if state == "vscode":
180
+ return RedirectResponse(
181
+ f"vscode://docforge.docforge/callback?token={token}&email={email}"
182
+ )
183
+ return RedirectResponse(
184
+ f"{FRONTEND_URL}/auth/callback?token={token}&email={email}"
185
+ )
backend/routers/search.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """GET /api/search?q={query} β€” search cached libraries."""
2
+
3
+ from fastapi import APIRouter, Query
4
+
5
+ from ..models.schemas import SearchResponse, SearchResult
6
+ from ..services.cache import search_cached_libraries
7
+
8
+ router = APIRouter(prefix="/api/search", tags=["search"])
9
+
10
+
11
+ @router.get("", response_model=SearchResponse)
12
+ async def search_libraries(q: str = Query("", description="Search query"), limit: int = Query(20)) -> SearchResponse:
13
+ """Search the DocForge cache for libraries matching a query string."""
14
+ raw_results = await search_cached_libraries(q)
15
+ results = [SearchResult(**r) for r in raw_results][:limit]
16
+ return SearchResponse(results=results)
backend/routers/versions.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """GET /api/versions/{package} β€” list cached versions for a package."""
2
+
3
+ from fastapi import APIRouter
4
+
5
+ from ..models.schemas import VersionsResponse
6
+ from ..services.cache import list_cached_versions
7
+
8
+ router = APIRouter(prefix="/api/versions", tags=["versions"])
9
+
10
+
11
+ @router.get("/{package:path}", response_model=VersionsResponse)
12
+ async def get_versions(package: str) -> VersionsResponse:
13
+ """List all cached versions of a package in the DocForge cache."""
14
+ versions = await list_cached_versions(package)
15
+ return VersionsResponse(package=package, versions=versions)
backend/services/__init__.py ADDED
File without changes
backend/services/auth_service.py ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Auth service: JWT creation/validation, password hashing, device-flow management,
3
+ and encrypted API key storage.
4
+
5
+ Storage strategy mirrors cache.py:
6
+ - Supabase when SUPABASE_URL + SUPABASE_KEY are set (production)
7
+ - In-memory dict otherwise (dev / CI)
8
+
9
+ Tables required in Supabase:
10
+ users (id uuid PK, email text UNIQUE, password_hash text, created_at timestamptz)
11
+ user_api_keys(user_id uuid FK->users.id, gemini_key_enc text, groq_key_enc text)
12
+ device_codes (code text PK, user_code text UNIQUE, user_id uuid nullable,
13
+ expires_at timestamptz, verified bool default false)
14
+ """
15
+
16
+ import os
17
+ import uuid
18
+ import string
19
+ import random
20
+ import time
21
+ from datetime import datetime, timezone, timedelta
22
+ from typing import Optional
23
+
24
+ from jose import jwt, JWTError
25
+ from passlib.context import CryptContext
26
+ from cryptography.fernet import Fernet
27
+ import base64
28
+
29
+ # ── Config ─────────────────────────────────────────────────────────────────────
30
+
31
+ SECRET_KEY = os.getenv("SECRET_KEY", "CHANGE_ME_IN_PRODUCTION_" + "x" * 32)
32
+ ALGORITHM = "HS256"
33
+ ACCESS_TOKEN_EXPIRE_HOURS = 24 * 30 # 30 days
34
+
35
+ # Key for encrypting stored API keys (Fernet requires 32-byte URL-safe base64)
36
+ _raw_enc_key = os.getenv("ENCRYPTION_KEY", "")
37
+ if _raw_enc_key:
38
+ FERNET_KEY = _raw_enc_key.encode()
39
+ else:
40
+ # Derive a stable key from SECRET_KEY so dev works without .env
41
+ import hashlib
42
+ _derived = hashlib.sha256(SECRET_KEY.encode()).digest()
43
+ FERNET_KEY = base64.urlsafe_b64encode(_derived)
44
+
45
+ _fernet = Fernet(FERNET_KEY)
46
+ _pwd_ctx = CryptContext(schemes=["bcrypt"], deprecated="auto")
47
+
48
+ # ── Supabase helpers ──────────────────────────────────────────────────────────
49
+
50
+ def _get_supabase():
51
+ url = os.getenv("SUPABASE_URL")
52
+ key = os.getenv("SUPABASE_KEY")
53
+ if url and key:
54
+ from supabase import create_client
55
+ return create_client(url, key)
56
+ return None
57
+
58
+ # ── In-memory fallback ────────────────────────────────────────────────────────
59
+ # { email β†’ { id, email, password_hash } }
60
+ _mem_users: dict = {}
61
+ # { user_id β†’ { gemini_key_enc, groq_key_enc } }
62
+ _mem_keys: dict = {}
63
+ # { device_code β†’ { user_code, user_id, expires_at, verified } }
64
+ _mem_devices: dict = {}
65
+
66
+ # ── Password helpers ──────────────────────────────────────────────────────────
67
+
68
+ def hash_password(plain: str) -> str:
69
+ return _pwd_ctx.hash(plain)
70
+
71
+ def verify_password(plain: str, hashed: str) -> bool:
72
+ return _pwd_ctx.verify(plain, hashed)
73
+
74
+ # ── Encryption helpers ────────────────────────────────────────────────────────
75
+
76
+ def encrypt_key(value: str) -> str:
77
+ return _fernet.encrypt(value.encode()).decode()
78
+
79
+ def decrypt_key(enc: str) -> str:
80
+ return _fernet.decrypt(enc.encode()).decode()
81
+
82
+ # ── JWT ───────────────────────────────────────────────────────────────────────
83
+
84
+ def create_access_token(user_id: str, email: str) -> str:
85
+ expire = datetime.now(timezone.utc) + timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS)
86
+ return jwt.encode(
87
+ {"sub": user_id, "email": email, "exp": expire},
88
+ SECRET_KEY,
89
+ algorithm=ALGORITHM,
90
+ )
91
+
92
+ def decode_token(token: str) -> Optional[dict]:
93
+ try:
94
+ return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
95
+ except JWTError:
96
+ return None
97
+
98
+ # ── User CRUD ─────────────────────────────────────────────────────────────────
99
+
100
+ async def create_user(email: str, password_hash: str) -> Optional[str]:
101
+ """Create a user, return their UUID. Returns None if email exists."""
102
+ sb = _get_supabase()
103
+ if sb:
104
+ existing = sb.table("users").select("id").eq("email", email).execute()
105
+ if existing.data:
106
+ return None
107
+ uid = str(uuid.uuid4())
108
+ sb.table("users").insert({"id": uid, "email": email, "password_hash": password_hash}).execute()
109
+ return uid
110
+ else:
111
+ if email in _mem_users:
112
+ return None
113
+ uid = str(uuid.uuid4())
114
+ _mem_users[email] = {"id": uid, "email": email, "password_hash": password_hash}
115
+ return uid
116
+
117
+ async def get_user_by_email(email: str) -> Optional[dict]:
118
+ sb = _get_supabase()
119
+ if sb:
120
+ res = sb.table("users").select("id, email, password_hash").eq("email", email).execute()
121
+ return res.data[0] if res.data else None
122
+ return _mem_users.get(email)
123
+
124
+ async def get_or_create_oauth_user(email: str) -> dict:
125
+ """Find user by email or create one without a password (OAuth users)."""
126
+ existing = await get_user_by_email(email)
127
+ if existing:
128
+ return existing
129
+ uid = str(uuid.uuid4())
130
+ sb = _get_supabase()
131
+ if sb:
132
+ sb.table("users").insert({"id": uid, "email": email, "password_hash": ""}).execute()
133
+ else:
134
+ _mem_users[email] = {"id": uid, "email": email, "password_hash": ""}
135
+ return {"id": uid, "email": email}
136
+
137
+
138
+ async def get_user_by_id(user_id: str) -> Optional[dict]:
139
+ sb = _get_supabase()
140
+ if sb:
141
+ res = sb.table("users").select("id, email").eq("id", user_id).execute()
142
+ return res.data[0] if res.data else None
143
+ for u in _mem_users.values():
144
+ if u["id"] == user_id:
145
+ return u
146
+ return None
147
+
148
+ # ── API Key CRUD ──────────────────────────────────────────────────────────────
149
+
150
+ async def save_api_keys(user_id: str, gemini_key: Optional[str], groq_key: Optional[str]):
151
+ sb = _get_supabase()
152
+ row: dict = {"user_id": user_id}
153
+ if gemini_key is not None:
154
+ row["gemini_key_enc"] = encrypt_key(gemini_key)
155
+ if groq_key is not None:
156
+ row["groq_key_enc"] = encrypt_key(groq_key)
157
+
158
+ if sb:
159
+ existing = sb.table("user_api_keys").select("user_id").eq("user_id", user_id).execute()
160
+ if existing.data:
161
+ sb.table("user_api_keys").update(row).eq("user_id", user_id).execute()
162
+ else:
163
+ sb.table("user_api_keys").insert(row).execute()
164
+ else:
165
+ if user_id not in _mem_keys:
166
+ _mem_keys[user_id] = {}
167
+ _mem_keys[user_id].update(row)
168
+
169
+ async def get_api_keys(user_id: str) -> dict:
170
+ """Returns { gemini_key: str|None, groq_key: str|None }"""
171
+ sb = _get_supabase()
172
+ if sb:
173
+ res = sb.table("user_api_keys").select("gemini_key_enc, groq_key_enc").eq("user_id", user_id).execute()
174
+ row = res.data[0] if res.data else {}
175
+ else:
176
+ row = _mem_keys.get(user_id, {})
177
+
178
+ def _dec(enc):
179
+ try:
180
+ return decrypt_key(enc) if enc else None
181
+ except Exception:
182
+ return None
183
+
184
+ return {
185
+ "gemini_key": _dec(row.get("gemini_key_enc")),
186
+ "groq_key": _dec(row.get("groq_key_enc")),
187
+ }
188
+
189
+ # ── Device flow ───────────────────────────────────────────────────────────────
190
+
191
+ def _random_code(length=8, chars=string.ascii_uppercase + string.digits) -> str:
192
+ return "".join(random.choices(chars, k=length))
193
+
194
+ def _user_code_format(code: str) -> str:
195
+ return f"{code[:4]}-{code[4:]}"
196
+
197
+ async def create_device_code() -> dict:
198
+ device_code = str(uuid.uuid4())
199
+ raw = _random_code(8)
200
+ user_code = _user_code_format(raw)
201
+ expires_at = time.time() + 900 # 15 min
202
+
203
+ sb = _get_supabase()
204
+ if sb:
205
+ sb.table("device_codes").insert({
206
+ "code": device_code,
207
+ "user_code": user_code,
208
+ "user_id": None,
209
+ "expires_at": datetime.fromtimestamp(expires_at, tz=timezone.utc).isoformat(),
210
+ "verified": False,
211
+ }).execute()
212
+ else:
213
+ _mem_devices[device_code] = {
214
+ "code": device_code,
215
+ "user_code": user_code,
216
+ "user_id": None,
217
+ "expires_at": expires_at,
218
+ "verified": False,
219
+ }
220
+
221
+ return {"device_code": device_code, "user_code": user_code}
222
+
223
+ async def verify_device_code(user_code: str, user_id: str) -> bool:
224
+ """Called from the web UI when user confirms the code. Returns True on success."""
225
+ sb = _get_supabase()
226
+ if sb:
227
+ res = sb.table("device_codes").select("code, expires_at, verified") \
228
+ .eq("user_code", user_code).execute()
229
+ if not res.data:
230
+ return False
231
+ row = res.data[0]
232
+ exp = datetime.fromisoformat(row["expires_at"]).timestamp()
233
+ if row["verified"] or time.time() > exp:
234
+ return False
235
+ sb.table("device_codes").update({"user_id": user_id, "verified": True}) \
236
+ .eq("user_code", user_code).execute()
237
+ return True
238
+ else:
239
+ for entry in _mem_devices.values():
240
+ if entry["user_code"] == user_code:
241
+ if entry["verified"] or time.time() > entry["expires_at"]:
242
+ return False
243
+ entry["user_id"] = user_id
244
+ entry["verified"] = True
245
+ return True
246
+ return False
247
+
248
+ async def poll_device_code(device_code: str) -> dict:
249
+ """Called by CLI to poll status. Returns status + token when verified."""
250
+ sb = _get_supabase()
251
+ if sb:
252
+ res = sb.table("device_codes").select("user_id, expires_at, verified") \
253
+ .eq("code", device_code).execute()
254
+ if not res.data:
255
+ return {"status": "expired"}
256
+ row = res.data[0]
257
+ exp = datetime.fromisoformat(row["expires_at"]).timestamp()
258
+ if time.time() > exp:
259
+ return {"status": "expired"}
260
+ if not row["verified"] or not row["user_id"]:
261
+ return {"status": "pending"}
262
+ user = await get_user_by_id(row["user_id"])
263
+ else:
264
+ entry = _mem_devices.get(device_code)
265
+ if not entry or time.time() > entry["expires_at"]:
266
+ return {"status": "expired"}
267
+ if not entry["verified"] or not entry["user_id"]:
268
+ return {"status": "pending"}
269
+ user = await get_user_by_id(entry["user_id"])
270
+
271
+ if not user:
272
+ return {"status": "expired"}
273
+
274
+ token = create_access_token(user["id"], user["email"])
275
+ return {"status": "complete", "access_token": token, "email": user["email"]}
backend/services/cache.py ADDED
@@ -0,0 +1,338 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Cache layer for DocForge.
3
+
4
+ Priority:
5
+ 1. Supabase β€” if SUPABASE_URL + SUPABASE_KEY are set (persistent, cross-restart)
6
+ 2. Redis β€” if DEV_MODE=false and REDIS_URL is set
7
+ 3. In-memory β€” fallback for local dev
8
+
9
+ Cache invalidation strategy:
10
+ - Pinned versions (e.g. react@18.2.0): 15-day TTL. Docs for a fixed version never change.
11
+ - "latest" / unpinned entries: GitHub release check on cache hit, but at most once per hour
12
+ (stored as github_checked_at in the cached payload). If a newer GitHub release exists
13
+ since cached_at, the entry is deleted and the pipeline re-runs for the next request.
14
+ """
15
+
16
+ import json
17
+ import logging
18
+ import os
19
+ import time
20
+ from datetime import datetime, timedelta, timezone
21
+ from typing import Any
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ DEV_MODE = os.getenv("DEV_MODE", "true").lower() == "true"
26
+ REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
27
+ SUPABASE_URL = os.getenv("SUPABASE_URL", "")
28
+ SUPABASE_KEY = os.getenv("SUPABASE_KEY", "")
29
+ GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")
30
+
31
+ # Pinned versions live for 15 days; "latest" entries rely on GitHub commit checks
32
+ CACHE_TTL_PINNED = 60 * 60 * 24 * 15 # 15 days
33
+ CACHE_TTL_LATEST = 60 * 60 * 24 * 15 # same wall-clock TTL, but invalidated earlier by GitHub
34
+ CACHE_TTL_SECONDS = CACHE_TTL_PINNED # default (keep old name for job TTL callers)
35
+ # How often we're allowed to re-check GitHub for a given library (seconds)
36
+ GITHUB_CHECK_INTERVAL = 60 * 60 # 1 hour
37
+
38
+
39
+ class InMemoryCache:
40
+ """Fallback cache for local development without Redis."""
41
+
42
+ def __init__(self) -> None:
43
+ self._store: dict[str, tuple[str, float]] = {} # key -> (value, expiry_ts)
44
+
45
+ async def get(self, key: str) -> str | None:
46
+ entry = self._store.get(key)
47
+ if entry is None:
48
+ return None
49
+ value, expiry = entry
50
+ if time.time() > expiry:
51
+ del self._store[key]
52
+ return None
53
+ return value
54
+
55
+ async def set(self, key: str, value: str, ttl: int = CACHE_TTL_SECONDS) -> None:
56
+ self._store[key] = (value, time.time() + ttl)
57
+
58
+ async def keys_matching(self, pattern: str) -> list[str]:
59
+ """Return keys that contain the pattern substring (simplified glob)."""
60
+ needle = pattern.replace("*", "")
61
+ now = time.time()
62
+ return [k for k, (_, exp) in self._store.items() if needle in k and now < exp]
63
+
64
+ async def delete(self, key: str) -> None:
65
+ self._store.pop(key, None)
66
+
67
+
68
+ class RedisCache:
69
+ """Production Redis cache."""
70
+
71
+ def __init__(self, redis_url: str) -> None:
72
+ import redis.asyncio as aioredis # type: ignore[import]
73
+ self._redis = aioredis.from_url(redis_url, decode_responses=True)
74
+
75
+ async def get(self, key: str) -> str | None:
76
+ return await self._redis.get(key)
77
+
78
+ async def set(self, key: str, value: str, ttl: int = CACHE_TTL_SECONDS) -> None:
79
+ await self._redis.setex(key, ttl, value)
80
+
81
+ async def keys_matching(self, pattern: str) -> list[str]:
82
+ return await self._redis.keys(pattern)
83
+
84
+ async def delete(self, key: str) -> None:
85
+ await self._redis.delete(key)
86
+
87
+
88
+ class SupabaseCache:
89
+ """Supabase-backed persistent cache using a `cache_entries` table."""
90
+
91
+ def __init__(self, url: str, key: str) -> None:
92
+ import asyncio
93
+ from supabase import create_client # type: ignore[import]
94
+ self._client = create_client(url, key)
95
+ self._loop = asyncio.get_event_loop
96
+
97
+ async def get(self, key: str) -> str | None:
98
+ import asyncio
99
+ now = datetime.now(timezone.utc).isoformat()
100
+ result = await asyncio.to_thread(
101
+ lambda: self._client.table("cache_entries")
102
+ .select("value")
103
+ .eq("key", key)
104
+ .gt("expires_at", now)
105
+ .execute()
106
+ )
107
+ if result.data:
108
+ return result.data[0]["value"]
109
+ return None
110
+
111
+ async def set(self, key: str, value: str, ttl: int = CACHE_TTL_SECONDS) -> None:
112
+ import asyncio
113
+ expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl)).isoformat()
114
+ await asyncio.to_thread(
115
+ lambda: self._client.table("cache_entries")
116
+ .upsert({"key": key, "value": value, "expires_at": expires_at})
117
+ .execute()
118
+ )
119
+
120
+ async def keys_matching(self, pattern: str) -> list[str]:
121
+ import asyncio
122
+ needle = pattern.replace("*", "")
123
+ now = datetime.now(timezone.utc).isoformat()
124
+ result = await asyncio.to_thread(
125
+ lambda: self._client.table("cache_entries")
126
+ .select("key")
127
+ .like("key", f"%{needle}%")
128
+ .gt("expires_at", now)
129
+ .execute()
130
+ )
131
+ return [row["key"] for row in result.data]
132
+
133
+ async def delete(self, key: str) -> None:
134
+ import asyncio
135
+ await asyncio.to_thread(
136
+ lambda: self._client.table("cache_entries")
137
+ .delete()
138
+ .eq("key", key)
139
+ .execute()
140
+ )
141
+
142
+
143
+ # Singleton cache instance
144
+ _cache: InMemoryCache | RedisCache | SupabaseCache | None = None
145
+
146
+
147
+ def get_cache() -> InMemoryCache | RedisCache | SupabaseCache:
148
+ global _cache
149
+ if _cache is None:
150
+ if SUPABASE_URL and SUPABASE_KEY:
151
+ logger.info("Using Supabase cache at %s", SUPABASE_URL)
152
+ try:
153
+ _cache = SupabaseCache(SUPABASE_URL, SUPABASE_KEY)
154
+ except Exception as exc:
155
+ logger.warning("Supabase cache init failed (%s), falling back to in-memory", exc)
156
+ _cache = InMemoryCache()
157
+ elif not DEV_MODE:
158
+ logger.info("Connecting to Redis at %s", REDIS_URL)
159
+ _cache = RedisCache(REDIS_URL)
160
+ else:
161
+ logger.info("DEV_MODE: using in-memory cache (no Redis)")
162
+ _cache = InMemoryCache()
163
+ return _cache
164
+
165
+
166
+ def make_context_key(library: str, version: str) -> str:
167
+ return f"docforge:context:{library}:{version}"
168
+
169
+
170
+ def make_job_key(job_id: str) -> str:
171
+ return f"docforge:job:{job_id}"
172
+
173
+
174
+ def _extract_github_owner_repo(repository: str) -> tuple[str, str] | None:
175
+ """Parse owner/repo from any GitHub URL variant."""
176
+ import re
177
+ m = re.search(r"github\.com[/:]([^/]+)/([^/.]+)", repository)
178
+ if m:
179
+ return m.group(1), m.group(2)
180
+ return None
181
+
182
+
183
+ async def _github_has_new_release(owner: str, repo: str, since: str) -> bool:
184
+ """
185
+ Return True if there is a GitHub release published after `since` (ISO date string).
186
+ Falls back to checking the latest commit on the default branch if no releases exist.
187
+ """
188
+ import httpx
189
+ headers = {"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"}
190
+ if GITHUB_TOKEN:
191
+ headers["Authorization"] = f"Bearer {GITHUB_TOKEN}"
192
+
193
+ try:
194
+ since_dt = datetime.fromisoformat(since).replace(tzinfo=timezone.utc)
195
+ except ValueError:
196
+ return False
197
+
198
+ async with httpx.AsyncClient(timeout=8) as client:
199
+ # 1. Check releases first
200
+ try:
201
+ r = await client.get(
202
+ f"https://api.github.com/repos/{owner}/{repo}/releases/latest",
203
+ headers=headers,
204
+ )
205
+ if r.status_code == 200:
206
+ published = r.json().get("published_at", "")
207
+ if published:
208
+ pub_dt = datetime.fromisoformat(published.rstrip("Z")).replace(tzinfo=timezone.utc)
209
+ if pub_dt > since_dt:
210
+ logger.info("New release on %s/%s (%s > %s) β€” cache stale", owner, repo, published, since)
211
+ return True
212
+ return False # release exists but is older β€” cache is fresh
213
+ except Exception:
214
+ pass # fall through to commit check
215
+
216
+ # 2. No releases β€” check latest commit on default branch
217
+ try:
218
+ r = await client.get(
219
+ f"https://api.github.com/repos/{owner}/{repo}/commits?per_page=1",
220
+ headers=headers,
221
+ )
222
+ if r.status_code == 200 and r.json():
223
+ commit_date = (
224
+ r.json()[0]
225
+ .get("commit", {})
226
+ .get("committer", {})
227
+ .get("date", "")
228
+ )
229
+ if commit_date:
230
+ commit_dt = datetime.fromisoformat(commit_date.rstrip("Z")).replace(tzinfo=timezone.utc)
231
+ if commit_dt > since_dt:
232
+ logger.info("New commit on %s/%s (%s > %s) β€” cache stale", owner, repo, commit_date, since)
233
+ return True
234
+ except Exception:
235
+ pass
236
+
237
+ return False
238
+
239
+
240
+ async def get_cached_context(library: str, version: str) -> dict[str, Any] | None:
241
+ cache = get_cache()
242
+ key = make_context_key(library, version)
243
+ raw = await cache.get(key)
244
+ if not raw:
245
+ logger.info("Cache MISS for %s@%s", library, version)
246
+ return None
247
+
248
+ data: dict[str, Any] = json.loads(raw)
249
+ logger.info("Cache HIT for %s@%s", library, version)
250
+
251
+ # Pinned versions (e.g. react@18.2.0) never need GitHub staleness checks β€”
252
+ # the docs for that exact version don't change. Only "latest" or similar.
253
+ is_pinned = version not in ("latest", "latest", "") and version[0].isdigit()
254
+ if is_pinned:
255
+ return data
256
+
257
+ # For "latest" entries: check GitHub at most once per GITHUB_CHECK_INTERVAL
258
+ last_checked = data.get("github_checked_at")
259
+ now_ts = time.time()
260
+ if last_checked and (now_ts - last_checked) < GITHUB_CHECK_INTERVAL:
261
+ return data # checked recently β€” trust the cache
262
+
263
+ repo_url: str = data.get("repository", "")
264
+ parsed = _extract_github_owner_repo(repo_url) if repo_url else None
265
+
266
+ if parsed:
267
+ owner, repo = parsed
268
+ cached_at: str = data.get("cached_at", "")
269
+ try:
270
+ stale = await _github_has_new_release(owner, repo, cached_at)
271
+ except Exception as exc:
272
+ logger.warning("GitHub staleness check failed for %s/%s: %s β€” keeping cache", owner, repo, exc)
273
+ stale = False
274
+
275
+ if stale:
276
+ logger.info("Invalidating cache for %s@%s β€” new GitHub release detected", library, version)
277
+ await cache.delete(key)
278
+ return None # caller will re-run the pipeline
279
+
280
+ # Mark the time we last checked so we don't re-check for another hour
281
+ data["github_checked_at"] = now_ts
282
+ await cache.set(key, json.dumps(data), ttl=CACHE_TTL_LATEST)
283
+ return data
284
+
285
+
286
+ async def set_cached_context(library: str, version: str, data: dict[str, Any]) -> None:
287
+ cache = get_cache()
288
+ # Pinned versions get the full 15-day TTL; "latest" also gets 15 days but
289
+ # GitHub staleness checks will evict it earlier if a new release lands.
290
+ is_pinned = version not in ("latest", "") and version[:1].isdigit()
291
+ ttl = CACHE_TTL_PINNED if is_pinned else CACHE_TTL_LATEST
292
+ await cache.set(make_context_key(library, version), json.dumps(data), ttl=ttl)
293
+
294
+
295
+ async def get_job(job_id: str) -> dict[str, Any] | None:
296
+ cache = get_cache()
297
+ raw = await cache.get(make_job_key(job_id))
298
+ if raw:
299
+ return json.loads(raw)
300
+ return None
301
+
302
+
303
+ async def set_job(job_id: str, data: dict[str, Any]) -> None:
304
+ cache = get_cache()
305
+ # Jobs are stored for 1 hour β€” enough to poll results
306
+ await cache.set(make_job_key(job_id), json.dumps(data), ttl=3600)
307
+
308
+
309
+ async def search_cached_libraries(query: str) -> list[dict[str, str]]:
310
+ """Search in-memory index for libraries matching query."""
311
+ cache = get_cache()
312
+ pattern = f"docforge:context:*{query.lower()}*"
313
+ keys = await cache.keys_matching(pattern)
314
+ results = []
315
+ for key in keys:
316
+ raw = await cache.get(key)
317
+ if raw:
318
+ data = json.loads(raw)
319
+ results.append({
320
+ "name": data.get("library", ""),
321
+ "version": data.get("version", ""),
322
+ "cached_at": data.get("cached_at", ""),
323
+ })
324
+ return results
325
+
326
+
327
+ async def list_cached_versions(package: str) -> list[str]:
328
+ """List all cached versions for a given package."""
329
+ cache = get_cache()
330
+ pattern = f"docforge:context:{package.lower()}:*"
331
+ keys = await cache.keys_matching(pattern)
332
+ versions = []
333
+ for key in keys:
334
+ # key format: docforge:context:{library}:{version}
335
+ parts = key.split(":")
336
+ if len(parts) >= 4:
337
+ versions.append(parts[3])
338
+ return sorted(versions, reverse=True)
backend/services/formatter.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Formatter: converts structured JSON β†’ .context.md
3
+
4
+ The output format is specifically designed to:
5
+ 1. Be readable by humans reviewing what was generated
6
+ 2. Maximize signal density for AI context windows (no fluff)
7
+ 3. Prominently feature gotchas β€” the main reason this tool exists
8
+ """
9
+
10
+ from datetime import date
11
+
12
+
13
+ def format_context_md(data: dict) -> str:
14
+ """Convert a structured library dict to a .context.md string."""
15
+ library = data.get("library", "unknown")
16
+ version = data.get("version", "unknown")
17
+ description = data.get("description", "")
18
+ components = data.get("components", [])
19
+ today = date.today().isoformat()
20
+
21
+ lines: list[str] = []
22
+
23
+ # Header
24
+ lines.append(f"# {library}@{version} β€” DocForge Context File")
25
+ lines.append(f"> Generated by DocForge | Last verified: {today}")
26
+ lines.append("> Drop this file into your project and reference it with @context.md in Cursor/Claude")
27
+ if description:
28
+ lines.append(f"\n{description}")
29
+ lines.append("")
30
+
31
+ # Component sections
32
+ for comp in components:
33
+ lines.append(f"---\n\n## {comp.get('name', 'Unknown')}")
34
+
35
+ # Install command
36
+ install = comp.get("install_command")
37
+ if install:
38
+ lines.append("\n**Install:**")
39
+ lines.append("```bash")
40
+ lines.append(install)
41
+ lines.append("```")
42
+
43
+ # Import path
44
+ import_path = comp.get("import_path")
45
+ if import_path:
46
+ lines.append("\n**Import:**")
47
+ lines.append("```typescript")
48
+ lines.append(import_path)
49
+ lines.append("```")
50
+
51
+ # Peer dependencies
52
+ peer_deps = comp.get("peer_dependencies", [])
53
+ if peer_deps:
54
+ lines.append("\n**Peer Dependencies:**")
55
+ for dep in peer_deps:
56
+ lines.append(f"- `{dep}`")
57
+
58
+ # Props table
59
+ props = comp.get("props", [])
60
+ if props:
61
+ lines.append("\n**Props:**")
62
+ lines.append("| Prop | Type | Required | Default |")
63
+ lines.append("|------|------|----------|---------|")
64
+ for prop in props:
65
+ name = prop.get("name", "")
66
+ ptype = prop.get("type", "")
67
+ required = "βœ“" if prop.get("required") else "β€”"
68
+ default = prop.get("default") or "β€”"
69
+ lines.append(f"| {name} | {ptype} | {required} | {default} |")
70
+
71
+ # Gotchas β€” the most important section
72
+ gotchas = comp.get("gotchas", [])
73
+ if gotchas:
74
+ lines.append("\n**⚠️ Gotchas (Common AI Codegen Errors):**")
75
+ for gotcha in gotchas:
76
+ lines.append(f"- {gotcha}")
77
+
78
+ # Usage example
79
+ usage = comp.get("usage_example")
80
+ if usage:
81
+ lines.append("\n**Usage:**")
82
+ # Detect language hint from content
83
+ lang = "tsx" if ("<" in usage and ">" in usage) else "typescript"
84
+ lines.append(f"```{lang}")
85
+ lines.append(usage.strip())
86
+ lines.append("```")
87
+
88
+ lines.append("")
89
+
90
+ return "\n".join(lines)
backend/services/ingestion/__init__.py ADDED
File without changes
backend/services/ingestion/copypaste_ingester.py ADDED
@@ -0,0 +1,819 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Detector and ingester for "copy-paste" style component libraries.
3
+
4
+ These libraries are NOT published as installable npm packages.
5
+ Developers copy component source code into their project via a CLI tool
6
+ (shadcn, jsrepo) or manually from the library's website.
7
+
8
+ Detection cascade (in priority order):
9
+ 1. jsrepo.dev β€” standard registry for Svelte/TS copy-paste libs
10
+ 2. shadcn registry format β€” homepage exposes /registry.json or /r/*.json
11
+ 3. GitHub README heuristics β€” README contains copy-paste CLI phrases
12
+ 4. npm package has no dist exports β€” source-only repo misidentified as package
13
+
14
+ If any pattern matches, ingest() returns pre-built documentation so the
15
+ pipeline can skip the normal URL crawl step entirely.
16
+ """
17
+
18
+ import logging
19
+ import re
20
+ from datetime import date
21
+ from urllib.parse import urlparse
22
+
23
+ import httpx
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+ _TIMEOUT = 15
28
+ _HEADERS = {"User-Agent": "docforge/1.0 (documentation indexer)"}
29
+ _GH_HEADERS = {**_HEADERS, "Accept": "application/vnd.github+json"}
30
+
31
+ # Regex patterns that appear in README files of copy-paste libraries
32
+ _README_SIGNALS = [
33
+ r"npx\s+shadcn",
34
+ r"npx\s+@shadcn",
35
+ r"pnpm\s+dlx\s+shadcn",
36
+ r"bunx\s+shadcn",
37
+ r"npx\s+jsrepo",
38
+ r"jsrepo\s+add",
39
+ r"not\s+published\s+to\s+npm",
40
+ r"not\s+available\s+on\s+npm",
41
+ r"no\s+npm\s+package",
42
+ r"copy\s+and\s+paste",
43
+ r"copy-paste\s+component",
44
+ ]
45
+
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # Public API
49
+ # ---------------------------------------------------------------------------
50
+
51
+ async def detect_and_ingest(
52
+ name: str,
53
+ github_url: str | None = None,
54
+ homepage: str | None = None,
55
+ npm_data: dict | None = None,
56
+ ) -> dict | None:
57
+ """
58
+ Run all detection patterns against the given signals.
59
+
60
+ Returns a metadata dict with _is_copypaste=True and _raw_docs set
61
+ if the library is identified as copy-paste style, otherwise None.
62
+ """
63
+ # ── Pattern 1: jsrepo.dev ─────────────────────────────────────────────────
64
+ docs = await check_jsrepo(name, github_url)
65
+ if docs:
66
+ logger.info("[copypaste] jsrepo match for '%s'", name)
67
+ return _make_result(name, github_url, docs, "jsrepo")
68
+
69
+ # ── Pattern 2: shadcn registry format ────────────────────────────────────
70
+ if homepage:
71
+ shadcn_result = await _check_shadcn_registry(homepage, name)
72
+ if shadcn_result is not None:
73
+ logger.info("[copypaste] shadcn registry match for '%s'", name)
74
+ result = _make_result(name, github_url, "", "shadcn")
75
+ result["_structured"] = shadcn_result
76
+ return result
77
+
78
+ # ── Pattern 3: npm package has no dist exports ───────────────────────────
79
+ # (fall through β€” if npm_data has no exports, README check below will
80
+ # confirm whether it's a copy-paste lib or just a poorly configured package)
81
+ no_dist = npm_data is not None and _has_no_dist_exports(npm_data)
82
+
83
+ # ── Pattern 4: GitHub README heuristics ──────────────────────────────────
84
+ if github_url:
85
+ docs = await _check_readme(github_url, name, require_signal=not no_dist)
86
+ if docs:
87
+ logger.info("[copypaste] README match for '%s'", name)
88
+ return _make_result(name, github_url, docs, "readme")
89
+
90
+ return None
91
+
92
+
93
+ # ---------------------------------------------------------------------------
94
+ # Pattern 1 β€” jsrepo.dev
95
+ # ---------------------------------------------------------------------------
96
+
97
+ async def check_jsrepo(name: str, github_url: str | None) -> str | None:
98
+ """
99
+ Check jsrepo.dev and raw GitHub paths for a blocks/registry JSON.
100
+ jsrepo registries are GitHub repos that publish a blocks.json or registry.json.
101
+ """
102
+ clean = name.lstrip("@").replace("/", "-")
103
+ candidates: list[str] = []
104
+ owner: str | None = None
105
+ repo: str | None = None
106
+
107
+ # Derive candidates from the known GitHub repo
108
+ if github_url:
109
+ m = re.search(r"github\.com[:/]([^/]+)/([^/.#]+)", github_url)
110
+ if m:
111
+ owner, repo = m.group(1), m.group(2)
112
+ for branch in ("main", "master"):
113
+ candidates += [
114
+ f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/blocks.json",
115
+ f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/registry.json",
116
+ f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/src/registry.json",
117
+ ]
118
+ candidates += [
119
+ f"https://jsrepo.dev/api/registries/{owner}/{repo}",
120
+ ]
121
+
122
+ # Generic jsrepo search by package name
123
+ candidates.append(f"https://jsrepo.dev/api/registries?q={clean}")
124
+
125
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS, follow_redirects=True) as client:
126
+ for url in candidates:
127
+ try:
128
+ resp = await client.get(url)
129
+ if not resp.is_success:
130
+ continue
131
+ if "json" not in resp.headers.get("content-type", ""):
132
+ continue
133
+
134
+ data = resp.json()
135
+ docs = _parse_blocks_json(data, name, owner=owner, repo=repo)
136
+ if docs:
137
+ return docs
138
+ except Exception as exc:
139
+ logger.debug("jsrepo check failed at %s: %s", url, exc)
140
+
141
+ return None
142
+
143
+
144
+ # ---------------------------------------------------------------------------
145
+ # Pattern 2 β€” shadcn registry format
146
+ # ---------------------------------------------------------------------------
147
+
148
+ async def check_shadcn_registry(homepage: str, name: str) -> "dict | None":
149
+ """Public alias β€” same as _check_shadcn_registry, called from context.py."""
150
+ return await _check_shadcn_registry(homepage, name)
151
+
152
+
153
+ async def _check_shadcn_registry(homepage: str, name: str) -> "dict | None":
154
+ """
155
+ Check if the homepage exposes a shadcn-compatible component registry.
156
+
157
+ React Bits (and similar) expose their registry at /r/registry.json using the
158
+ shadcn registry schema ({$schema, name, items:[]}). The jsrepo CLI treats the
159
+ /r path as its registry root: `npx jsrepo add https://site.dev/r ComponentName`.
160
+ """
161
+ parsed = urlparse(homepage)
162
+ base = f"{parsed.scheme}://{parsed.netloc}"
163
+
164
+ candidates = [
165
+ # /r/registry.json β€” the pattern used by react-bits and jsrepo-compatible sites
166
+ f"{base}/r/registry.json",
167
+ f"{base}/r/blocks.json",
168
+ f"{base}/r/index.json",
169
+ # root-level registry
170
+ f"{base}/registry.json",
171
+ f"{homepage.rstrip('/')}/registry.json",
172
+ f"{base}/api/registry.json",
173
+ ]
174
+
175
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS, follow_redirects=True) as client:
176
+ for url in candidates:
177
+ try:
178
+ resp = await client.get(url)
179
+ if not resp.is_success:
180
+ continue
181
+ if "json" not in resp.headers.get("content-type", ""):
182
+ continue
183
+
184
+ data = resp.json()
185
+ items = None
186
+ if isinstance(data, list):
187
+ items = data
188
+ elif isinstance(data, dict):
189
+ items = (
190
+ data.get("items")
191
+ or data.get("components")
192
+ or data.get("registry")
193
+ )
194
+
195
+ if items and isinstance(items, list) and len(items) > 0:
196
+ logger.info("shadcn registry found at %s (%d items)", url, len(items))
197
+ version = (data.get("version") if isinstance(data, dict) else None) or "latest"
198
+ return await _format_shadcn_registry(items, name, url, version)
199
+
200
+ except Exception as exc:
201
+ logger.debug("shadcn registry check failed at %s: %s", url, exc)
202
+
203
+ return None
204
+
205
+
206
+ # ---------------------------------------------------------------------------
207
+ # Pattern 3 β€” npm package with no dist exports (checked in detect_and_ingest)
208
+ # ---------------------------------------------------------------------------
209
+
210
+ def _has_no_dist_exports(npm_data: dict) -> bool:
211
+ """
212
+ Return True if the npm package.json has no built distribution artefacts.
213
+ A package with no main/module/exports and no dist/ in files[] is likely
214
+ a source-only repo published for reference rather than installation.
215
+ """
216
+ has_entry = bool(
217
+ npm_data.get("main")
218
+ or npm_data.get("module")
219
+ or npm_data.get("exports")
220
+ )
221
+ has_dist_files = any(
222
+ "dist/" in str(f) or "build/" in str(f)
223
+ for f in (npm_data.get("files") or [])
224
+ )
225
+ return not has_entry and not has_dist_files
226
+
227
+
228
+ # ---------------------------------------------------------------------------
229
+ # Pattern 4 β€” GitHub README heuristics
230
+ # ---------------------------------------------------------------------------
231
+
232
+ async def _check_readme(
233
+ github_url: str,
234
+ name: str,
235
+ require_signal: bool = True,
236
+ ) -> str | None:
237
+ """
238
+ Fetch the GitHub README.
239
+ If require_signal=True, only return content when a copy-paste phrase is found.
240
+ If require_signal=False (npm_data had no exports), return content regardless
241
+ since we already have a structural signal β€” just confirm it's not a normal lib.
242
+ """
243
+ m = re.search(r"github\.com[:/]([^/]+)/([^/.#]+)", github_url)
244
+ if not m:
245
+ return None
246
+
247
+ owner, repo = m.group(1), m.group(2)
248
+ readme_content: str | None = None
249
+
250
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
251
+ for branch in ("main", "master"):
252
+ for filename in ("README.md", "readme.md", "Readme.md"):
253
+ url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{filename}"
254
+ try:
255
+ resp = await client.get(url)
256
+ if resp.is_success and resp.text.strip():
257
+ readme_content = resp.text
258
+ break
259
+ except Exception:
260
+ continue
261
+ if readme_content:
262
+ break
263
+
264
+ if not readme_content:
265
+ return None
266
+
267
+ readme_lower = readme_content.lower()
268
+ has_signal = any(
269
+ re.search(pattern, readme_lower)
270
+ for pattern in _README_SIGNALS
271
+ )
272
+
273
+ if require_signal and not has_signal:
274
+ return None
275
+
276
+ # Enrich with component listing from the repo tree
277
+ component_list = await _fetch_component_listing(client, owner, repo)
278
+
279
+ m2 = re.search(r"github\.com[:/]([^/]+)/([^/.#]+)", github_url)
280
+ gh_slug = f"{m2.group(1)}/{m2.group(2)}" if m2 else None
281
+
282
+ lines = [
283
+ f"# {name} β€” Copy-Paste Component Library\n\n",
284
+ "⚠️ IMPORTANT FOR AI CODE GENERATION:\n",
285
+ f"This library ({name}) is a COPY-PASTE component library β€” it is NOT published as an npm package.\n",
286
+ "DO NOT generate 'npm install {name}' or any npm/yarn/pnpm install command for this library.\n",
287
+ "Components must be copied into the project manually or via a CLI tool.\n\n",
288
+ "## How to install components\n\n",
289
+ ]
290
+ if gh_slug:
291
+ lines.append(
292
+ f"```bash\n# Option A β€” jsrepo CLI\nnpx jsrepo add {gh_slug}/<ComponentName>\n\n"
293
+ f"# Option B β€” shadcn CLI (if the project uses shadcn)\nnpx shadcn@latest add <ComponentName>\n```\n\n"
294
+ "After adding, import from your local components directory:\n"
295
+ "`import { ComponentName } from '@/components/ComponentName'`\n\n"
296
+ )
297
+ lines.append("## README\n\n")
298
+ lines.append(readme_content[:10_000]) # README covers install + overview
299
+
300
+ if component_list:
301
+ lines.append(f"\n\n## Available Components\n\n{component_list}")
302
+
303
+ return "".join(lines)
304
+
305
+
306
+ async def _fetch_component_listing(
307
+ client: httpx.AsyncClient,
308
+ owner: str,
309
+ repo: str,
310
+ ) -> str | None:
311
+ """Try common component directory paths to list available components."""
312
+ paths = [
313
+ f"https://api.github.com/repos/{owner}/{repo}/contents/src/components",
314
+ f"https://api.github.com/repos/{owner}/{repo}/contents/components",
315
+ f"https://api.github.com/repos/{owner}/{repo}/contents/src/lib/components",
316
+ f"https://api.github.com/repos/{owner}/{repo}/contents/packages",
317
+ f"https://api.github.com/repos/{owner}/{repo}/contents/src",
318
+ ]
319
+
320
+ for url in paths:
321
+ try:
322
+ resp = await client.get(url, headers=_GH_HEADERS)
323
+ if not resp.is_success:
324
+ continue
325
+ items = resp.json()
326
+ if not isinstance(items, list):
327
+ continue
328
+
329
+ names = [
330
+ re.sub(r"\.(svelte|tsx|vue|jsx|ts|js)$", "", item["name"])
331
+ for item in items
332
+ if item.get("type") in ("file", "dir")
333
+ and not item["name"].startswith(".")
334
+ and item["name"][0].isupper() # components are usually PascalCase
335
+ ]
336
+ if len(names) >= 3:
337
+ return "\n".join(f"- {n}" for n in names[:60])
338
+ except Exception:
339
+ continue
340
+
341
+ return None
342
+
343
+
344
+ # ---------------------------------------------------------------------------
345
+ # Helpers
346
+ # ---------------------------------------------------------------------------
347
+
348
+ def _make_result(
349
+ name: str,
350
+ github_url: str | None,
351
+ raw_docs: str,
352
+ method: str,
353
+ ) -> dict:
354
+ return {
355
+ "name": name,
356
+ "version": "latest",
357
+ "description": f"Copy-paste component library (detected via {method})",
358
+ "homepage": github_url,
359
+ "repository": github_url,
360
+ "_is_copypaste": True,
361
+ "_copypaste_method": method,
362
+ "_raw_docs": raw_docs,
363
+ }
364
+
365
+
366
+ def _parse_blocks_json(
367
+ data: object,
368
+ name: str,
369
+ owner: str | None = None,
370
+ repo: str | None = None,
371
+ ) -> str | None:
372
+ """Parse a blocks.json / registry.json payload into readable docs."""
373
+ items: list | None = None
374
+
375
+ if isinstance(data, list) and data:
376
+ items = data
377
+ elif isinstance(data, dict):
378
+ items = (
379
+ data.get("blocks")
380
+ or data.get("components")
381
+ or data.get("items")
382
+ or data.get("registry")
383
+ )
384
+ # jsrepo search results wrapper
385
+ if not items and "results" in data and data["results"]:
386
+ items = data["results"]
387
+
388
+ if not items or not isinstance(items, list) or len(items) == 0:
389
+ return None
390
+
391
+ gh_slug = f"{owner}/{repo}" if owner and repo else None
392
+
393
+ lines = [
394
+ f"# {name} β€” Copy-Paste Component Library\n\n",
395
+ "⚠️ IMPORTANT FOR AI CODE GENERATION:\n",
396
+ f"This library ({name}) is a COPY-PASTE component library β€” it is NOT published as an npm package.\n",
397
+ "DO NOT generate 'npm install {name}' or any npm/yarn/pnpm install command for this library.\n",
398
+ "Components are added to a project via the jsrepo CLI, NOT by installing a package.\n\n",
399
+ "## How to install components\n\n",
400
+ ]
401
+
402
+ if gh_slug:
403
+ lines.append(f"Use the jsrepo CLI to add individual components:\n")
404
+ lines.append(f"```bash\n# Install jsrepo CLI (one-time)\nnpm install -g jsrepo\n\n")
405
+ lines.append(f"# Add a specific component (example)\nnpx jsrepo add {gh_slug}/<type>/<ComponentName>\n```\n\n")
406
+ else:
407
+ lines.append(f"Use the jsrepo CLI: `npx jsrepo add <registry>/<type>/<ComponentName>`\n\n")
408
+
409
+ lines.append(
410
+ "After adding, components live in your local project directory.\n"
411
+ "Import them from your local path, e.g. `import {{ ComponentName }} from '@/components/ComponentName'`\n\n"
412
+ f"Total components available: {len(items)}\n"
413
+ )
414
+
415
+ for block in items:
416
+ if not isinstance(block, dict):
417
+ continue
418
+ bname = block.get("name") or block.get("title") or ""
419
+ if not bname:
420
+ continue
421
+
422
+ desc = block.get("description") or ""
423
+ category = block.get("type") or block.get("category") or ""
424
+
425
+ entry = f"\n## {bname}"
426
+ if category:
427
+ entry += f" ({category})"
428
+ entry += "\n"
429
+ if desc:
430
+ entry += f"{desc}\n"
431
+
432
+ # jsrepo add command per component
433
+ if gh_slug and category:
434
+ entry += f"Install: `npx jsrepo add {gh_slug}/{category}/{bname}`\n"
435
+ elif gh_slug:
436
+ entry += f"Install: `npx jsrepo add {gh_slug}/{bname}`\n"
437
+
438
+ deps = block.get("dependencies") or block.get("registryDependencies") or []
439
+ if deps:
440
+ entry += f"npm peer dependencies (install these): {', '.join(str(d) for d in deps)}\n"
441
+
442
+ lines.append(entry)
443
+
444
+ return "".join(lines)
445
+
446
+
447
+ def _parse_ts_props(snippet: str) -> list[dict]:
448
+ """Parse a TypeScript interface/type snippet into a list of prop dicts."""
449
+ brace_start = snippet.find("{")
450
+ if brace_start == -1:
451
+ return []
452
+
453
+ props: list[dict] = []
454
+ depth = 0
455
+ i = brace_start
456
+ buf = ""
457
+
458
+ while i < len(snippet):
459
+ ch = snippet[i]
460
+ if ch == "{":
461
+ depth += 1
462
+ if depth > 1:
463
+ buf += ch
464
+ elif ch == "}":
465
+ depth -= 1
466
+ if depth == 0:
467
+ break
468
+ buf += ch
469
+ elif ch == ";" and depth == 1:
470
+ prop = _parse_prop_line(buf)
471
+ if prop:
472
+ props.append(prop)
473
+ buf = ""
474
+ elif ch == "\n" and depth == 1 and buf.strip():
475
+ prop = _parse_prop_line(buf)
476
+ if prop:
477
+ props.append(prop)
478
+ buf = ""
479
+ else:
480
+ buf += ch
481
+ i += 1
482
+
483
+ if buf.strip():
484
+ prop = _parse_prop_line(buf)
485
+ if prop:
486
+ props.append(prop)
487
+
488
+ return props
489
+
490
+
491
+ def _parse_prop_line(line: str) -> "dict | None":
492
+ """Parse a single TypeScript property declaration."""
493
+ line = line.strip().rstrip(";").strip()
494
+ if not line or line.startswith("//") or line.startswith("*") or line.startswith("["):
495
+ return None
496
+ m = re.match(r"^(\w+)(\?)?\s*:\s*(.+)$", line)
497
+ if not m:
498
+ return None
499
+ prop_type = m.group(3).strip()
500
+ if len(prop_type) > 80:
501
+ prop_type = prop_type[:77] + "..."
502
+ return {
503
+ "name": m.group(1),
504
+ "type": prop_type,
505
+ "required": m.group(2) != "?",
506
+ "default": None,
507
+ }
508
+
509
+
510
+ def _example_value(prop: dict) -> str:
511
+ """Return a realistic example value for a prop based on its name and type."""
512
+ name = prop["name"].lower()
513
+ t = prop["type"].lower()
514
+
515
+ # Name-driven heuristics (more specific than type alone)
516
+ if any(k in name for k in ("text", "label", "title", "sentence", "placeholder")):
517
+ return '"Hello World"'
518
+ if any(k in name for k in ("src", "url", "href", "path", "image", "logo")):
519
+ return '"/path/to/file"'
520
+ if "color" in name:
521
+ return '"#3b82f6"'
522
+ if name in ("classname", "class"):
523
+ return '"your-class"'
524
+ if any(k in name for k in ("duration", "delay", "speed", "time")):
525
+ return "{1.5}" if "duration" in name else "{300}"
526
+ if any(k in name for k in ("count", "max", "min", "size", "radius", "width", "height")):
527
+ return "{100}"
528
+ if "items" in name or "logos" in name or "images" in name:
529
+ return "{[]}"
530
+ if name in ("to", "target"):
531
+ return "{1000}"
532
+ if name in ("from"):
533
+ return "{0}"
534
+
535
+ # Type-driven fallback
536
+ if "string" in t:
537
+ return '"value"'
538
+ if "number" in t:
539
+ return "{0}"
540
+ if "boolean" in t:
541
+ return "{true}"
542
+ if "reactnode" in t or "element" in t:
543
+ return "{<div>content</div>}"
544
+ if "[]" in t or t.startswith("array"):
545
+ return "{[]}"
546
+ if "=>" in t or "function" in t:
547
+ return "{() => {}}"
548
+ return "{/* value */}"
549
+
550
+
551
+ # Props to skip in commented optional list β€” too generic to be useful examples
552
+ _BORING_PROPS = frozenset({"classname", "style", "id", "ref", "key", "tabindex", "role"})
553
+
554
+
555
+ def _generate_usage(title: str, props: list[dict]) -> str:
556
+ """
557
+ Generate a proper runnable usage example with import + function wrapper.
558
+ Required props are shown with realistic values.
559
+ A selection of interesting optional props are shown commented out.
560
+ """
561
+ import_line = f"import {{ {title} }} from '@/components/{title}';"
562
+
563
+ required = [p for p in props if p.get("required") and p["name"] != "children"]
564
+ has_children = any(p["name"] == "children" for p in props)
565
+ needs_children = any(p["name"] == "children" and p.get("required") for p in props)
566
+
567
+ # Pick up to 5 interesting optional props to show commented (skip boring ones)
568
+ optional_shown = [
569
+ p for p in props
570
+ if not p.get("required")
571
+ and p["name"].lower() not in _BORING_PROPS
572
+ and p["name"] != "children"
573
+ ][:5]
574
+
575
+ indent = " " # 6 spaces inside return ( <Component )
576
+ attr_lines: list[str] = []
577
+
578
+ for p in required:
579
+ attr_lines.append(f"{indent}{p['name']}={_example_value(p)}")
580
+
581
+ for p in optional_shown:
582
+ attr_lines.append(f"{indent}// {p['name']}={_example_value(p)}")
583
+
584
+ attrs = "\n".join(attr_lines)
585
+ open_tag = f"<{title}"
586
+ if attrs:
587
+ open_tag += f"\n{attrs}\n "
588
+
589
+ if needs_children or (has_children and not required):
590
+ if attrs:
591
+ jsx = f"<{title}\n{attrs}\n >\n {{/* content */}}\n </{title}>"
592
+ else:
593
+ jsx = f"<{title}>\n {{/* content */}}\n </{title}>"
594
+ else:
595
+ jsx = f"<{title}\n{attrs}\n />" if attrs else f"<{title} />"
596
+
597
+ return (
598
+ f"{import_line}\n\n"
599
+ f"export default function Example() {{\n"
600
+ f" return (\n"
601
+ f" {jsx}\n"
602
+ f" );\n"
603
+ f"}}"
604
+ )
605
+
606
+
607
+ async def _format_shadcn_registry(
608
+ items: list, name: str, registry_url: str, version: str = "latest"
609
+ ) -> dict:
610
+ """
611
+ Build a pre-structured JSON dict from a shadcn/jsrepo registry.
612
+ Returns a dict matching the schema expected by format_context_md(),
613
+ bypassing the LLM structurer entirely.
614
+ """
615
+ from urllib.parse import urlparse as _urlparse
616
+ _p = _urlparse(registry_url)
617
+ registry_base = f"{_p.scheme}://{_p.netloc}{_p.path.rsplit('/', 1)[0]}"
618
+
619
+ # Deduplicate by title β€” registries like react-bits expose 4 variants per
620
+ # component (JS/TS Γ— CSS/TW). Prefer the TypeScript variant because its
621
+ # source file has proper interface declarations we can extract for props.
622
+ seen_titles: dict[str, dict] = {}
623
+ for item in items:
624
+ if not isinstance(item, dict):
625
+ continue
626
+ title = item.get("title") or item.get("name") or ""
627
+ if not title:
628
+ continue
629
+ item_name = item.get("name") or ""
630
+ is_ts = "-TS-" in item_name or item_name.endswith("-TS")
631
+ if title not in seen_titles:
632
+ seen_titles[title] = dict(item)
633
+ else:
634
+ existing = seen_titles[title]
635
+ existing_name = existing.get("name") or ""
636
+ if is_ts and "-TS-" not in existing_name and not existing_name.endswith("-TS"):
637
+ merged_deps = sorted(
638
+ set(existing.get("dependencies") or [])
639
+ | set(item.get("dependencies") or [])
640
+ )
641
+ seen_titles[title] = {**item, "dependencies": merged_deps}
642
+ else:
643
+ existing_deps = set(existing.get("dependencies") or [])
644
+ existing_deps.update(item.get("dependencies") or [])
645
+ existing["dependencies"] = sorted(existing_deps)
646
+
647
+ unique_items = list(seen_titles.values())
648
+
649
+ # Fetch individual component JSON files in parallel to extract prop types
650
+ props_map = await _fetch_all_component_props(registry_base, unique_items)
651
+
652
+ today = date.today().isoformat()
653
+ components = []
654
+
655
+ for item in unique_items:
656
+ title = item.get("title") or item.get("name") or ""
657
+ if not title:
658
+ continue
659
+
660
+ deps = item.get("dependencies") or []
661
+ prop_snippet = props_map.get(title)
662
+ props = _parse_ts_props(prop_snippet) if prop_snippet else []
663
+ usage = _generate_usage(title, props)
664
+
665
+ gotchas = [
666
+ f"Install with `npx jsrepo add {registry_base} {title}` β€” do NOT use npm install",
667
+ f"Import from your local project: `import {{ {title} }} from '@/components/{title}'`",
668
+ ]
669
+ if deps:
670
+ gotchas.append(f"Peer dependencies must be installed separately: {', '.join(deps)}")
671
+
672
+ components.append({
673
+ "name": title,
674
+ "install_command": f"npx jsrepo add {registry_base} {title}",
675
+ "import_path": f"import {{ {title} }} from '@/components/{title}'",
676
+ "props": props,
677
+ "peer_dependencies": deps,
678
+ "gotchas": gotchas,
679
+ "usage_example": usage,
680
+ "last_verified": today,
681
+ })
682
+
683
+ return {
684
+ "library": name,
685
+ "version": version,
686
+ "description": f"Copy-Paste Component Library (jsrepo/shadcn registry) β€” {len(components)} components",
687
+ "components": components,
688
+ }
689
+
690
+
691
+ async def _fetch_all_component_props(
692
+ registry_base: str,
693
+ items: list,
694
+ max_concurrent: int = 15,
695
+ ) -> dict[str, str]:
696
+ """
697
+ Fetch individual component JSON endpoints in parallel and extract TypeScript
698
+ prop interfaces from the embedded file content.
699
+
700
+ Individual component JSONs live at {registry_base}/{item_name}.json and
701
+ include a `files[]` array with the full source `content` field.
702
+ """
703
+ import asyncio
704
+
705
+ sem = asyncio.Semaphore(max_concurrent)
706
+
707
+ async def fetch_one(
708
+ client: httpx.AsyncClient, item: dict
709
+ ) -> tuple[str, str | None]:
710
+ title = item.get("title") or item.get("name") or ""
711
+ item_name = item.get("name") or title
712
+ if not title:
713
+ return title, None
714
+
715
+ # Build candidate URLs β€” prefer TypeScript variants (have proper interfaces)
716
+ ts_name = item_name.replace("-JS-", "-TS-") if "-JS-" in item_name else item_name
717
+ candidate_urls = []
718
+ if ts_name != item_name:
719
+ candidate_urls.append(f"{registry_base}/{ts_name}.json")
720
+ candidate_urls += [
721
+ f"{registry_base}/{item_name}.json",
722
+ f"{registry_base}/{title}-TS-CSS.json",
723
+ f"{registry_base}/{title}-TS-TW.json",
724
+ f"{registry_base}/{title}.json",
725
+ ]
726
+
727
+ async with sem:
728
+ for url in candidate_urls:
729
+ try:
730
+ resp = await client.get(url, timeout=6)
731
+ if not resp.is_success:
732
+ continue
733
+ if "json" not in resp.headers.get("content-type", ""):
734
+ continue
735
+ data = resp.json()
736
+ # The individual JSON may wrap items[] or be the item directly
737
+ files = data.get("files") or []
738
+ if not files and isinstance(data.get("items"), list):
739
+ files = data["items"][0].get("files") or []
740
+ for f in files:
741
+ content = f.get("content") or ""
742
+ if content:
743
+ snippet = _extract_ts_interface(content)
744
+ if snippet:
745
+ return title, snippet
746
+ except Exception:
747
+ pass
748
+ return title, None
749
+
750
+ async with httpx.AsyncClient(
751
+ timeout=8, headers=_HEADERS, follow_redirects=True
752
+ ) as client:
753
+ results = await asyncio.gather(
754
+ *[fetch_one(client, item) for item in items],
755
+ return_exceptions=False,
756
+ )
757
+
758
+ return {t: p for t, p in results if p}
759
+
760
+
761
+ def _extract_ts_interface(content: str) -> str | None:
762
+ """
763
+ Extract TypeScript prop types from source code.
764
+ Handles:
765
+ - interface AnimatedContentProps { ... } (named interface)
766
+ - type AnimatedContentProps = { ... } (named type alias)
767
+ - function Foo({ a, b }: { a: string; b?: number }) (inline params)
768
+
769
+ Uses brace balancing so nested types like `config?: { tension: number }`
770
+ are captured correctly.
771
+ """
772
+ # ── Pattern A: named interface or type alias ──────────────────────────────
773
+ header = re.search(
774
+ r'(?:interface|type)\s+\w*[Pp]rops?\w*\s*(?:=\s*)?(?:<[^>]*>)?\s*\{',
775
+ content,
776
+ )
777
+ if header:
778
+ snippet = _extract_balanced_braces(content, header.start())
779
+ if snippet:
780
+ return _trim(snippet)
781
+
782
+ # ── Pattern B: inline destructured params ────────────────────────────────
783
+ # Matches: ComponentName({ propA, propB }: { propA: type; propB?: type })
784
+ inline = re.search(
785
+ r'(?:function\s+\w+|const\s+\w+\s*=\s*(?:React\.)?(?:forwardRef\()?(?:async\s+)?function\s*\w*|\(\s*)\(\s*\{[^}]*\}\s*:\s*\{',
786
+ content,
787
+ )
788
+ if inline:
789
+ # Find the opening brace of the type annotation
790
+ brace_pos = content.find(': {', inline.start())
791
+ if brace_pos != -1:
792
+ snippet = _extract_balanced_braces(content, brace_pos + 2)
793
+ if snippet:
794
+ return _trim(snippet)
795
+
796
+ return None
797
+
798
+
799
+ def _extract_balanced_braces(content: str, start: int) -> str | None:
800
+ """Return the substring from `start` through the matching closing brace."""
801
+ brace_start = content.find('{', start)
802
+ if brace_start == -1:
803
+ return None
804
+ depth = 0
805
+ for i in range(brace_start, min(brace_start + 2000, len(content))):
806
+ if content[i] == '{':
807
+ depth += 1
808
+ elif content[i] == '}':
809
+ depth -= 1
810
+ if depth == 0:
811
+ return content[start: i + 1]
812
+ return None
813
+
814
+
815
+ def _trim(snippet: str, max_len: int = 700) -> str:
816
+ snippet = snippet.strip()
817
+ if len(snippet) > max_len:
818
+ snippet = snippet[:max_len - 3] + "..."
819
+ return snippet
backend/services/ingestion/github_ingester.py ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ GitHub repository ingester.
3
+
4
+ Primary path β€” GitHub Contents API (no clone, no disk I/O, fast):
5
+ Fetches README, docs/, and .d.ts files via authenticated HTTPS API calls.
6
+ Works for repos of any size; no git toolchain required.
7
+
8
+ Fallback path β€” shallow git clone:
9
+ Used only when the repo is not on github.com (e.g. GitLab, self-hosted).
10
+ 120 s timeout kept for compatibility.
11
+ """
12
+
13
+ import base64
14
+ import logging
15
+ import os
16
+ import re
17
+ import subprocess
18
+ import tempfile
19
+ from pathlib import Path
20
+
21
+ import httpx
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")
26
+ _GH_API = "https://api.github.com"
27
+
28
+
29
+ def _gh_headers() -> dict[str, str]:
30
+ h = {"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"}
31
+ if GITHUB_TOKEN:
32
+ h["Authorization"] = f"Bearer {GITHUB_TOKEN}"
33
+ return h
34
+
35
+
36
+ def _parse_owner_repo(url: str) -> tuple[str, str] | None:
37
+ """Extract (owner, repo) from any github.com URL variant."""
38
+ m = re.search(r"github\.com[/:]([^/]+)/([^/.]+)", url)
39
+ if m:
40
+ return m.group(1), m.group(2)
41
+ return None
42
+
43
+
44
+ async def _api_file_content(client: httpx.AsyncClient, owner: str, repo: str, path: str, ref: str | None) -> str | None:
45
+ """Fetch a single file from the GitHub contents API. Returns decoded text or None."""
46
+ url = f"{_GH_API}/repos/{owner}/{repo}/contents/{path}"
47
+ params = {"ref": ref} if ref else {}
48
+ try:
49
+ r = await client.get(url, headers=_gh_headers(), params=params, timeout=15)
50
+ if r.status_code != 200:
51
+ return None
52
+ data = r.json()
53
+ if isinstance(data, list):
54
+ return None # it's a directory
55
+ encoded = data.get("content", "")
56
+ return base64.b64decode(encoded).decode("utf-8", errors="ignore")
57
+ except Exception as exc:
58
+ logger.debug("API fetch failed for %s/%s/%s: %s", owner, repo, path, exc)
59
+ return None
60
+
61
+
62
+ async def _api_list_dir(client: httpx.AsyncClient, owner: str, repo: str, path: str, ref: str | None) -> list[dict] | None:
63
+ """List directory contents via the GitHub contents API."""
64
+ url = f"{_GH_API}/repos/{owner}/{repo}/contents/{path}"
65
+ params = {"ref": ref} if ref else {}
66
+ try:
67
+ r = await client.get(url, headers=_gh_headers(), params=params, timeout=15)
68
+ if r.status_code != 200:
69
+ return None
70
+ data = r.json()
71
+ return data if isinstance(data, list) else None
72
+ except Exception:
73
+ return None
74
+
75
+
76
+ async def _api_readme(client: httpx.AsyncClient, owner: str, repo: str, ref: str | None) -> str | None:
77
+ """Fetch the repo README via the dedicated GitHub endpoint (handles any casing/extension)."""
78
+ url = f"{_GH_API}/repos/{owner}/{repo}/readme"
79
+ params = {"ref": ref} if ref else {}
80
+ try:
81
+ r = await client.get(url, headers=_gh_headers(), params=params, timeout=15)
82
+ if r.status_code != 200:
83
+ return None
84
+ encoded = r.json().get("content", "")
85
+ return base64.b64decode(encoded).decode("utf-8", errors="ignore")
86
+ except Exception:
87
+ return None
88
+
89
+
90
+ async def _api_ingest(owner: str, repo: str, ref: str | None) -> str:
91
+ """
92
+ Fetch documentation from GitHub using the Contents API β€” no git clone.
93
+ Retrieves README, docs/ folder (up to 30 .md/.mdx files),
94
+ top-level .d.ts files, and package.json.
95
+ """
96
+ content_parts: list[str] = []
97
+
98
+ async with httpx.AsyncClient(timeout=20) as client:
99
+ # 1. README
100
+ readme = await _api_readme(client, owner, repo, ref)
101
+ if readme:
102
+ content_parts.append(f"# README\n\n{readme}")
103
+ logger.debug("API ingest: README fetched (%d chars)", len(readme))
104
+
105
+ # 2. docs/ folder
106
+ for docs_dir in ["docs", "documentation", "doc"]:
107
+ entries = await _api_list_dir(client, owner, repo, docs_dir, ref)
108
+ if entries is None:
109
+ continue
110
+ # Collect .md / .mdx files recursively (one level for now)
111
+ md_entries = [e for e in entries if e.get("type") == "file"
112
+ and e.get("name", "").endswith((".md", ".mdx"))][:30]
113
+ for entry in md_entries:
114
+ text = await _api_file_content(client, owner, repo, entry["path"], ref)
115
+ if text:
116
+ content_parts.append(f"\n\n<!-- FILE: {entry['path']} -->\n{text}")
117
+ if md_entries:
118
+ logger.debug("API ingest: %d doc files from %s/", len(md_entries), docs_dir)
119
+ break
120
+
121
+ # 3. Top-level .d.ts files (type definitions)
122
+ root_entries = await _api_list_dir(client, owner, repo, "", ref)
123
+ if root_entries:
124
+ dts_entries = [e for e in root_entries if e.get("type") == "file"
125
+ and e.get("name", "").endswith(".d.ts")][:20]
126
+ dts_parts: list[str] = []
127
+ for entry in dts_entries:
128
+ text = await _api_file_content(client, owner, repo, entry["path"], ref)
129
+ if text:
130
+ dts_parts.append(f"// FILE: {entry['path']}\n{text}")
131
+ if dts_parts:
132
+ content_parts.append(
133
+ "\n\n## TypeScript Type Definitions\n\n```typescript\n"
134
+ + "\n\n".join(dts_parts) + "\n```"
135
+ )
136
+
137
+ # 4. package.json
138
+ pkg_entry = next((e for e in root_entries if e.get("name") == "package.json"), None)
139
+ if pkg_entry:
140
+ pkg_text = await _api_file_content(client, owner, repo, "package.json", ref)
141
+ if pkg_text:
142
+ content_parts.append(f"\n\n## package.json\n\n```json\n{pkg_text}\n```")
143
+
144
+ combined = "\n".join(content_parts)
145
+ logger.info("GitHub API ingest done for %s/%s@%s: %d chars", owner, repo, ref or "default", len(combined))
146
+ return combined
147
+
148
+
149
+ async def ingest_github_repo(repo_url: str, version_tag: str | None = None) -> str:
150
+ """
151
+ Ingest documentation from a GitHub repository.
152
+
153
+ For github.com repos: uses the GitHub Contents API (fast, no clone).
154
+ For other hosts: falls back to shallow git clone (120 s timeout).
155
+ """
156
+ parsed = _parse_owner_repo(repo_url)
157
+
158
+ if parsed:
159
+ owner, repo = parsed
160
+ ref: str | None = None
161
+
162
+ # Try version tag refs in order: v1.2.3, then 1.2.3
163
+ if version_tag:
164
+ async with httpx.AsyncClient(timeout=10) as client:
165
+ for candidate in [f"v{version_tag}", version_tag]:
166
+ tag_resp = await client.get(
167
+ f"{_GH_API}/repos/{owner}/{repo}/git/ref/tags/{candidate}",
168
+ headers=_gh_headers(),
169
+ )
170
+ if tag_resp.status_code == 200:
171
+ ref = candidate
172
+ logger.info("Using tag %s for %s/%s", candidate, owner, repo)
173
+ break
174
+ else:
175
+ logger.warning("Tag %s not found in %s/%s β€” using default branch", version_tag, owner, repo)
176
+
177
+ return await _api_ingest(owner, repo, ref)
178
+
179
+ # ── Fallback: non-GitHub host β€” shallow clone ──────────────────────────
180
+ clone_url = _normalize_clone_url(repo_url)
181
+ return _clone_and_extract(clone_url, version_tag)
182
+
183
+
184
+ def _clone_and_extract(repo_url: str, version_tag: str | None) -> str:
185
+ """Shallow clone fallback for non-GitHub hosts."""
186
+ with tempfile.TemporaryDirectory() as tmpdir:
187
+ result = None
188
+
189
+ if version_tag:
190
+ for tag in [f"v{version_tag}", version_tag]:
191
+ logger.info("Cloning %s at tag %s", repo_url, tag)
192
+ result = subprocess.run(
193
+ ["git", "clone", "--depth=1", "--quiet", "--branch", tag, repo_url, tmpdir],
194
+ capture_output=True, text=True, timeout=120,
195
+ )
196
+ if result.returncode == 0:
197
+ break
198
+ else:
199
+ result = None
200
+
201
+ if result is None or result.returncode != 0:
202
+ logger.info("Cloning %s (default branch)", repo_url)
203
+ result = subprocess.run(
204
+ ["git", "clone", "--depth=1", "--quiet", repo_url, tmpdir],
205
+ capture_output=True, text=True, timeout=120,
206
+ )
207
+
208
+ if result.returncode != 0:
209
+ raise RuntimeError(f"git clone failed: {result.stderr}")
210
+
211
+ content_parts: list[str] = []
212
+ repo_path = Path(tmpdir)
213
+
214
+ for readme_name in ["README.md", "README.mdx", "readme.md"]:
215
+ readme = repo_path / readme_name
216
+ if readme.exists():
217
+ content_parts.append(f"# README\n\n{readme.read_text(errors='ignore')}")
218
+ break
219
+
220
+ for docs_dir_name in ["docs", "documentation", "doc"]:
221
+ docs_dir = repo_path / docs_dir_name
222
+ if docs_dir.is_dir():
223
+ md_files = sorted(docs_dir.rglob("*.md")) + sorted(docs_dir.rglob("*.mdx"))
224
+ for md_file in md_files[:30]:
225
+ try:
226
+ text = md_file.read_text(errors="ignore")
227
+ rel = md_file.relative_to(repo_path)
228
+ content_parts.append(f"\n\n<!-- FILE: {rel} -->\n{text}")
229
+ except Exception as exc:
230
+ logger.debug("Skipping %s: %s", md_file, exc)
231
+ break
232
+
233
+ dts_files = list(repo_path.rglob("*.d.ts"))[:20]
234
+ dts_parts = []
235
+ for dts in dts_files:
236
+ try:
237
+ text = dts.read_text(errors="ignore")
238
+ rel = dts.relative_to(repo_path)
239
+ dts_parts.append(f"// FILE: {rel}\n{text}")
240
+ except Exception:
241
+ pass
242
+ if dts_parts:
243
+ content_parts.append(
244
+ "\n\n## TypeScript Type Definitions\n\n```typescript\n"
245
+ + "\n\n".join(dts_parts) + "\n```"
246
+ )
247
+
248
+ pkg_json = repo_path / "package.json"
249
+ if pkg_json.exists():
250
+ content_parts.append(f"\n\n## package.json\n\n```json\n{pkg_json.read_text()}\n```")
251
+
252
+ combined = "\n".join(content_parts)
253
+ logger.info("Clone ingest done: %d chars", len(combined))
254
+ return combined
255
+
256
+
257
+ def _normalize_clone_url(url: str) -> str:
258
+ """Convert various URL formats to a cloneable HTTPS URL."""
259
+ url = re.sub(r"^git\+", "", url)
260
+ url = re.sub(r"^git://", "https://", url)
261
+ url = url.split("#")[0]
262
+ url = re.sub(r"(github\.com/[^/]+/[^/]+)/(tree|blob)/.*", r"\1", url)
263
+ url = url.rstrip("/")
264
+ if not url.endswith(".git"):
265
+ url += ".git"
266
+ return url
backend/services/ingestion/npm_ingester.py ADDED
@@ -0,0 +1,379 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ npm/PyPI package ingester.
3
+
4
+ Resolves a package name + version to its documentation URL,
5
+ then fetches and returns raw Markdown content.
6
+
7
+ Phase 1: httpx-based static crawl (fast, works for most docs).
8
+ Phase 2 (Playwright) is in url_ingester.py for JS-rendered sites.
9
+ """
10
+
11
+ import logging
12
+ import re
13
+ from urllib.parse import urljoin, urlparse
14
+
15
+ import httpx
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ NPM_REGISTRY = "https://registry.npmjs.org"
20
+ PYPI_REGISTRY = "https://pypi.org/pypi"
21
+
22
+
23
+ async def resolve_npm_package(package_input: str) -> dict:
24
+ """
25
+ Parse 'react-bits@2.1.4' or '@tanstack/react-query@5.0.0' and
26
+ fetch metadata from the npm registry.
27
+ """
28
+ name, version = _parse_npm_input(package_input)
29
+
30
+ async with httpx.AsyncClient(timeout=30) as client:
31
+ # Scoped packages (@scope/name) need the slash preserved in the URL.
32
+ # Packages like "@react-bits" with no sub-name are invalid scoped names β€”
33
+ # strip the @ and try as a plain package.
34
+ resolved_name = name
35
+ if name.startswith("@") and "/" not in name:
36
+ resolved_name = name.lstrip("@")
37
+ logger.warning("Invalid scoped name %r (no slash) β€” trying as plain package %r", name, resolved_name)
38
+
39
+ url = f"{NPM_REGISTRY}/{resolved_name}/{version}"
40
+ resp = await client.get(url)
41
+
42
+ if resp.status_code == 404:
43
+ result = await _resolve_not_found(client, resolved_name, version)
44
+ if result:
45
+ return result
46
+ suffix = f"@{version}" if version != "latest" else ""
47
+ raise ValueError(f"'{resolved_name}{suffix}' was not found on npm or any known registry.")
48
+
49
+ resp.raise_for_status()
50
+ data = resp.json()
51
+
52
+ homepage = data.get("homepage") or _extract_github_pages(data)
53
+ repository_url = (data.get("repository") or {}).get("url", "")
54
+
55
+ return {
56
+ "name": data["name"],
57
+ "version": data["version"],
58
+ "description": data.get("description"),
59
+ "homepage": homepage,
60
+ "repository": repository_url,
61
+ "keywords": data.get("keywords", []),
62
+ }
63
+
64
+
65
+ async def resolve_pypi_package(package_input: str) -> dict:
66
+ """
67
+ Parse 'fastapi' or 'fastapi==0.110.0' and fetch metadata from PyPI.
68
+ """
69
+ if "==" in package_input:
70
+ name, version = package_input.split("==", 1)
71
+ else:
72
+ name, version = package_input, None
73
+
74
+ async with httpx.AsyncClient(timeout=30) as client:
75
+ if version:
76
+ url = f"{PYPI_REGISTRY}/{name}/{version}/json"
77
+ else:
78
+ url = f"{PYPI_REGISTRY}/{name}/json"
79
+ resp = await client.get(url)
80
+ resp.raise_for_status()
81
+ data = resp.json()
82
+
83
+ info = data["info"]
84
+ homepage = info.get("home_page") or info.get("project_urls", {}).get("Documentation")
85
+ resolved_version = info["version"]
86
+
87
+ return {
88
+ "name": info["name"],
89
+ "version": resolved_version,
90
+ "description": info.get("summary"),
91
+ "homepage": homepage,
92
+ "repository": info.get("project_urls", {}).get("Source"),
93
+ }
94
+
95
+
96
+ async def crawl_static_docs(url: str, max_pages: int = 10) -> str:
97
+ """
98
+ Crawl a documentation site using httpx (fast, works for static HTML).
99
+ Returns concatenated Markdown-ish text extracted from the HTML.
100
+ Falls back to url_ingester's Playwright crawler for JS-rendered sites.
101
+ """
102
+ logger.info("Static crawl starting at %s", url)
103
+ visited: set[str] = set()
104
+ pages: list[str] = []
105
+
106
+ base = urlparse(url)
107
+ base_origin = f"{base.scheme}://{base.netloc}"
108
+
109
+ to_visit = [url]
110
+
111
+ async with httpx.AsyncClient(
112
+ timeout=30,
113
+ headers={"User-Agent": "DocForge/1.0 (documentation indexer)"},
114
+ follow_redirects=True,
115
+ ) as client:
116
+ while to_visit and len(pages) < max_pages:
117
+ current_url = to_visit.pop(0)
118
+ if current_url in visited:
119
+ continue
120
+ visited.add(current_url)
121
+
122
+ try:
123
+ resp = await client.get(current_url)
124
+ if resp.status_code != 200:
125
+ continue
126
+ ct = resp.headers.get("content-type", "")
127
+ if "html" not in ct:
128
+ continue
129
+
130
+ text = _html_to_text(resp.text)
131
+ pages.append(f"\n\n<!-- SOURCE: {current_url} -->\n{text}")
132
+ logger.debug("Crawled %s (%d chars)", current_url, len(text))
133
+
134
+ # Discover links on the same origin that look like doc pages
135
+ new_links = _extract_doc_links(resp.text, base_origin, current_url)
136
+ for link in new_links:
137
+ if link not in visited:
138
+ to_visit.append(link)
139
+
140
+ except Exception as exc:
141
+ logger.warning("Failed to fetch %s: %s", current_url, exc)
142
+
143
+ combined = "\n".join(pages)
144
+ logger.info("Static crawl done: %d pages, %d chars total", len(pages), len(combined))
145
+ return combined
146
+
147
+
148
+ # ---------------------------------------------------------------------------
149
+ # Internal helpers
150
+ # ---------------------------------------------------------------------------
151
+
152
+ def _parse_npm_input(package_input: str) -> tuple[str, str]:
153
+ """Handle both 'pkg@version' and '@scope/pkg@version' forms.
154
+
155
+ Strips surrounding quotes/whitespace and semver range prefixes so raw
156
+ package.json values like '@emotion/styled": "^11.14.0' are handled cleanly.
157
+ """
158
+ package_input = package_input.strip().strip("\"'")
159
+
160
+ # Reject URL-based dependency specs (e.g. "https://www.reactbits.dev/")
161
+ if package_input.startswith(("http://", "https://", "git+", "git://", "github:", "file:")):
162
+ raise ValueError(
163
+ f"Input looks like a URL, not an npm package name: {package_input!r}. "
164
+ "Use input_type='url' instead."
165
+ )
166
+
167
+ # Reject filenames mistaken for package names (package.json, tsconfig.json, etc.)
168
+ _KNOWN_FILENAMES = {
169
+ "package.json", "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
170
+ "npm-shrinkwrap.json", "tsconfig.json", ".npmrc", ".nvmrc",
171
+ }
172
+ if package_input.lower() in _KNOWN_FILENAMES:
173
+ raise ValueError(
174
+ f"'{package_input}' is a filename, not an npm package name. "
175
+ "Pass the actual package name (e.g. 'react', 'lodash')."
176
+ )
177
+
178
+ if package_input.startswith("@"):
179
+ # Scoped package: extract @scope/name, then optionally @version
180
+ m = re.match(r'^(@[A-Za-z0-9._-]+/[A-Za-z0-9._-]+)(?:@([^\s"\':]*))?', package_input)
181
+ if m:
182
+ name = m.group(1)
183
+ raw_version = m.group(2) or ""
184
+ else:
185
+ # Fallback: split on last '@' if there are two or more
186
+ at_count = package_input.count("@")
187
+ if at_count >= 2:
188
+ last_at = package_input.rfind("@")
189
+ name = package_input[:last_at]
190
+ raw_version = package_input[last_at + 1:]
191
+ else:
192
+ name = package_input
193
+ raw_version = ""
194
+ else:
195
+ if "@" in package_input:
196
+ name, raw_version = package_input.rsplit("@", 1)
197
+ else:
198
+ name = package_input
199
+ raw_version = ""
200
+
201
+ # Strip semver range prefixes (^, ~, >=, <=, >, <, =) from version
202
+ version = re.sub(r"^[^0-9]*", "", raw_version.strip().strip("\"'"))
203
+
204
+ # If still no version, scan the original input for any semver-like number
205
+ # (handles raw package.json values like '@scope/pkg": "^6.4.8"')
206
+ if not version:
207
+ fallback = re.search(r'(\d+\.\d+[\.\d]*)', package_input)
208
+ version = fallback.group(1) if fallback else "latest"
209
+
210
+ return name.strip(), version
211
+
212
+
213
+ async def _resolve_not_found(
214
+ client: httpx.AsyncClient,
215
+ name: str,
216
+ version: str,
217
+ ) -> dict | None:
218
+ """
219
+ Full cascade for when npm returns 404 on a package.
220
+
221
+ 1. Fetch npm package root (exists even when a specific version doesn't)
222
+ β†’ version-specific 404: return GitHub tag metadata so pipeline can
223
+ clone the exact git tag.
224
+ 2. Extract GitHub URL from npm root data (or search GitHub if not found).
225
+ 3. Run copy-paste detection (jsrepo / shadcn registry / README patterns /
226
+ no-dist-exports).
227
+ 4. Fall back to plain GitHub ingestion if the repo is found but isn't
228
+ a copy-paste library.
229
+ """
230
+ github_url: str | None = None
231
+ homepage: str | None = None
232
+ npm_root: dict | None = None
233
+
234
+ # ── Step 1: npm package root ──────────────────────────────────────────────
235
+ pkg_resp = await client.get(f"{NPM_REGISTRY}/{name}")
236
+ if pkg_resp.status_code == 200:
237
+ npm_root = pkg_resp.json()
238
+ dist_tags = npm_root.get("dist-tags") or {}
239
+ latest_ver = dist_tags.get("latest", "")
240
+ homepage = npm_root.get("homepage")
241
+
242
+ # For a version-specific 404: try GitHub git-tag ingestion first
243
+ if version != "latest" and latest_ver:
244
+ latest_meta = (npm_root.get("versions") or {}).get(latest_ver, {})
245
+ repo_url = (latest_meta.get("repository") or {}).get("url", "")
246
+ if repo_url and "github.com" in repo_url:
247
+ logger.info(
248
+ "npm %s@%s not on registry β€” GitHub repo found (%s), will try git tag",
249
+ name, version, repo_url,
250
+ )
251
+ return {
252
+ "name": name,
253
+ "version": version,
254
+ "description": npm_root.get("description"),
255
+ "homepage": None,
256
+ "repository": repo_url,
257
+ "keywords": npm_root.get("keywords", []),
258
+ "_version_tag": version,
259
+ }
260
+ latest_hint = f" Latest on npm: {latest_ver}." if latest_ver else ""
261
+ logger.warning("npm %s@%s not found.%s Running copy-paste detection.", name, version, latest_hint)
262
+
263
+ # Extract GitHub URL from latest version metadata
264
+ if latest_ver:
265
+ latest_meta = (npm_root.get("versions") or {}).get(latest_ver, {})
266
+ repo_url = (latest_meta.get("repository") or {}).get("url", "")
267
+ if "github.com" in repo_url:
268
+ github_url = repo_url
269
+
270
+ # ── Step 2: GitHub search if repo still unknown ───────────────────────────
271
+ if not github_url:
272
+ github_url = await _github_search(client, name)
273
+
274
+ # ── Step 3: Copy-paste detection ──────────────────────────────────────────
275
+ if github_url or homepage:
276
+ from .copypaste_ingester import detect_and_ingest
277
+ result = await detect_and_ingest(
278
+ name=name,
279
+ github_url=github_url,
280
+ homepage=homepage or github_url,
281
+ npm_data=npm_root,
282
+ )
283
+ if result:
284
+ return result
285
+
286
+ # ── Step 4: Plain GitHub fallback (repo found, not copy-paste) ───────────
287
+ if github_url:
288
+ logger.info("npm '%s' not found β€” falling back to GitHub ingestion: %s", name, github_url)
289
+ return {
290
+ "name": name,
291
+ "version": version if version != "latest" else "latest",
292
+ "description": (npm_root or {}).get("description"),
293
+ "homepage": homepage or github_url,
294
+ "repository": github_url,
295
+ "keywords": (npm_root or {}).get("keywords", []),
296
+ }
297
+
298
+ return None
299
+
300
+
301
+ async def _github_search(client: httpx.AsyncClient, name: str) -> str | None:
302
+ """Search GitHub repositories for the most likely match for this package name."""
303
+ # Clean up scoped names (@scope/pkg β†’ scope pkg) for better search results
304
+ query = name.lstrip("@").replace("/", " ").replace("-", " ")
305
+ url = (
306
+ f"https://api.github.com/search/repositories"
307
+ f"?q={query}&per_page=3&sort=stars&order=desc"
308
+ )
309
+ try:
310
+ resp = await client.get(
311
+ url,
312
+ headers={"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"},
313
+ )
314
+ if resp.is_success:
315
+ items = resp.json().get("items", [])
316
+ if items:
317
+ # Prefer repos whose name closely matches the package name
318
+ clean_name = name.lstrip("@").split("/")[-1].lower()
319
+ for item in items:
320
+ if clean_name in item.get("name", "").lower():
321
+ return item["html_url"]
322
+ return items[0]["html_url"]
323
+ except Exception as exc:
324
+ logger.warning("GitHub search failed for '%s': %s", name, exc)
325
+ return None
326
+
327
+
328
+ def _extract_github_pages(data: dict) -> str | None:
329
+ """Try to derive a docs URL from the GitHub repository URL."""
330
+ repo = (data.get("repository") or {}).get("url", "")
331
+ if "github.com" in repo:
332
+ # Convert git+https://github.com/user/repo.git β†’ https://user.github.io/repo
333
+ match = re.search(r"github\.com[:/]([^/]+)/([^/.]+)", repo)
334
+ if match:
335
+ user, repo_name = match.group(1), match.group(2)
336
+ return f"https://{user}.github.io/{repo_name}"
337
+ return None
338
+
339
+
340
+ def _html_to_text(html: str) -> str:
341
+ """
342
+ Very lightweight HTML β†’ text converter.
343
+ We strip tags and normalize whitespace so Gemini can process the content.
344
+ For production, Docling by IBM does a much better job with complex layouts.
345
+ """
346
+ # Remove scripts and styles completely
347
+ html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
348
+ # Replace block-level tags with newlines
349
+ html = re.sub(r"<(h[1-6]|p|div|section|article|li|tr|br)[^>]*>", "\n", html, flags=re.IGNORECASE)
350
+ # Remove remaining tags
351
+ html = re.sub(r"<[^>]+>", "", html)
352
+ # Decode basic HTML entities
353
+ html = html.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&").replace("&nbsp;", " ").replace("&#39;", "'").replace("&quot;", '"')
354
+ # Collapse excessive whitespace
355
+ html = re.sub(r"\n{3,}", "\n\n", html)
356
+ html = re.sub(r" {2,}", " ", html)
357
+ return html.strip()
358
+
359
+
360
+ def _extract_doc_links(html: str, base_origin: str, current_url: str) -> list[str]:
361
+ """Extract same-origin links that look like documentation pages."""
362
+ hrefs = re.findall(r'href=["\']([^"\'#?]+)["\']', html)
363
+ links = []
364
+ for href in hrefs:
365
+ if href.startswith("http"):
366
+ full = href
367
+ elif href.startswith("/"):
368
+ full = base_origin + href
369
+ else:
370
+ full = urljoin(current_url, href)
371
+
372
+ parsed = urlparse(full)
373
+ if parsed.netloc != urlparse(base_origin).netloc:
374
+ continue
375
+ # Skip obviously non-doc paths
376
+ if any(ext in parsed.path for ext in [".png", ".jpg", ".css", ".js", ".svg", ".ico", ".woff"]):
377
+ continue
378
+ links.append(full)
379
+ return links[:20] # Safety cap per page
backend/services/ingestion/paste_ingester.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Paste/upload ingester.
3
+
4
+ Accepts raw HTML, PDF (via Docling), or plain Markdown from the user
5
+ and converts it to clean Markdown suitable for Gemini structuring.
6
+ """
7
+
8
+ import logging
9
+ import re
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ async def ingest_paste(content: str, content_type: str = "auto") -> str:
15
+ """
16
+ Process pasted or uploaded content into clean Markdown.
17
+
18
+ content_type: "html" | "markdown" | "pdf_base64" | "auto"
19
+ """
20
+ if content_type == "auto":
21
+ content_type = _detect_content_type(content)
22
+
23
+ logger.info("Processing paste content as %s (%d chars)", content_type, len(content))
24
+
25
+ if content_type == "html":
26
+ return _html_to_markdown(content)
27
+ elif content_type == "pdf_base64":
28
+ return await _pdf_to_markdown(content)
29
+ else:
30
+ # Already Markdown or plain text β€” pass through with minimal cleanup
31
+ return _clean_markdown(content)
32
+
33
+
34
+ def _detect_content_type(content: str) -> str:
35
+ """Heuristic detection of content format."""
36
+ stripped = content.strip()
37
+ if stripped.startswith("<!DOCTYPE") or stripped.startswith("<html") or stripped.count("<div") > 3:
38
+ return "html"
39
+ # Base64-encoded PDF starts with JVBERi0
40
+ if stripped.startswith("JVBERi0"):
41
+ return "pdf_base64"
42
+ return "markdown"
43
+
44
+
45
+ def _html_to_markdown(html: str) -> str:
46
+ """
47
+ Convert HTML to Markdown.
48
+ Uses html2text if available, otherwise falls back to regex stripping.
49
+ """
50
+ try:
51
+ import html2text # type: ignore[import]
52
+ h = html2text.HTML2Text()
53
+ h.ignore_links = False
54
+ h.ignore_images = True
55
+ h.body_width = 0 # Don't wrap lines
56
+ return h.handle(html)
57
+ except ImportError:
58
+ logger.debug("html2text not installed, using regex fallback")
59
+ return _regex_html_strip(html)
60
+
61
+
62
+ def _regex_html_strip(html: str) -> str:
63
+ """Minimal HTML β†’ text via regex (fallback when html2text unavailable)."""
64
+ html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
65
+ # Convert headings
66
+ for i in range(6, 0, -1):
67
+ html = re.sub(rf"<h{i}[^>]*>(.*?)</h{i}>", rf"{'#' * i} \1\n", html, flags=re.IGNORECASE | re.DOTALL)
68
+ # Code blocks
69
+ html = re.sub(r"<pre[^>]*>(.*?)</pre>", r"```\n\1\n```", html, flags=re.IGNORECASE | re.DOTALL)
70
+ html = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", html, flags=re.IGNORECASE | re.DOTALL)
71
+ # Lists
72
+ html = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1", html, flags=re.IGNORECASE | re.DOTALL)
73
+ # Paragraphs and divs
74
+ html = re.sub(r"<(p|div|br)[^>]*>", "\n", html, flags=re.IGNORECASE)
75
+ # Strip remaining tags
76
+ html = re.sub(r"<[^>]+>", "", html)
77
+ # Decode entities
78
+ html = html.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&").replace("&nbsp;", " ")
79
+ return re.sub(r"\n{3,}", "\n\n", html).strip()
80
+
81
+
82
+ async def _pdf_to_markdown(pdf_base64: str) -> str:
83
+ """
84
+ Convert base64-encoded PDF to Markdown using Docling by IBM.
85
+ Falls back to a message if Docling is not installed.
86
+ """
87
+ try:
88
+ import base64
89
+ import io
90
+
91
+ from docling.document_converter import DocumentConverter # type: ignore[import]
92
+
93
+ pdf_bytes = base64.b64decode(pdf_base64)
94
+ converter = DocumentConverter()
95
+ # Docling expects a file path or URL; write to temp file
96
+ import tempfile
97
+ with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
98
+ f.write(pdf_bytes)
99
+ tmp_path = f.name
100
+
101
+ result = converter.convert(tmp_path)
102
+ return result.document.export_to_markdown()
103
+ except ImportError:
104
+ logger.warning("Docling not installed, cannot convert PDF")
105
+ return "[PDF conversion requires Docling: pip install docling]"
106
+ except Exception as exc:
107
+ logger.error("PDF conversion failed: %s", exc)
108
+ return f"[PDF conversion failed: {exc}]"
109
+
110
+
111
+ def _clean_markdown(text: str) -> str:
112
+ """Remove excessive blank lines and leading/trailing whitespace."""
113
+ return re.sub(r"\n{4,}", "\n\n\n", text).strip()
backend/services/ingestion/registries_ingester.py ADDED
@@ -0,0 +1,236 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Multi-registry package resolver.
3
+
4
+ Supports: crates.io, RubyGems, pub.dev, NuGet, Maven Central, Hex (Elixir), CRAN (R).
5
+ Each resolver returns a dict with keys: name, version, homepage, repository.
6
+ """
7
+
8
+ import logging
9
+ import re
10
+
11
+ import httpx
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ _HEADERS = {"User-Agent": "docforge/1.0 (documentation indexer)"}
16
+ _TIMEOUT = 15
17
+
18
+
19
+ def _parse_at(input_str: str) -> tuple[str, str]:
20
+ """
21
+ Parse 'name@version' or just 'name'. Strip any 'prefix:' first.
22
+ Returns (name, version) where version defaults to 'latest'.
23
+ """
24
+ if ":" in input_str:
25
+ input_str = input_str.split(":", 1)[1]
26
+ input_str = input_str.strip().strip("\"'")
27
+ if "@" in input_str:
28
+ name, ver = input_str.rsplit("@", 1)
29
+ return name.strip(), ver.strip() or "latest"
30
+ return input_str.strip(), "latest"
31
+
32
+
33
+ # ── crates.io ─────────────────────────────────────────────────────────────────
34
+
35
+ async def resolve_crates(input_str: str) -> dict:
36
+ name, version = _parse_at(input_str)
37
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
38
+ if version == "latest":
39
+ url = f"https://crates.io/api/v1/crates/{name}"
40
+ else:
41
+ url = f"https://crates.io/api/v1/crates/{name}/{version}"
42
+
43
+ resp = await client.get(url)
44
+ if not resp.is_success:
45
+ raise ValueError(f"crates.io: '{name}' not found")
46
+
47
+ data = resp.json()
48
+
49
+ if version == "latest":
50
+ krate = data.get("crate", {})
51
+ resolved = krate.get("newest_version", "latest")
52
+ homepage = krate.get("homepage") or krate.get("repository")
53
+ repository = krate.get("repository")
54
+ else:
55
+ ver_obj = data.get("version", {})
56
+ resolved = ver_obj.get("num", version)
57
+ # Fetch crate root for homepage/repo
58
+ try:
59
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
60
+ root = await client.get(f"https://crates.io/api/v1/crates/{name}")
61
+ root_data = root.json() if root.is_success else {}
62
+ except Exception:
63
+ root_data = {}
64
+ krate = root_data.get("crate", {})
65
+ homepage = krate.get("homepage") or krate.get("repository")
66
+ repository = krate.get("repository")
67
+
68
+ # docs.rs is the canonical Rust docs site
69
+ docs_url = homepage or f"https://docs.rs/{name}/{resolved}"
70
+ return {"name": name, "version": resolved, "homepage": docs_url, "repository": repository}
71
+
72
+
73
+ # ── RubyGems ──────────────────────────────────────────────────────────────────
74
+
75
+ async def resolve_rubygems(input_str: str) -> dict:
76
+ name, version = _parse_at(input_str)
77
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
78
+ resp = await client.get(f"https://rubygems.org/api/v1/gems/{name}.json")
79
+ if not resp.is_success:
80
+ raise ValueError(f"RubyGems: '{name}' not found")
81
+ data = resp.json()
82
+
83
+ resolved = version if version != "latest" else data.get("version", "latest")
84
+ homepage = (
85
+ data.get("homepage_uri")
86
+ or data.get("documentation_uri")
87
+ or data.get("project_uri")
88
+ )
89
+ repository = data.get("source_code_uri")
90
+ return {"name": data.get("name", name), "version": resolved, "homepage": homepage, "repository": repository}
91
+
92
+
93
+ # ── pub.dev (Dart / Flutter) ──────────────────────────────────────────────────
94
+
95
+ async def resolve_pubdev(input_str: str) -> dict:
96
+ name, version = _parse_at(input_str)
97
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
98
+ resp = await client.get(f"https://pub.dev/api/packages/{name}")
99
+ if not resp.is_success:
100
+ raise ValueError(f"pub.dev: '{name}' not found")
101
+ data = resp.json()
102
+
103
+ resolved = version if version != "latest" else data.get("latest", {}).get("version", "latest")
104
+ pubspec = data.get("latest", {}).get("pubspec", {})
105
+ homepage = (
106
+ pubspec.get("homepage")
107
+ or pubspec.get("documentation")
108
+ or f"https://pub.dev/packages/{name}"
109
+ )
110
+ repository = pubspec.get("repository")
111
+ return {"name": name, "version": resolved, "homepage": homepage, "repository": repository}
112
+
113
+
114
+ # ── NuGet (.NET) ──────────────────────────────────────────────────────────────
115
+
116
+ async def resolve_nuget(input_str: str) -> dict:
117
+ name, version = _parse_at(input_str)
118
+ lower = name.lower()
119
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
120
+ resp = await client.get(f"https://api.nuget.org/v3/registration5/{lower}/index.json")
121
+ if not resp.is_success:
122
+ raise ValueError(f"NuGet: '{name}' not found")
123
+ data = resp.json()
124
+
125
+ # Flatten pages β†’ items β†’ catalogEntry
126
+ pages = data.get("items", [])
127
+ last_entry = None
128
+ resolved = version
129
+ if pages:
130
+ last_page = pages[-1]
131
+ items = last_page.get("items", [])
132
+ if items:
133
+ last_item = items[-1]
134
+ last_entry = last_item.get("catalogEntry", {})
135
+ if version == "latest":
136
+ resolved = last_entry.get("version", "latest")
137
+
138
+ homepage = (last_entry or {}).get("projectUrl") or f"https://www.nuget.org/packages/{name}"
139
+ return {"name": name, "version": resolved, "homepage": homepage, "repository": None}
140
+
141
+
142
+ # ── Maven Central ─────────────────────────────────────────────────────────────
143
+ # Input format (after prefix strip): "com.google.guava:guava@32.1.3"
144
+
145
+ async def resolve_maven(input_str: str) -> dict:
146
+ # Strip "mvn:" prefix if present
147
+ stripped = input_str[4:] if input_str.startswith("mvn:") else input_str
148
+
149
+ if "@" in stripped:
150
+ ga, version = stripped.rsplit("@", 1)
151
+ version = version.strip()
152
+ else:
153
+ ga, version = stripped, "latest"
154
+
155
+ if ":" not in ga:
156
+ raise ValueError(
157
+ f"Maven input must be groupId:artifactId[@version], e.g. com.google.guava:guava. Got: {input_str!r}"
158
+ )
159
+
160
+ group_id, artifact_id = ga.split(":", 1)
161
+ group_id = group_id.strip()
162
+ artifact_id = artifact_id.strip()
163
+
164
+ from urllib.parse import quote
165
+ if version == "latest":
166
+ query = f"g:{group_id} AND a:{artifact_id}"
167
+ search_url = (
168
+ f"https://search.maven.org/solrsearch/select"
169
+ f"?q={quote(query)}&core=gav&rows=1&wt=json"
170
+ )
171
+ else:
172
+ query = f"g:{group_id} AND a:{artifact_id} AND v:{version}"
173
+ search_url = (
174
+ f"https://search.maven.org/solrsearch/select"
175
+ f"?q={quote(query)}&core=gav&rows=1&wt=json"
176
+ )
177
+
178
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
179
+ resp = await client.get(search_url)
180
+ if not resp.is_success:
181
+ raise ValueError(f"Maven Central: '{group_id}:{artifact_id}' not found")
182
+ data = resp.json()
183
+
184
+ docs = data.get("response", {}).get("docs", [])
185
+ if not docs:
186
+ raise ValueError(f"Maven Central: '{group_id}:{artifact_id}' not found")
187
+
188
+ resolved = docs[0].get("v", version)
189
+ name = f"{group_id}:{artifact_id}"
190
+ homepage = f"https://mvnrepository.com/artifact/{group_id}/{artifact_id}/{resolved}"
191
+ return {"name": name, "version": resolved, "homepage": homepage, "repository": None}
192
+
193
+
194
+ # ── Hex (Elixir) ──────────────────────────────────────────────────────────────
195
+
196
+ async def resolve_hex(input_str: str) -> dict:
197
+ name, version = _parse_at(input_str)
198
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
199
+ resp = await client.get(
200
+ f"https://hex.pm/api/packages/{name}",
201
+ headers={**_HEADERS, "Accept": "application/json"},
202
+ )
203
+ if not resp.is_success:
204
+ raise ValueError(f"Hex: '{name}' not found")
205
+ data = resp.json()
206
+
207
+ resolved = version if version != "latest" else (
208
+ data.get("latest_stable_version") or data.get("latest_version", "latest")
209
+ )
210
+ meta = data.get("meta", {})
211
+ links = meta.get("links", {})
212
+ homepage = (
213
+ links.get("Website")
214
+ or links.get("Documentation")
215
+ or f"https://hexdocs.pm/{name}/{resolved}"
216
+ )
217
+ repository = links.get("GitHub") or links.get("Repository")
218
+ return {"name": data.get("name", name), "version": resolved, "homepage": homepage, "repository": repository}
219
+
220
+
221
+ # ── CRAN (R) ──────────────────────────────────────────────────────────────────
222
+
223
+ async def resolve_cran(input_str: str) -> dict:
224
+ name, _ = _parse_at(input_str)
225
+ async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
226
+ resp = await client.get(f"https://crandb.r-pkg.org/{name}")
227
+ if not resp.is_success:
228
+ raise ValueError(f"CRAN: '{name}' not found")
229
+ data = resp.json()
230
+
231
+ resolved = data.get("Version", "latest")
232
+ raw_url = data.get("URL", "")
233
+ homepage = raw_url.split(",")[0].strip() if raw_url else f"https://cran.r-project.org/package={name}"
234
+ bug_reports = data.get("BugReports", "")
235
+ repository = re.sub(r"/issues.*", "", bug_reports) if "github.com" in bug_reports else None
236
+ return {"name": data.get("Package", name), "version": resolved, "homepage": homepage, "repository": repository}
backend/services/ingestion/search_ingester.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Web search fallback using DuckDuckGo HTML endpoint.
3
+
4
+ When a library isn't found in any known registry (or has no docs URL),
5
+ this module searches DuckDuckGo for "{library} documentation" and returns
6
+ the first organic result URL for the crawling pipeline to consume.
7
+
8
+ No API key needed β€” uses the public HTML endpoint.
9
+ """
10
+
11
+ import logging
12
+ import re
13
+ from urllib.parse import unquote
14
+
15
+ import httpx
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ _TIMEOUT = 20
20
+ _USER_AGENT = (
21
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
22
+ "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
23
+ )
24
+
25
+
26
+ async def find_docs_url(query: str) -> str | None:
27
+ """
28
+ Search DuckDuckGo for '{query} documentation' and return the first result URL.
29
+ Returns None if nothing useful is found.
30
+ """
31
+ search_query = f"{query} documentation"
32
+ from urllib.parse import quote_plus
33
+ url = f"https://html.duckduckgo.com/html/?q={quote_plus(search_query)}"
34
+
35
+ logger.info("Web search fallback: searching for '%s'", query)
36
+
37
+ try:
38
+ async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
39
+ resp = await client.get(
40
+ url,
41
+ headers={
42
+ "User-Agent": _USER_AGENT,
43
+ "Accept": "text/html,application/xhtml+xml",
44
+ },
45
+ follow_redirects=True,
46
+ )
47
+ if not resp.is_success:
48
+ logger.warning("DuckDuckGo search returned %s", resp.status_code)
49
+ return None
50
+ html = resp.text
51
+ except Exception as exc:
52
+ logger.warning("DuckDuckGo search failed: %s", exc)
53
+ return None
54
+
55
+ # DuckDuckGo HTML results encode actual URLs as uddg= query param:
56
+ # href="//duckduckgo.com/l/?uddg=https%3A%2F%2Freact.dev%2F&rut=..."
57
+ matches = re.findall(r"uddg=([^&\"'>\s]+)", html)
58
+ for encoded in matches:
59
+ try:
60
+ decoded = unquote(encoded)
61
+ except Exception:
62
+ decoded = encoded
63
+
64
+ if decoded.startswith("http") and "duckduckgo.com" not in decoded:
65
+ logger.info("Web search found docs URL: %s", decoded)
66
+ return decoded
67
+
68
+ logger.warning("No usable docs URL found via web search for '%s'", query)
69
+ return None
backend/services/ingestion/url_ingester.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ URL ingester using Playwright for JS-rendered documentation sites.
3
+
4
+ Many modern doc sites (React-based, Next.js, VitePress, etc.) render content
5
+ client-side, so httpx alone returns empty shells. Playwright runs a real browser
6
+ to get the fully rendered DOM.
7
+
8
+ Falls back to static crawl if Playwright is not installed.
9
+ """
10
+
11
+ import asyncio
12
+ import logging
13
+ import os
14
+ import random
15
+ import re
16
+ from urllib.parse import urlparse
17
+
18
+ import httpx
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+ PLAYWRIGHT_TIMEOUT = int(os.getenv("PLAYWRIGHT_TIMEOUT", "60000")) # ms
23
+
24
+ # Realistic Chrome user agent β€” rotated per session to avoid fingerprinting
25
+ _USER_AGENTS = [
26
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
27
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
28
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
29
+ ]
30
+
31
+
32
+ async def crawl_with_playwright(url: str, max_pages: int = 15) -> str:
33
+ """
34
+ Use Playwright Chromium to crawl a JS-rendered documentation site.
35
+ Returns concatenated text content from all discovered pages.
36
+ """
37
+ try:
38
+ from playwright.async_api import async_playwright # type: ignore[import]
39
+ except ImportError:
40
+ logger.warning("Playwright not installed, falling back to static crawl")
41
+ from .npm_ingester import crawl_static_docs
42
+ return await crawl_static_docs(url, max_pages)
43
+
44
+ logger.info("Playwright crawl starting at %s", url)
45
+ pages_text: list[str] = []
46
+ visited: set[str] = set()
47
+ to_visit = [url]
48
+
49
+ base = urlparse(url)
50
+ base_origin = f"{base.scheme}://{base.netloc}"
51
+
52
+ async with async_playwright() as pw:
53
+ browser = await pw.chromium.launch(
54
+ headless=True,
55
+ args=[
56
+ "--disable-blink-features=AutomationControlled",
57
+ "--no-sandbox",
58
+ "--disable-dev-shm-usage",
59
+ ],
60
+ )
61
+ context = await browser.new_context(
62
+ user_agent=random.choice(_USER_AGENTS),
63
+ viewport={"width": 1280, "height": 800},
64
+ locale="en-US",
65
+ timezone_id="America/New_York",
66
+ extra_http_headers={"Accept-Language": "en-US,en;q=0.9"},
67
+ )
68
+ # Remove the webdriver property that sites use to detect Playwright
69
+ await context.add_init_script(
70
+ "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
71
+ )
72
+ page = await context.new_page()
73
+ page.set_default_timeout(PLAYWRIGHT_TIMEOUT)
74
+
75
+ while to_visit and len(pages_text) < max_pages:
76
+ current_url = to_visit.pop(0)
77
+ if current_url in visited:
78
+ continue
79
+ visited.add(current_url)
80
+
81
+ try:
82
+ await page.goto(current_url, wait_until="domcontentloaded", timeout=15000)
83
+ # Human-like random delay between pages (1–2.5s)
84
+ await asyncio.sleep(random.uniform(1.0, 2.5))
85
+
86
+ text = await page.evaluate("""() => {
87
+ // Remove nav, footer, scripts and extract meaningful text
88
+ const remove = ['script', 'style', 'nav', 'footer', 'header', '.sidebar', '.navigation'];
89
+ remove.forEach(sel => document.querySelectorAll(sel).forEach(el => el.remove()));
90
+ return document.body ? document.body.innerText : '';
91
+ }""")
92
+
93
+ if text.strip():
94
+ pages_text.append(f"\n\n<!-- SOURCE: {current_url} -->\n{text}")
95
+ logger.debug("Playwright crawled %s (%d chars)", current_url, len(text))
96
+
97
+ # Collect doc links
98
+ links = await page.evaluate(f"""() => {{
99
+ return Array.from(document.querySelectorAll('a[href]'))
100
+ .map(a => a.href)
101
+ .filter(href => href.startsWith('{base_origin}'));
102
+ }}""")
103
+
104
+ for link in links[:20]:
105
+ clean = link.split("#")[0].split("?")[0]
106
+ if clean not in visited and _is_doc_link(clean):
107
+ to_visit.append(clean)
108
+
109
+ except Exception as exc:
110
+ logger.warning("Playwright failed on %s: %s", current_url, exc)
111
+
112
+ await browser.close()
113
+
114
+ combined = "\n".join(pages_text)
115
+ logger.info("Playwright crawl done: %d pages, %d chars total", len(pages_text), len(combined))
116
+ return combined
117
+
118
+
119
+ async def crawl_url(url: str, max_pages: int = 15) -> str:
120
+ """
121
+ Entry point: try Playwright first (handles JS sites), fall back to static.
122
+ """
123
+ return await crawl_with_playwright(url, max_pages)
124
+
125
+
126
+ # Repo path segments that are not a real repo root
127
+ _GH_EXCLUDED_OWNERS = frozenset({
128
+ "sponsors", "features", "pricing", "enterprise", "about",
129
+ "login", "join", "organizations", "apps", "marketplace",
130
+ })
131
+ _GH_EXCLUDED_REPOS = frozenset({
132
+ "issues", "pulls", "discussions", "actions", "releases",
133
+ "commits", "blob", "tree", "wiki", "pulse",
134
+ })
135
+
136
+
137
+ async def extract_github_url(url: str) -> str | None:
138
+ """
139
+ Quick static fetch of a URL to extract a GitHub repo link from its HTML.
140
+
141
+ Most library/component sites link to their GitHub repo in the header,
142
+ footer, or meta tags. This is fast (no browser), works even on JS-heavy
143
+ sites because meta tags are server-rendered.
144
+
145
+ Returns the first clean 'https://github.com/{owner}/{repo}' found,
146
+ or None if nothing is found.
147
+ """
148
+ try:
149
+ async with httpx.AsyncClient(
150
+ timeout=10,
151
+ follow_redirects=True,
152
+ headers={"User-Agent": "docforge/1.0 (documentation indexer)"},
153
+ ) as client:
154
+ resp = await client.get(url)
155
+ if not resp.is_success:
156
+ return None
157
+ html = resp.text
158
+ except Exception as exc:
159
+ logger.debug("extract_github_url fetch failed for %s: %s", url, exc)
160
+ return None
161
+
162
+ # Match all github.com/{owner}/{repo} occurrences in the raw HTML
163
+ matches = re.findall(
164
+ r'https?://github\.com/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)',
165
+ html,
166
+ )
167
+
168
+ seen: set[str] = set()
169
+ for owner, repo in matches:
170
+ if owner.lower() in _GH_EXCLUDED_OWNERS:
171
+ continue
172
+ repo = repo.rstrip("/").split("/")[0].removesuffix(".git")
173
+ if repo.lower() in _GH_EXCLUDED_REPOS:
174
+ continue
175
+ key = f"{owner}/{repo}"
176
+ if key not in seen:
177
+ seen.add(key)
178
+ return f"https://github.com/{owner}/{repo}"
179
+
180
+ return None
181
+
182
+
183
+ def _is_doc_link(url: str) -> bool:
184
+ """Filter out non-documentation links."""
185
+ bad_exts = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico",
186
+ ".css", ".js", ".woff", ".woff2", ".ttf", ".map")
187
+ bad_paths = ("/api/", "/cdn-cgi/", "/_next/", "/__vite", "/assets/")
188
+ path = urlparse(url).path.lower()
189
+ return (
190
+ not any(path.endswith(ext) for ext in bad_exts)
191
+ and not any(bad in path for bad in bad_paths)
192
+ )
backend/services/structurer.py ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Structuring engine: Gemini 2.0 Flash (primary) β†’ Groq LLaMA (fallback).
3
+
4
+ Takes raw Markdown documentation and produces a structured JSON representation
5
+ with components, props, gotchas, and version-specific details.
6
+
7
+ The prompt is engineered to hunt specifically for AI codegen failure patterns
8
+ (gotchas) that training data tends to get wrong.
9
+ """
10
+
11
+ import json
12
+ import logging
13
+ import os
14
+ import re
15
+ import time
16
+ from datetime import date
17
+
18
+ import asyncio
19
+
20
+ import httpx
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ class _QuotaError(Exception):
26
+ """Raised internally when an LLM provider returns 429."""
27
+
28
+ GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
29
+ GEMINI_MODEL = "gemini-2.0-flash"
30
+ GEMINI_ENDPOINT = (
31
+ f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent"
32
+ )
33
+
34
+ GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
35
+ GROQ_MODEL = "llama-3.3-70b-versatile"
36
+ GROQ_ENDPOINT = "https://api.groq.com/openai/v1/chat/completions"
37
+ # Groq's context window is 128k tokens; keep well under to avoid truncation errors
38
+ GROQ_MAX_DOC_CHARS = 40_000
39
+
40
+ STRUCTURED_SCHEMA = """{
41
+ "library": "string",
42
+ "version": "string",
43
+ "description": "string",
44
+ "components": [
45
+ {
46
+ "name": "string",
47
+ "import_path": "string (exact import statement, e.g. import X from '@/components/X')",
48
+ "install_command": "string (exact command from docs, NOT generic 'npm install')",
49
+ "props": [
50
+ {
51
+ "name": "string",
52
+ "type": "string",
53
+ "required": "boolean",
54
+ "default": "string or null"
55
+ }
56
+ ],
57
+ "gotchas": ["string (specific AI codegen failure patterns)"],
58
+ "peer_dependencies": ["string"],
59
+ "usage_example": "string (complete, runnable code snippet)",
60
+ "last_verified": "string (ISO date)"
61
+ }
62
+ ]
63
+ }"""
64
+
65
+ STRUCTURING_PROMPT = """You are a documentation structuring engine for DocForge. Your job is to extract \
66
+ precise, developer-critical information from raw library documentation.
67
+
68
+ Given the following raw documentation for {library}@{version}, extract structured information.
69
+
70
+ CRITICAL β€” You must specifically hunt for:
71
+ 1. GOTCHAS: Things that commonly break AI code generation. Examples:
72
+ - Required directives ("use client", "use server" in Next.js App Router)
73
+ - Deprecated props or methods from previous versions
74
+ - Breaking changes between major versions (e.g. v4 β†’ v5 API changes)
75
+ - Required peer dependencies that must be installed separately
76
+ - Common misuse patterns (wrong import path, wrong prop name)
77
+ - Framework-specific requirements (Next.js App Router vs Pages Router)
78
+ - CSS imports that must be added manually
79
+ - Configuration file changes required
80
+
81
+ 2. EXACT install commands β€” NOT generic "npm install {library}". Look for npx, dlx, or custom CLI commands.
82
+ 3. EXACT import paths β€” these change between major versions and are the #1 AI codegen failure point.
83
+ 4. All props with their TypeScript types, whether they are required, and default values.
84
+ 5. Version-specific API differences if mentioned in the docs.
85
+
86
+ Return ONLY valid JSON (no markdown code fences, no explanation, no extra whitespace) in EXACTLY this schema:
87
+ {schema}
88
+
89
+ If the documentation mentions multiple components or modules, include each as a separate entry in "components".
90
+ If there is only one main export, use a single component entry with the library name as the component name.
91
+ Set last_verified to today's date: {today}.
92
+
93
+ Raw documentation:
94
+ {raw_docs}"""
95
+
96
+
97
+ async def structure_with_gemini(
98
+ library: str,
99
+ version: str,
100
+ raw_docs: str,
101
+ gemini_key: str | None = None,
102
+ groq_key: str | None = None,
103
+ ) -> dict:
104
+ """
105
+ Call Gemini 2.0 Flash to structure raw documentation into our JSON schema.
106
+
107
+ gemini_key / groq_key β€” per-user keys that override environment variables.
108
+ Falls back to env vars when not provided.
109
+ """
110
+ effective_gemini = gemini_key or GEMINI_API_KEY
111
+ effective_groq = groq_key or GROQ_API_KEY
112
+
113
+ if not effective_gemini:
114
+ logger.warning("GEMINI_API_KEY not set, using mock structurer")
115
+ return _mock_structure(library, version)
116
+
117
+ # Truncate to ~80k chars β€” keeps Gemini token usage around 20k per request,
118
+ # which is much friendlier to the free tier daily quota (1M tokens/day).
119
+ # Quality is maintained: the most relevant documentation is always first.
120
+ max_doc_chars = 80_000
121
+ truncated = raw_docs[:max_doc_chars]
122
+ if len(raw_docs) > max_doc_chars:
123
+ logger.info("Truncated docs from %d to %d chars for Gemini", len(raw_docs), max_doc_chars)
124
+
125
+ prompt = STRUCTURING_PROMPT.format(
126
+ library=library,
127
+ version=version,
128
+ schema=STRUCTURED_SCHEMA,
129
+ today=date.today().isoformat(),
130
+ raw_docs=truncated,
131
+ )
132
+
133
+ t0 = time.time()
134
+ payload = {
135
+ "contents": [{"parts": [{"text": prompt}]}],
136
+ "generationConfig": {
137
+ "temperature": 0.1,
138
+ "maxOutputTokens": 8192,
139
+ "responseMimeType": "application/json",
140
+ },
141
+ }
142
+
143
+ try:
144
+ async with httpx.AsyncClient(timeout=120) as client:
145
+ resp = await client.post(
146
+ GEMINI_ENDPOINT,
147
+ params={"key": effective_gemini},
148
+ json=payload,
149
+ )
150
+ if resp.status_code == 429:
151
+ raise _QuotaError("Gemini")
152
+ resp.raise_for_status()
153
+ data = resp.json()
154
+
155
+ elapsed = time.time() - t0
156
+ logger.info("Gemini structuring took %.1fs for %s@%s", elapsed, library, version)
157
+ raw_output = data["candidates"][0]["content"]["parts"][0]["text"]
158
+ return _parse_gemini_json(raw_output, library, version)
159
+
160
+ except _QuotaError:
161
+ logger.warning("Gemini quota hit for %s@%s β€” falling back to Groq", library, version)
162
+ except httpx.HTTPStatusError as exc:
163
+ if exc.response.status_code == 429:
164
+ logger.warning("Gemini quota hit for %s@%s β€” falling back to Groq", library, version)
165
+ else:
166
+ logger.error("Gemini failed for %s@%s: %s", library, version, exc)
167
+ raise
168
+ except Exception as exc:
169
+ logger.error("Gemini failed for %s@%s: %s", library, version, exc)
170
+ raise
171
+
172
+ # ── Groq fallback ────────────────────────────────────────────────────────
173
+ return await _structure_with_groq(library, version, raw_docs, t0, groq_key=effective_groq)
174
+
175
+
176
+ async def _structure_with_groq(
177
+ library: str, version: str, raw_docs: str, t0: float, groq_key: str | None = None
178
+ ) -> dict:
179
+ """Call Groq (LLaMA 3.3 70B) as a fallback when Gemini quota is exhausted."""
180
+ effective_groq = groq_key or GROQ_API_KEY
181
+ if not effective_groq:
182
+ raise RuntimeError(
183
+ "Gemini quota exceeded and no Groq key available. "
184
+ "Add a Groq key in your DocForge settings (free at console.groq.com)."
185
+ )
186
+
187
+ truncated = raw_docs[:GROQ_MAX_DOC_CHARS]
188
+ if len(raw_docs) > GROQ_MAX_DOC_CHARS:
189
+ logger.info("Truncated docs from %d to %d chars for Groq", len(raw_docs), GROQ_MAX_DOC_CHARS)
190
+
191
+ prompt = STRUCTURING_PROMPT.format(
192
+ library=library,
193
+ version=version,
194
+ schema=STRUCTURED_SCHEMA,
195
+ today=date.today().isoformat(),
196
+ raw_docs=truncated,
197
+ )
198
+
199
+ payload = {
200
+ "model": GROQ_MODEL,
201
+ "messages": [{"role": "user", "content": prompt}],
202
+ "temperature": 0.1,
203
+ "max_tokens": 32768,
204
+ }
205
+
206
+ # Retry up to 3 times on 429 with exponential backoff
207
+ retry_delays = [10, 30, 60]
208
+ for attempt, delay in enumerate(retry_delays):
209
+ async with httpx.AsyncClient(timeout=120) as client:
210
+ resp = await client.post(
211
+ GROQ_ENDPOINT,
212
+ headers={"Authorization": f"Bearer {effective_groq}"},
213
+ json=payload,
214
+ )
215
+
216
+ if resp.status_code == 429:
217
+ if attempt < len(retry_delays) - 1:
218
+ logger.warning(
219
+ "Groq rate limited for %s@%s β€” waiting %ds before retry %d/3",
220
+ library, version, delay, attempt + 1,
221
+ )
222
+ await asyncio.sleep(delay)
223
+ continue
224
+ raise RuntimeError(
225
+ f"Groq rate limit exceeded for {library}@{version}. "
226
+ "Both Gemini and Groq are rate-limited. Wait a minute and try again, "
227
+ "or add a fresh API key in DocForge settings."
228
+ )
229
+
230
+ resp.raise_for_status()
231
+ data = resp.json()
232
+ break
233
+
234
+ elapsed = time.time() - t0
235
+ logger.info("Groq structuring took %.1fs for %s@%s", elapsed, library, version)
236
+ raw_output = data["choices"][0]["message"]["content"]
237
+ return _parse_gemini_json(raw_output, library, version)
238
+
239
+
240
+ def _parse_gemini_json(raw: str, library: str, version: str) -> dict:
241
+ """
242
+ Parse Gemini's response, handling common formatting issues.
243
+ Gemini sometimes wraps JSON in markdown code fences even when asked not to.
244
+ """
245
+ # Strip markdown code fences if present
246
+ raw = re.sub(r"^```(?:json)?\s*", "", raw.strip(), flags=re.MULTILINE)
247
+ raw = re.sub(r"\s*```$", "", raw.strip(), flags=re.MULTILINE)
248
+ raw = raw.strip()
249
+
250
+ try:
251
+ result = json.loads(raw)
252
+ except json.JSONDecodeError as exc:
253
+ logger.error("Failed to parse Gemini JSON output: %s\nRaw: %s...", exc, raw[:500])
254
+ # Return a minimal valid structure rather than crashing
255
+ return _mock_structure(library, version)
256
+
257
+ # Ensure required top-level fields
258
+ result.setdefault("library", library)
259
+ result.setdefault("version", version)
260
+ result.setdefault("components", [])
261
+
262
+ # Normalize component fields
263
+ today = date.today().isoformat()
264
+ for comp in result.get("components", []):
265
+ comp.setdefault("gotchas", [])
266
+ comp.setdefault("props", [])
267
+ comp.setdefault("peer_dependencies", [])
268
+ comp.setdefault("last_verified", today)
269
+
270
+ return result
271
+
272
+
273
+ def _mock_structure(library: str, version: str) -> dict:
274
+ """
275
+ Fallback when Gemini API key is not configured.
276
+ Returns a minimal but valid structure for local testing.
277
+ """
278
+ return {
279
+ "library": library,
280
+ "version": version,
281
+ "description": f"Documentation for {library}@{version}",
282
+ "components": [
283
+ {
284
+ "name": library,
285
+ "import_path": f"import {{ {library} }} from '{library}'",
286
+ "install_command": f"npm install {library}",
287
+ "props": [],
288
+ "gotchas": [
289
+ "⚠️ Gemini API key not configured β€” run with GEMINI_API_KEY set for real extraction"
290
+ ],
291
+ "peer_dependencies": [],
292
+ "usage_example": f"<{library} />",
293
+ "last_verified": date.today().isoformat(),
294
+ }
295
+ ],
296
+ }