Spaces:
Sleeping
Sleeping
Commit Β·
6ca8e48
0
Parent(s):
fix: set PLAYWRIGHT_BROWSERS_PATH so chromium is found at runtime by user
Browse files- Dockerfile +31 -0
- README.md +13 -0
- backend/.env.example +51 -0
- backend/Dockerfile +24 -0
- backend/__init__.py +0 -0
- backend/main.py +101 -0
- backend/models/__init__.py +0 -0
- backend/models/schemas.py +102 -0
- backend/models/user_models.py +55 -0
- backend/requirements.txt +21 -0
- backend/routers/__init__.py +0 -0
- backend/routers/auth.py +130 -0
- backend/routers/context.py +468 -0
- backend/routers/corrections.py +96 -0
- backend/routers/oauth.py +185 -0
- backend/routers/search.py +16 -0
- backend/routers/versions.py +15 -0
- backend/services/__init__.py +0 -0
- backend/services/auth_service.py +275 -0
- backend/services/cache.py +338 -0
- backend/services/formatter.py +90 -0
- backend/services/ingestion/__init__.py +0 -0
- backend/services/ingestion/copypaste_ingester.py +819 -0
- backend/services/ingestion/github_ingester.py +266 -0
- backend/services/ingestion/npm_ingester.py +379 -0
- backend/services/ingestion/paste_ingester.py +113 -0
- backend/services/ingestion/registries_ingester.py +236 -0
- backend/services/ingestion/search_ingester.py +69 -0
- backend/services/ingestion/url_ingester.py +192 -0
- backend/services/structurer.py +296 -0
Dockerfile
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
RUN useradd -m -u 1000 user
|
| 4 |
+
ENV PATH="/home/user/.local/bin:$PATH"
|
| 5 |
+
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
|
| 6 |
+
|
| 7 |
+
WORKDIR /app
|
| 8 |
+
|
| 9 |
+
RUN apt-get update && apt-get install -y \
|
| 10 |
+
wget curl gnupg \
|
| 11 |
+
libnss3 libatk1.0-0 libatk-bridge2.0-0 \
|
| 12 |
+
libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 \
|
| 13 |
+
libxfixes3 libxrandr2 libgbm1 libasound2 \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 15 |
+
|
| 16 |
+
COPY backend/requirements.txt .
|
| 17 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 18 |
+
|
| 19 |
+
RUN mkdir /ms-playwright && chmod 777 /ms-playwright
|
| 20 |
+
RUN playwright install chromium --with-deps
|
| 21 |
+
|
| 22 |
+
COPY --chown=user backend/ ./backend/
|
| 23 |
+
|
| 24 |
+
USER user
|
| 25 |
+
|
| 26 |
+
ENV DEV_MODE=false
|
| 27 |
+
ENV PORT=7860
|
| 28 |
+
|
| 29 |
+
EXPOSE 7860
|
| 30 |
+
|
| 31 |
+
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: DocForge API
|
| 3 |
+
emoji: π¨
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
pinned: false
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
# DocForge API
|
| 12 |
+
|
| 13 |
+
Backend API for DocForge β AI-powered documentation context file generator.
|
backend/.env.example
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# DocForge Backend Environment Variables
|
| 2 |
+
# Copy this to .env and fill in your values
|
| 3 |
+
|
| 4 |
+
# ββ Auth ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 5 |
+
# Secret key for signing JWTs β change this in production!
|
| 6 |
+
SECRET_KEY=change_me_to_a_long_random_string_in_production
|
| 7 |
+
|
| 8 |
+
# Fernet key for encrypting stored API keys (32-byte URL-safe base64)
|
| 9 |
+
# Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
| 10 |
+
# If omitted, derived from SECRET_KEY (fine for dev, use a real key in prod)
|
| 11 |
+
ENCRYPTION_KEY=
|
| 12 |
+
|
| 13 |
+
# URL of your frontend (used in device-flow verification_uri + OAuth redirects)
|
| 14 |
+
FRONTEND_URL=http://localhost:3000
|
| 15 |
+
|
| 16 |
+
# URL of this backend (used to build OAuth callback URIs)
|
| 17 |
+
BACKEND_URL=http://localhost:8000
|
| 18 |
+
|
| 19 |
+
# ββ OAuth (Google + GitHub sign-in) ββββββββββββββββββββββββββββββββββββββββββ
|
| 20 |
+
# Google: https://console.cloud.google.com/apis/credentials
|
| 21 |
+
# Authorised redirect URI to add: BACKEND_URL/api/auth/oauth/google/callback
|
| 22 |
+
GOOGLE_CLIENT_ID=
|
| 23 |
+
GOOGLE_CLIENT_SECRET=
|
| 24 |
+
|
| 25 |
+
# GitHub: https://github.com/settings/developers β OAuth Apps
|
| 26 |
+
# Callback URL to set: BACKEND_URL/api/auth/oauth/github/callback
|
| 27 |
+
GITHUB_CLIENT_ID=
|
| 28 |
+
GITHUB_CLIENT_SECRET=
|
| 29 |
+
|
| 30 |
+
# ββ Fallback AI keys (used when user has no keys stored) βββββββββββββββββββββ
|
| 31 |
+
# Optional: lets unauthenticated / keyless users still get a result.
|
| 32 |
+
# Leave blank to require every user to supply their own key.
|
| 33 |
+
GEMINI_API_KEY=
|
| 34 |
+
GROQ_API_KEY=
|
| 35 |
+
|
| 36 |
+
# ββ Storage βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 37 |
+
# Redis URL (optional β uses in-memory cache if not set)
|
| 38 |
+
REDIS_URL=redis://localhost:6379
|
| 39 |
+
|
| 40 |
+
# Supabase (recommended for production β stores users, API keys, cache, device codes)
|
| 41 |
+
# Required tables: users, user_api_keys, device_codes, cache_entries, corrections
|
| 42 |
+
SUPABASE_URL=your_supabase_url
|
| 43 |
+
SUPABASE_KEY=your_supabase_key
|
| 44 |
+
|
| 45 |
+
# ββ Misc ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 46 |
+
# Playwright timeout in milliseconds
|
| 47 |
+
PLAYWRIGHT_TIMEOUT=60000
|
| 48 |
+
|
| 49 |
+
# Set to true to use in-memory fallbacks (no Redis/Supabase required)
|
| 50 |
+
# Perfect for local development
|
| 51 |
+
DEV_MODE=true
|
backend/Dockerfile
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
# Install system dependencies for Playwright
|
| 6 |
+
RUN apt-get update && apt-get install -y \
|
| 7 |
+
wget \
|
| 8 |
+
gnupg \
|
| 9 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 10 |
+
|
| 11 |
+
COPY requirements.txt .
|
| 12 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 13 |
+
|
| 14 |
+
# Install Playwright browser binaries
|
| 15 |
+
RUN playwright install chromium --with-deps
|
| 16 |
+
|
| 17 |
+
COPY . .
|
| 18 |
+
|
| 19 |
+
ENV DEV_MODE=false
|
| 20 |
+
ENV PORT=8000
|
| 21 |
+
|
| 22 |
+
EXPOSE 8000
|
| 23 |
+
|
| 24 |
+
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
backend/__init__.py
ADDED
|
File without changes
|
backend/main.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
DocForge FastAPI Backend
|
| 3 |
+
|
| 4 |
+
Thin client / fat backend architecture:
|
| 5 |
+
- All heavy lifting (crawling, structuring, caching) happens here
|
| 6 |
+
- VS Code extension and MCP server are thin HTTP clients to this API
|
| 7 |
+
|
| 8 |
+
Run locally:
|
| 9 |
+
uvicorn backend.main:app --reload --port 8000
|
| 10 |
+
|
| 11 |
+
Or via Makefile:
|
| 12 |
+
make dev-backend
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import logging
|
| 16 |
+
import os
|
| 17 |
+
import time
|
| 18 |
+
|
| 19 |
+
from dotenv import load_dotenv
|
| 20 |
+
from fastapi import FastAPI, Request
|
| 21 |
+
|
| 22 |
+
load_dotenv()
|
| 23 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 24 |
+
from fastapi.responses import JSONResponse
|
| 25 |
+
|
| 26 |
+
from .routers import auth, context, corrections, search, versions, oauth
|
| 27 |
+
|
| 28 |
+
# ββ Logging setup βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 29 |
+
logging.basicConfig(
|
| 30 |
+
level=logging.INFO,
|
| 31 |
+
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
| 32 |
+
)
|
| 33 |
+
logger = logging.getLogger(__name__)
|
| 34 |
+
|
| 35 |
+
# ββ App initialization ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 36 |
+
app = FastAPI(
|
| 37 |
+
title="DocForge API",
|
| 38 |
+
description=(
|
| 39 |
+
"Turn any library's documentation into a prompt-ready .context.md file, "
|
| 40 |
+
"pinned to the exact version you're using."
|
| 41 |
+
),
|
| 42 |
+
version="1.0.0",
|
| 43 |
+
docs_url="/docs",
|
| 44 |
+
redoc_url="/redoc",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
# Allow VS Code extension and MCP server to call from any origin during dev
|
| 48 |
+
app.add_middleware(
|
| 49 |
+
CORSMiddleware,
|
| 50 |
+
allow_origins=["*"],
|
| 51 |
+
allow_methods=["*"],
|
| 52 |
+
allow_headers=["*"],
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
# ββ Request timing middleware βββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 56 |
+
@app.middleware("http")
|
| 57 |
+
async def log_request_time(request: Request, call_next):
|
| 58 |
+
t0 = time.time()
|
| 59 |
+
response = await call_next(request)
|
| 60 |
+
elapsed_ms = (time.time() - t0) * 1000
|
| 61 |
+
logger.info("%s %s β %d (%.0fms)", request.method, request.url.path, response.status_code, elapsed_ms)
|
| 62 |
+
return response
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
# ββ Routes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 66 |
+
app.include_router(auth.router)
|
| 67 |
+
app.include_router(oauth.router)
|
| 68 |
+
app.include_router(context.router)
|
| 69 |
+
app.include_router(corrections.router)
|
| 70 |
+
app.include_router(search.router)
|
| 71 |
+
app.include_router(versions.router)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@app.get("/api/health", tags=["health"])
|
| 75 |
+
async def health_check() -> dict:
|
| 76 |
+
"""Health check endpoint β used by VS Code extension to verify backend connectivity."""
|
| 77 |
+
return {
|
| 78 |
+
"status": "ok",
|
| 79 |
+
"service": "docforge-api",
|
| 80 |
+
"version": "1.0.0",
|
| 81 |
+
"dev_mode": os.getenv("DEV_MODE", "true").lower() == "true",
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@app.get("/", tags=["root"])
|
| 86 |
+
async def root() -> dict:
|
| 87 |
+
return {
|
| 88 |
+
"message": "DocForge API is running",
|
| 89 |
+
"docs": "/docs",
|
| 90 |
+
"health": "/api/health",
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# ββ Global error handler ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 95 |
+
@app.exception_handler(Exception)
|
| 96 |
+
async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
| 97 |
+
logger.exception("Unhandled exception on %s: %s", request.url, exc)
|
| 98 |
+
return JSONResponse(
|
| 99 |
+
status_code=500,
|
| 100 |
+
content={"detail": f"Internal server error: {str(exc)}"},
|
| 101 |
+
)
|
backend/models/__init__.py
ADDED
|
File without changes
|
backend/models/schemas.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pydantic models for DocForge API request/response schemas."""
|
| 2 |
+
|
| 3 |
+
from datetime import date
|
| 4 |
+
from enum import Enum
|
| 5 |
+
from typing import Any
|
| 6 |
+
from uuid import UUID
|
| 7 |
+
|
| 8 |
+
from pydantic import BaseModel, Field
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class InputType(str, Enum):
|
| 12 |
+
npm = "npm"
|
| 13 |
+
pypi = "pypi"
|
| 14 |
+
url = "url"
|
| 15 |
+
github = "github"
|
| 16 |
+
paste = "paste"
|
| 17 |
+
crates = "crates"
|
| 18 |
+
rubygems = "rubygems"
|
| 19 |
+
pubdev = "pubdev"
|
| 20 |
+
nuget = "nuget"
|
| 21 |
+
maven = "maven"
|
| 22 |
+
hex = "hex"
|
| 23 |
+
cran = "cran"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class OutputFormat(str, Enum):
|
| 27 |
+
context_md = "context_md"
|
| 28 |
+
json = "json"
|
| 29 |
+
mcp = "mcp"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class JobStatus(str, Enum):
|
| 33 |
+
queued = "queued"
|
| 34 |
+
processing = "processing"
|
| 35 |
+
complete = "complete"
|
| 36 |
+
failed = "failed"
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class ContextRequest(BaseModel):
|
| 40 |
+
input: str = Field(..., description="Package name, URL, or GitHub repo")
|
| 41 |
+
input_type: InputType = Field(InputType.npm, description="Type of input")
|
| 42 |
+
components: list[str] | None = Field(None, description="Specific components to extract")
|
| 43 |
+
output_format: OutputFormat = Field(OutputFormat.context_md, description="Output format")
|
| 44 |
+
content: str | None = Field(None, description="Raw content for paste input type")
|
| 45 |
+
# Client-supplied API keys (used when no Bearer token is present)
|
| 46 |
+
gemini_key: str | None = Field(None, description="Gemini API key (CLI local-key mode)")
|
| 47 |
+
groq_key: str | None = Field(None, description="Groq API key (CLI local-key mode)")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class JobCreatedResponse(BaseModel):
|
| 51 |
+
job_id: str
|
| 52 |
+
status: JobStatus = JobStatus.queued
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class PropSchema(BaseModel):
|
| 56 |
+
name: str
|
| 57 |
+
type: str
|
| 58 |
+
required: bool = False
|
| 59 |
+
default: str | None = None
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class ComponentSchema(BaseModel):
|
| 63 |
+
name: str
|
| 64 |
+
import_path: str | None = None
|
| 65 |
+
install_command: str | None = None
|
| 66 |
+
props: list[PropSchema] = Field(default_factory=list)
|
| 67 |
+
gotchas: list[str] = Field(default_factory=list)
|
| 68 |
+
peer_dependencies: list[str] = Field(default_factory=list)
|
| 69 |
+
usage_example: str | None = None
|
| 70 |
+
last_verified: str = Field(default_factory=lambda: date.today().isoformat())
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class StructuredLibrary(BaseModel):
|
| 74 |
+
library: str
|
| 75 |
+
version: str
|
| 76 |
+
description: str | None = None
|
| 77 |
+
components: list[ComponentSchema] = Field(default_factory=list)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class ContextJobResult(BaseModel):
|
| 81 |
+
status: JobStatus
|
| 82 |
+
job_id: str
|
| 83 |
+
library: str | None = None
|
| 84 |
+
version: str | None = None
|
| 85 |
+
output: str | None = None
|
| 86 |
+
components: list[dict[str, Any]] = Field(default_factory=list)
|
| 87 |
+
error: str | None = None
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class SearchResult(BaseModel):
|
| 91 |
+
name: str
|
| 92 |
+
version: str
|
| 93 |
+
cached_at: str
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class SearchResponse(BaseModel):
|
| 97 |
+
results: list[SearchResult]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class VersionsResponse(BaseModel):
|
| 101 |
+
package: str
|
| 102 |
+
versions: list[str]
|
backend/models/user_models.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel, EmailStr
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
# ββ Request / Response models ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 6 |
+
|
| 7 |
+
class RegisterRequest(BaseModel):
|
| 8 |
+
email: str
|
| 9 |
+
password: str
|
| 10 |
+
gemini_key: Optional[str] = None
|
| 11 |
+
groq_key: Optional[str] = None
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class LoginRequest(BaseModel):
|
| 15 |
+
email: str
|
| 16 |
+
password: str
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class UpdateKeysRequest(BaseModel):
|
| 20 |
+
gemini_key: Optional[str] = None
|
| 21 |
+
groq_key: Optional[str] = None
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class AuthResponse(BaseModel):
|
| 25 |
+
access_token: str
|
| 26 |
+
token_type: str = "bearer"
|
| 27 |
+
user_id: str
|
| 28 |
+
email: str
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class UserProfile(BaseModel):
|
| 32 |
+
user_id: str
|
| 33 |
+
email: str
|
| 34 |
+
has_gemini_key: bool
|
| 35 |
+
has_groq_key: bool
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# ββ Device flow models (for CLI / extension auth) βββββββββββββββββββββββββββββ
|
| 39 |
+
|
| 40 |
+
class DeviceInitResponse(BaseModel):
|
| 41 |
+
device_code: str # opaque, sent by CLI to poll
|
| 42 |
+
user_code: str # human-readable code shown in terminal
|
| 43 |
+
verification_uri: str # URL user opens in browser
|
| 44 |
+
expires_in: int = 900 # seconds
|
| 45 |
+
interval: int = 5 # polling interval seconds
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class DeviceVerifyRequest(BaseModel):
|
| 49 |
+
user_code: str # code the user types on the website
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class DevicePollResponse(BaseModel):
|
| 53 |
+
status: str # "pending" | "complete" | "expired"
|
| 54 |
+
access_token: Optional[str] = None
|
| 55 |
+
email: Optional[str] = None
|
backend/requirements.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi==0.115.0
|
| 2 |
+
uvicorn[standard]==0.30.6
|
| 3 |
+
httpx==0.27.2
|
| 4 |
+
pydantic>=2.11.0
|
| 5 |
+
python-dotenv==1.0.1
|
| 6 |
+
# Playwright for JS-rendered docs sites
|
| 7 |
+
playwright>=1.50.0
|
| 8 |
+
# HTML to Markdown conversion
|
| 9 |
+
html2text==2024.2.26
|
| 10 |
+
# Redis (production cache)
|
| 11 |
+
redis==5.1.0
|
| 12 |
+
# Supabase (production storage)
|
| 13 |
+
supabase==2.9.0
|
| 14 |
+
# PDF/document conversion via IBM Docling (optional β install separately)
|
| 15 |
+
# docling==2.0.0
|
| 16 |
+
# Google Gemini
|
| 17 |
+
google-generativeai==0.8.3
|
| 18 |
+
# Auth
|
| 19 |
+
python-jose[cryptography]==3.5.0
|
| 20 |
+
passlib[bcrypt]==1.7.4
|
| 21 |
+
cryptography==46.0.5
|
backend/routers/__init__.py
ADDED
|
File without changes
|
backend/routers/auth.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Auth router β all endpoints under /api/auth
|
| 3 |
+
|
| 4 |
+
Public:
|
| 5 |
+
POST /api/auth/register create account (+ optional API keys)
|
| 6 |
+
POST /api/auth/login email + password β JWT
|
| 7 |
+
POST /api/auth/device/init CLI / extension requests a device code
|
| 8 |
+
GET /api/auth/device/poll CLI polls until user verifies on web
|
| 9 |
+
POST /api/auth/device/verify web UI posts user_code to mark verified
|
| 10 |
+
|
| 11 |
+
Protected (requires Bearer token):
|
| 12 |
+
GET /api/auth/me current user profile
|
| 13 |
+
PUT /api/auth/keys save / update API keys
|
| 14 |
+
POST /api/auth/logout (client-side: just discard the token)
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import os
|
| 18 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 19 |
+
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
| 20 |
+
|
| 21 |
+
from backend.models.user_models import (
|
| 22 |
+
RegisterRequest, LoginRequest, UpdateKeysRequest,
|
| 23 |
+
AuthResponse, UserProfile,
|
| 24 |
+
DeviceInitResponse, DeviceVerifyRequest, DevicePollResponse,
|
| 25 |
+
)
|
| 26 |
+
from backend.services import auth_service
|
| 27 |
+
|
| 28 |
+
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
| 29 |
+
_bearer = HTTPBearer(auto_error=False)
|
| 30 |
+
|
| 31 |
+
FRONTEND_URL = os.getenv("FRONTEND_URL", "http://localhost:3000")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# ββ Auth dependency βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 35 |
+
|
| 36 |
+
async def current_user(
|
| 37 |
+
creds: HTTPAuthorizationCredentials = Depends(_bearer),
|
| 38 |
+
) -> dict:
|
| 39 |
+
if not creds:
|
| 40 |
+
raise HTTPException(status_code=401, detail="Not authenticated")
|
| 41 |
+
payload = auth_service.decode_token(creds.credentials)
|
| 42 |
+
if not payload:
|
| 43 |
+
raise HTTPException(status_code=401, detail="Invalid or expired token")
|
| 44 |
+
user = await auth_service.get_user_by_id(payload["sub"])
|
| 45 |
+
if not user:
|
| 46 |
+
raise HTTPException(status_code=401, detail="User not found")
|
| 47 |
+
return user
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
# ββ Public endpoints ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 51 |
+
|
| 52 |
+
@router.post("/register", response_model=AuthResponse)
|
| 53 |
+
async def register(body: RegisterRequest):
|
| 54 |
+
hashed = auth_service.hash_password(body.password)
|
| 55 |
+
user_id = await auth_service.create_user(body.email, hashed)
|
| 56 |
+
if not user_id:
|
| 57 |
+
raise HTTPException(status_code=409, detail="Email already registered")
|
| 58 |
+
|
| 59 |
+
# Optionally save API keys during registration
|
| 60 |
+
if body.gemini_key or body.groq_key:
|
| 61 |
+
await auth_service.save_api_keys(user_id, body.gemini_key, body.groq_key)
|
| 62 |
+
|
| 63 |
+
token = auth_service.create_access_token(user_id, body.email)
|
| 64 |
+
return AuthResponse(access_token=token, user_id=user_id, email=body.email)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@router.post("/login", response_model=AuthResponse)
|
| 68 |
+
async def login(body: LoginRequest):
|
| 69 |
+
user = await auth_service.get_user_by_email(body.email)
|
| 70 |
+
if not user or not auth_service.verify_password(body.password, user["password_hash"]):
|
| 71 |
+
raise HTTPException(status_code=401, detail="Invalid email or password")
|
| 72 |
+
|
| 73 |
+
token = auth_service.create_access_token(user["id"], user["email"])
|
| 74 |
+
return AuthResponse(access_token=token, user_id=user["id"], email=user["email"])
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# ββ Device flow βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 78 |
+
|
| 79 |
+
@router.post("/device/init", response_model=DeviceInitResponse)
|
| 80 |
+
async def device_init():
|
| 81 |
+
"""CLI / extension calls this to get a device code."""
|
| 82 |
+
result = await auth_service.create_device_code()
|
| 83 |
+
return DeviceInitResponse(
|
| 84 |
+
device_code=result["device_code"],
|
| 85 |
+
user_code=result["user_code"],
|
| 86 |
+
verification_uri=f"{FRONTEND_URL}/auth/device",
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@router.get("/device/poll", response_model=DevicePollResponse)
|
| 91 |
+
async def device_poll(device_code: str):
|
| 92 |
+
"""CLI polls this every 5 s until status == 'complete'."""
|
| 93 |
+
result = await auth_service.poll_device_code(device_code)
|
| 94 |
+
return DevicePollResponse(**result)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
@router.post("/device/verify")
|
| 98 |
+
async def device_verify(
|
| 99 |
+
body: DeviceVerifyRequest,
|
| 100 |
+
user: dict = Depends(current_user),
|
| 101 |
+
):
|
| 102 |
+
"""Web UI posts this after the logged-in user enters their user_code."""
|
| 103 |
+
ok = await auth_service.verify_device_code(body.user_code, user["id"])
|
| 104 |
+
if not ok:
|
| 105 |
+
raise HTTPException(status_code=400, detail="Invalid or expired code")
|
| 106 |
+
return {"ok": True}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
# ββ Protected endpoints βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 110 |
+
|
| 111 |
+
@router.get("/me", response_model=UserProfile)
|
| 112 |
+
async def me(user: dict = Depends(current_user)):
|
| 113 |
+
keys = await auth_service.get_api_keys(user["id"])
|
| 114 |
+
return UserProfile(
|
| 115 |
+
user_id=user["id"],
|
| 116 |
+
email=user["email"],
|
| 117 |
+
has_gemini_key=bool(keys.get("gemini_key")),
|
| 118 |
+
has_groq_key=bool(keys.get("groq_key")),
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@router.put("/keys")
|
| 123 |
+
async def update_keys(
|
| 124 |
+
body: UpdateKeysRequest,
|
| 125 |
+
user: dict = Depends(current_user),
|
| 126 |
+
):
|
| 127 |
+
if not body.gemini_key and not body.groq_key:
|
| 128 |
+
raise HTTPException(status_code=400, detail="Provide at least one key")
|
| 129 |
+
await auth_service.save_api_keys(user["id"], body.gemini_key, body.groq_key)
|
| 130 |
+
return {"ok": True, "message": "API keys saved"}
|
backend/routers/context.py
ADDED
|
@@ -0,0 +1,468 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
/api/context routes β the core DocForge pipeline endpoint.
|
| 3 |
+
|
| 4 |
+
POST /api/context β queues a job, returns job_id immediately
|
| 5 |
+
GET /api/context/{job_id} β returns job status and result when done
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
import time
|
| 10 |
+
import uuid
|
| 11 |
+
from datetime import date
|
| 12 |
+
from urllib.parse import urlparse
|
| 13 |
+
|
| 14 |
+
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
| 15 |
+
|
| 16 |
+
from ..models.schemas import (
|
| 17 |
+
ContextJobResult,
|
| 18 |
+
ContextRequest,
|
| 19 |
+
InputType,
|
| 20 |
+
JobCreatedResponse,
|
| 21 |
+
JobStatus,
|
| 22 |
+
OutputFormat,
|
| 23 |
+
)
|
| 24 |
+
from ..services import cache
|
| 25 |
+
from ..services.formatter import format_context_md
|
| 26 |
+
from ..services.ingestion.npm_ingester import (
|
| 27 |
+
resolve_npm_package,
|
| 28 |
+
resolve_pypi_package,
|
| 29 |
+
)
|
| 30 |
+
from ..services.ingestion.paste_ingester import ingest_paste
|
| 31 |
+
from ..services.ingestion.registries_ingester import (
|
| 32 |
+
resolve_crates,
|
| 33 |
+
resolve_cran,
|
| 34 |
+
resolve_hex,
|
| 35 |
+
resolve_maven,
|
| 36 |
+
resolve_nuget,
|
| 37 |
+
resolve_pubdev,
|
| 38 |
+
resolve_rubygems,
|
| 39 |
+
)
|
| 40 |
+
from ..services.ingestion.search_ingester import find_docs_url
|
| 41 |
+
from ..services.ingestion.url_ingester import crawl_url
|
| 42 |
+
from ..services.structurer import structure_with_gemini
|
| 43 |
+
|
| 44 |
+
logger = logging.getLogger(__name__)
|
| 45 |
+
router = APIRouter(prefix="/api/context", tags=["context"])
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@router.post("", response_model=JobCreatedResponse)
|
| 49 |
+
async def create_context_job(
|
| 50 |
+
request: ContextRequest,
|
| 51 |
+
background_tasks: BackgroundTasks,
|
| 52 |
+
http_request: Request,
|
| 53 |
+
) -> JobCreatedResponse:
|
| 54 |
+
"""
|
| 55 |
+
Queue a documentation context generation job.
|
| 56 |
+
Returns immediately with a job_id for polling.
|
| 57 |
+
|
| 58 |
+
If a valid Bearer token is provided the user's stored API keys are used for
|
| 59 |
+
the AI step; otherwise falls back to env-var keys (dev / open access).
|
| 60 |
+
"""
|
| 61 |
+
# ββ Auth: prefer stored keys from Bearer token, fall back to request body β
|
| 62 |
+
gemini_key: str | None = None
|
| 63 |
+
groq_key: str | None = None
|
| 64 |
+
|
| 65 |
+
auth_header = http_request.headers.get("authorization", "")
|
| 66 |
+
if auth_header.startswith("Bearer "):
|
| 67 |
+
from ..services.auth_service import decode_token, get_api_keys
|
| 68 |
+
payload = decode_token(auth_header.split(" ", 1)[1])
|
| 69 |
+
if payload:
|
| 70 |
+
keys = await get_api_keys(payload["sub"])
|
| 71 |
+
gemini_key = keys.get("gemini_key")
|
| 72 |
+
groq_key = keys.get("groq_key")
|
| 73 |
+
|
| 74 |
+
# CLI local-key mode: keys sent directly in the request body
|
| 75 |
+
if not gemini_key and not groq_key:
|
| 76 |
+
gemini_key = request.gemini_key or None
|
| 77 |
+
groq_key = request.groq_key or None
|
| 78 |
+
|
| 79 |
+
# Enforce user-supplied key β DocForge does not maintain a server-level key.
|
| 80 |
+
# DEV_MODE bypasses this so local dev works without a key configured.
|
| 81 |
+
import os as _os
|
| 82 |
+
if not gemini_key and not _os.getenv("DEV_MODE", "false").lower() == "true":
|
| 83 |
+
raise HTTPException(
|
| 84 |
+
status_code=400,
|
| 85 |
+
detail=(
|
| 86 |
+
"Gemini API key required. "
|
| 87 |
+
"Get a free key at https://aistudio.google.com/app/apikey "
|
| 88 |
+
"then add it in DocForge Settings."
|
| 89 |
+
),
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
job_id = str(uuid.uuid4())
|
| 93 |
+
|
| 94 |
+
# Store initial job state
|
| 95 |
+
await cache.set_job(job_id, {
|
| 96 |
+
"status": JobStatus.queued,
|
| 97 |
+
"job_id": job_id,
|
| 98 |
+
"input": request.input,
|
| 99 |
+
"input_type": request.input_type,
|
| 100 |
+
})
|
| 101 |
+
|
| 102 |
+
# Run the heavy pipeline work in the background so the HTTP response
|
| 103 |
+
# returns instantly (crawling takes 30β90s)
|
| 104 |
+
background_tasks.add_task(
|
| 105 |
+
_run_pipeline,
|
| 106 |
+
job_id=job_id,
|
| 107 |
+
request=request,
|
| 108 |
+
gemini_key=gemini_key,
|
| 109 |
+
groq_key=groq_key,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
logger.info("Queued job %s for input=%s type=%s", job_id, request.input, request.input_type)
|
| 113 |
+
return JobCreatedResponse(job_id=job_id, status=JobStatus.queued)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@router.get("/{job_id}", response_model=ContextJobResult)
|
| 117 |
+
async def get_context_job(job_id: str) -> ContextJobResult:
|
| 118 |
+
"""Poll job status. Returns output when status == 'complete'."""
|
| 119 |
+
job = await cache.get_job(job_id)
|
| 120 |
+
if job is None:
|
| 121 |
+
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
| 122 |
+
return ContextJobResult(**job)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# ---------------------------------------------------------------------------
|
| 126 |
+
# Background pipeline
|
| 127 |
+
# ---------------------------------------------------------------------------
|
| 128 |
+
|
| 129 |
+
async def _run_pipeline(
|
| 130 |
+
job_id: str,
|
| 131 |
+
request: ContextRequest,
|
| 132 |
+
gemini_key: str | None = None,
|
| 133 |
+
groq_key: str | None = None,
|
| 134 |
+
) -> None:
|
| 135 |
+
"""
|
| 136 |
+
Full DocForge pipeline:
|
| 137 |
+
1. Resolve package / fetch URL / clone repo
|
| 138 |
+
2. Crawl documentation
|
| 139 |
+
3. Structure with Gemini (using user's stored key if authenticated)
|
| 140 |
+
4. Format as .context.md
|
| 141 |
+
5. Cache result
|
| 142 |
+
"""
|
| 143 |
+
t_start = time.time()
|
| 144 |
+
|
| 145 |
+
async def update_status(status: str, **extra: object) -> None:
|
| 146 |
+
job = await cache.get_job(job_id) or {}
|
| 147 |
+
job.update({"status": status, "job_id": job_id, **extra})
|
| 148 |
+
await cache.set_job(job_id, job)
|
| 149 |
+
|
| 150 |
+
try:
|
| 151 |
+
await update_status(JobStatus.processing)
|
| 152 |
+
|
| 153 |
+
# ββ Step 1: Resolve metadata ββββββββββββββββββββββββββββββββββββββ
|
| 154 |
+
library = request.input
|
| 155 |
+
version = "latest"
|
| 156 |
+
docs_url: str | None = None
|
| 157 |
+
version_tag: str | None = None
|
| 158 |
+
copypaste_docs: str | None = None # pre-fetched docs for copy-paste libs
|
| 159 |
+
pre_structured: dict | None = None # pre-built structured dict (skips LLM)
|
| 160 |
+
|
| 161 |
+
if request.input_type == InputType.npm:
|
| 162 |
+
# If the user somehow passed a URL as an npm package, treat it as a URL crawl
|
| 163 |
+
if request.input.startswith(("http://", "https://")):
|
| 164 |
+
logger.warning("npm input looks like a URL β switching to URL crawl: %s", request.input)
|
| 165 |
+
raw_docs = await crawl_url(request.input)
|
| 166 |
+
structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key)
|
| 167 |
+
output = format_context_md(structured) if request.output_format == OutputFormat.context_md else __import__("json").dumps(structured, indent=2)
|
| 168 |
+
structured["cached_at"] = date.today().isoformat()
|
| 169 |
+
await cache.set_cached_context(library, version, structured)
|
| 170 |
+
await update_status(JobStatus.complete, library=library, version=version, output=output, components=structured.get("components", []))
|
| 171 |
+
return
|
| 172 |
+
meta = await resolve_npm_package(request.input)
|
| 173 |
+
library = meta["name"]
|
| 174 |
+
version = meta["version"]
|
| 175 |
+
docs_url = meta.get("homepage")
|
| 176 |
+
version_tag = meta.get("_version_tag") # set when version not on npm but GitHub exists
|
| 177 |
+
# Copy-paste library: docs already fetched during resolution
|
| 178 |
+
if meta.get("_is_copypaste"):
|
| 179 |
+
copypaste_docs = meta.get("_raw_docs", "")
|
| 180 |
+
pre_structured = meta.get("_structured") or None
|
| 181 |
+
logger.info(
|
| 182 |
+
"Copy-paste library '%s' detected via %s β skipping crawl (%d chars pre-fetched)",
|
| 183 |
+
library, meta.get("_copypaste_method"), len(copypaste_docs),
|
| 184 |
+
)
|
| 185 |
+
else:
|
| 186 |
+
logger.info("Resolved npm: %s@%s docs=%s tag=%s", library, version, docs_url, version_tag)
|
| 187 |
+
|
| 188 |
+
elif request.input_type == InputType.pypi:
|
| 189 |
+
meta = await resolve_pypi_package(request.input)
|
| 190 |
+
library = meta["name"]
|
| 191 |
+
version = meta["version"]
|
| 192 |
+
docs_url = meta.get("homepage")
|
| 193 |
+
logger.info("Resolved PyPI: %s@%s docs=%s", library, version, docs_url)
|
| 194 |
+
|
| 195 |
+
elif request.input_type == InputType.crates:
|
| 196 |
+
meta = await resolve_crates(request.input)
|
| 197 |
+
library = meta["name"]
|
| 198 |
+
version = meta["version"]
|
| 199 |
+
docs_url = meta.get("homepage")
|
| 200 |
+
logger.info("Resolved crates.io: %s@%s docs=%s", library, version, docs_url)
|
| 201 |
+
|
| 202 |
+
elif request.input_type == InputType.rubygems:
|
| 203 |
+
meta = await resolve_rubygems(request.input)
|
| 204 |
+
library = meta["name"]
|
| 205 |
+
version = meta["version"]
|
| 206 |
+
docs_url = meta.get("homepage")
|
| 207 |
+
logger.info("Resolved RubyGems: %s@%s docs=%s", library, version, docs_url)
|
| 208 |
+
|
| 209 |
+
elif request.input_type == InputType.pubdev:
|
| 210 |
+
meta = await resolve_pubdev(request.input)
|
| 211 |
+
library = meta["name"]
|
| 212 |
+
version = meta["version"]
|
| 213 |
+
docs_url = meta.get("homepage")
|
| 214 |
+
logger.info("Resolved pub.dev: %s@%s docs=%s", library, version, docs_url)
|
| 215 |
+
|
| 216 |
+
elif request.input_type == InputType.nuget:
|
| 217 |
+
meta = await resolve_nuget(request.input)
|
| 218 |
+
library = meta["name"]
|
| 219 |
+
version = meta["version"]
|
| 220 |
+
docs_url = meta.get("homepage")
|
| 221 |
+
logger.info("Resolved NuGet: %s@%s docs=%s", library, version, docs_url)
|
| 222 |
+
|
| 223 |
+
elif request.input_type == InputType.maven:
|
| 224 |
+
meta = await resolve_maven(request.input)
|
| 225 |
+
library = meta["name"]
|
| 226 |
+
version = meta["version"]
|
| 227 |
+
docs_url = meta.get("homepage")
|
| 228 |
+
logger.info("Resolved Maven: %s@%s docs=%s", library, version, docs_url)
|
| 229 |
+
|
| 230 |
+
elif request.input_type == InputType.hex:
|
| 231 |
+
meta = await resolve_hex(request.input)
|
| 232 |
+
library = meta["name"]
|
| 233 |
+
version = meta["version"]
|
| 234 |
+
docs_url = meta.get("homepage")
|
| 235 |
+
logger.info("Resolved Hex: %s@%s docs=%s", library, version, docs_url)
|
| 236 |
+
|
| 237 |
+
elif request.input_type == InputType.cran:
|
| 238 |
+
meta = await resolve_cran(request.input)
|
| 239 |
+
library = meta["name"]
|
| 240 |
+
version = meta["version"]
|
| 241 |
+
docs_url = meta.get("homepage")
|
| 242 |
+
logger.info("Resolved CRAN: %s@%s docs=%s", library, version, docs_url)
|
| 243 |
+
|
| 244 |
+
elif request.input_type == InputType.url:
|
| 245 |
+
# Derive a clean library name from the domain so the cache key is
|
| 246 |
+
# human-readable ("reactbits") rather than the raw URL.
|
| 247 |
+
# GitHub URLs keep their repo name; plain URLs use the domain stem.
|
| 248 |
+
if "github.com" in request.input:
|
| 249 |
+
# e.g. https://github.com/DavidHDev/react-bits β react-bits
|
| 250 |
+
parts = [p for p in urlparse(request.input).path.split("/") if p]
|
| 251 |
+
library = parts[1] if len(parts) >= 2 else request.input
|
| 252 |
+
else:
|
| 253 |
+
domain = urlparse(request.input).netloc.removeprefix("www.")
|
| 254 |
+
library = domain.split(".")[0] # "reactbits" from "reactbits.dev"
|
| 255 |
+
logger.info("URL input '%s' β library name '%s' for cache/structuring", request.input, library)
|
| 256 |
+
|
| 257 |
+
# ββ Step 2: Check cache (skip expensive crawl if already done) ββββ
|
| 258 |
+
cached = await cache.get_cached_context(library, version)
|
| 259 |
+
if cached:
|
| 260 |
+
output = format_context_md(cached) if request.output_format == OutputFormat.context_md else str(cached)
|
| 261 |
+
await update_status(
|
| 262 |
+
JobStatus.complete,
|
| 263 |
+
library=library,
|
| 264 |
+
version=version,
|
| 265 |
+
output=output,
|
| 266 |
+
components=cached.get("components", []),
|
| 267 |
+
)
|
| 268 |
+
logger.info("Cache hit for %s@%s, job %s done in %.1fs", library, version, job_id, time.time() - t_start)
|
| 269 |
+
return
|
| 270 |
+
|
| 271 |
+
# ββ Step 3: Crawl documentation βββββββββββββββββββββββββββββββββββ
|
| 272 |
+
raw_docs = ""
|
| 273 |
+
|
| 274 |
+
# Copy-paste library β docs were fetched during resolution, skip crawl
|
| 275 |
+
if copypaste_docs is not None:
|
| 276 |
+
raw_docs = copypaste_docs
|
| 277 |
+
|
| 278 |
+
elif request.input_type == InputType.paste:
|
| 279 |
+
raw_docs = await ingest_paste(request.content or request.input)
|
| 280 |
+
|
| 281 |
+
elif request.input_type == InputType.github:
|
| 282 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 283 |
+
raw_docs = await ingest_github_repo(request.input)
|
| 284 |
+
|
| 285 |
+
elif request.input_type == InputType.url:
|
| 286 |
+
if "github.com" in request.input:
|
| 287 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 288 |
+
raw_docs = await ingest_github_repo(request.input)
|
| 289 |
+
else:
|
| 290 |
+
import asyncio as _asyncio
|
| 291 |
+
from ..services.ingestion.url_ingester import extract_github_url
|
| 292 |
+
from ..services.ingestion.copypaste_ingester import (
|
| 293 |
+
check_jsrepo, check_shadcn_registry, detect_and_ingest,
|
| 294 |
+
)
|
| 295 |
+
|
| 296 |
+
# library is already set to the clean name_hint in Step 1
|
| 297 |
+
# ββ Parallel: all three lightweight checks at once βββββββββββββ
|
| 298 |
+
# check_shadcn_registry hits /r/registry.json etc. β it does NOT
|
| 299 |
+
# need a GitHub URL, so it runs alongside the other two.
|
| 300 |
+
github_url, jsrepo_name_docs, shadcn_result = await _asyncio.gather(
|
| 301 |
+
extract_github_url(request.input),
|
| 302 |
+
check_jsrepo(library, None),
|
| 303 |
+
check_shadcn_registry(request.input, library),
|
| 304 |
+
)
|
| 305 |
+
|
| 306 |
+
# shadcn registry returns a pre-built structured dict β skip LLM entirely
|
| 307 |
+
if shadcn_result is not None:
|
| 308 |
+
pre_structured = shadcn_result
|
| 309 |
+
logger.info(
|
| 310 |
+
"URL '%s' β shadcn registry match for '%s' (%d components, skipping LLM)",
|
| 311 |
+
request.input, library, len(pre_structured.get("components", [])),
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
else:
|
| 315 |
+
# Prefer jsrepo registry hits for the raw-docs path
|
| 316 |
+
quick_docs = jsrepo_name_docs
|
| 317 |
+
|
| 318 |
+
# If we also have a GitHub URL, do a targeted jsrepo check
|
| 319 |
+
# (raw GitHub paths + jsrepo/{owner}/{repo}).
|
| 320 |
+
jsrepo_targeted: str | None = None
|
| 321 |
+
if github_url and not quick_docs:
|
| 322 |
+
jsrepo_targeted = await check_jsrepo(library, github_url)
|
| 323 |
+
|
| 324 |
+
cp_docs = jsrepo_targeted or quick_docs
|
| 325 |
+
|
| 326 |
+
if cp_docs:
|
| 327 |
+
raw_docs = cp_docs
|
| 328 |
+
logger.info("URL '%s' β registry match for '%s' (%d chars)", request.input, library, len(raw_docs))
|
| 329 |
+
|
| 330 |
+
elif github_url:
|
| 331 |
+
logger.info("URL '%s' β GitHub repo found: %s", request.input, github_url)
|
| 332 |
+
# copy-paste detection with full github_url context
|
| 333 |
+
cp_result = await detect_and_ingest(
|
| 334 |
+
name=library,
|
| 335 |
+
github_url=github_url,
|
| 336 |
+
homepage=request.input,
|
| 337 |
+
)
|
| 338 |
+
if cp_result:
|
| 339 |
+
raw_docs = cp_result.get("_raw_docs", "")
|
| 340 |
+
logger.info(
|
| 341 |
+
"URL '%s' β copy-paste lib (method=%s) %d chars",
|
| 342 |
+
request.input, cp_result.get("_copypaste_method"), len(raw_docs),
|
| 343 |
+
)
|
| 344 |
+
else:
|
| 345 |
+
# Not copy-paste β GitHub ingester still beats Playwright
|
| 346 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 347 |
+
raw_docs = await ingest_github_repo(github_url)
|
| 348 |
+
logger.info("URL '%s' β GitHub ingestion (%d chars)", request.input, len(raw_docs))
|
| 349 |
+
|
| 350 |
+
else:
|
| 351 |
+
# No GitHub found β fall back to Playwright
|
| 352 |
+
raw_docs = await crawl_url(request.input)
|
| 353 |
+
logger.info("URL '%s' β Playwright crawl (%d chars)", request.input, len(raw_docs))
|
| 354 |
+
|
| 355 |
+
# Sparse content β try GitHub search by domain name.
|
| 356 |
+
# Only accept a fallback repo whose homepage points back to the
|
| 357 |
+
# same domain we started from (prevents picking up unrelated
|
| 358 |
+
# packages like MCP wrappers that happen to mention the name).
|
| 359 |
+
if len(raw_docs) < 20_000:
|
| 360 |
+
import httpx
|
| 361 |
+
input_domain = urlparse(request.input).netloc.removeprefix("www.")
|
| 362 |
+
async with httpx.AsyncClient(timeout=10) as _client:
|
| 363 |
+
from ..services.ingestion.npm_ingester import _github_search
|
| 364 |
+
fallback = await _github_search(_client, library)
|
| 365 |
+
if fallback:
|
| 366 |
+
# Quick check: does the GitHub repo's homepage match?
|
| 367 |
+
try:
|
| 368 |
+
meta_resp = await _client.get(
|
| 369 |
+
f"https://api.github.com/repos/{fallback.removeprefix('https://github.com/')}",
|
| 370 |
+
headers={"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"},
|
| 371 |
+
timeout=5,
|
| 372 |
+
)
|
| 373 |
+
if meta_resp.is_success:
|
| 374 |
+
repo_homepage = meta_resp.json().get("homepage") or ""
|
| 375 |
+
if input_domain not in repo_homepage:
|
| 376 |
+
logger.info(
|
| 377 |
+
"GitHub fallback '%s' homepage '%s' doesn't match domain '%s' β skipping",
|
| 378 |
+
fallback, repo_homepage, input_domain,
|
| 379 |
+
)
|
| 380 |
+
fallback = None
|
| 381 |
+
except Exception:
|
| 382 |
+
fallback = None # if we can't verify, skip it
|
| 383 |
+
|
| 384 |
+
if fallback:
|
| 385 |
+
logger.info(
|
| 386 |
+
"Sparse Playwright content (%d chars) for '%s' β trying GitHub: %s",
|
| 387 |
+
len(raw_docs), request.input, fallback,
|
| 388 |
+
)
|
| 389 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 390 |
+
github_docs = await ingest_github_repo(fallback)
|
| 391 |
+
if len(github_docs) > len(raw_docs):
|
| 392 |
+
raw_docs = github_docs
|
| 393 |
+
|
| 394 |
+
else:
|
| 395 |
+
# npm or pypi: crawl the resolved docs URL
|
| 396 |
+
if version_tag and not docs_url:
|
| 397 |
+
# Version not on npm but GitHub repo known β use git tag directly
|
| 398 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 399 |
+
repo_url = meta.get("repository", "")
|
| 400 |
+
import re as _re
|
| 401 |
+
repo_url = _re.sub(r"^git\+", "", repo_url)
|
| 402 |
+
logger.info("Using GitHub tag %s for %s", version_tag, repo_url)
|
| 403 |
+
raw_docs = await ingest_github_repo(repo_url, version_tag=version_tag)
|
| 404 |
+
elif docs_url:
|
| 405 |
+
if "github.com" in docs_url:
|
| 406 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 407 |
+
raw_docs = await ingest_github_repo(docs_url, version_tag=version_tag)
|
| 408 |
+
else:
|
| 409 |
+
raw_docs = await crawl_url(docs_url)
|
| 410 |
+
else:
|
| 411 |
+
# Web search fallback: query DuckDuckGo for docs
|
| 412 |
+
found_url = await find_docs_url(f"{library} {version}")
|
| 413 |
+
if found_url:
|
| 414 |
+
logger.info("Web search found docs for %s: %s", library, found_url)
|
| 415 |
+
if "github.com" in found_url:
|
| 416 |
+
from ..services.ingestion.github_ingester import ingest_github_repo
|
| 417 |
+
raw_docs = await ingest_github_repo(found_url)
|
| 418 |
+
else:
|
| 419 |
+
raw_docs = await crawl_url(found_url)
|
| 420 |
+
else:
|
| 421 |
+
logger.warning("No docs URL found for %s, sending minimal stub", library)
|
| 422 |
+
raw_docs = f"Library: {library}\nVersion: {version}\nNo documentation URL found."
|
| 423 |
+
|
| 424 |
+
logger.info("Crawl done for %s@%s: %d chars in %.1fs", library, version, len(raw_docs), time.time() - t_start)
|
| 425 |
+
|
| 426 |
+
# ββ Step 4: Structure with Gemini βββββββββββββββββββββββββββββββββ
|
| 427 |
+
# Skip LLM for copy-paste libraries with pre-built structured data
|
| 428 |
+
if pre_structured is not None:
|
| 429 |
+
structured = pre_structured
|
| 430 |
+
structured.setdefault("library", library)
|
| 431 |
+
structured.setdefault("version", version)
|
| 432 |
+
logger.info("Using pre-built structure for %s@%s (%d components)", library, version, len(structured.get("components", [])))
|
| 433 |
+
else:
|
| 434 |
+
structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key)
|
| 435 |
+
|
| 436 |
+
# Filter to requested components if specified
|
| 437 |
+
if request.components:
|
| 438 |
+
requested = {c.lower() for c in request.components}
|
| 439 |
+
structured["components"] = [
|
| 440 |
+
c for c in structured.get("components", [])
|
| 441 |
+
if c.get("name", "").lower() in requested
|
| 442 |
+
]
|
| 443 |
+
|
| 444 |
+
# ββ Step 5: Format output βββββββββββββββββββββββββββββββββββββββββ
|
| 445 |
+
if request.output_format == OutputFormat.context_md:
|
| 446 |
+
output = format_context_md(structured)
|
| 447 |
+
else:
|
| 448 |
+
import json
|
| 449 |
+
output = json.dumps(structured, indent=2)
|
| 450 |
+
|
| 451 |
+
# ββ Step 6: Cache and complete ββββββββββββββββββββββββββββββββββββ
|
| 452 |
+
structured["cached_at"] = date.today().isoformat()
|
| 453 |
+
await cache.set_cached_context(library, version, structured)
|
| 454 |
+
|
| 455 |
+
total = time.time() - t_start
|
| 456 |
+
logger.info("Pipeline complete for %s@%s in %.1fs", library, version, total)
|
| 457 |
+
|
| 458 |
+
await update_status(
|
| 459 |
+
JobStatus.complete,
|
| 460 |
+
library=library,
|
| 461 |
+
version=version,
|
| 462 |
+
output=output,
|
| 463 |
+
components=structured.get("components", []),
|
| 464 |
+
)
|
| 465 |
+
|
| 466 |
+
except Exception as exc:
|
| 467 |
+
logger.exception("Pipeline failed for job %s: %s", job_id, exc)
|
| 468 |
+
await update_status(JobStatus.failed, error=str(exc))
|
backend/routers/corrections.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""POST /api/corrections β submit a correction for a library's extracted docs."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import uuid
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
from fastapi import APIRouter
|
| 8 |
+
from pydantic import BaseModel
|
| 9 |
+
|
| 10 |
+
router = APIRouter(prefix="/api/corrections", tags=["corrections"])
|
| 11 |
+
|
| 12 |
+
DEV_MODE = os.getenv("DEV_MODE", "true").lower() == "true"
|
| 13 |
+
SUPABASE_URL = os.getenv("SUPABASE_URL", "")
|
| 14 |
+
SUPABASE_KEY = os.getenv("SUPABASE_KEY", "")
|
| 15 |
+
|
| 16 |
+
# In-memory store for DEV_MODE
|
| 17 |
+
_corrections: dict[str, dict] = {}
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class CorrectionRequest(BaseModel):
|
| 21 |
+
library: str
|
| 22 |
+
version: str
|
| 23 |
+
component_name: str
|
| 24 |
+
field: str # "gotcha" | "prop" | "usage_example" | "import_path"
|
| 25 |
+
correction: str
|
| 26 |
+
reporter_email: Optional[str] = None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class CorrectionResponse(BaseModel):
|
| 30 |
+
correction_id: str
|
| 31 |
+
status: str
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@router.post("", response_model=CorrectionResponse)
|
| 35 |
+
async def submit_correction(req: CorrectionRequest) -> CorrectionResponse:
|
| 36 |
+
"""Submit a correction for a library's extracted documentation."""
|
| 37 |
+
correction_id = str(uuid.uuid4())
|
| 38 |
+
|
| 39 |
+
if DEV_MODE or not (SUPABASE_URL and SUPABASE_KEY):
|
| 40 |
+
_corrections[correction_id] = {
|
| 41 |
+
"id": correction_id,
|
| 42 |
+
"library": req.library,
|
| 43 |
+
"version": req.version,
|
| 44 |
+
"component_name": req.component_name,
|
| 45 |
+
"field": req.field,
|
| 46 |
+
"correction": req.correction,
|
| 47 |
+
"reporter_email": req.reporter_email,
|
| 48 |
+
"status": "pending",
|
| 49 |
+
}
|
| 50 |
+
else:
|
| 51 |
+
import httpx
|
| 52 |
+
async with httpx.AsyncClient(timeout=10) as client:
|
| 53 |
+
await client.post(
|
| 54 |
+
f"{SUPABASE_URL}/rest/v1/corrections",
|
| 55 |
+
headers={
|
| 56 |
+
"apikey": SUPABASE_KEY,
|
| 57 |
+
"Authorization": f"Bearer {SUPABASE_KEY}",
|
| 58 |
+
"Content-Type": "application/json",
|
| 59 |
+
"Prefer": "return=minimal",
|
| 60 |
+
},
|
| 61 |
+
json={
|
| 62 |
+
"id": correction_id,
|
| 63 |
+
"library": req.library,
|
| 64 |
+
"version": req.version,
|
| 65 |
+
"component_name": req.component_name,
|
| 66 |
+
"field": req.field,
|
| 67 |
+
"correction": req.correction,
|
| 68 |
+
"reporter_email": req.reporter_email,
|
| 69 |
+
"status": "pending",
|
| 70 |
+
},
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
return CorrectionResponse(correction_id=correction_id, status="pending")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@router.get("/{library}")
|
| 77 |
+
async def get_corrections(library: str, version: Optional[str] = None) -> dict:
|
| 78 |
+
"""Get approved corrections for a library."""
|
| 79 |
+
if DEV_MODE or not (SUPABASE_URL and SUPABASE_KEY):
|
| 80 |
+
results = [
|
| 81 |
+
c for c in _corrections.values()
|
| 82 |
+
if c["library"] == library and c["status"] == "approved"
|
| 83 |
+
and (version is None or c["version"] == version)
|
| 84 |
+
]
|
| 85 |
+
return {"corrections": results}
|
| 86 |
+
|
| 87 |
+
import httpx
|
| 88 |
+
params = f"library=eq.{library}&status=eq.approved"
|
| 89 |
+
if version:
|
| 90 |
+
params += f"&version=eq.{version}"
|
| 91 |
+
async with httpx.AsyncClient(timeout=10) as client:
|
| 92 |
+
resp = await client.get(
|
| 93 |
+
f"{SUPABASE_URL}/rest/v1/corrections?{params}",
|
| 94 |
+
headers={"apikey": SUPABASE_KEY, "Authorization": f"Bearer {SUPABASE_KEY}"},
|
| 95 |
+
)
|
| 96 |
+
return {"corrections": resp.json() if resp.status_code == 200 else []}
|
backend/routers/oauth.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OAuth router β Google and GitHub sign-in / sign-up
|
| 3 |
+
|
| 4 |
+
GET /api/auth/oauth/google β redirect to Google consent page
|
| 5 |
+
GET /api/auth/oauth/google/callback β exchange code, create/find user, redirect to frontend
|
| 6 |
+
GET /api/auth/oauth/github β redirect to GitHub consent page
|
| 7 |
+
GET /api/auth/oauth/github/callback β exchange code, create/find user, redirect to frontend
|
| 8 |
+
|
| 9 |
+
After successful auth the user is sent to:
|
| 10 |
+
FRONTEND_URL/auth/callback?token=<jwt>&email=<email>
|
| 11 |
+
On failure they land at:
|
| 12 |
+
FRONTEND_URL/auth/login?error=<message>
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import httpx
|
| 17 |
+
from urllib.parse import urlencode
|
| 18 |
+
|
| 19 |
+
from fastapi import APIRouter
|
| 20 |
+
from fastapi.responses import RedirectResponse
|
| 21 |
+
|
| 22 |
+
from backend.services import auth_service
|
| 23 |
+
|
| 24 |
+
router = APIRouter(prefix="/api/auth/oauth", tags=["oauth"])
|
| 25 |
+
|
| 26 |
+
FRONTEND_URL = os.getenv("FRONTEND_URL", "http://localhost:3000")
|
| 27 |
+
BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8000")
|
| 28 |
+
GOOGLE_CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID", "")
|
| 29 |
+
GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET", "")
|
| 30 |
+
GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "")
|
| 31 |
+
GITHUB_CLIENT_SECRET = os.getenv("GITHUB_CLIENT_SECRET", "")
|
| 32 |
+
|
| 33 |
+
GOOGLE_REDIRECT_URI = f"{BACKEND_URL}/api/auth/oauth/google/callback"
|
| 34 |
+
GITHUB_REDIRECT_URI = f"{BACKEND_URL}/api/auth/oauth/github/callback"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _error_redirect(msg: str) -> RedirectResponse:
|
| 38 |
+
return RedirectResponse(f"{FRONTEND_URL}/auth/login?error={msg.replace(' ', '+')}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ββ Google ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 42 |
+
|
| 43 |
+
@router.get("/google")
|
| 44 |
+
async def google_login(source: str = "web", callback_port: str = ""):
|
| 45 |
+
if not GOOGLE_CLIENT_ID:
|
| 46 |
+
return _error_redirect("Google OAuth not configured")
|
| 47 |
+
state = f"vscode_local:{callback_port}" if source == "vscode_local" and callback_port else source
|
| 48 |
+
params = urlencode({
|
| 49 |
+
"client_id": GOOGLE_CLIENT_ID,
|
| 50 |
+
"redirect_uri": GOOGLE_REDIRECT_URI,
|
| 51 |
+
"response_type": "code",
|
| 52 |
+
"scope": "openid email profile",
|
| 53 |
+
"access_type": "offline",
|
| 54 |
+
"state": state,
|
| 55 |
+
})
|
| 56 |
+
return RedirectResponse(f"https://accounts.google.com/o/oauth2/v2/auth?{params}")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@router.get("/google/callback")
|
| 60 |
+
async def google_callback(code: str = "", error: str = "", state: str = "web"):
|
| 61 |
+
if error or not code:
|
| 62 |
+
return _error_redirect("Google sign-in cancelled")
|
| 63 |
+
|
| 64 |
+
async with httpx.AsyncClient() as client:
|
| 65 |
+
token_res = await client.post(
|
| 66 |
+
"https://oauth2.googleapis.com/token",
|
| 67 |
+
data={
|
| 68 |
+
"code": code,
|
| 69 |
+
"client_id": GOOGLE_CLIENT_ID,
|
| 70 |
+
"client_secret": GOOGLE_CLIENT_SECRET,
|
| 71 |
+
"redirect_uri": GOOGLE_REDIRECT_URI,
|
| 72 |
+
"grant_type": "authorization_code",
|
| 73 |
+
},
|
| 74 |
+
)
|
| 75 |
+
if not token_res.is_success:
|
| 76 |
+
return _error_redirect("Google token exchange failed")
|
| 77 |
+
|
| 78 |
+
access_token = token_res.json().get("access_token")
|
| 79 |
+
if not access_token:
|
| 80 |
+
return _error_redirect("No access token from Google")
|
| 81 |
+
|
| 82 |
+
info_res = await client.get(
|
| 83 |
+
"https://www.googleapis.com/oauth2/v2/userinfo",
|
| 84 |
+
headers={"Authorization": f"Bearer {access_token}"},
|
| 85 |
+
)
|
| 86 |
+
if not info_res.is_success:
|
| 87 |
+
return _error_redirect("Could not fetch Google profile")
|
| 88 |
+
|
| 89 |
+
email = info_res.json().get("email")
|
| 90 |
+
|
| 91 |
+
if not email:
|
| 92 |
+
return _error_redirect("No email in Google profile")
|
| 93 |
+
|
| 94 |
+
user = await auth_service.get_or_create_oauth_user(email)
|
| 95 |
+
token = auth_service.create_access_token(user["id"], user["email"])
|
| 96 |
+
if state.startswith("vscode_local:"):
|
| 97 |
+
port = state.split(":", 1)[1]
|
| 98 |
+
return RedirectResponse(f"http://localhost:{port}/?token={token}&email={email}")
|
| 99 |
+
if state == "vscode":
|
| 100 |
+
return RedirectResponse(
|
| 101 |
+
f"vscode://docforge.docforge/callback?token={token}&email={email}"
|
| 102 |
+
)
|
| 103 |
+
return RedirectResponse(
|
| 104 |
+
f"{FRONTEND_URL}/auth/callback?token={token}&email={email}"
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# ββ GitHub ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 109 |
+
|
| 110 |
+
@router.get("/github")
|
| 111 |
+
async def github_login(source: str = "web", callback_port: str = ""):
|
| 112 |
+
if not GITHUB_CLIENT_ID:
|
| 113 |
+
return _error_redirect("GitHub OAuth not configured")
|
| 114 |
+
state = f"vscode_local:{callback_port}" if source == "vscode_local" and callback_port else source
|
| 115 |
+
params = urlencode({
|
| 116 |
+
"client_id": GITHUB_CLIENT_ID,
|
| 117 |
+
"redirect_uri": GITHUB_REDIRECT_URI,
|
| 118 |
+
"scope": "user:email",
|
| 119 |
+
"state": state,
|
| 120 |
+
})
|
| 121 |
+
return RedirectResponse(f"https://github.com/login/oauth/authorize?{params}")
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@router.get("/github/callback")
|
| 125 |
+
async def github_callback(code: str = "", error: str = "", state: str = "web"):
|
| 126 |
+
if error or not code:
|
| 127 |
+
return _error_redirect("GitHub sign-in cancelled")
|
| 128 |
+
|
| 129 |
+
async with httpx.AsyncClient() as client:
|
| 130 |
+
token_res = await client.post(
|
| 131 |
+
"https://github.com/login/oauth/access_token",
|
| 132 |
+
headers={"Accept": "application/json"},
|
| 133 |
+
data={
|
| 134 |
+
"client_id": GITHUB_CLIENT_ID,
|
| 135 |
+
"client_secret": GITHUB_CLIENT_SECRET,
|
| 136 |
+
"code": code,
|
| 137 |
+
"redirect_uri": GITHUB_REDIRECT_URI,
|
| 138 |
+
},
|
| 139 |
+
)
|
| 140 |
+
if not token_res.is_success:
|
| 141 |
+
return _error_redirect("GitHub token exchange failed")
|
| 142 |
+
|
| 143 |
+
gh_token = token_res.json().get("access_token")
|
| 144 |
+
if not gh_token:
|
| 145 |
+
return _error_redirect("No access token from GitHub")
|
| 146 |
+
|
| 147 |
+
# Prefer verified primary email from /user/emails
|
| 148 |
+
emails_res = await client.get(
|
| 149 |
+
"https://api.github.com/user/emails",
|
| 150 |
+
headers={
|
| 151 |
+
"Authorization": f"Bearer {gh_token}",
|
| 152 |
+
"Accept": "application/vnd.github+json",
|
| 153 |
+
},
|
| 154 |
+
)
|
| 155 |
+
email = None
|
| 156 |
+
if emails_res.is_success:
|
| 157 |
+
for entry in emails_res.json():
|
| 158 |
+
if entry.get("primary") and entry.get("verified"):
|
| 159 |
+
email = entry["email"]
|
| 160 |
+
break
|
| 161 |
+
|
| 162 |
+
# Fallback to public profile email
|
| 163 |
+
if not email:
|
| 164 |
+
user_res = await client.get(
|
| 165 |
+
"https://api.github.com/user",
|
| 166 |
+
headers={"Authorization": f"Bearer {gh_token}"},
|
| 167 |
+
)
|
| 168 |
+
if user_res.is_success:
|
| 169 |
+
email = user_res.json().get("email")
|
| 170 |
+
|
| 171 |
+
if not email:
|
| 172 |
+
return _error_redirect("No verified email on GitHub account")
|
| 173 |
+
|
| 174 |
+
user = await auth_service.get_or_create_oauth_user(email)
|
| 175 |
+
token = auth_service.create_access_token(user["id"], user["email"])
|
| 176 |
+
if state.startswith("vscode_local:"):
|
| 177 |
+
port = state.split(":", 1)[1]
|
| 178 |
+
return RedirectResponse(f"http://localhost:{port}/?token={token}&email={email}")
|
| 179 |
+
if state == "vscode":
|
| 180 |
+
return RedirectResponse(
|
| 181 |
+
f"vscode://docforge.docforge/callback?token={token}&email={email}"
|
| 182 |
+
)
|
| 183 |
+
return RedirectResponse(
|
| 184 |
+
f"{FRONTEND_URL}/auth/callback?token={token}&email={email}"
|
| 185 |
+
)
|
backend/routers/search.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GET /api/search?q={query} β search cached libraries."""
|
| 2 |
+
|
| 3 |
+
from fastapi import APIRouter, Query
|
| 4 |
+
|
| 5 |
+
from ..models.schemas import SearchResponse, SearchResult
|
| 6 |
+
from ..services.cache import search_cached_libraries
|
| 7 |
+
|
| 8 |
+
router = APIRouter(prefix="/api/search", tags=["search"])
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@router.get("", response_model=SearchResponse)
|
| 12 |
+
async def search_libraries(q: str = Query("", description="Search query"), limit: int = Query(20)) -> SearchResponse:
|
| 13 |
+
"""Search the DocForge cache for libraries matching a query string."""
|
| 14 |
+
raw_results = await search_cached_libraries(q)
|
| 15 |
+
results = [SearchResult(**r) for r in raw_results][:limit]
|
| 16 |
+
return SearchResponse(results=results)
|
backend/routers/versions.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GET /api/versions/{package} β list cached versions for a package."""
|
| 2 |
+
|
| 3 |
+
from fastapi import APIRouter
|
| 4 |
+
|
| 5 |
+
from ..models.schemas import VersionsResponse
|
| 6 |
+
from ..services.cache import list_cached_versions
|
| 7 |
+
|
| 8 |
+
router = APIRouter(prefix="/api/versions", tags=["versions"])
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@router.get("/{package:path}", response_model=VersionsResponse)
|
| 12 |
+
async def get_versions(package: str) -> VersionsResponse:
|
| 13 |
+
"""List all cached versions of a package in the DocForge cache."""
|
| 14 |
+
versions = await list_cached_versions(package)
|
| 15 |
+
return VersionsResponse(package=package, versions=versions)
|
backend/services/__init__.py
ADDED
|
File without changes
|
backend/services/auth_service.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Auth service: JWT creation/validation, password hashing, device-flow management,
|
| 3 |
+
and encrypted API key storage.
|
| 4 |
+
|
| 5 |
+
Storage strategy mirrors cache.py:
|
| 6 |
+
- Supabase when SUPABASE_URL + SUPABASE_KEY are set (production)
|
| 7 |
+
- In-memory dict otherwise (dev / CI)
|
| 8 |
+
|
| 9 |
+
Tables required in Supabase:
|
| 10 |
+
users (id uuid PK, email text UNIQUE, password_hash text, created_at timestamptz)
|
| 11 |
+
user_api_keys(user_id uuid FK->users.id, gemini_key_enc text, groq_key_enc text)
|
| 12 |
+
device_codes (code text PK, user_code text UNIQUE, user_id uuid nullable,
|
| 13 |
+
expires_at timestamptz, verified bool default false)
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
import uuid
|
| 18 |
+
import string
|
| 19 |
+
import random
|
| 20 |
+
import time
|
| 21 |
+
from datetime import datetime, timezone, timedelta
|
| 22 |
+
from typing import Optional
|
| 23 |
+
|
| 24 |
+
from jose import jwt, JWTError
|
| 25 |
+
from passlib.context import CryptContext
|
| 26 |
+
from cryptography.fernet import Fernet
|
| 27 |
+
import base64
|
| 28 |
+
|
| 29 |
+
# ββ Config βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 30 |
+
|
| 31 |
+
SECRET_KEY = os.getenv("SECRET_KEY", "CHANGE_ME_IN_PRODUCTION_" + "x" * 32)
|
| 32 |
+
ALGORITHM = "HS256"
|
| 33 |
+
ACCESS_TOKEN_EXPIRE_HOURS = 24 * 30 # 30 days
|
| 34 |
+
|
| 35 |
+
# Key for encrypting stored API keys (Fernet requires 32-byte URL-safe base64)
|
| 36 |
+
_raw_enc_key = os.getenv("ENCRYPTION_KEY", "")
|
| 37 |
+
if _raw_enc_key:
|
| 38 |
+
FERNET_KEY = _raw_enc_key.encode()
|
| 39 |
+
else:
|
| 40 |
+
# Derive a stable key from SECRET_KEY so dev works without .env
|
| 41 |
+
import hashlib
|
| 42 |
+
_derived = hashlib.sha256(SECRET_KEY.encode()).digest()
|
| 43 |
+
FERNET_KEY = base64.urlsafe_b64encode(_derived)
|
| 44 |
+
|
| 45 |
+
_fernet = Fernet(FERNET_KEY)
|
| 46 |
+
_pwd_ctx = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
| 47 |
+
|
| 48 |
+
# ββ Supabase helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 49 |
+
|
| 50 |
+
def _get_supabase():
|
| 51 |
+
url = os.getenv("SUPABASE_URL")
|
| 52 |
+
key = os.getenv("SUPABASE_KEY")
|
| 53 |
+
if url and key:
|
| 54 |
+
from supabase import create_client
|
| 55 |
+
return create_client(url, key)
|
| 56 |
+
return None
|
| 57 |
+
|
| 58 |
+
# ββ In-memory fallback ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 59 |
+
# { email β { id, email, password_hash } }
|
| 60 |
+
_mem_users: dict = {}
|
| 61 |
+
# { user_id β { gemini_key_enc, groq_key_enc } }
|
| 62 |
+
_mem_keys: dict = {}
|
| 63 |
+
# { device_code β { user_code, user_id, expires_at, verified } }
|
| 64 |
+
_mem_devices: dict = {}
|
| 65 |
+
|
| 66 |
+
# ββ Password helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 67 |
+
|
| 68 |
+
def hash_password(plain: str) -> str:
|
| 69 |
+
return _pwd_ctx.hash(plain)
|
| 70 |
+
|
| 71 |
+
def verify_password(plain: str, hashed: str) -> bool:
|
| 72 |
+
return _pwd_ctx.verify(plain, hashed)
|
| 73 |
+
|
| 74 |
+
# ββ Encryption helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 75 |
+
|
| 76 |
+
def encrypt_key(value: str) -> str:
|
| 77 |
+
return _fernet.encrypt(value.encode()).decode()
|
| 78 |
+
|
| 79 |
+
def decrypt_key(enc: str) -> str:
|
| 80 |
+
return _fernet.decrypt(enc.encode()).decode()
|
| 81 |
+
|
| 82 |
+
# ββ JWT βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 83 |
+
|
| 84 |
+
def create_access_token(user_id: str, email: str) -> str:
|
| 85 |
+
expire = datetime.now(timezone.utc) + timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS)
|
| 86 |
+
return jwt.encode(
|
| 87 |
+
{"sub": user_id, "email": email, "exp": expire},
|
| 88 |
+
SECRET_KEY,
|
| 89 |
+
algorithm=ALGORITHM,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
def decode_token(token: str) -> Optional[dict]:
|
| 93 |
+
try:
|
| 94 |
+
return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
| 95 |
+
except JWTError:
|
| 96 |
+
return None
|
| 97 |
+
|
| 98 |
+
# ββ User CRUD βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 99 |
+
|
| 100 |
+
async def create_user(email: str, password_hash: str) -> Optional[str]:
|
| 101 |
+
"""Create a user, return their UUID. Returns None if email exists."""
|
| 102 |
+
sb = _get_supabase()
|
| 103 |
+
if sb:
|
| 104 |
+
existing = sb.table("users").select("id").eq("email", email).execute()
|
| 105 |
+
if existing.data:
|
| 106 |
+
return None
|
| 107 |
+
uid = str(uuid.uuid4())
|
| 108 |
+
sb.table("users").insert({"id": uid, "email": email, "password_hash": password_hash}).execute()
|
| 109 |
+
return uid
|
| 110 |
+
else:
|
| 111 |
+
if email in _mem_users:
|
| 112 |
+
return None
|
| 113 |
+
uid = str(uuid.uuid4())
|
| 114 |
+
_mem_users[email] = {"id": uid, "email": email, "password_hash": password_hash}
|
| 115 |
+
return uid
|
| 116 |
+
|
| 117 |
+
async def get_user_by_email(email: str) -> Optional[dict]:
|
| 118 |
+
sb = _get_supabase()
|
| 119 |
+
if sb:
|
| 120 |
+
res = sb.table("users").select("id, email, password_hash").eq("email", email).execute()
|
| 121 |
+
return res.data[0] if res.data else None
|
| 122 |
+
return _mem_users.get(email)
|
| 123 |
+
|
| 124 |
+
async def get_or_create_oauth_user(email: str) -> dict:
|
| 125 |
+
"""Find user by email or create one without a password (OAuth users)."""
|
| 126 |
+
existing = await get_user_by_email(email)
|
| 127 |
+
if existing:
|
| 128 |
+
return existing
|
| 129 |
+
uid = str(uuid.uuid4())
|
| 130 |
+
sb = _get_supabase()
|
| 131 |
+
if sb:
|
| 132 |
+
sb.table("users").insert({"id": uid, "email": email, "password_hash": ""}).execute()
|
| 133 |
+
else:
|
| 134 |
+
_mem_users[email] = {"id": uid, "email": email, "password_hash": ""}
|
| 135 |
+
return {"id": uid, "email": email}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
async def get_user_by_id(user_id: str) -> Optional[dict]:
|
| 139 |
+
sb = _get_supabase()
|
| 140 |
+
if sb:
|
| 141 |
+
res = sb.table("users").select("id, email").eq("id", user_id).execute()
|
| 142 |
+
return res.data[0] if res.data else None
|
| 143 |
+
for u in _mem_users.values():
|
| 144 |
+
if u["id"] == user_id:
|
| 145 |
+
return u
|
| 146 |
+
return None
|
| 147 |
+
|
| 148 |
+
# ββ API Key CRUD ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 149 |
+
|
| 150 |
+
async def save_api_keys(user_id: str, gemini_key: Optional[str], groq_key: Optional[str]):
|
| 151 |
+
sb = _get_supabase()
|
| 152 |
+
row: dict = {"user_id": user_id}
|
| 153 |
+
if gemini_key is not None:
|
| 154 |
+
row["gemini_key_enc"] = encrypt_key(gemini_key)
|
| 155 |
+
if groq_key is not None:
|
| 156 |
+
row["groq_key_enc"] = encrypt_key(groq_key)
|
| 157 |
+
|
| 158 |
+
if sb:
|
| 159 |
+
existing = sb.table("user_api_keys").select("user_id").eq("user_id", user_id).execute()
|
| 160 |
+
if existing.data:
|
| 161 |
+
sb.table("user_api_keys").update(row).eq("user_id", user_id).execute()
|
| 162 |
+
else:
|
| 163 |
+
sb.table("user_api_keys").insert(row).execute()
|
| 164 |
+
else:
|
| 165 |
+
if user_id not in _mem_keys:
|
| 166 |
+
_mem_keys[user_id] = {}
|
| 167 |
+
_mem_keys[user_id].update(row)
|
| 168 |
+
|
| 169 |
+
async def get_api_keys(user_id: str) -> dict:
|
| 170 |
+
"""Returns { gemini_key: str|None, groq_key: str|None }"""
|
| 171 |
+
sb = _get_supabase()
|
| 172 |
+
if sb:
|
| 173 |
+
res = sb.table("user_api_keys").select("gemini_key_enc, groq_key_enc").eq("user_id", user_id).execute()
|
| 174 |
+
row = res.data[0] if res.data else {}
|
| 175 |
+
else:
|
| 176 |
+
row = _mem_keys.get(user_id, {})
|
| 177 |
+
|
| 178 |
+
def _dec(enc):
|
| 179 |
+
try:
|
| 180 |
+
return decrypt_key(enc) if enc else None
|
| 181 |
+
except Exception:
|
| 182 |
+
return None
|
| 183 |
+
|
| 184 |
+
return {
|
| 185 |
+
"gemini_key": _dec(row.get("gemini_key_enc")),
|
| 186 |
+
"groq_key": _dec(row.get("groq_key_enc")),
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
# ββ Device flow βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 190 |
+
|
| 191 |
+
def _random_code(length=8, chars=string.ascii_uppercase + string.digits) -> str:
|
| 192 |
+
return "".join(random.choices(chars, k=length))
|
| 193 |
+
|
| 194 |
+
def _user_code_format(code: str) -> str:
|
| 195 |
+
return f"{code[:4]}-{code[4:]}"
|
| 196 |
+
|
| 197 |
+
async def create_device_code() -> dict:
|
| 198 |
+
device_code = str(uuid.uuid4())
|
| 199 |
+
raw = _random_code(8)
|
| 200 |
+
user_code = _user_code_format(raw)
|
| 201 |
+
expires_at = time.time() + 900 # 15 min
|
| 202 |
+
|
| 203 |
+
sb = _get_supabase()
|
| 204 |
+
if sb:
|
| 205 |
+
sb.table("device_codes").insert({
|
| 206 |
+
"code": device_code,
|
| 207 |
+
"user_code": user_code,
|
| 208 |
+
"user_id": None,
|
| 209 |
+
"expires_at": datetime.fromtimestamp(expires_at, tz=timezone.utc).isoformat(),
|
| 210 |
+
"verified": False,
|
| 211 |
+
}).execute()
|
| 212 |
+
else:
|
| 213 |
+
_mem_devices[device_code] = {
|
| 214 |
+
"code": device_code,
|
| 215 |
+
"user_code": user_code,
|
| 216 |
+
"user_id": None,
|
| 217 |
+
"expires_at": expires_at,
|
| 218 |
+
"verified": False,
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
return {"device_code": device_code, "user_code": user_code}
|
| 222 |
+
|
| 223 |
+
async def verify_device_code(user_code: str, user_id: str) -> bool:
|
| 224 |
+
"""Called from the web UI when user confirms the code. Returns True on success."""
|
| 225 |
+
sb = _get_supabase()
|
| 226 |
+
if sb:
|
| 227 |
+
res = sb.table("device_codes").select("code, expires_at, verified") \
|
| 228 |
+
.eq("user_code", user_code).execute()
|
| 229 |
+
if not res.data:
|
| 230 |
+
return False
|
| 231 |
+
row = res.data[0]
|
| 232 |
+
exp = datetime.fromisoformat(row["expires_at"]).timestamp()
|
| 233 |
+
if row["verified"] or time.time() > exp:
|
| 234 |
+
return False
|
| 235 |
+
sb.table("device_codes").update({"user_id": user_id, "verified": True}) \
|
| 236 |
+
.eq("user_code", user_code).execute()
|
| 237 |
+
return True
|
| 238 |
+
else:
|
| 239 |
+
for entry in _mem_devices.values():
|
| 240 |
+
if entry["user_code"] == user_code:
|
| 241 |
+
if entry["verified"] or time.time() > entry["expires_at"]:
|
| 242 |
+
return False
|
| 243 |
+
entry["user_id"] = user_id
|
| 244 |
+
entry["verified"] = True
|
| 245 |
+
return True
|
| 246 |
+
return False
|
| 247 |
+
|
| 248 |
+
async def poll_device_code(device_code: str) -> dict:
|
| 249 |
+
"""Called by CLI to poll status. Returns status + token when verified."""
|
| 250 |
+
sb = _get_supabase()
|
| 251 |
+
if sb:
|
| 252 |
+
res = sb.table("device_codes").select("user_id, expires_at, verified") \
|
| 253 |
+
.eq("code", device_code).execute()
|
| 254 |
+
if not res.data:
|
| 255 |
+
return {"status": "expired"}
|
| 256 |
+
row = res.data[0]
|
| 257 |
+
exp = datetime.fromisoformat(row["expires_at"]).timestamp()
|
| 258 |
+
if time.time() > exp:
|
| 259 |
+
return {"status": "expired"}
|
| 260 |
+
if not row["verified"] or not row["user_id"]:
|
| 261 |
+
return {"status": "pending"}
|
| 262 |
+
user = await get_user_by_id(row["user_id"])
|
| 263 |
+
else:
|
| 264 |
+
entry = _mem_devices.get(device_code)
|
| 265 |
+
if not entry or time.time() > entry["expires_at"]:
|
| 266 |
+
return {"status": "expired"}
|
| 267 |
+
if not entry["verified"] or not entry["user_id"]:
|
| 268 |
+
return {"status": "pending"}
|
| 269 |
+
user = await get_user_by_id(entry["user_id"])
|
| 270 |
+
|
| 271 |
+
if not user:
|
| 272 |
+
return {"status": "expired"}
|
| 273 |
+
|
| 274 |
+
token = create_access_token(user["id"], user["email"])
|
| 275 |
+
return {"status": "complete", "access_token": token, "email": user["email"]}
|
backend/services/cache.py
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Cache layer for DocForge.
|
| 3 |
+
|
| 4 |
+
Priority:
|
| 5 |
+
1. Supabase β if SUPABASE_URL + SUPABASE_KEY are set (persistent, cross-restart)
|
| 6 |
+
2. Redis β if DEV_MODE=false and REDIS_URL is set
|
| 7 |
+
3. In-memory β fallback for local dev
|
| 8 |
+
|
| 9 |
+
Cache invalidation strategy:
|
| 10 |
+
- Pinned versions (e.g. react@18.2.0): 15-day TTL. Docs for a fixed version never change.
|
| 11 |
+
- "latest" / unpinned entries: GitHub release check on cache hit, but at most once per hour
|
| 12 |
+
(stored as github_checked_at in the cached payload). If a newer GitHub release exists
|
| 13 |
+
since cached_at, the entry is deleted and the pipeline re-runs for the next request.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import logging
|
| 18 |
+
import os
|
| 19 |
+
import time
|
| 20 |
+
from datetime import datetime, timedelta, timezone
|
| 21 |
+
from typing import Any
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
|
| 25 |
+
DEV_MODE = os.getenv("DEV_MODE", "true").lower() == "true"
|
| 26 |
+
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
|
| 27 |
+
SUPABASE_URL = os.getenv("SUPABASE_URL", "")
|
| 28 |
+
SUPABASE_KEY = os.getenv("SUPABASE_KEY", "")
|
| 29 |
+
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")
|
| 30 |
+
|
| 31 |
+
# Pinned versions live for 15 days; "latest" entries rely on GitHub commit checks
|
| 32 |
+
CACHE_TTL_PINNED = 60 * 60 * 24 * 15 # 15 days
|
| 33 |
+
CACHE_TTL_LATEST = 60 * 60 * 24 * 15 # same wall-clock TTL, but invalidated earlier by GitHub
|
| 34 |
+
CACHE_TTL_SECONDS = CACHE_TTL_PINNED # default (keep old name for job TTL callers)
|
| 35 |
+
# How often we're allowed to re-check GitHub for a given library (seconds)
|
| 36 |
+
GITHUB_CHECK_INTERVAL = 60 * 60 # 1 hour
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class InMemoryCache:
|
| 40 |
+
"""Fallback cache for local development without Redis."""
|
| 41 |
+
|
| 42 |
+
def __init__(self) -> None:
|
| 43 |
+
self._store: dict[str, tuple[str, float]] = {} # key -> (value, expiry_ts)
|
| 44 |
+
|
| 45 |
+
async def get(self, key: str) -> str | None:
|
| 46 |
+
entry = self._store.get(key)
|
| 47 |
+
if entry is None:
|
| 48 |
+
return None
|
| 49 |
+
value, expiry = entry
|
| 50 |
+
if time.time() > expiry:
|
| 51 |
+
del self._store[key]
|
| 52 |
+
return None
|
| 53 |
+
return value
|
| 54 |
+
|
| 55 |
+
async def set(self, key: str, value: str, ttl: int = CACHE_TTL_SECONDS) -> None:
|
| 56 |
+
self._store[key] = (value, time.time() + ttl)
|
| 57 |
+
|
| 58 |
+
async def keys_matching(self, pattern: str) -> list[str]:
|
| 59 |
+
"""Return keys that contain the pattern substring (simplified glob)."""
|
| 60 |
+
needle = pattern.replace("*", "")
|
| 61 |
+
now = time.time()
|
| 62 |
+
return [k for k, (_, exp) in self._store.items() if needle in k and now < exp]
|
| 63 |
+
|
| 64 |
+
async def delete(self, key: str) -> None:
|
| 65 |
+
self._store.pop(key, None)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
class RedisCache:
|
| 69 |
+
"""Production Redis cache."""
|
| 70 |
+
|
| 71 |
+
def __init__(self, redis_url: str) -> None:
|
| 72 |
+
import redis.asyncio as aioredis # type: ignore[import]
|
| 73 |
+
self._redis = aioredis.from_url(redis_url, decode_responses=True)
|
| 74 |
+
|
| 75 |
+
async def get(self, key: str) -> str | None:
|
| 76 |
+
return await self._redis.get(key)
|
| 77 |
+
|
| 78 |
+
async def set(self, key: str, value: str, ttl: int = CACHE_TTL_SECONDS) -> None:
|
| 79 |
+
await self._redis.setex(key, ttl, value)
|
| 80 |
+
|
| 81 |
+
async def keys_matching(self, pattern: str) -> list[str]:
|
| 82 |
+
return await self._redis.keys(pattern)
|
| 83 |
+
|
| 84 |
+
async def delete(self, key: str) -> None:
|
| 85 |
+
await self._redis.delete(key)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class SupabaseCache:
|
| 89 |
+
"""Supabase-backed persistent cache using a `cache_entries` table."""
|
| 90 |
+
|
| 91 |
+
def __init__(self, url: str, key: str) -> None:
|
| 92 |
+
import asyncio
|
| 93 |
+
from supabase import create_client # type: ignore[import]
|
| 94 |
+
self._client = create_client(url, key)
|
| 95 |
+
self._loop = asyncio.get_event_loop
|
| 96 |
+
|
| 97 |
+
async def get(self, key: str) -> str | None:
|
| 98 |
+
import asyncio
|
| 99 |
+
now = datetime.now(timezone.utc).isoformat()
|
| 100 |
+
result = await asyncio.to_thread(
|
| 101 |
+
lambda: self._client.table("cache_entries")
|
| 102 |
+
.select("value")
|
| 103 |
+
.eq("key", key)
|
| 104 |
+
.gt("expires_at", now)
|
| 105 |
+
.execute()
|
| 106 |
+
)
|
| 107 |
+
if result.data:
|
| 108 |
+
return result.data[0]["value"]
|
| 109 |
+
return None
|
| 110 |
+
|
| 111 |
+
async def set(self, key: str, value: str, ttl: int = CACHE_TTL_SECONDS) -> None:
|
| 112 |
+
import asyncio
|
| 113 |
+
expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl)).isoformat()
|
| 114 |
+
await asyncio.to_thread(
|
| 115 |
+
lambda: self._client.table("cache_entries")
|
| 116 |
+
.upsert({"key": key, "value": value, "expires_at": expires_at})
|
| 117 |
+
.execute()
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
async def keys_matching(self, pattern: str) -> list[str]:
|
| 121 |
+
import asyncio
|
| 122 |
+
needle = pattern.replace("*", "")
|
| 123 |
+
now = datetime.now(timezone.utc).isoformat()
|
| 124 |
+
result = await asyncio.to_thread(
|
| 125 |
+
lambda: self._client.table("cache_entries")
|
| 126 |
+
.select("key")
|
| 127 |
+
.like("key", f"%{needle}%")
|
| 128 |
+
.gt("expires_at", now)
|
| 129 |
+
.execute()
|
| 130 |
+
)
|
| 131 |
+
return [row["key"] for row in result.data]
|
| 132 |
+
|
| 133 |
+
async def delete(self, key: str) -> None:
|
| 134 |
+
import asyncio
|
| 135 |
+
await asyncio.to_thread(
|
| 136 |
+
lambda: self._client.table("cache_entries")
|
| 137 |
+
.delete()
|
| 138 |
+
.eq("key", key)
|
| 139 |
+
.execute()
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
# Singleton cache instance
|
| 144 |
+
_cache: InMemoryCache | RedisCache | SupabaseCache | None = None
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def get_cache() -> InMemoryCache | RedisCache | SupabaseCache:
|
| 148 |
+
global _cache
|
| 149 |
+
if _cache is None:
|
| 150 |
+
if SUPABASE_URL and SUPABASE_KEY:
|
| 151 |
+
logger.info("Using Supabase cache at %s", SUPABASE_URL)
|
| 152 |
+
try:
|
| 153 |
+
_cache = SupabaseCache(SUPABASE_URL, SUPABASE_KEY)
|
| 154 |
+
except Exception as exc:
|
| 155 |
+
logger.warning("Supabase cache init failed (%s), falling back to in-memory", exc)
|
| 156 |
+
_cache = InMemoryCache()
|
| 157 |
+
elif not DEV_MODE:
|
| 158 |
+
logger.info("Connecting to Redis at %s", REDIS_URL)
|
| 159 |
+
_cache = RedisCache(REDIS_URL)
|
| 160 |
+
else:
|
| 161 |
+
logger.info("DEV_MODE: using in-memory cache (no Redis)")
|
| 162 |
+
_cache = InMemoryCache()
|
| 163 |
+
return _cache
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def make_context_key(library: str, version: str) -> str:
|
| 167 |
+
return f"docforge:context:{library}:{version}"
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def make_job_key(job_id: str) -> str:
|
| 171 |
+
return f"docforge:job:{job_id}"
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _extract_github_owner_repo(repository: str) -> tuple[str, str] | None:
|
| 175 |
+
"""Parse owner/repo from any GitHub URL variant."""
|
| 176 |
+
import re
|
| 177 |
+
m = re.search(r"github\.com[/:]([^/]+)/([^/.]+)", repository)
|
| 178 |
+
if m:
|
| 179 |
+
return m.group(1), m.group(2)
|
| 180 |
+
return None
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
async def _github_has_new_release(owner: str, repo: str, since: str) -> bool:
|
| 184 |
+
"""
|
| 185 |
+
Return True if there is a GitHub release published after `since` (ISO date string).
|
| 186 |
+
Falls back to checking the latest commit on the default branch if no releases exist.
|
| 187 |
+
"""
|
| 188 |
+
import httpx
|
| 189 |
+
headers = {"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"}
|
| 190 |
+
if GITHUB_TOKEN:
|
| 191 |
+
headers["Authorization"] = f"Bearer {GITHUB_TOKEN}"
|
| 192 |
+
|
| 193 |
+
try:
|
| 194 |
+
since_dt = datetime.fromisoformat(since).replace(tzinfo=timezone.utc)
|
| 195 |
+
except ValueError:
|
| 196 |
+
return False
|
| 197 |
+
|
| 198 |
+
async with httpx.AsyncClient(timeout=8) as client:
|
| 199 |
+
# 1. Check releases first
|
| 200 |
+
try:
|
| 201 |
+
r = await client.get(
|
| 202 |
+
f"https://api.github.com/repos/{owner}/{repo}/releases/latest",
|
| 203 |
+
headers=headers,
|
| 204 |
+
)
|
| 205 |
+
if r.status_code == 200:
|
| 206 |
+
published = r.json().get("published_at", "")
|
| 207 |
+
if published:
|
| 208 |
+
pub_dt = datetime.fromisoformat(published.rstrip("Z")).replace(tzinfo=timezone.utc)
|
| 209 |
+
if pub_dt > since_dt:
|
| 210 |
+
logger.info("New release on %s/%s (%s > %s) β cache stale", owner, repo, published, since)
|
| 211 |
+
return True
|
| 212 |
+
return False # release exists but is older β cache is fresh
|
| 213 |
+
except Exception:
|
| 214 |
+
pass # fall through to commit check
|
| 215 |
+
|
| 216 |
+
# 2. No releases β check latest commit on default branch
|
| 217 |
+
try:
|
| 218 |
+
r = await client.get(
|
| 219 |
+
f"https://api.github.com/repos/{owner}/{repo}/commits?per_page=1",
|
| 220 |
+
headers=headers,
|
| 221 |
+
)
|
| 222 |
+
if r.status_code == 200 and r.json():
|
| 223 |
+
commit_date = (
|
| 224 |
+
r.json()[0]
|
| 225 |
+
.get("commit", {})
|
| 226 |
+
.get("committer", {})
|
| 227 |
+
.get("date", "")
|
| 228 |
+
)
|
| 229 |
+
if commit_date:
|
| 230 |
+
commit_dt = datetime.fromisoformat(commit_date.rstrip("Z")).replace(tzinfo=timezone.utc)
|
| 231 |
+
if commit_dt > since_dt:
|
| 232 |
+
logger.info("New commit on %s/%s (%s > %s) β cache stale", owner, repo, commit_date, since)
|
| 233 |
+
return True
|
| 234 |
+
except Exception:
|
| 235 |
+
pass
|
| 236 |
+
|
| 237 |
+
return False
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
async def get_cached_context(library: str, version: str) -> dict[str, Any] | None:
|
| 241 |
+
cache = get_cache()
|
| 242 |
+
key = make_context_key(library, version)
|
| 243 |
+
raw = await cache.get(key)
|
| 244 |
+
if not raw:
|
| 245 |
+
logger.info("Cache MISS for %s@%s", library, version)
|
| 246 |
+
return None
|
| 247 |
+
|
| 248 |
+
data: dict[str, Any] = json.loads(raw)
|
| 249 |
+
logger.info("Cache HIT for %s@%s", library, version)
|
| 250 |
+
|
| 251 |
+
# Pinned versions (e.g. react@18.2.0) never need GitHub staleness checks β
|
| 252 |
+
# the docs for that exact version don't change. Only "latest" or similar.
|
| 253 |
+
is_pinned = version not in ("latest", "latest", "") and version[0].isdigit()
|
| 254 |
+
if is_pinned:
|
| 255 |
+
return data
|
| 256 |
+
|
| 257 |
+
# For "latest" entries: check GitHub at most once per GITHUB_CHECK_INTERVAL
|
| 258 |
+
last_checked = data.get("github_checked_at")
|
| 259 |
+
now_ts = time.time()
|
| 260 |
+
if last_checked and (now_ts - last_checked) < GITHUB_CHECK_INTERVAL:
|
| 261 |
+
return data # checked recently β trust the cache
|
| 262 |
+
|
| 263 |
+
repo_url: str = data.get("repository", "")
|
| 264 |
+
parsed = _extract_github_owner_repo(repo_url) if repo_url else None
|
| 265 |
+
|
| 266 |
+
if parsed:
|
| 267 |
+
owner, repo = parsed
|
| 268 |
+
cached_at: str = data.get("cached_at", "")
|
| 269 |
+
try:
|
| 270 |
+
stale = await _github_has_new_release(owner, repo, cached_at)
|
| 271 |
+
except Exception as exc:
|
| 272 |
+
logger.warning("GitHub staleness check failed for %s/%s: %s β keeping cache", owner, repo, exc)
|
| 273 |
+
stale = False
|
| 274 |
+
|
| 275 |
+
if stale:
|
| 276 |
+
logger.info("Invalidating cache for %s@%s β new GitHub release detected", library, version)
|
| 277 |
+
await cache.delete(key)
|
| 278 |
+
return None # caller will re-run the pipeline
|
| 279 |
+
|
| 280 |
+
# Mark the time we last checked so we don't re-check for another hour
|
| 281 |
+
data["github_checked_at"] = now_ts
|
| 282 |
+
await cache.set(key, json.dumps(data), ttl=CACHE_TTL_LATEST)
|
| 283 |
+
return data
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
async def set_cached_context(library: str, version: str, data: dict[str, Any]) -> None:
|
| 287 |
+
cache = get_cache()
|
| 288 |
+
# Pinned versions get the full 15-day TTL; "latest" also gets 15 days but
|
| 289 |
+
# GitHub staleness checks will evict it earlier if a new release lands.
|
| 290 |
+
is_pinned = version not in ("latest", "") and version[:1].isdigit()
|
| 291 |
+
ttl = CACHE_TTL_PINNED if is_pinned else CACHE_TTL_LATEST
|
| 292 |
+
await cache.set(make_context_key(library, version), json.dumps(data), ttl=ttl)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
async def get_job(job_id: str) -> dict[str, Any] | None:
|
| 296 |
+
cache = get_cache()
|
| 297 |
+
raw = await cache.get(make_job_key(job_id))
|
| 298 |
+
if raw:
|
| 299 |
+
return json.loads(raw)
|
| 300 |
+
return None
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
async def set_job(job_id: str, data: dict[str, Any]) -> None:
|
| 304 |
+
cache = get_cache()
|
| 305 |
+
# Jobs are stored for 1 hour β enough to poll results
|
| 306 |
+
await cache.set(make_job_key(job_id), json.dumps(data), ttl=3600)
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
async def search_cached_libraries(query: str) -> list[dict[str, str]]:
|
| 310 |
+
"""Search in-memory index for libraries matching query."""
|
| 311 |
+
cache = get_cache()
|
| 312 |
+
pattern = f"docforge:context:*{query.lower()}*"
|
| 313 |
+
keys = await cache.keys_matching(pattern)
|
| 314 |
+
results = []
|
| 315 |
+
for key in keys:
|
| 316 |
+
raw = await cache.get(key)
|
| 317 |
+
if raw:
|
| 318 |
+
data = json.loads(raw)
|
| 319 |
+
results.append({
|
| 320 |
+
"name": data.get("library", ""),
|
| 321 |
+
"version": data.get("version", ""),
|
| 322 |
+
"cached_at": data.get("cached_at", ""),
|
| 323 |
+
})
|
| 324 |
+
return results
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
async def list_cached_versions(package: str) -> list[str]:
|
| 328 |
+
"""List all cached versions for a given package."""
|
| 329 |
+
cache = get_cache()
|
| 330 |
+
pattern = f"docforge:context:{package.lower()}:*"
|
| 331 |
+
keys = await cache.keys_matching(pattern)
|
| 332 |
+
versions = []
|
| 333 |
+
for key in keys:
|
| 334 |
+
# key format: docforge:context:{library}:{version}
|
| 335 |
+
parts = key.split(":")
|
| 336 |
+
if len(parts) >= 4:
|
| 337 |
+
versions.append(parts[3])
|
| 338 |
+
return sorted(versions, reverse=True)
|
backend/services/formatter.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Formatter: converts structured JSON β .context.md
|
| 3 |
+
|
| 4 |
+
The output format is specifically designed to:
|
| 5 |
+
1. Be readable by humans reviewing what was generated
|
| 6 |
+
2. Maximize signal density for AI context windows (no fluff)
|
| 7 |
+
3. Prominently feature gotchas β the main reason this tool exists
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from datetime import date
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def format_context_md(data: dict) -> str:
|
| 14 |
+
"""Convert a structured library dict to a .context.md string."""
|
| 15 |
+
library = data.get("library", "unknown")
|
| 16 |
+
version = data.get("version", "unknown")
|
| 17 |
+
description = data.get("description", "")
|
| 18 |
+
components = data.get("components", [])
|
| 19 |
+
today = date.today().isoformat()
|
| 20 |
+
|
| 21 |
+
lines: list[str] = []
|
| 22 |
+
|
| 23 |
+
# Header
|
| 24 |
+
lines.append(f"# {library}@{version} β DocForge Context File")
|
| 25 |
+
lines.append(f"> Generated by DocForge | Last verified: {today}")
|
| 26 |
+
lines.append("> Drop this file into your project and reference it with @context.md in Cursor/Claude")
|
| 27 |
+
if description:
|
| 28 |
+
lines.append(f"\n{description}")
|
| 29 |
+
lines.append("")
|
| 30 |
+
|
| 31 |
+
# Component sections
|
| 32 |
+
for comp in components:
|
| 33 |
+
lines.append(f"---\n\n## {comp.get('name', 'Unknown')}")
|
| 34 |
+
|
| 35 |
+
# Install command
|
| 36 |
+
install = comp.get("install_command")
|
| 37 |
+
if install:
|
| 38 |
+
lines.append("\n**Install:**")
|
| 39 |
+
lines.append("```bash")
|
| 40 |
+
lines.append(install)
|
| 41 |
+
lines.append("```")
|
| 42 |
+
|
| 43 |
+
# Import path
|
| 44 |
+
import_path = comp.get("import_path")
|
| 45 |
+
if import_path:
|
| 46 |
+
lines.append("\n**Import:**")
|
| 47 |
+
lines.append("```typescript")
|
| 48 |
+
lines.append(import_path)
|
| 49 |
+
lines.append("```")
|
| 50 |
+
|
| 51 |
+
# Peer dependencies
|
| 52 |
+
peer_deps = comp.get("peer_dependencies", [])
|
| 53 |
+
if peer_deps:
|
| 54 |
+
lines.append("\n**Peer Dependencies:**")
|
| 55 |
+
for dep in peer_deps:
|
| 56 |
+
lines.append(f"- `{dep}`")
|
| 57 |
+
|
| 58 |
+
# Props table
|
| 59 |
+
props = comp.get("props", [])
|
| 60 |
+
if props:
|
| 61 |
+
lines.append("\n**Props:**")
|
| 62 |
+
lines.append("| Prop | Type | Required | Default |")
|
| 63 |
+
lines.append("|------|------|----------|---------|")
|
| 64 |
+
for prop in props:
|
| 65 |
+
name = prop.get("name", "")
|
| 66 |
+
ptype = prop.get("type", "")
|
| 67 |
+
required = "β" if prop.get("required") else "β"
|
| 68 |
+
default = prop.get("default") or "β"
|
| 69 |
+
lines.append(f"| {name} | {ptype} | {required} | {default} |")
|
| 70 |
+
|
| 71 |
+
# Gotchas β the most important section
|
| 72 |
+
gotchas = comp.get("gotchas", [])
|
| 73 |
+
if gotchas:
|
| 74 |
+
lines.append("\n**β οΈ Gotchas (Common AI Codegen Errors):**")
|
| 75 |
+
for gotcha in gotchas:
|
| 76 |
+
lines.append(f"- {gotcha}")
|
| 77 |
+
|
| 78 |
+
# Usage example
|
| 79 |
+
usage = comp.get("usage_example")
|
| 80 |
+
if usage:
|
| 81 |
+
lines.append("\n**Usage:**")
|
| 82 |
+
# Detect language hint from content
|
| 83 |
+
lang = "tsx" if ("<" in usage and ">" in usage) else "typescript"
|
| 84 |
+
lines.append(f"```{lang}")
|
| 85 |
+
lines.append(usage.strip())
|
| 86 |
+
lines.append("```")
|
| 87 |
+
|
| 88 |
+
lines.append("")
|
| 89 |
+
|
| 90 |
+
return "\n".join(lines)
|
backend/services/ingestion/__init__.py
ADDED
|
File without changes
|
backend/services/ingestion/copypaste_ingester.py
ADDED
|
@@ -0,0 +1,819 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Detector and ingester for "copy-paste" style component libraries.
|
| 3 |
+
|
| 4 |
+
These libraries are NOT published as installable npm packages.
|
| 5 |
+
Developers copy component source code into their project via a CLI tool
|
| 6 |
+
(shadcn, jsrepo) or manually from the library's website.
|
| 7 |
+
|
| 8 |
+
Detection cascade (in priority order):
|
| 9 |
+
1. jsrepo.dev β standard registry for Svelte/TS copy-paste libs
|
| 10 |
+
2. shadcn registry format β homepage exposes /registry.json or /r/*.json
|
| 11 |
+
3. GitHub README heuristics β README contains copy-paste CLI phrases
|
| 12 |
+
4. npm package has no dist exports β source-only repo misidentified as package
|
| 13 |
+
|
| 14 |
+
If any pattern matches, ingest() returns pre-built documentation so the
|
| 15 |
+
pipeline can skip the normal URL crawl step entirely.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import logging
|
| 19 |
+
import re
|
| 20 |
+
from datetime import date
|
| 21 |
+
from urllib.parse import urlparse
|
| 22 |
+
|
| 23 |
+
import httpx
|
| 24 |
+
|
| 25 |
+
logger = logging.getLogger(__name__)
|
| 26 |
+
|
| 27 |
+
_TIMEOUT = 15
|
| 28 |
+
_HEADERS = {"User-Agent": "docforge/1.0 (documentation indexer)"}
|
| 29 |
+
_GH_HEADERS = {**_HEADERS, "Accept": "application/vnd.github+json"}
|
| 30 |
+
|
| 31 |
+
# Regex patterns that appear in README files of copy-paste libraries
|
| 32 |
+
_README_SIGNALS = [
|
| 33 |
+
r"npx\s+shadcn",
|
| 34 |
+
r"npx\s+@shadcn",
|
| 35 |
+
r"pnpm\s+dlx\s+shadcn",
|
| 36 |
+
r"bunx\s+shadcn",
|
| 37 |
+
r"npx\s+jsrepo",
|
| 38 |
+
r"jsrepo\s+add",
|
| 39 |
+
r"not\s+published\s+to\s+npm",
|
| 40 |
+
r"not\s+available\s+on\s+npm",
|
| 41 |
+
r"no\s+npm\s+package",
|
| 42 |
+
r"copy\s+and\s+paste",
|
| 43 |
+
r"copy-paste\s+component",
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# ---------------------------------------------------------------------------
|
| 48 |
+
# Public API
|
| 49 |
+
# ---------------------------------------------------------------------------
|
| 50 |
+
|
| 51 |
+
async def detect_and_ingest(
|
| 52 |
+
name: str,
|
| 53 |
+
github_url: str | None = None,
|
| 54 |
+
homepage: str | None = None,
|
| 55 |
+
npm_data: dict | None = None,
|
| 56 |
+
) -> dict | None:
|
| 57 |
+
"""
|
| 58 |
+
Run all detection patterns against the given signals.
|
| 59 |
+
|
| 60 |
+
Returns a metadata dict with _is_copypaste=True and _raw_docs set
|
| 61 |
+
if the library is identified as copy-paste style, otherwise None.
|
| 62 |
+
"""
|
| 63 |
+
# ββ Pattern 1: jsrepo.dev βββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 64 |
+
docs = await check_jsrepo(name, github_url)
|
| 65 |
+
if docs:
|
| 66 |
+
logger.info("[copypaste] jsrepo match for '%s'", name)
|
| 67 |
+
return _make_result(name, github_url, docs, "jsrepo")
|
| 68 |
+
|
| 69 |
+
# ββ Pattern 2: shadcn registry format ββββββββββββββββββββββββββββββββββββ
|
| 70 |
+
if homepage:
|
| 71 |
+
shadcn_result = await _check_shadcn_registry(homepage, name)
|
| 72 |
+
if shadcn_result is not None:
|
| 73 |
+
logger.info("[copypaste] shadcn registry match for '%s'", name)
|
| 74 |
+
result = _make_result(name, github_url, "", "shadcn")
|
| 75 |
+
result["_structured"] = shadcn_result
|
| 76 |
+
return result
|
| 77 |
+
|
| 78 |
+
# ββ Pattern 3: npm package has no dist exports βββββββββββββββββββββββββββ
|
| 79 |
+
# (fall through β if npm_data has no exports, README check below will
|
| 80 |
+
# confirm whether it's a copy-paste lib or just a poorly configured package)
|
| 81 |
+
no_dist = npm_data is not None and _has_no_dist_exports(npm_data)
|
| 82 |
+
|
| 83 |
+
# ββ Pattern 4: GitHub README heuristics ββββββββββββββββββββββββββββββββββ
|
| 84 |
+
if github_url:
|
| 85 |
+
docs = await _check_readme(github_url, name, require_signal=not no_dist)
|
| 86 |
+
if docs:
|
| 87 |
+
logger.info("[copypaste] README match for '%s'", name)
|
| 88 |
+
return _make_result(name, github_url, docs, "readme")
|
| 89 |
+
|
| 90 |
+
return None
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ---------------------------------------------------------------------------
|
| 94 |
+
# Pattern 1 β jsrepo.dev
|
| 95 |
+
# ---------------------------------------------------------------------------
|
| 96 |
+
|
| 97 |
+
async def check_jsrepo(name: str, github_url: str | None) -> str | None:
|
| 98 |
+
"""
|
| 99 |
+
Check jsrepo.dev and raw GitHub paths for a blocks/registry JSON.
|
| 100 |
+
jsrepo registries are GitHub repos that publish a blocks.json or registry.json.
|
| 101 |
+
"""
|
| 102 |
+
clean = name.lstrip("@").replace("/", "-")
|
| 103 |
+
candidates: list[str] = []
|
| 104 |
+
owner: str | None = None
|
| 105 |
+
repo: str | None = None
|
| 106 |
+
|
| 107 |
+
# Derive candidates from the known GitHub repo
|
| 108 |
+
if github_url:
|
| 109 |
+
m = re.search(r"github\.com[:/]([^/]+)/([^/.#]+)", github_url)
|
| 110 |
+
if m:
|
| 111 |
+
owner, repo = m.group(1), m.group(2)
|
| 112 |
+
for branch in ("main", "master"):
|
| 113 |
+
candidates += [
|
| 114 |
+
f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/blocks.json",
|
| 115 |
+
f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/registry.json",
|
| 116 |
+
f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/src/registry.json",
|
| 117 |
+
]
|
| 118 |
+
candidates += [
|
| 119 |
+
f"https://jsrepo.dev/api/registries/{owner}/{repo}",
|
| 120 |
+
]
|
| 121 |
+
|
| 122 |
+
# Generic jsrepo search by package name
|
| 123 |
+
candidates.append(f"https://jsrepo.dev/api/registries?q={clean}")
|
| 124 |
+
|
| 125 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS, follow_redirects=True) as client:
|
| 126 |
+
for url in candidates:
|
| 127 |
+
try:
|
| 128 |
+
resp = await client.get(url)
|
| 129 |
+
if not resp.is_success:
|
| 130 |
+
continue
|
| 131 |
+
if "json" not in resp.headers.get("content-type", ""):
|
| 132 |
+
continue
|
| 133 |
+
|
| 134 |
+
data = resp.json()
|
| 135 |
+
docs = _parse_blocks_json(data, name, owner=owner, repo=repo)
|
| 136 |
+
if docs:
|
| 137 |
+
return docs
|
| 138 |
+
except Exception as exc:
|
| 139 |
+
logger.debug("jsrepo check failed at %s: %s", url, exc)
|
| 140 |
+
|
| 141 |
+
return None
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
# ---------------------------------------------------------------------------
|
| 145 |
+
# Pattern 2 β shadcn registry format
|
| 146 |
+
# ---------------------------------------------------------------------------
|
| 147 |
+
|
| 148 |
+
async def check_shadcn_registry(homepage: str, name: str) -> "dict | None":
|
| 149 |
+
"""Public alias β same as _check_shadcn_registry, called from context.py."""
|
| 150 |
+
return await _check_shadcn_registry(homepage, name)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
async def _check_shadcn_registry(homepage: str, name: str) -> "dict | None":
|
| 154 |
+
"""
|
| 155 |
+
Check if the homepage exposes a shadcn-compatible component registry.
|
| 156 |
+
|
| 157 |
+
React Bits (and similar) expose their registry at /r/registry.json using the
|
| 158 |
+
shadcn registry schema ({$schema, name, items:[]}). The jsrepo CLI treats the
|
| 159 |
+
/r path as its registry root: `npx jsrepo add https://site.dev/r ComponentName`.
|
| 160 |
+
"""
|
| 161 |
+
parsed = urlparse(homepage)
|
| 162 |
+
base = f"{parsed.scheme}://{parsed.netloc}"
|
| 163 |
+
|
| 164 |
+
candidates = [
|
| 165 |
+
# /r/registry.json β the pattern used by react-bits and jsrepo-compatible sites
|
| 166 |
+
f"{base}/r/registry.json",
|
| 167 |
+
f"{base}/r/blocks.json",
|
| 168 |
+
f"{base}/r/index.json",
|
| 169 |
+
# root-level registry
|
| 170 |
+
f"{base}/registry.json",
|
| 171 |
+
f"{homepage.rstrip('/')}/registry.json",
|
| 172 |
+
f"{base}/api/registry.json",
|
| 173 |
+
]
|
| 174 |
+
|
| 175 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS, follow_redirects=True) as client:
|
| 176 |
+
for url in candidates:
|
| 177 |
+
try:
|
| 178 |
+
resp = await client.get(url)
|
| 179 |
+
if not resp.is_success:
|
| 180 |
+
continue
|
| 181 |
+
if "json" not in resp.headers.get("content-type", ""):
|
| 182 |
+
continue
|
| 183 |
+
|
| 184 |
+
data = resp.json()
|
| 185 |
+
items = None
|
| 186 |
+
if isinstance(data, list):
|
| 187 |
+
items = data
|
| 188 |
+
elif isinstance(data, dict):
|
| 189 |
+
items = (
|
| 190 |
+
data.get("items")
|
| 191 |
+
or data.get("components")
|
| 192 |
+
or data.get("registry")
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
if items and isinstance(items, list) and len(items) > 0:
|
| 196 |
+
logger.info("shadcn registry found at %s (%d items)", url, len(items))
|
| 197 |
+
version = (data.get("version") if isinstance(data, dict) else None) or "latest"
|
| 198 |
+
return await _format_shadcn_registry(items, name, url, version)
|
| 199 |
+
|
| 200 |
+
except Exception as exc:
|
| 201 |
+
logger.debug("shadcn registry check failed at %s: %s", url, exc)
|
| 202 |
+
|
| 203 |
+
return None
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
# ---------------------------------------------------------------------------
|
| 207 |
+
# Pattern 3 β npm package with no dist exports (checked in detect_and_ingest)
|
| 208 |
+
# ---------------------------------------------------------------------------
|
| 209 |
+
|
| 210 |
+
def _has_no_dist_exports(npm_data: dict) -> bool:
|
| 211 |
+
"""
|
| 212 |
+
Return True if the npm package.json has no built distribution artefacts.
|
| 213 |
+
A package with no main/module/exports and no dist/ in files[] is likely
|
| 214 |
+
a source-only repo published for reference rather than installation.
|
| 215 |
+
"""
|
| 216 |
+
has_entry = bool(
|
| 217 |
+
npm_data.get("main")
|
| 218 |
+
or npm_data.get("module")
|
| 219 |
+
or npm_data.get("exports")
|
| 220 |
+
)
|
| 221 |
+
has_dist_files = any(
|
| 222 |
+
"dist/" in str(f) or "build/" in str(f)
|
| 223 |
+
for f in (npm_data.get("files") or [])
|
| 224 |
+
)
|
| 225 |
+
return not has_entry and not has_dist_files
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
# ---------------------------------------------------------------------------
|
| 229 |
+
# Pattern 4 β GitHub README heuristics
|
| 230 |
+
# ---------------------------------------------------------------------------
|
| 231 |
+
|
| 232 |
+
async def _check_readme(
|
| 233 |
+
github_url: str,
|
| 234 |
+
name: str,
|
| 235 |
+
require_signal: bool = True,
|
| 236 |
+
) -> str | None:
|
| 237 |
+
"""
|
| 238 |
+
Fetch the GitHub README.
|
| 239 |
+
If require_signal=True, only return content when a copy-paste phrase is found.
|
| 240 |
+
If require_signal=False (npm_data had no exports), return content regardless
|
| 241 |
+
since we already have a structural signal β just confirm it's not a normal lib.
|
| 242 |
+
"""
|
| 243 |
+
m = re.search(r"github\.com[:/]([^/]+)/([^/.#]+)", github_url)
|
| 244 |
+
if not m:
|
| 245 |
+
return None
|
| 246 |
+
|
| 247 |
+
owner, repo = m.group(1), m.group(2)
|
| 248 |
+
readme_content: str | None = None
|
| 249 |
+
|
| 250 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 251 |
+
for branch in ("main", "master"):
|
| 252 |
+
for filename in ("README.md", "readme.md", "Readme.md"):
|
| 253 |
+
url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{filename}"
|
| 254 |
+
try:
|
| 255 |
+
resp = await client.get(url)
|
| 256 |
+
if resp.is_success and resp.text.strip():
|
| 257 |
+
readme_content = resp.text
|
| 258 |
+
break
|
| 259 |
+
except Exception:
|
| 260 |
+
continue
|
| 261 |
+
if readme_content:
|
| 262 |
+
break
|
| 263 |
+
|
| 264 |
+
if not readme_content:
|
| 265 |
+
return None
|
| 266 |
+
|
| 267 |
+
readme_lower = readme_content.lower()
|
| 268 |
+
has_signal = any(
|
| 269 |
+
re.search(pattern, readme_lower)
|
| 270 |
+
for pattern in _README_SIGNALS
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
if require_signal and not has_signal:
|
| 274 |
+
return None
|
| 275 |
+
|
| 276 |
+
# Enrich with component listing from the repo tree
|
| 277 |
+
component_list = await _fetch_component_listing(client, owner, repo)
|
| 278 |
+
|
| 279 |
+
m2 = re.search(r"github\.com[:/]([^/]+)/([^/.#]+)", github_url)
|
| 280 |
+
gh_slug = f"{m2.group(1)}/{m2.group(2)}" if m2 else None
|
| 281 |
+
|
| 282 |
+
lines = [
|
| 283 |
+
f"# {name} β Copy-Paste Component Library\n\n",
|
| 284 |
+
"β οΈ IMPORTANT FOR AI CODE GENERATION:\n",
|
| 285 |
+
f"This library ({name}) is a COPY-PASTE component library β it is NOT published as an npm package.\n",
|
| 286 |
+
"DO NOT generate 'npm install {name}' or any npm/yarn/pnpm install command for this library.\n",
|
| 287 |
+
"Components must be copied into the project manually or via a CLI tool.\n\n",
|
| 288 |
+
"## How to install components\n\n",
|
| 289 |
+
]
|
| 290 |
+
if gh_slug:
|
| 291 |
+
lines.append(
|
| 292 |
+
f"```bash\n# Option A β jsrepo CLI\nnpx jsrepo add {gh_slug}/<ComponentName>\n\n"
|
| 293 |
+
f"# Option B β shadcn CLI (if the project uses shadcn)\nnpx shadcn@latest add <ComponentName>\n```\n\n"
|
| 294 |
+
"After adding, import from your local components directory:\n"
|
| 295 |
+
"`import { ComponentName } from '@/components/ComponentName'`\n\n"
|
| 296 |
+
)
|
| 297 |
+
lines.append("## README\n\n")
|
| 298 |
+
lines.append(readme_content[:10_000]) # README covers install + overview
|
| 299 |
+
|
| 300 |
+
if component_list:
|
| 301 |
+
lines.append(f"\n\n## Available Components\n\n{component_list}")
|
| 302 |
+
|
| 303 |
+
return "".join(lines)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
async def _fetch_component_listing(
|
| 307 |
+
client: httpx.AsyncClient,
|
| 308 |
+
owner: str,
|
| 309 |
+
repo: str,
|
| 310 |
+
) -> str | None:
|
| 311 |
+
"""Try common component directory paths to list available components."""
|
| 312 |
+
paths = [
|
| 313 |
+
f"https://api.github.com/repos/{owner}/{repo}/contents/src/components",
|
| 314 |
+
f"https://api.github.com/repos/{owner}/{repo}/contents/components",
|
| 315 |
+
f"https://api.github.com/repos/{owner}/{repo}/contents/src/lib/components",
|
| 316 |
+
f"https://api.github.com/repos/{owner}/{repo}/contents/packages",
|
| 317 |
+
f"https://api.github.com/repos/{owner}/{repo}/contents/src",
|
| 318 |
+
]
|
| 319 |
+
|
| 320 |
+
for url in paths:
|
| 321 |
+
try:
|
| 322 |
+
resp = await client.get(url, headers=_GH_HEADERS)
|
| 323 |
+
if not resp.is_success:
|
| 324 |
+
continue
|
| 325 |
+
items = resp.json()
|
| 326 |
+
if not isinstance(items, list):
|
| 327 |
+
continue
|
| 328 |
+
|
| 329 |
+
names = [
|
| 330 |
+
re.sub(r"\.(svelte|tsx|vue|jsx|ts|js)$", "", item["name"])
|
| 331 |
+
for item in items
|
| 332 |
+
if item.get("type") in ("file", "dir")
|
| 333 |
+
and not item["name"].startswith(".")
|
| 334 |
+
and item["name"][0].isupper() # components are usually PascalCase
|
| 335 |
+
]
|
| 336 |
+
if len(names) >= 3:
|
| 337 |
+
return "\n".join(f"- {n}" for n in names[:60])
|
| 338 |
+
except Exception:
|
| 339 |
+
continue
|
| 340 |
+
|
| 341 |
+
return None
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
# ---------------------------------------------------------------------------
|
| 345 |
+
# Helpers
|
| 346 |
+
# ---------------------------------------------------------------------------
|
| 347 |
+
|
| 348 |
+
def _make_result(
|
| 349 |
+
name: str,
|
| 350 |
+
github_url: str | None,
|
| 351 |
+
raw_docs: str,
|
| 352 |
+
method: str,
|
| 353 |
+
) -> dict:
|
| 354 |
+
return {
|
| 355 |
+
"name": name,
|
| 356 |
+
"version": "latest",
|
| 357 |
+
"description": f"Copy-paste component library (detected via {method})",
|
| 358 |
+
"homepage": github_url,
|
| 359 |
+
"repository": github_url,
|
| 360 |
+
"_is_copypaste": True,
|
| 361 |
+
"_copypaste_method": method,
|
| 362 |
+
"_raw_docs": raw_docs,
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def _parse_blocks_json(
|
| 367 |
+
data: object,
|
| 368 |
+
name: str,
|
| 369 |
+
owner: str | None = None,
|
| 370 |
+
repo: str | None = None,
|
| 371 |
+
) -> str | None:
|
| 372 |
+
"""Parse a blocks.json / registry.json payload into readable docs."""
|
| 373 |
+
items: list | None = None
|
| 374 |
+
|
| 375 |
+
if isinstance(data, list) and data:
|
| 376 |
+
items = data
|
| 377 |
+
elif isinstance(data, dict):
|
| 378 |
+
items = (
|
| 379 |
+
data.get("blocks")
|
| 380 |
+
or data.get("components")
|
| 381 |
+
or data.get("items")
|
| 382 |
+
or data.get("registry")
|
| 383 |
+
)
|
| 384 |
+
# jsrepo search results wrapper
|
| 385 |
+
if not items and "results" in data and data["results"]:
|
| 386 |
+
items = data["results"]
|
| 387 |
+
|
| 388 |
+
if not items or not isinstance(items, list) or len(items) == 0:
|
| 389 |
+
return None
|
| 390 |
+
|
| 391 |
+
gh_slug = f"{owner}/{repo}" if owner and repo else None
|
| 392 |
+
|
| 393 |
+
lines = [
|
| 394 |
+
f"# {name} β Copy-Paste Component Library\n\n",
|
| 395 |
+
"β οΈ IMPORTANT FOR AI CODE GENERATION:\n",
|
| 396 |
+
f"This library ({name}) is a COPY-PASTE component library β it is NOT published as an npm package.\n",
|
| 397 |
+
"DO NOT generate 'npm install {name}' or any npm/yarn/pnpm install command for this library.\n",
|
| 398 |
+
"Components are added to a project via the jsrepo CLI, NOT by installing a package.\n\n",
|
| 399 |
+
"## How to install components\n\n",
|
| 400 |
+
]
|
| 401 |
+
|
| 402 |
+
if gh_slug:
|
| 403 |
+
lines.append(f"Use the jsrepo CLI to add individual components:\n")
|
| 404 |
+
lines.append(f"```bash\n# Install jsrepo CLI (one-time)\nnpm install -g jsrepo\n\n")
|
| 405 |
+
lines.append(f"# Add a specific component (example)\nnpx jsrepo add {gh_slug}/<type>/<ComponentName>\n```\n\n")
|
| 406 |
+
else:
|
| 407 |
+
lines.append(f"Use the jsrepo CLI: `npx jsrepo add <registry>/<type>/<ComponentName>`\n\n")
|
| 408 |
+
|
| 409 |
+
lines.append(
|
| 410 |
+
"After adding, components live in your local project directory.\n"
|
| 411 |
+
"Import them from your local path, e.g. `import {{ ComponentName }} from '@/components/ComponentName'`\n\n"
|
| 412 |
+
f"Total components available: {len(items)}\n"
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
for block in items:
|
| 416 |
+
if not isinstance(block, dict):
|
| 417 |
+
continue
|
| 418 |
+
bname = block.get("name") or block.get("title") or ""
|
| 419 |
+
if not bname:
|
| 420 |
+
continue
|
| 421 |
+
|
| 422 |
+
desc = block.get("description") or ""
|
| 423 |
+
category = block.get("type") or block.get("category") or ""
|
| 424 |
+
|
| 425 |
+
entry = f"\n## {bname}"
|
| 426 |
+
if category:
|
| 427 |
+
entry += f" ({category})"
|
| 428 |
+
entry += "\n"
|
| 429 |
+
if desc:
|
| 430 |
+
entry += f"{desc}\n"
|
| 431 |
+
|
| 432 |
+
# jsrepo add command per component
|
| 433 |
+
if gh_slug and category:
|
| 434 |
+
entry += f"Install: `npx jsrepo add {gh_slug}/{category}/{bname}`\n"
|
| 435 |
+
elif gh_slug:
|
| 436 |
+
entry += f"Install: `npx jsrepo add {gh_slug}/{bname}`\n"
|
| 437 |
+
|
| 438 |
+
deps = block.get("dependencies") or block.get("registryDependencies") or []
|
| 439 |
+
if deps:
|
| 440 |
+
entry += f"npm peer dependencies (install these): {', '.join(str(d) for d in deps)}\n"
|
| 441 |
+
|
| 442 |
+
lines.append(entry)
|
| 443 |
+
|
| 444 |
+
return "".join(lines)
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def _parse_ts_props(snippet: str) -> list[dict]:
|
| 448 |
+
"""Parse a TypeScript interface/type snippet into a list of prop dicts."""
|
| 449 |
+
brace_start = snippet.find("{")
|
| 450 |
+
if brace_start == -1:
|
| 451 |
+
return []
|
| 452 |
+
|
| 453 |
+
props: list[dict] = []
|
| 454 |
+
depth = 0
|
| 455 |
+
i = brace_start
|
| 456 |
+
buf = ""
|
| 457 |
+
|
| 458 |
+
while i < len(snippet):
|
| 459 |
+
ch = snippet[i]
|
| 460 |
+
if ch == "{":
|
| 461 |
+
depth += 1
|
| 462 |
+
if depth > 1:
|
| 463 |
+
buf += ch
|
| 464 |
+
elif ch == "}":
|
| 465 |
+
depth -= 1
|
| 466 |
+
if depth == 0:
|
| 467 |
+
break
|
| 468 |
+
buf += ch
|
| 469 |
+
elif ch == ";" and depth == 1:
|
| 470 |
+
prop = _parse_prop_line(buf)
|
| 471 |
+
if prop:
|
| 472 |
+
props.append(prop)
|
| 473 |
+
buf = ""
|
| 474 |
+
elif ch == "\n" and depth == 1 and buf.strip():
|
| 475 |
+
prop = _parse_prop_line(buf)
|
| 476 |
+
if prop:
|
| 477 |
+
props.append(prop)
|
| 478 |
+
buf = ""
|
| 479 |
+
else:
|
| 480 |
+
buf += ch
|
| 481 |
+
i += 1
|
| 482 |
+
|
| 483 |
+
if buf.strip():
|
| 484 |
+
prop = _parse_prop_line(buf)
|
| 485 |
+
if prop:
|
| 486 |
+
props.append(prop)
|
| 487 |
+
|
| 488 |
+
return props
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
def _parse_prop_line(line: str) -> "dict | None":
|
| 492 |
+
"""Parse a single TypeScript property declaration."""
|
| 493 |
+
line = line.strip().rstrip(";").strip()
|
| 494 |
+
if not line or line.startswith("//") or line.startswith("*") or line.startswith("["):
|
| 495 |
+
return None
|
| 496 |
+
m = re.match(r"^(\w+)(\?)?\s*:\s*(.+)$", line)
|
| 497 |
+
if not m:
|
| 498 |
+
return None
|
| 499 |
+
prop_type = m.group(3).strip()
|
| 500 |
+
if len(prop_type) > 80:
|
| 501 |
+
prop_type = prop_type[:77] + "..."
|
| 502 |
+
return {
|
| 503 |
+
"name": m.group(1),
|
| 504 |
+
"type": prop_type,
|
| 505 |
+
"required": m.group(2) != "?",
|
| 506 |
+
"default": None,
|
| 507 |
+
}
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def _example_value(prop: dict) -> str:
|
| 511 |
+
"""Return a realistic example value for a prop based on its name and type."""
|
| 512 |
+
name = prop["name"].lower()
|
| 513 |
+
t = prop["type"].lower()
|
| 514 |
+
|
| 515 |
+
# Name-driven heuristics (more specific than type alone)
|
| 516 |
+
if any(k in name for k in ("text", "label", "title", "sentence", "placeholder")):
|
| 517 |
+
return '"Hello World"'
|
| 518 |
+
if any(k in name for k in ("src", "url", "href", "path", "image", "logo")):
|
| 519 |
+
return '"/path/to/file"'
|
| 520 |
+
if "color" in name:
|
| 521 |
+
return '"#3b82f6"'
|
| 522 |
+
if name in ("classname", "class"):
|
| 523 |
+
return '"your-class"'
|
| 524 |
+
if any(k in name for k in ("duration", "delay", "speed", "time")):
|
| 525 |
+
return "{1.5}" if "duration" in name else "{300}"
|
| 526 |
+
if any(k in name for k in ("count", "max", "min", "size", "radius", "width", "height")):
|
| 527 |
+
return "{100}"
|
| 528 |
+
if "items" in name or "logos" in name or "images" in name:
|
| 529 |
+
return "{[]}"
|
| 530 |
+
if name in ("to", "target"):
|
| 531 |
+
return "{1000}"
|
| 532 |
+
if name in ("from"):
|
| 533 |
+
return "{0}"
|
| 534 |
+
|
| 535 |
+
# Type-driven fallback
|
| 536 |
+
if "string" in t:
|
| 537 |
+
return '"value"'
|
| 538 |
+
if "number" in t:
|
| 539 |
+
return "{0}"
|
| 540 |
+
if "boolean" in t:
|
| 541 |
+
return "{true}"
|
| 542 |
+
if "reactnode" in t or "element" in t:
|
| 543 |
+
return "{<div>content</div>}"
|
| 544 |
+
if "[]" in t or t.startswith("array"):
|
| 545 |
+
return "{[]}"
|
| 546 |
+
if "=>" in t or "function" in t:
|
| 547 |
+
return "{() => {}}"
|
| 548 |
+
return "{/* value */}"
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
# Props to skip in commented optional list β too generic to be useful examples
|
| 552 |
+
_BORING_PROPS = frozenset({"classname", "style", "id", "ref", "key", "tabindex", "role"})
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def _generate_usage(title: str, props: list[dict]) -> str:
|
| 556 |
+
"""
|
| 557 |
+
Generate a proper runnable usage example with import + function wrapper.
|
| 558 |
+
Required props are shown with realistic values.
|
| 559 |
+
A selection of interesting optional props are shown commented out.
|
| 560 |
+
"""
|
| 561 |
+
import_line = f"import {{ {title} }} from '@/components/{title}';"
|
| 562 |
+
|
| 563 |
+
required = [p for p in props if p.get("required") and p["name"] != "children"]
|
| 564 |
+
has_children = any(p["name"] == "children" for p in props)
|
| 565 |
+
needs_children = any(p["name"] == "children" and p.get("required") for p in props)
|
| 566 |
+
|
| 567 |
+
# Pick up to 5 interesting optional props to show commented (skip boring ones)
|
| 568 |
+
optional_shown = [
|
| 569 |
+
p for p in props
|
| 570 |
+
if not p.get("required")
|
| 571 |
+
and p["name"].lower() not in _BORING_PROPS
|
| 572 |
+
and p["name"] != "children"
|
| 573 |
+
][:5]
|
| 574 |
+
|
| 575 |
+
indent = " " # 6 spaces inside return ( <Component )
|
| 576 |
+
attr_lines: list[str] = []
|
| 577 |
+
|
| 578 |
+
for p in required:
|
| 579 |
+
attr_lines.append(f"{indent}{p['name']}={_example_value(p)}")
|
| 580 |
+
|
| 581 |
+
for p in optional_shown:
|
| 582 |
+
attr_lines.append(f"{indent}// {p['name']}={_example_value(p)}")
|
| 583 |
+
|
| 584 |
+
attrs = "\n".join(attr_lines)
|
| 585 |
+
open_tag = f"<{title}"
|
| 586 |
+
if attrs:
|
| 587 |
+
open_tag += f"\n{attrs}\n "
|
| 588 |
+
|
| 589 |
+
if needs_children or (has_children and not required):
|
| 590 |
+
if attrs:
|
| 591 |
+
jsx = f"<{title}\n{attrs}\n >\n {{/* content */}}\n </{title}>"
|
| 592 |
+
else:
|
| 593 |
+
jsx = f"<{title}>\n {{/* content */}}\n </{title}>"
|
| 594 |
+
else:
|
| 595 |
+
jsx = f"<{title}\n{attrs}\n />" if attrs else f"<{title} />"
|
| 596 |
+
|
| 597 |
+
return (
|
| 598 |
+
f"{import_line}\n\n"
|
| 599 |
+
f"export default function Example() {{\n"
|
| 600 |
+
f" return (\n"
|
| 601 |
+
f" {jsx}\n"
|
| 602 |
+
f" );\n"
|
| 603 |
+
f"}}"
|
| 604 |
+
)
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
async def _format_shadcn_registry(
|
| 608 |
+
items: list, name: str, registry_url: str, version: str = "latest"
|
| 609 |
+
) -> dict:
|
| 610 |
+
"""
|
| 611 |
+
Build a pre-structured JSON dict from a shadcn/jsrepo registry.
|
| 612 |
+
Returns a dict matching the schema expected by format_context_md(),
|
| 613 |
+
bypassing the LLM structurer entirely.
|
| 614 |
+
"""
|
| 615 |
+
from urllib.parse import urlparse as _urlparse
|
| 616 |
+
_p = _urlparse(registry_url)
|
| 617 |
+
registry_base = f"{_p.scheme}://{_p.netloc}{_p.path.rsplit('/', 1)[0]}"
|
| 618 |
+
|
| 619 |
+
# Deduplicate by title β registries like react-bits expose 4 variants per
|
| 620 |
+
# component (JS/TS Γ CSS/TW). Prefer the TypeScript variant because its
|
| 621 |
+
# source file has proper interface declarations we can extract for props.
|
| 622 |
+
seen_titles: dict[str, dict] = {}
|
| 623 |
+
for item in items:
|
| 624 |
+
if not isinstance(item, dict):
|
| 625 |
+
continue
|
| 626 |
+
title = item.get("title") or item.get("name") or ""
|
| 627 |
+
if not title:
|
| 628 |
+
continue
|
| 629 |
+
item_name = item.get("name") or ""
|
| 630 |
+
is_ts = "-TS-" in item_name or item_name.endswith("-TS")
|
| 631 |
+
if title not in seen_titles:
|
| 632 |
+
seen_titles[title] = dict(item)
|
| 633 |
+
else:
|
| 634 |
+
existing = seen_titles[title]
|
| 635 |
+
existing_name = existing.get("name") or ""
|
| 636 |
+
if is_ts and "-TS-" not in existing_name and not existing_name.endswith("-TS"):
|
| 637 |
+
merged_deps = sorted(
|
| 638 |
+
set(existing.get("dependencies") or [])
|
| 639 |
+
| set(item.get("dependencies") or [])
|
| 640 |
+
)
|
| 641 |
+
seen_titles[title] = {**item, "dependencies": merged_deps}
|
| 642 |
+
else:
|
| 643 |
+
existing_deps = set(existing.get("dependencies") or [])
|
| 644 |
+
existing_deps.update(item.get("dependencies") or [])
|
| 645 |
+
existing["dependencies"] = sorted(existing_deps)
|
| 646 |
+
|
| 647 |
+
unique_items = list(seen_titles.values())
|
| 648 |
+
|
| 649 |
+
# Fetch individual component JSON files in parallel to extract prop types
|
| 650 |
+
props_map = await _fetch_all_component_props(registry_base, unique_items)
|
| 651 |
+
|
| 652 |
+
today = date.today().isoformat()
|
| 653 |
+
components = []
|
| 654 |
+
|
| 655 |
+
for item in unique_items:
|
| 656 |
+
title = item.get("title") or item.get("name") or ""
|
| 657 |
+
if not title:
|
| 658 |
+
continue
|
| 659 |
+
|
| 660 |
+
deps = item.get("dependencies") or []
|
| 661 |
+
prop_snippet = props_map.get(title)
|
| 662 |
+
props = _parse_ts_props(prop_snippet) if prop_snippet else []
|
| 663 |
+
usage = _generate_usage(title, props)
|
| 664 |
+
|
| 665 |
+
gotchas = [
|
| 666 |
+
f"Install with `npx jsrepo add {registry_base} {title}` β do NOT use npm install",
|
| 667 |
+
f"Import from your local project: `import {{ {title} }} from '@/components/{title}'`",
|
| 668 |
+
]
|
| 669 |
+
if deps:
|
| 670 |
+
gotchas.append(f"Peer dependencies must be installed separately: {', '.join(deps)}")
|
| 671 |
+
|
| 672 |
+
components.append({
|
| 673 |
+
"name": title,
|
| 674 |
+
"install_command": f"npx jsrepo add {registry_base} {title}",
|
| 675 |
+
"import_path": f"import {{ {title} }} from '@/components/{title}'",
|
| 676 |
+
"props": props,
|
| 677 |
+
"peer_dependencies": deps,
|
| 678 |
+
"gotchas": gotchas,
|
| 679 |
+
"usage_example": usage,
|
| 680 |
+
"last_verified": today,
|
| 681 |
+
})
|
| 682 |
+
|
| 683 |
+
return {
|
| 684 |
+
"library": name,
|
| 685 |
+
"version": version,
|
| 686 |
+
"description": f"Copy-Paste Component Library (jsrepo/shadcn registry) β {len(components)} components",
|
| 687 |
+
"components": components,
|
| 688 |
+
}
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
async def _fetch_all_component_props(
|
| 692 |
+
registry_base: str,
|
| 693 |
+
items: list,
|
| 694 |
+
max_concurrent: int = 15,
|
| 695 |
+
) -> dict[str, str]:
|
| 696 |
+
"""
|
| 697 |
+
Fetch individual component JSON endpoints in parallel and extract TypeScript
|
| 698 |
+
prop interfaces from the embedded file content.
|
| 699 |
+
|
| 700 |
+
Individual component JSONs live at {registry_base}/{item_name}.json and
|
| 701 |
+
include a `files[]` array with the full source `content` field.
|
| 702 |
+
"""
|
| 703 |
+
import asyncio
|
| 704 |
+
|
| 705 |
+
sem = asyncio.Semaphore(max_concurrent)
|
| 706 |
+
|
| 707 |
+
async def fetch_one(
|
| 708 |
+
client: httpx.AsyncClient, item: dict
|
| 709 |
+
) -> tuple[str, str | None]:
|
| 710 |
+
title = item.get("title") or item.get("name") or ""
|
| 711 |
+
item_name = item.get("name") or title
|
| 712 |
+
if not title:
|
| 713 |
+
return title, None
|
| 714 |
+
|
| 715 |
+
# Build candidate URLs β prefer TypeScript variants (have proper interfaces)
|
| 716 |
+
ts_name = item_name.replace("-JS-", "-TS-") if "-JS-" in item_name else item_name
|
| 717 |
+
candidate_urls = []
|
| 718 |
+
if ts_name != item_name:
|
| 719 |
+
candidate_urls.append(f"{registry_base}/{ts_name}.json")
|
| 720 |
+
candidate_urls += [
|
| 721 |
+
f"{registry_base}/{item_name}.json",
|
| 722 |
+
f"{registry_base}/{title}-TS-CSS.json",
|
| 723 |
+
f"{registry_base}/{title}-TS-TW.json",
|
| 724 |
+
f"{registry_base}/{title}.json",
|
| 725 |
+
]
|
| 726 |
+
|
| 727 |
+
async with sem:
|
| 728 |
+
for url in candidate_urls:
|
| 729 |
+
try:
|
| 730 |
+
resp = await client.get(url, timeout=6)
|
| 731 |
+
if not resp.is_success:
|
| 732 |
+
continue
|
| 733 |
+
if "json" not in resp.headers.get("content-type", ""):
|
| 734 |
+
continue
|
| 735 |
+
data = resp.json()
|
| 736 |
+
# The individual JSON may wrap items[] or be the item directly
|
| 737 |
+
files = data.get("files") or []
|
| 738 |
+
if not files and isinstance(data.get("items"), list):
|
| 739 |
+
files = data["items"][0].get("files") or []
|
| 740 |
+
for f in files:
|
| 741 |
+
content = f.get("content") or ""
|
| 742 |
+
if content:
|
| 743 |
+
snippet = _extract_ts_interface(content)
|
| 744 |
+
if snippet:
|
| 745 |
+
return title, snippet
|
| 746 |
+
except Exception:
|
| 747 |
+
pass
|
| 748 |
+
return title, None
|
| 749 |
+
|
| 750 |
+
async with httpx.AsyncClient(
|
| 751 |
+
timeout=8, headers=_HEADERS, follow_redirects=True
|
| 752 |
+
) as client:
|
| 753 |
+
results = await asyncio.gather(
|
| 754 |
+
*[fetch_one(client, item) for item in items],
|
| 755 |
+
return_exceptions=False,
|
| 756 |
+
)
|
| 757 |
+
|
| 758 |
+
return {t: p for t, p in results if p}
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
def _extract_ts_interface(content: str) -> str | None:
|
| 762 |
+
"""
|
| 763 |
+
Extract TypeScript prop types from source code.
|
| 764 |
+
Handles:
|
| 765 |
+
- interface AnimatedContentProps { ... } (named interface)
|
| 766 |
+
- type AnimatedContentProps = { ... } (named type alias)
|
| 767 |
+
- function Foo({ a, b }: { a: string; b?: number }) (inline params)
|
| 768 |
+
|
| 769 |
+
Uses brace balancing so nested types like `config?: { tension: number }`
|
| 770 |
+
are captured correctly.
|
| 771 |
+
"""
|
| 772 |
+
# ββ Pattern A: named interface or type alias ββββββββββββββββββββββββββββββ
|
| 773 |
+
header = re.search(
|
| 774 |
+
r'(?:interface|type)\s+\w*[Pp]rops?\w*\s*(?:=\s*)?(?:<[^>]*>)?\s*\{',
|
| 775 |
+
content,
|
| 776 |
+
)
|
| 777 |
+
if header:
|
| 778 |
+
snippet = _extract_balanced_braces(content, header.start())
|
| 779 |
+
if snippet:
|
| 780 |
+
return _trim(snippet)
|
| 781 |
+
|
| 782 |
+
# ββ Pattern B: inline destructured params ββββββββββββββββββββββββββββββββ
|
| 783 |
+
# Matches: ComponentName({ propA, propB }: { propA: type; propB?: type })
|
| 784 |
+
inline = re.search(
|
| 785 |
+
r'(?:function\s+\w+|const\s+\w+\s*=\s*(?:React\.)?(?:forwardRef\()?(?:async\s+)?function\s*\w*|\(\s*)\(\s*\{[^}]*\}\s*:\s*\{',
|
| 786 |
+
content,
|
| 787 |
+
)
|
| 788 |
+
if inline:
|
| 789 |
+
# Find the opening brace of the type annotation
|
| 790 |
+
brace_pos = content.find(': {', inline.start())
|
| 791 |
+
if brace_pos != -1:
|
| 792 |
+
snippet = _extract_balanced_braces(content, brace_pos + 2)
|
| 793 |
+
if snippet:
|
| 794 |
+
return _trim(snippet)
|
| 795 |
+
|
| 796 |
+
return None
|
| 797 |
+
|
| 798 |
+
|
| 799 |
+
def _extract_balanced_braces(content: str, start: int) -> str | None:
|
| 800 |
+
"""Return the substring from `start` through the matching closing brace."""
|
| 801 |
+
brace_start = content.find('{', start)
|
| 802 |
+
if brace_start == -1:
|
| 803 |
+
return None
|
| 804 |
+
depth = 0
|
| 805 |
+
for i in range(brace_start, min(brace_start + 2000, len(content))):
|
| 806 |
+
if content[i] == '{':
|
| 807 |
+
depth += 1
|
| 808 |
+
elif content[i] == '}':
|
| 809 |
+
depth -= 1
|
| 810 |
+
if depth == 0:
|
| 811 |
+
return content[start: i + 1]
|
| 812 |
+
return None
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
def _trim(snippet: str, max_len: int = 700) -> str:
|
| 816 |
+
snippet = snippet.strip()
|
| 817 |
+
if len(snippet) > max_len:
|
| 818 |
+
snippet = snippet[:max_len - 3] + "..."
|
| 819 |
+
return snippet
|
backend/services/ingestion/github_ingester.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
GitHub repository ingester.
|
| 3 |
+
|
| 4 |
+
Primary path β GitHub Contents API (no clone, no disk I/O, fast):
|
| 5 |
+
Fetches README, docs/, and .d.ts files via authenticated HTTPS API calls.
|
| 6 |
+
Works for repos of any size; no git toolchain required.
|
| 7 |
+
|
| 8 |
+
Fallback path β shallow git clone:
|
| 9 |
+
Used only when the repo is not on github.com (e.g. GitLab, self-hosted).
|
| 10 |
+
120 s timeout kept for compatibility.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import base64
|
| 14 |
+
import logging
|
| 15 |
+
import os
|
| 16 |
+
import re
|
| 17 |
+
import subprocess
|
| 18 |
+
import tempfile
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import httpx
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
|
| 25 |
+
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")
|
| 26 |
+
_GH_API = "https://api.github.com"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _gh_headers() -> dict[str, str]:
|
| 30 |
+
h = {"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"}
|
| 31 |
+
if GITHUB_TOKEN:
|
| 32 |
+
h["Authorization"] = f"Bearer {GITHUB_TOKEN}"
|
| 33 |
+
return h
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _parse_owner_repo(url: str) -> tuple[str, str] | None:
|
| 37 |
+
"""Extract (owner, repo) from any github.com URL variant."""
|
| 38 |
+
m = re.search(r"github\.com[/:]([^/]+)/([^/.]+)", url)
|
| 39 |
+
if m:
|
| 40 |
+
return m.group(1), m.group(2)
|
| 41 |
+
return None
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
async def _api_file_content(client: httpx.AsyncClient, owner: str, repo: str, path: str, ref: str | None) -> str | None:
|
| 45 |
+
"""Fetch a single file from the GitHub contents API. Returns decoded text or None."""
|
| 46 |
+
url = f"{_GH_API}/repos/{owner}/{repo}/contents/{path}"
|
| 47 |
+
params = {"ref": ref} if ref else {}
|
| 48 |
+
try:
|
| 49 |
+
r = await client.get(url, headers=_gh_headers(), params=params, timeout=15)
|
| 50 |
+
if r.status_code != 200:
|
| 51 |
+
return None
|
| 52 |
+
data = r.json()
|
| 53 |
+
if isinstance(data, list):
|
| 54 |
+
return None # it's a directory
|
| 55 |
+
encoded = data.get("content", "")
|
| 56 |
+
return base64.b64decode(encoded).decode("utf-8", errors="ignore")
|
| 57 |
+
except Exception as exc:
|
| 58 |
+
logger.debug("API fetch failed for %s/%s/%s: %s", owner, repo, path, exc)
|
| 59 |
+
return None
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
async def _api_list_dir(client: httpx.AsyncClient, owner: str, repo: str, path: str, ref: str | None) -> list[dict] | None:
|
| 63 |
+
"""List directory contents via the GitHub contents API."""
|
| 64 |
+
url = f"{_GH_API}/repos/{owner}/{repo}/contents/{path}"
|
| 65 |
+
params = {"ref": ref} if ref else {}
|
| 66 |
+
try:
|
| 67 |
+
r = await client.get(url, headers=_gh_headers(), params=params, timeout=15)
|
| 68 |
+
if r.status_code != 200:
|
| 69 |
+
return None
|
| 70 |
+
data = r.json()
|
| 71 |
+
return data if isinstance(data, list) else None
|
| 72 |
+
except Exception:
|
| 73 |
+
return None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
async def _api_readme(client: httpx.AsyncClient, owner: str, repo: str, ref: str | None) -> str | None:
|
| 77 |
+
"""Fetch the repo README via the dedicated GitHub endpoint (handles any casing/extension)."""
|
| 78 |
+
url = f"{_GH_API}/repos/{owner}/{repo}/readme"
|
| 79 |
+
params = {"ref": ref} if ref else {}
|
| 80 |
+
try:
|
| 81 |
+
r = await client.get(url, headers=_gh_headers(), params=params, timeout=15)
|
| 82 |
+
if r.status_code != 200:
|
| 83 |
+
return None
|
| 84 |
+
encoded = r.json().get("content", "")
|
| 85 |
+
return base64.b64decode(encoded).decode("utf-8", errors="ignore")
|
| 86 |
+
except Exception:
|
| 87 |
+
return None
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
async def _api_ingest(owner: str, repo: str, ref: str | None) -> str:
|
| 91 |
+
"""
|
| 92 |
+
Fetch documentation from GitHub using the Contents API β no git clone.
|
| 93 |
+
Retrieves README, docs/ folder (up to 30 .md/.mdx files),
|
| 94 |
+
top-level .d.ts files, and package.json.
|
| 95 |
+
"""
|
| 96 |
+
content_parts: list[str] = []
|
| 97 |
+
|
| 98 |
+
async with httpx.AsyncClient(timeout=20) as client:
|
| 99 |
+
# 1. README
|
| 100 |
+
readme = await _api_readme(client, owner, repo, ref)
|
| 101 |
+
if readme:
|
| 102 |
+
content_parts.append(f"# README\n\n{readme}")
|
| 103 |
+
logger.debug("API ingest: README fetched (%d chars)", len(readme))
|
| 104 |
+
|
| 105 |
+
# 2. docs/ folder
|
| 106 |
+
for docs_dir in ["docs", "documentation", "doc"]:
|
| 107 |
+
entries = await _api_list_dir(client, owner, repo, docs_dir, ref)
|
| 108 |
+
if entries is None:
|
| 109 |
+
continue
|
| 110 |
+
# Collect .md / .mdx files recursively (one level for now)
|
| 111 |
+
md_entries = [e for e in entries if e.get("type") == "file"
|
| 112 |
+
and e.get("name", "").endswith((".md", ".mdx"))][:30]
|
| 113 |
+
for entry in md_entries:
|
| 114 |
+
text = await _api_file_content(client, owner, repo, entry["path"], ref)
|
| 115 |
+
if text:
|
| 116 |
+
content_parts.append(f"\n\n<!-- FILE: {entry['path']} -->\n{text}")
|
| 117 |
+
if md_entries:
|
| 118 |
+
logger.debug("API ingest: %d doc files from %s/", len(md_entries), docs_dir)
|
| 119 |
+
break
|
| 120 |
+
|
| 121 |
+
# 3. Top-level .d.ts files (type definitions)
|
| 122 |
+
root_entries = await _api_list_dir(client, owner, repo, "", ref)
|
| 123 |
+
if root_entries:
|
| 124 |
+
dts_entries = [e for e in root_entries if e.get("type") == "file"
|
| 125 |
+
and e.get("name", "").endswith(".d.ts")][:20]
|
| 126 |
+
dts_parts: list[str] = []
|
| 127 |
+
for entry in dts_entries:
|
| 128 |
+
text = await _api_file_content(client, owner, repo, entry["path"], ref)
|
| 129 |
+
if text:
|
| 130 |
+
dts_parts.append(f"// FILE: {entry['path']}\n{text}")
|
| 131 |
+
if dts_parts:
|
| 132 |
+
content_parts.append(
|
| 133 |
+
"\n\n## TypeScript Type Definitions\n\n```typescript\n"
|
| 134 |
+
+ "\n\n".join(dts_parts) + "\n```"
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
# 4. package.json
|
| 138 |
+
pkg_entry = next((e for e in root_entries if e.get("name") == "package.json"), None)
|
| 139 |
+
if pkg_entry:
|
| 140 |
+
pkg_text = await _api_file_content(client, owner, repo, "package.json", ref)
|
| 141 |
+
if pkg_text:
|
| 142 |
+
content_parts.append(f"\n\n## package.json\n\n```json\n{pkg_text}\n```")
|
| 143 |
+
|
| 144 |
+
combined = "\n".join(content_parts)
|
| 145 |
+
logger.info("GitHub API ingest done for %s/%s@%s: %d chars", owner, repo, ref or "default", len(combined))
|
| 146 |
+
return combined
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
async def ingest_github_repo(repo_url: str, version_tag: str | None = None) -> str:
|
| 150 |
+
"""
|
| 151 |
+
Ingest documentation from a GitHub repository.
|
| 152 |
+
|
| 153 |
+
For github.com repos: uses the GitHub Contents API (fast, no clone).
|
| 154 |
+
For other hosts: falls back to shallow git clone (120 s timeout).
|
| 155 |
+
"""
|
| 156 |
+
parsed = _parse_owner_repo(repo_url)
|
| 157 |
+
|
| 158 |
+
if parsed:
|
| 159 |
+
owner, repo = parsed
|
| 160 |
+
ref: str | None = None
|
| 161 |
+
|
| 162 |
+
# Try version tag refs in order: v1.2.3, then 1.2.3
|
| 163 |
+
if version_tag:
|
| 164 |
+
async with httpx.AsyncClient(timeout=10) as client:
|
| 165 |
+
for candidate in [f"v{version_tag}", version_tag]:
|
| 166 |
+
tag_resp = await client.get(
|
| 167 |
+
f"{_GH_API}/repos/{owner}/{repo}/git/ref/tags/{candidate}",
|
| 168 |
+
headers=_gh_headers(),
|
| 169 |
+
)
|
| 170 |
+
if tag_resp.status_code == 200:
|
| 171 |
+
ref = candidate
|
| 172 |
+
logger.info("Using tag %s for %s/%s", candidate, owner, repo)
|
| 173 |
+
break
|
| 174 |
+
else:
|
| 175 |
+
logger.warning("Tag %s not found in %s/%s β using default branch", version_tag, owner, repo)
|
| 176 |
+
|
| 177 |
+
return await _api_ingest(owner, repo, ref)
|
| 178 |
+
|
| 179 |
+
# ββ Fallback: non-GitHub host β shallow clone ββββββββββββββββββββββββββ
|
| 180 |
+
clone_url = _normalize_clone_url(repo_url)
|
| 181 |
+
return _clone_and_extract(clone_url, version_tag)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def _clone_and_extract(repo_url: str, version_tag: str | None) -> str:
|
| 185 |
+
"""Shallow clone fallback for non-GitHub hosts."""
|
| 186 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 187 |
+
result = None
|
| 188 |
+
|
| 189 |
+
if version_tag:
|
| 190 |
+
for tag in [f"v{version_tag}", version_tag]:
|
| 191 |
+
logger.info("Cloning %s at tag %s", repo_url, tag)
|
| 192 |
+
result = subprocess.run(
|
| 193 |
+
["git", "clone", "--depth=1", "--quiet", "--branch", tag, repo_url, tmpdir],
|
| 194 |
+
capture_output=True, text=True, timeout=120,
|
| 195 |
+
)
|
| 196 |
+
if result.returncode == 0:
|
| 197 |
+
break
|
| 198 |
+
else:
|
| 199 |
+
result = None
|
| 200 |
+
|
| 201 |
+
if result is None or result.returncode != 0:
|
| 202 |
+
logger.info("Cloning %s (default branch)", repo_url)
|
| 203 |
+
result = subprocess.run(
|
| 204 |
+
["git", "clone", "--depth=1", "--quiet", repo_url, tmpdir],
|
| 205 |
+
capture_output=True, text=True, timeout=120,
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
if result.returncode != 0:
|
| 209 |
+
raise RuntimeError(f"git clone failed: {result.stderr}")
|
| 210 |
+
|
| 211 |
+
content_parts: list[str] = []
|
| 212 |
+
repo_path = Path(tmpdir)
|
| 213 |
+
|
| 214 |
+
for readme_name in ["README.md", "README.mdx", "readme.md"]:
|
| 215 |
+
readme = repo_path / readme_name
|
| 216 |
+
if readme.exists():
|
| 217 |
+
content_parts.append(f"# README\n\n{readme.read_text(errors='ignore')}")
|
| 218 |
+
break
|
| 219 |
+
|
| 220 |
+
for docs_dir_name in ["docs", "documentation", "doc"]:
|
| 221 |
+
docs_dir = repo_path / docs_dir_name
|
| 222 |
+
if docs_dir.is_dir():
|
| 223 |
+
md_files = sorted(docs_dir.rglob("*.md")) + sorted(docs_dir.rglob("*.mdx"))
|
| 224 |
+
for md_file in md_files[:30]:
|
| 225 |
+
try:
|
| 226 |
+
text = md_file.read_text(errors="ignore")
|
| 227 |
+
rel = md_file.relative_to(repo_path)
|
| 228 |
+
content_parts.append(f"\n\n<!-- FILE: {rel} -->\n{text}")
|
| 229 |
+
except Exception as exc:
|
| 230 |
+
logger.debug("Skipping %s: %s", md_file, exc)
|
| 231 |
+
break
|
| 232 |
+
|
| 233 |
+
dts_files = list(repo_path.rglob("*.d.ts"))[:20]
|
| 234 |
+
dts_parts = []
|
| 235 |
+
for dts in dts_files:
|
| 236 |
+
try:
|
| 237 |
+
text = dts.read_text(errors="ignore")
|
| 238 |
+
rel = dts.relative_to(repo_path)
|
| 239 |
+
dts_parts.append(f"// FILE: {rel}\n{text}")
|
| 240 |
+
except Exception:
|
| 241 |
+
pass
|
| 242 |
+
if dts_parts:
|
| 243 |
+
content_parts.append(
|
| 244 |
+
"\n\n## TypeScript Type Definitions\n\n```typescript\n"
|
| 245 |
+
+ "\n\n".join(dts_parts) + "\n```"
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
pkg_json = repo_path / "package.json"
|
| 249 |
+
if pkg_json.exists():
|
| 250 |
+
content_parts.append(f"\n\n## package.json\n\n```json\n{pkg_json.read_text()}\n```")
|
| 251 |
+
|
| 252 |
+
combined = "\n".join(content_parts)
|
| 253 |
+
logger.info("Clone ingest done: %d chars", len(combined))
|
| 254 |
+
return combined
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _normalize_clone_url(url: str) -> str:
|
| 258 |
+
"""Convert various URL formats to a cloneable HTTPS URL."""
|
| 259 |
+
url = re.sub(r"^git\+", "", url)
|
| 260 |
+
url = re.sub(r"^git://", "https://", url)
|
| 261 |
+
url = url.split("#")[0]
|
| 262 |
+
url = re.sub(r"(github\.com/[^/]+/[^/]+)/(tree|blob)/.*", r"\1", url)
|
| 263 |
+
url = url.rstrip("/")
|
| 264 |
+
if not url.endswith(".git"):
|
| 265 |
+
url += ".git"
|
| 266 |
+
return url
|
backend/services/ingestion/npm_ingester.py
ADDED
|
@@ -0,0 +1,379 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
npm/PyPI package ingester.
|
| 3 |
+
|
| 4 |
+
Resolves a package name + version to its documentation URL,
|
| 5 |
+
then fetches and returns raw Markdown content.
|
| 6 |
+
|
| 7 |
+
Phase 1: httpx-based static crawl (fast, works for most docs).
|
| 8 |
+
Phase 2 (Playwright) is in url_ingester.py for JS-rendered sites.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import logging
|
| 12 |
+
import re
|
| 13 |
+
from urllib.parse import urljoin, urlparse
|
| 14 |
+
|
| 15 |
+
import httpx
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
NPM_REGISTRY = "https://registry.npmjs.org"
|
| 20 |
+
PYPI_REGISTRY = "https://pypi.org/pypi"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
async def resolve_npm_package(package_input: str) -> dict:
|
| 24 |
+
"""
|
| 25 |
+
Parse 'react-bits@2.1.4' or '@tanstack/react-query@5.0.0' and
|
| 26 |
+
fetch metadata from the npm registry.
|
| 27 |
+
"""
|
| 28 |
+
name, version = _parse_npm_input(package_input)
|
| 29 |
+
|
| 30 |
+
async with httpx.AsyncClient(timeout=30) as client:
|
| 31 |
+
# Scoped packages (@scope/name) need the slash preserved in the URL.
|
| 32 |
+
# Packages like "@react-bits" with no sub-name are invalid scoped names β
|
| 33 |
+
# strip the @ and try as a plain package.
|
| 34 |
+
resolved_name = name
|
| 35 |
+
if name.startswith("@") and "/" not in name:
|
| 36 |
+
resolved_name = name.lstrip("@")
|
| 37 |
+
logger.warning("Invalid scoped name %r (no slash) β trying as plain package %r", name, resolved_name)
|
| 38 |
+
|
| 39 |
+
url = f"{NPM_REGISTRY}/{resolved_name}/{version}"
|
| 40 |
+
resp = await client.get(url)
|
| 41 |
+
|
| 42 |
+
if resp.status_code == 404:
|
| 43 |
+
result = await _resolve_not_found(client, resolved_name, version)
|
| 44 |
+
if result:
|
| 45 |
+
return result
|
| 46 |
+
suffix = f"@{version}" if version != "latest" else ""
|
| 47 |
+
raise ValueError(f"'{resolved_name}{suffix}' was not found on npm or any known registry.")
|
| 48 |
+
|
| 49 |
+
resp.raise_for_status()
|
| 50 |
+
data = resp.json()
|
| 51 |
+
|
| 52 |
+
homepage = data.get("homepage") or _extract_github_pages(data)
|
| 53 |
+
repository_url = (data.get("repository") or {}).get("url", "")
|
| 54 |
+
|
| 55 |
+
return {
|
| 56 |
+
"name": data["name"],
|
| 57 |
+
"version": data["version"],
|
| 58 |
+
"description": data.get("description"),
|
| 59 |
+
"homepage": homepage,
|
| 60 |
+
"repository": repository_url,
|
| 61 |
+
"keywords": data.get("keywords", []),
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
async def resolve_pypi_package(package_input: str) -> dict:
|
| 66 |
+
"""
|
| 67 |
+
Parse 'fastapi' or 'fastapi==0.110.0' and fetch metadata from PyPI.
|
| 68 |
+
"""
|
| 69 |
+
if "==" in package_input:
|
| 70 |
+
name, version = package_input.split("==", 1)
|
| 71 |
+
else:
|
| 72 |
+
name, version = package_input, None
|
| 73 |
+
|
| 74 |
+
async with httpx.AsyncClient(timeout=30) as client:
|
| 75 |
+
if version:
|
| 76 |
+
url = f"{PYPI_REGISTRY}/{name}/{version}/json"
|
| 77 |
+
else:
|
| 78 |
+
url = f"{PYPI_REGISTRY}/{name}/json"
|
| 79 |
+
resp = await client.get(url)
|
| 80 |
+
resp.raise_for_status()
|
| 81 |
+
data = resp.json()
|
| 82 |
+
|
| 83 |
+
info = data["info"]
|
| 84 |
+
homepage = info.get("home_page") or info.get("project_urls", {}).get("Documentation")
|
| 85 |
+
resolved_version = info["version"]
|
| 86 |
+
|
| 87 |
+
return {
|
| 88 |
+
"name": info["name"],
|
| 89 |
+
"version": resolved_version,
|
| 90 |
+
"description": info.get("summary"),
|
| 91 |
+
"homepage": homepage,
|
| 92 |
+
"repository": info.get("project_urls", {}).get("Source"),
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
async def crawl_static_docs(url: str, max_pages: int = 10) -> str:
|
| 97 |
+
"""
|
| 98 |
+
Crawl a documentation site using httpx (fast, works for static HTML).
|
| 99 |
+
Returns concatenated Markdown-ish text extracted from the HTML.
|
| 100 |
+
Falls back to url_ingester's Playwright crawler for JS-rendered sites.
|
| 101 |
+
"""
|
| 102 |
+
logger.info("Static crawl starting at %s", url)
|
| 103 |
+
visited: set[str] = set()
|
| 104 |
+
pages: list[str] = []
|
| 105 |
+
|
| 106 |
+
base = urlparse(url)
|
| 107 |
+
base_origin = f"{base.scheme}://{base.netloc}"
|
| 108 |
+
|
| 109 |
+
to_visit = [url]
|
| 110 |
+
|
| 111 |
+
async with httpx.AsyncClient(
|
| 112 |
+
timeout=30,
|
| 113 |
+
headers={"User-Agent": "DocForge/1.0 (documentation indexer)"},
|
| 114 |
+
follow_redirects=True,
|
| 115 |
+
) as client:
|
| 116 |
+
while to_visit and len(pages) < max_pages:
|
| 117 |
+
current_url = to_visit.pop(0)
|
| 118 |
+
if current_url in visited:
|
| 119 |
+
continue
|
| 120 |
+
visited.add(current_url)
|
| 121 |
+
|
| 122 |
+
try:
|
| 123 |
+
resp = await client.get(current_url)
|
| 124 |
+
if resp.status_code != 200:
|
| 125 |
+
continue
|
| 126 |
+
ct = resp.headers.get("content-type", "")
|
| 127 |
+
if "html" not in ct:
|
| 128 |
+
continue
|
| 129 |
+
|
| 130 |
+
text = _html_to_text(resp.text)
|
| 131 |
+
pages.append(f"\n\n<!-- SOURCE: {current_url} -->\n{text}")
|
| 132 |
+
logger.debug("Crawled %s (%d chars)", current_url, len(text))
|
| 133 |
+
|
| 134 |
+
# Discover links on the same origin that look like doc pages
|
| 135 |
+
new_links = _extract_doc_links(resp.text, base_origin, current_url)
|
| 136 |
+
for link in new_links:
|
| 137 |
+
if link not in visited:
|
| 138 |
+
to_visit.append(link)
|
| 139 |
+
|
| 140 |
+
except Exception as exc:
|
| 141 |
+
logger.warning("Failed to fetch %s: %s", current_url, exc)
|
| 142 |
+
|
| 143 |
+
combined = "\n".join(pages)
|
| 144 |
+
logger.info("Static crawl done: %d pages, %d chars total", len(pages), len(combined))
|
| 145 |
+
return combined
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
# ---------------------------------------------------------------------------
|
| 149 |
+
# Internal helpers
|
| 150 |
+
# ---------------------------------------------------------------------------
|
| 151 |
+
|
| 152 |
+
def _parse_npm_input(package_input: str) -> tuple[str, str]:
|
| 153 |
+
"""Handle both 'pkg@version' and '@scope/pkg@version' forms.
|
| 154 |
+
|
| 155 |
+
Strips surrounding quotes/whitespace and semver range prefixes so raw
|
| 156 |
+
package.json values like '@emotion/styled": "^11.14.0' are handled cleanly.
|
| 157 |
+
"""
|
| 158 |
+
package_input = package_input.strip().strip("\"'")
|
| 159 |
+
|
| 160 |
+
# Reject URL-based dependency specs (e.g. "https://www.reactbits.dev/")
|
| 161 |
+
if package_input.startswith(("http://", "https://", "git+", "git://", "github:", "file:")):
|
| 162 |
+
raise ValueError(
|
| 163 |
+
f"Input looks like a URL, not an npm package name: {package_input!r}. "
|
| 164 |
+
"Use input_type='url' instead."
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
# Reject filenames mistaken for package names (package.json, tsconfig.json, etc.)
|
| 168 |
+
_KNOWN_FILENAMES = {
|
| 169 |
+
"package.json", "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
|
| 170 |
+
"npm-shrinkwrap.json", "tsconfig.json", ".npmrc", ".nvmrc",
|
| 171 |
+
}
|
| 172 |
+
if package_input.lower() in _KNOWN_FILENAMES:
|
| 173 |
+
raise ValueError(
|
| 174 |
+
f"'{package_input}' is a filename, not an npm package name. "
|
| 175 |
+
"Pass the actual package name (e.g. 'react', 'lodash')."
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
if package_input.startswith("@"):
|
| 179 |
+
# Scoped package: extract @scope/name, then optionally @version
|
| 180 |
+
m = re.match(r'^(@[A-Za-z0-9._-]+/[A-Za-z0-9._-]+)(?:@([^\s"\':]*))?', package_input)
|
| 181 |
+
if m:
|
| 182 |
+
name = m.group(1)
|
| 183 |
+
raw_version = m.group(2) or ""
|
| 184 |
+
else:
|
| 185 |
+
# Fallback: split on last '@' if there are two or more
|
| 186 |
+
at_count = package_input.count("@")
|
| 187 |
+
if at_count >= 2:
|
| 188 |
+
last_at = package_input.rfind("@")
|
| 189 |
+
name = package_input[:last_at]
|
| 190 |
+
raw_version = package_input[last_at + 1:]
|
| 191 |
+
else:
|
| 192 |
+
name = package_input
|
| 193 |
+
raw_version = ""
|
| 194 |
+
else:
|
| 195 |
+
if "@" in package_input:
|
| 196 |
+
name, raw_version = package_input.rsplit("@", 1)
|
| 197 |
+
else:
|
| 198 |
+
name = package_input
|
| 199 |
+
raw_version = ""
|
| 200 |
+
|
| 201 |
+
# Strip semver range prefixes (^, ~, >=, <=, >, <, =) from version
|
| 202 |
+
version = re.sub(r"^[^0-9]*", "", raw_version.strip().strip("\"'"))
|
| 203 |
+
|
| 204 |
+
# If still no version, scan the original input for any semver-like number
|
| 205 |
+
# (handles raw package.json values like '@scope/pkg": "^6.4.8"')
|
| 206 |
+
if not version:
|
| 207 |
+
fallback = re.search(r'(\d+\.\d+[\.\d]*)', package_input)
|
| 208 |
+
version = fallback.group(1) if fallback else "latest"
|
| 209 |
+
|
| 210 |
+
return name.strip(), version
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
async def _resolve_not_found(
|
| 214 |
+
client: httpx.AsyncClient,
|
| 215 |
+
name: str,
|
| 216 |
+
version: str,
|
| 217 |
+
) -> dict | None:
|
| 218 |
+
"""
|
| 219 |
+
Full cascade for when npm returns 404 on a package.
|
| 220 |
+
|
| 221 |
+
1. Fetch npm package root (exists even when a specific version doesn't)
|
| 222 |
+
β version-specific 404: return GitHub tag metadata so pipeline can
|
| 223 |
+
clone the exact git tag.
|
| 224 |
+
2. Extract GitHub URL from npm root data (or search GitHub if not found).
|
| 225 |
+
3. Run copy-paste detection (jsrepo / shadcn registry / README patterns /
|
| 226 |
+
no-dist-exports).
|
| 227 |
+
4. Fall back to plain GitHub ingestion if the repo is found but isn't
|
| 228 |
+
a copy-paste library.
|
| 229 |
+
"""
|
| 230 |
+
github_url: str | None = None
|
| 231 |
+
homepage: str | None = None
|
| 232 |
+
npm_root: dict | None = None
|
| 233 |
+
|
| 234 |
+
# ββ Step 1: npm package root ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 235 |
+
pkg_resp = await client.get(f"{NPM_REGISTRY}/{name}")
|
| 236 |
+
if pkg_resp.status_code == 200:
|
| 237 |
+
npm_root = pkg_resp.json()
|
| 238 |
+
dist_tags = npm_root.get("dist-tags") or {}
|
| 239 |
+
latest_ver = dist_tags.get("latest", "")
|
| 240 |
+
homepage = npm_root.get("homepage")
|
| 241 |
+
|
| 242 |
+
# For a version-specific 404: try GitHub git-tag ingestion first
|
| 243 |
+
if version != "latest" and latest_ver:
|
| 244 |
+
latest_meta = (npm_root.get("versions") or {}).get(latest_ver, {})
|
| 245 |
+
repo_url = (latest_meta.get("repository") or {}).get("url", "")
|
| 246 |
+
if repo_url and "github.com" in repo_url:
|
| 247 |
+
logger.info(
|
| 248 |
+
"npm %s@%s not on registry β GitHub repo found (%s), will try git tag",
|
| 249 |
+
name, version, repo_url,
|
| 250 |
+
)
|
| 251 |
+
return {
|
| 252 |
+
"name": name,
|
| 253 |
+
"version": version,
|
| 254 |
+
"description": npm_root.get("description"),
|
| 255 |
+
"homepage": None,
|
| 256 |
+
"repository": repo_url,
|
| 257 |
+
"keywords": npm_root.get("keywords", []),
|
| 258 |
+
"_version_tag": version,
|
| 259 |
+
}
|
| 260 |
+
latest_hint = f" Latest on npm: {latest_ver}." if latest_ver else ""
|
| 261 |
+
logger.warning("npm %s@%s not found.%s Running copy-paste detection.", name, version, latest_hint)
|
| 262 |
+
|
| 263 |
+
# Extract GitHub URL from latest version metadata
|
| 264 |
+
if latest_ver:
|
| 265 |
+
latest_meta = (npm_root.get("versions") or {}).get(latest_ver, {})
|
| 266 |
+
repo_url = (latest_meta.get("repository") or {}).get("url", "")
|
| 267 |
+
if "github.com" in repo_url:
|
| 268 |
+
github_url = repo_url
|
| 269 |
+
|
| 270 |
+
# ββ Step 2: GitHub search if repo still unknown βββββββββββββββββββββββββββ
|
| 271 |
+
if not github_url:
|
| 272 |
+
github_url = await _github_search(client, name)
|
| 273 |
+
|
| 274 |
+
# ββ Step 3: Copy-paste detection ββββββββββββββββββββββββββββββββββββββββββ
|
| 275 |
+
if github_url or homepage:
|
| 276 |
+
from .copypaste_ingester import detect_and_ingest
|
| 277 |
+
result = await detect_and_ingest(
|
| 278 |
+
name=name,
|
| 279 |
+
github_url=github_url,
|
| 280 |
+
homepage=homepage or github_url,
|
| 281 |
+
npm_data=npm_root,
|
| 282 |
+
)
|
| 283 |
+
if result:
|
| 284 |
+
return result
|
| 285 |
+
|
| 286 |
+
# ββ Step 4: Plain GitHub fallback (repo found, not copy-paste) βββββββββββ
|
| 287 |
+
if github_url:
|
| 288 |
+
logger.info("npm '%s' not found β falling back to GitHub ingestion: %s", name, github_url)
|
| 289 |
+
return {
|
| 290 |
+
"name": name,
|
| 291 |
+
"version": version if version != "latest" else "latest",
|
| 292 |
+
"description": (npm_root or {}).get("description"),
|
| 293 |
+
"homepage": homepage or github_url,
|
| 294 |
+
"repository": github_url,
|
| 295 |
+
"keywords": (npm_root or {}).get("keywords", []),
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
return None
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
async def _github_search(client: httpx.AsyncClient, name: str) -> str | None:
|
| 302 |
+
"""Search GitHub repositories for the most likely match for this package name."""
|
| 303 |
+
# Clean up scoped names (@scope/pkg β scope pkg) for better search results
|
| 304 |
+
query = name.lstrip("@").replace("/", " ").replace("-", " ")
|
| 305 |
+
url = (
|
| 306 |
+
f"https://api.github.com/search/repositories"
|
| 307 |
+
f"?q={query}&per_page=3&sort=stars&order=desc"
|
| 308 |
+
)
|
| 309 |
+
try:
|
| 310 |
+
resp = await client.get(
|
| 311 |
+
url,
|
| 312 |
+
headers={"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"},
|
| 313 |
+
)
|
| 314 |
+
if resp.is_success:
|
| 315 |
+
items = resp.json().get("items", [])
|
| 316 |
+
if items:
|
| 317 |
+
# Prefer repos whose name closely matches the package name
|
| 318 |
+
clean_name = name.lstrip("@").split("/")[-1].lower()
|
| 319 |
+
for item in items:
|
| 320 |
+
if clean_name in item.get("name", "").lower():
|
| 321 |
+
return item["html_url"]
|
| 322 |
+
return items[0]["html_url"]
|
| 323 |
+
except Exception as exc:
|
| 324 |
+
logger.warning("GitHub search failed for '%s': %s", name, exc)
|
| 325 |
+
return None
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def _extract_github_pages(data: dict) -> str | None:
|
| 329 |
+
"""Try to derive a docs URL from the GitHub repository URL."""
|
| 330 |
+
repo = (data.get("repository") or {}).get("url", "")
|
| 331 |
+
if "github.com" in repo:
|
| 332 |
+
# Convert git+https://github.com/user/repo.git β https://user.github.io/repo
|
| 333 |
+
match = re.search(r"github\.com[:/]([^/]+)/([^/.]+)", repo)
|
| 334 |
+
if match:
|
| 335 |
+
user, repo_name = match.group(1), match.group(2)
|
| 336 |
+
return f"https://{user}.github.io/{repo_name}"
|
| 337 |
+
return None
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def _html_to_text(html: str) -> str:
|
| 341 |
+
"""
|
| 342 |
+
Very lightweight HTML β text converter.
|
| 343 |
+
We strip tags and normalize whitespace so Gemini can process the content.
|
| 344 |
+
For production, Docling by IBM does a much better job with complex layouts.
|
| 345 |
+
"""
|
| 346 |
+
# Remove scripts and styles completely
|
| 347 |
+
html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
|
| 348 |
+
# Replace block-level tags with newlines
|
| 349 |
+
html = re.sub(r"<(h[1-6]|p|div|section|article|li|tr|br)[^>]*>", "\n", html, flags=re.IGNORECASE)
|
| 350 |
+
# Remove remaining tags
|
| 351 |
+
html = re.sub(r"<[^>]+>", "", html)
|
| 352 |
+
# Decode basic HTML entities
|
| 353 |
+
html = html.replace("<", "<").replace(">", ">").replace("&", "&").replace(" ", " ").replace("'", "'").replace(""", '"')
|
| 354 |
+
# Collapse excessive whitespace
|
| 355 |
+
html = re.sub(r"\n{3,}", "\n\n", html)
|
| 356 |
+
html = re.sub(r" {2,}", " ", html)
|
| 357 |
+
return html.strip()
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def _extract_doc_links(html: str, base_origin: str, current_url: str) -> list[str]:
|
| 361 |
+
"""Extract same-origin links that look like documentation pages."""
|
| 362 |
+
hrefs = re.findall(r'href=["\']([^"\'#?]+)["\']', html)
|
| 363 |
+
links = []
|
| 364 |
+
for href in hrefs:
|
| 365 |
+
if href.startswith("http"):
|
| 366 |
+
full = href
|
| 367 |
+
elif href.startswith("/"):
|
| 368 |
+
full = base_origin + href
|
| 369 |
+
else:
|
| 370 |
+
full = urljoin(current_url, href)
|
| 371 |
+
|
| 372 |
+
parsed = urlparse(full)
|
| 373 |
+
if parsed.netloc != urlparse(base_origin).netloc:
|
| 374 |
+
continue
|
| 375 |
+
# Skip obviously non-doc paths
|
| 376 |
+
if any(ext in parsed.path for ext in [".png", ".jpg", ".css", ".js", ".svg", ".ico", ".woff"]):
|
| 377 |
+
continue
|
| 378 |
+
links.append(full)
|
| 379 |
+
return links[:20] # Safety cap per page
|
backend/services/ingestion/paste_ingester.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Paste/upload ingester.
|
| 3 |
+
|
| 4 |
+
Accepts raw HTML, PDF (via Docling), or plain Markdown from the user
|
| 5 |
+
and converts it to clean Markdown suitable for Gemini structuring.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
import re
|
| 10 |
+
|
| 11 |
+
logger = logging.getLogger(__name__)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
async def ingest_paste(content: str, content_type: str = "auto") -> str:
|
| 15 |
+
"""
|
| 16 |
+
Process pasted or uploaded content into clean Markdown.
|
| 17 |
+
|
| 18 |
+
content_type: "html" | "markdown" | "pdf_base64" | "auto"
|
| 19 |
+
"""
|
| 20 |
+
if content_type == "auto":
|
| 21 |
+
content_type = _detect_content_type(content)
|
| 22 |
+
|
| 23 |
+
logger.info("Processing paste content as %s (%d chars)", content_type, len(content))
|
| 24 |
+
|
| 25 |
+
if content_type == "html":
|
| 26 |
+
return _html_to_markdown(content)
|
| 27 |
+
elif content_type == "pdf_base64":
|
| 28 |
+
return await _pdf_to_markdown(content)
|
| 29 |
+
else:
|
| 30 |
+
# Already Markdown or plain text β pass through with minimal cleanup
|
| 31 |
+
return _clean_markdown(content)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _detect_content_type(content: str) -> str:
|
| 35 |
+
"""Heuristic detection of content format."""
|
| 36 |
+
stripped = content.strip()
|
| 37 |
+
if stripped.startswith("<!DOCTYPE") or stripped.startswith("<html") or stripped.count("<div") > 3:
|
| 38 |
+
return "html"
|
| 39 |
+
# Base64-encoded PDF starts with JVBERi0
|
| 40 |
+
if stripped.startswith("JVBERi0"):
|
| 41 |
+
return "pdf_base64"
|
| 42 |
+
return "markdown"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _html_to_markdown(html: str) -> str:
|
| 46 |
+
"""
|
| 47 |
+
Convert HTML to Markdown.
|
| 48 |
+
Uses html2text if available, otherwise falls back to regex stripping.
|
| 49 |
+
"""
|
| 50 |
+
try:
|
| 51 |
+
import html2text # type: ignore[import]
|
| 52 |
+
h = html2text.HTML2Text()
|
| 53 |
+
h.ignore_links = False
|
| 54 |
+
h.ignore_images = True
|
| 55 |
+
h.body_width = 0 # Don't wrap lines
|
| 56 |
+
return h.handle(html)
|
| 57 |
+
except ImportError:
|
| 58 |
+
logger.debug("html2text not installed, using regex fallback")
|
| 59 |
+
return _regex_html_strip(html)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _regex_html_strip(html: str) -> str:
|
| 63 |
+
"""Minimal HTML β text via regex (fallback when html2text unavailable)."""
|
| 64 |
+
html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
|
| 65 |
+
# Convert headings
|
| 66 |
+
for i in range(6, 0, -1):
|
| 67 |
+
html = re.sub(rf"<h{i}[^>]*>(.*?)</h{i}>", rf"{'#' * i} \1\n", html, flags=re.IGNORECASE | re.DOTALL)
|
| 68 |
+
# Code blocks
|
| 69 |
+
html = re.sub(r"<pre[^>]*>(.*?)</pre>", r"```\n\1\n```", html, flags=re.IGNORECASE | re.DOTALL)
|
| 70 |
+
html = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", html, flags=re.IGNORECASE | re.DOTALL)
|
| 71 |
+
# Lists
|
| 72 |
+
html = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1", html, flags=re.IGNORECASE | re.DOTALL)
|
| 73 |
+
# Paragraphs and divs
|
| 74 |
+
html = re.sub(r"<(p|div|br)[^>]*>", "\n", html, flags=re.IGNORECASE)
|
| 75 |
+
# Strip remaining tags
|
| 76 |
+
html = re.sub(r"<[^>]+>", "", html)
|
| 77 |
+
# Decode entities
|
| 78 |
+
html = html.replace("<", "<").replace(">", ">").replace("&", "&").replace(" ", " ")
|
| 79 |
+
return re.sub(r"\n{3,}", "\n\n", html).strip()
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
async def _pdf_to_markdown(pdf_base64: str) -> str:
|
| 83 |
+
"""
|
| 84 |
+
Convert base64-encoded PDF to Markdown using Docling by IBM.
|
| 85 |
+
Falls back to a message if Docling is not installed.
|
| 86 |
+
"""
|
| 87 |
+
try:
|
| 88 |
+
import base64
|
| 89 |
+
import io
|
| 90 |
+
|
| 91 |
+
from docling.document_converter import DocumentConverter # type: ignore[import]
|
| 92 |
+
|
| 93 |
+
pdf_bytes = base64.b64decode(pdf_base64)
|
| 94 |
+
converter = DocumentConverter()
|
| 95 |
+
# Docling expects a file path or URL; write to temp file
|
| 96 |
+
import tempfile
|
| 97 |
+
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
| 98 |
+
f.write(pdf_bytes)
|
| 99 |
+
tmp_path = f.name
|
| 100 |
+
|
| 101 |
+
result = converter.convert(tmp_path)
|
| 102 |
+
return result.document.export_to_markdown()
|
| 103 |
+
except ImportError:
|
| 104 |
+
logger.warning("Docling not installed, cannot convert PDF")
|
| 105 |
+
return "[PDF conversion requires Docling: pip install docling]"
|
| 106 |
+
except Exception as exc:
|
| 107 |
+
logger.error("PDF conversion failed: %s", exc)
|
| 108 |
+
return f"[PDF conversion failed: {exc}]"
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _clean_markdown(text: str) -> str:
|
| 112 |
+
"""Remove excessive blank lines and leading/trailing whitespace."""
|
| 113 |
+
return re.sub(r"\n{4,}", "\n\n\n", text).strip()
|
backend/services/ingestion/registries_ingester.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Multi-registry package resolver.
|
| 3 |
+
|
| 4 |
+
Supports: crates.io, RubyGems, pub.dev, NuGet, Maven Central, Hex (Elixir), CRAN (R).
|
| 5 |
+
Each resolver returns a dict with keys: name, version, homepage, repository.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
import re
|
| 10 |
+
|
| 11 |
+
import httpx
|
| 12 |
+
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
|
| 15 |
+
_HEADERS = {"User-Agent": "docforge/1.0 (documentation indexer)"}
|
| 16 |
+
_TIMEOUT = 15
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _parse_at(input_str: str) -> tuple[str, str]:
|
| 20 |
+
"""
|
| 21 |
+
Parse 'name@version' or just 'name'. Strip any 'prefix:' first.
|
| 22 |
+
Returns (name, version) where version defaults to 'latest'.
|
| 23 |
+
"""
|
| 24 |
+
if ":" in input_str:
|
| 25 |
+
input_str = input_str.split(":", 1)[1]
|
| 26 |
+
input_str = input_str.strip().strip("\"'")
|
| 27 |
+
if "@" in input_str:
|
| 28 |
+
name, ver = input_str.rsplit("@", 1)
|
| 29 |
+
return name.strip(), ver.strip() or "latest"
|
| 30 |
+
return input_str.strip(), "latest"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# ββ crates.io βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 34 |
+
|
| 35 |
+
async def resolve_crates(input_str: str) -> dict:
|
| 36 |
+
name, version = _parse_at(input_str)
|
| 37 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 38 |
+
if version == "latest":
|
| 39 |
+
url = f"https://crates.io/api/v1/crates/{name}"
|
| 40 |
+
else:
|
| 41 |
+
url = f"https://crates.io/api/v1/crates/{name}/{version}"
|
| 42 |
+
|
| 43 |
+
resp = await client.get(url)
|
| 44 |
+
if not resp.is_success:
|
| 45 |
+
raise ValueError(f"crates.io: '{name}' not found")
|
| 46 |
+
|
| 47 |
+
data = resp.json()
|
| 48 |
+
|
| 49 |
+
if version == "latest":
|
| 50 |
+
krate = data.get("crate", {})
|
| 51 |
+
resolved = krate.get("newest_version", "latest")
|
| 52 |
+
homepage = krate.get("homepage") or krate.get("repository")
|
| 53 |
+
repository = krate.get("repository")
|
| 54 |
+
else:
|
| 55 |
+
ver_obj = data.get("version", {})
|
| 56 |
+
resolved = ver_obj.get("num", version)
|
| 57 |
+
# Fetch crate root for homepage/repo
|
| 58 |
+
try:
|
| 59 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 60 |
+
root = await client.get(f"https://crates.io/api/v1/crates/{name}")
|
| 61 |
+
root_data = root.json() if root.is_success else {}
|
| 62 |
+
except Exception:
|
| 63 |
+
root_data = {}
|
| 64 |
+
krate = root_data.get("crate", {})
|
| 65 |
+
homepage = krate.get("homepage") or krate.get("repository")
|
| 66 |
+
repository = krate.get("repository")
|
| 67 |
+
|
| 68 |
+
# docs.rs is the canonical Rust docs site
|
| 69 |
+
docs_url = homepage or f"https://docs.rs/{name}/{resolved}"
|
| 70 |
+
return {"name": name, "version": resolved, "homepage": docs_url, "repository": repository}
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# ββ RubyGems ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 74 |
+
|
| 75 |
+
async def resolve_rubygems(input_str: str) -> dict:
|
| 76 |
+
name, version = _parse_at(input_str)
|
| 77 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 78 |
+
resp = await client.get(f"https://rubygems.org/api/v1/gems/{name}.json")
|
| 79 |
+
if not resp.is_success:
|
| 80 |
+
raise ValueError(f"RubyGems: '{name}' not found")
|
| 81 |
+
data = resp.json()
|
| 82 |
+
|
| 83 |
+
resolved = version if version != "latest" else data.get("version", "latest")
|
| 84 |
+
homepage = (
|
| 85 |
+
data.get("homepage_uri")
|
| 86 |
+
or data.get("documentation_uri")
|
| 87 |
+
or data.get("project_uri")
|
| 88 |
+
)
|
| 89 |
+
repository = data.get("source_code_uri")
|
| 90 |
+
return {"name": data.get("name", name), "version": resolved, "homepage": homepage, "repository": repository}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ββ pub.dev (Dart / Flutter) ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 94 |
+
|
| 95 |
+
async def resolve_pubdev(input_str: str) -> dict:
|
| 96 |
+
name, version = _parse_at(input_str)
|
| 97 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 98 |
+
resp = await client.get(f"https://pub.dev/api/packages/{name}")
|
| 99 |
+
if not resp.is_success:
|
| 100 |
+
raise ValueError(f"pub.dev: '{name}' not found")
|
| 101 |
+
data = resp.json()
|
| 102 |
+
|
| 103 |
+
resolved = version if version != "latest" else data.get("latest", {}).get("version", "latest")
|
| 104 |
+
pubspec = data.get("latest", {}).get("pubspec", {})
|
| 105 |
+
homepage = (
|
| 106 |
+
pubspec.get("homepage")
|
| 107 |
+
or pubspec.get("documentation")
|
| 108 |
+
or f"https://pub.dev/packages/{name}"
|
| 109 |
+
)
|
| 110 |
+
repository = pubspec.get("repository")
|
| 111 |
+
return {"name": name, "version": resolved, "homepage": homepage, "repository": repository}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
# ββ NuGet (.NET) ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 115 |
+
|
| 116 |
+
async def resolve_nuget(input_str: str) -> dict:
|
| 117 |
+
name, version = _parse_at(input_str)
|
| 118 |
+
lower = name.lower()
|
| 119 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 120 |
+
resp = await client.get(f"https://api.nuget.org/v3/registration5/{lower}/index.json")
|
| 121 |
+
if not resp.is_success:
|
| 122 |
+
raise ValueError(f"NuGet: '{name}' not found")
|
| 123 |
+
data = resp.json()
|
| 124 |
+
|
| 125 |
+
# Flatten pages β items β catalogEntry
|
| 126 |
+
pages = data.get("items", [])
|
| 127 |
+
last_entry = None
|
| 128 |
+
resolved = version
|
| 129 |
+
if pages:
|
| 130 |
+
last_page = pages[-1]
|
| 131 |
+
items = last_page.get("items", [])
|
| 132 |
+
if items:
|
| 133 |
+
last_item = items[-1]
|
| 134 |
+
last_entry = last_item.get("catalogEntry", {})
|
| 135 |
+
if version == "latest":
|
| 136 |
+
resolved = last_entry.get("version", "latest")
|
| 137 |
+
|
| 138 |
+
homepage = (last_entry or {}).get("projectUrl") or f"https://www.nuget.org/packages/{name}"
|
| 139 |
+
return {"name": name, "version": resolved, "homepage": homepage, "repository": None}
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# ββ Maven Central βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 143 |
+
# Input format (after prefix strip): "com.google.guava:guava@32.1.3"
|
| 144 |
+
|
| 145 |
+
async def resolve_maven(input_str: str) -> dict:
|
| 146 |
+
# Strip "mvn:" prefix if present
|
| 147 |
+
stripped = input_str[4:] if input_str.startswith("mvn:") else input_str
|
| 148 |
+
|
| 149 |
+
if "@" in stripped:
|
| 150 |
+
ga, version = stripped.rsplit("@", 1)
|
| 151 |
+
version = version.strip()
|
| 152 |
+
else:
|
| 153 |
+
ga, version = stripped, "latest"
|
| 154 |
+
|
| 155 |
+
if ":" not in ga:
|
| 156 |
+
raise ValueError(
|
| 157 |
+
f"Maven input must be groupId:artifactId[@version], e.g. com.google.guava:guava. Got: {input_str!r}"
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
group_id, artifact_id = ga.split(":", 1)
|
| 161 |
+
group_id = group_id.strip()
|
| 162 |
+
artifact_id = artifact_id.strip()
|
| 163 |
+
|
| 164 |
+
from urllib.parse import quote
|
| 165 |
+
if version == "latest":
|
| 166 |
+
query = f"g:{group_id} AND a:{artifact_id}"
|
| 167 |
+
search_url = (
|
| 168 |
+
f"https://search.maven.org/solrsearch/select"
|
| 169 |
+
f"?q={quote(query)}&core=gav&rows=1&wt=json"
|
| 170 |
+
)
|
| 171 |
+
else:
|
| 172 |
+
query = f"g:{group_id} AND a:{artifact_id} AND v:{version}"
|
| 173 |
+
search_url = (
|
| 174 |
+
f"https://search.maven.org/solrsearch/select"
|
| 175 |
+
f"?q={quote(query)}&core=gav&rows=1&wt=json"
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 179 |
+
resp = await client.get(search_url)
|
| 180 |
+
if not resp.is_success:
|
| 181 |
+
raise ValueError(f"Maven Central: '{group_id}:{artifact_id}' not found")
|
| 182 |
+
data = resp.json()
|
| 183 |
+
|
| 184 |
+
docs = data.get("response", {}).get("docs", [])
|
| 185 |
+
if not docs:
|
| 186 |
+
raise ValueError(f"Maven Central: '{group_id}:{artifact_id}' not found")
|
| 187 |
+
|
| 188 |
+
resolved = docs[0].get("v", version)
|
| 189 |
+
name = f"{group_id}:{artifact_id}"
|
| 190 |
+
homepage = f"https://mvnrepository.com/artifact/{group_id}/{artifact_id}/{resolved}"
|
| 191 |
+
return {"name": name, "version": resolved, "homepage": homepage, "repository": None}
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
# ββ Hex (Elixir) ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 195 |
+
|
| 196 |
+
async def resolve_hex(input_str: str) -> dict:
|
| 197 |
+
name, version = _parse_at(input_str)
|
| 198 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 199 |
+
resp = await client.get(
|
| 200 |
+
f"https://hex.pm/api/packages/{name}",
|
| 201 |
+
headers={**_HEADERS, "Accept": "application/json"},
|
| 202 |
+
)
|
| 203 |
+
if not resp.is_success:
|
| 204 |
+
raise ValueError(f"Hex: '{name}' not found")
|
| 205 |
+
data = resp.json()
|
| 206 |
+
|
| 207 |
+
resolved = version if version != "latest" else (
|
| 208 |
+
data.get("latest_stable_version") or data.get("latest_version", "latest")
|
| 209 |
+
)
|
| 210 |
+
meta = data.get("meta", {})
|
| 211 |
+
links = meta.get("links", {})
|
| 212 |
+
homepage = (
|
| 213 |
+
links.get("Website")
|
| 214 |
+
or links.get("Documentation")
|
| 215 |
+
or f"https://hexdocs.pm/{name}/{resolved}"
|
| 216 |
+
)
|
| 217 |
+
repository = links.get("GitHub") or links.get("Repository")
|
| 218 |
+
return {"name": data.get("name", name), "version": resolved, "homepage": homepage, "repository": repository}
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# ββ CRAN (R) ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 222 |
+
|
| 223 |
+
async def resolve_cran(input_str: str) -> dict:
|
| 224 |
+
name, _ = _parse_at(input_str)
|
| 225 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT, headers=_HEADERS) as client:
|
| 226 |
+
resp = await client.get(f"https://crandb.r-pkg.org/{name}")
|
| 227 |
+
if not resp.is_success:
|
| 228 |
+
raise ValueError(f"CRAN: '{name}' not found")
|
| 229 |
+
data = resp.json()
|
| 230 |
+
|
| 231 |
+
resolved = data.get("Version", "latest")
|
| 232 |
+
raw_url = data.get("URL", "")
|
| 233 |
+
homepage = raw_url.split(",")[0].strip() if raw_url else f"https://cran.r-project.org/package={name}"
|
| 234 |
+
bug_reports = data.get("BugReports", "")
|
| 235 |
+
repository = re.sub(r"/issues.*", "", bug_reports) if "github.com" in bug_reports else None
|
| 236 |
+
return {"name": data.get("Package", name), "version": resolved, "homepage": homepage, "repository": repository}
|
backend/services/ingestion/search_ingester.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Web search fallback using DuckDuckGo HTML endpoint.
|
| 3 |
+
|
| 4 |
+
When a library isn't found in any known registry (or has no docs URL),
|
| 5 |
+
this module searches DuckDuckGo for "{library} documentation" and returns
|
| 6 |
+
the first organic result URL for the crawling pipeline to consume.
|
| 7 |
+
|
| 8 |
+
No API key needed β uses the public HTML endpoint.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import logging
|
| 12 |
+
import re
|
| 13 |
+
from urllib.parse import unquote
|
| 14 |
+
|
| 15 |
+
import httpx
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
_TIMEOUT = 20
|
| 20 |
+
_USER_AGENT = (
|
| 21 |
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
| 22 |
+
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
async def find_docs_url(query: str) -> str | None:
|
| 27 |
+
"""
|
| 28 |
+
Search DuckDuckGo for '{query} documentation' and return the first result URL.
|
| 29 |
+
Returns None if nothing useful is found.
|
| 30 |
+
"""
|
| 31 |
+
search_query = f"{query} documentation"
|
| 32 |
+
from urllib.parse import quote_plus
|
| 33 |
+
url = f"https://html.duckduckgo.com/html/?q={quote_plus(search_query)}"
|
| 34 |
+
|
| 35 |
+
logger.info("Web search fallback: searching for '%s'", query)
|
| 36 |
+
|
| 37 |
+
try:
|
| 38 |
+
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
| 39 |
+
resp = await client.get(
|
| 40 |
+
url,
|
| 41 |
+
headers={
|
| 42 |
+
"User-Agent": _USER_AGENT,
|
| 43 |
+
"Accept": "text/html,application/xhtml+xml",
|
| 44 |
+
},
|
| 45 |
+
follow_redirects=True,
|
| 46 |
+
)
|
| 47 |
+
if not resp.is_success:
|
| 48 |
+
logger.warning("DuckDuckGo search returned %s", resp.status_code)
|
| 49 |
+
return None
|
| 50 |
+
html = resp.text
|
| 51 |
+
except Exception as exc:
|
| 52 |
+
logger.warning("DuckDuckGo search failed: %s", exc)
|
| 53 |
+
return None
|
| 54 |
+
|
| 55 |
+
# DuckDuckGo HTML results encode actual URLs as uddg= query param:
|
| 56 |
+
# href="//duckduckgo.com/l/?uddg=https%3A%2F%2Freact.dev%2F&rut=..."
|
| 57 |
+
matches = re.findall(r"uddg=([^&\"'>\s]+)", html)
|
| 58 |
+
for encoded in matches:
|
| 59 |
+
try:
|
| 60 |
+
decoded = unquote(encoded)
|
| 61 |
+
except Exception:
|
| 62 |
+
decoded = encoded
|
| 63 |
+
|
| 64 |
+
if decoded.startswith("http") and "duckduckgo.com" not in decoded:
|
| 65 |
+
logger.info("Web search found docs URL: %s", decoded)
|
| 66 |
+
return decoded
|
| 67 |
+
|
| 68 |
+
logger.warning("No usable docs URL found via web search for '%s'", query)
|
| 69 |
+
return None
|
backend/services/ingestion/url_ingester.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
URL ingester using Playwright for JS-rendered documentation sites.
|
| 3 |
+
|
| 4 |
+
Many modern doc sites (React-based, Next.js, VitePress, etc.) render content
|
| 5 |
+
client-side, so httpx alone returns empty shells. Playwright runs a real browser
|
| 6 |
+
to get the fully rendered DOM.
|
| 7 |
+
|
| 8 |
+
Falls back to static crawl if Playwright is not installed.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import asyncio
|
| 12 |
+
import logging
|
| 13 |
+
import os
|
| 14 |
+
import random
|
| 15 |
+
import re
|
| 16 |
+
from urllib.parse import urlparse
|
| 17 |
+
|
| 18 |
+
import httpx
|
| 19 |
+
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
PLAYWRIGHT_TIMEOUT = int(os.getenv("PLAYWRIGHT_TIMEOUT", "60000")) # ms
|
| 23 |
+
|
| 24 |
+
# Realistic Chrome user agent β rotated per session to avoid fingerprinting
|
| 25 |
+
_USER_AGENTS = [
|
| 26 |
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
| 27 |
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
| 28 |
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
async def crawl_with_playwright(url: str, max_pages: int = 15) -> str:
|
| 33 |
+
"""
|
| 34 |
+
Use Playwright Chromium to crawl a JS-rendered documentation site.
|
| 35 |
+
Returns concatenated text content from all discovered pages.
|
| 36 |
+
"""
|
| 37 |
+
try:
|
| 38 |
+
from playwright.async_api import async_playwright # type: ignore[import]
|
| 39 |
+
except ImportError:
|
| 40 |
+
logger.warning("Playwright not installed, falling back to static crawl")
|
| 41 |
+
from .npm_ingester import crawl_static_docs
|
| 42 |
+
return await crawl_static_docs(url, max_pages)
|
| 43 |
+
|
| 44 |
+
logger.info("Playwright crawl starting at %s", url)
|
| 45 |
+
pages_text: list[str] = []
|
| 46 |
+
visited: set[str] = set()
|
| 47 |
+
to_visit = [url]
|
| 48 |
+
|
| 49 |
+
base = urlparse(url)
|
| 50 |
+
base_origin = f"{base.scheme}://{base.netloc}"
|
| 51 |
+
|
| 52 |
+
async with async_playwright() as pw:
|
| 53 |
+
browser = await pw.chromium.launch(
|
| 54 |
+
headless=True,
|
| 55 |
+
args=[
|
| 56 |
+
"--disable-blink-features=AutomationControlled",
|
| 57 |
+
"--no-sandbox",
|
| 58 |
+
"--disable-dev-shm-usage",
|
| 59 |
+
],
|
| 60 |
+
)
|
| 61 |
+
context = await browser.new_context(
|
| 62 |
+
user_agent=random.choice(_USER_AGENTS),
|
| 63 |
+
viewport={"width": 1280, "height": 800},
|
| 64 |
+
locale="en-US",
|
| 65 |
+
timezone_id="America/New_York",
|
| 66 |
+
extra_http_headers={"Accept-Language": "en-US,en;q=0.9"},
|
| 67 |
+
)
|
| 68 |
+
# Remove the webdriver property that sites use to detect Playwright
|
| 69 |
+
await context.add_init_script(
|
| 70 |
+
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
|
| 71 |
+
)
|
| 72 |
+
page = await context.new_page()
|
| 73 |
+
page.set_default_timeout(PLAYWRIGHT_TIMEOUT)
|
| 74 |
+
|
| 75 |
+
while to_visit and len(pages_text) < max_pages:
|
| 76 |
+
current_url = to_visit.pop(0)
|
| 77 |
+
if current_url in visited:
|
| 78 |
+
continue
|
| 79 |
+
visited.add(current_url)
|
| 80 |
+
|
| 81 |
+
try:
|
| 82 |
+
await page.goto(current_url, wait_until="domcontentloaded", timeout=15000)
|
| 83 |
+
# Human-like random delay between pages (1β2.5s)
|
| 84 |
+
await asyncio.sleep(random.uniform(1.0, 2.5))
|
| 85 |
+
|
| 86 |
+
text = await page.evaluate("""() => {
|
| 87 |
+
// Remove nav, footer, scripts and extract meaningful text
|
| 88 |
+
const remove = ['script', 'style', 'nav', 'footer', 'header', '.sidebar', '.navigation'];
|
| 89 |
+
remove.forEach(sel => document.querySelectorAll(sel).forEach(el => el.remove()));
|
| 90 |
+
return document.body ? document.body.innerText : '';
|
| 91 |
+
}""")
|
| 92 |
+
|
| 93 |
+
if text.strip():
|
| 94 |
+
pages_text.append(f"\n\n<!-- SOURCE: {current_url} -->\n{text}")
|
| 95 |
+
logger.debug("Playwright crawled %s (%d chars)", current_url, len(text))
|
| 96 |
+
|
| 97 |
+
# Collect doc links
|
| 98 |
+
links = await page.evaluate(f"""() => {{
|
| 99 |
+
return Array.from(document.querySelectorAll('a[href]'))
|
| 100 |
+
.map(a => a.href)
|
| 101 |
+
.filter(href => href.startsWith('{base_origin}'));
|
| 102 |
+
}}""")
|
| 103 |
+
|
| 104 |
+
for link in links[:20]:
|
| 105 |
+
clean = link.split("#")[0].split("?")[0]
|
| 106 |
+
if clean not in visited and _is_doc_link(clean):
|
| 107 |
+
to_visit.append(clean)
|
| 108 |
+
|
| 109 |
+
except Exception as exc:
|
| 110 |
+
logger.warning("Playwright failed on %s: %s", current_url, exc)
|
| 111 |
+
|
| 112 |
+
await browser.close()
|
| 113 |
+
|
| 114 |
+
combined = "\n".join(pages_text)
|
| 115 |
+
logger.info("Playwright crawl done: %d pages, %d chars total", len(pages_text), len(combined))
|
| 116 |
+
return combined
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
async def crawl_url(url: str, max_pages: int = 15) -> str:
|
| 120 |
+
"""
|
| 121 |
+
Entry point: try Playwright first (handles JS sites), fall back to static.
|
| 122 |
+
"""
|
| 123 |
+
return await crawl_with_playwright(url, max_pages)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
# Repo path segments that are not a real repo root
|
| 127 |
+
_GH_EXCLUDED_OWNERS = frozenset({
|
| 128 |
+
"sponsors", "features", "pricing", "enterprise", "about",
|
| 129 |
+
"login", "join", "organizations", "apps", "marketplace",
|
| 130 |
+
})
|
| 131 |
+
_GH_EXCLUDED_REPOS = frozenset({
|
| 132 |
+
"issues", "pulls", "discussions", "actions", "releases",
|
| 133 |
+
"commits", "blob", "tree", "wiki", "pulse",
|
| 134 |
+
})
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
async def extract_github_url(url: str) -> str | None:
|
| 138 |
+
"""
|
| 139 |
+
Quick static fetch of a URL to extract a GitHub repo link from its HTML.
|
| 140 |
+
|
| 141 |
+
Most library/component sites link to their GitHub repo in the header,
|
| 142 |
+
footer, or meta tags. This is fast (no browser), works even on JS-heavy
|
| 143 |
+
sites because meta tags are server-rendered.
|
| 144 |
+
|
| 145 |
+
Returns the first clean 'https://github.com/{owner}/{repo}' found,
|
| 146 |
+
or None if nothing is found.
|
| 147 |
+
"""
|
| 148 |
+
try:
|
| 149 |
+
async with httpx.AsyncClient(
|
| 150 |
+
timeout=10,
|
| 151 |
+
follow_redirects=True,
|
| 152 |
+
headers={"User-Agent": "docforge/1.0 (documentation indexer)"},
|
| 153 |
+
) as client:
|
| 154 |
+
resp = await client.get(url)
|
| 155 |
+
if not resp.is_success:
|
| 156 |
+
return None
|
| 157 |
+
html = resp.text
|
| 158 |
+
except Exception as exc:
|
| 159 |
+
logger.debug("extract_github_url fetch failed for %s: %s", url, exc)
|
| 160 |
+
return None
|
| 161 |
+
|
| 162 |
+
# Match all github.com/{owner}/{repo} occurrences in the raw HTML
|
| 163 |
+
matches = re.findall(
|
| 164 |
+
r'https?://github\.com/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)',
|
| 165 |
+
html,
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
seen: set[str] = set()
|
| 169 |
+
for owner, repo in matches:
|
| 170 |
+
if owner.lower() in _GH_EXCLUDED_OWNERS:
|
| 171 |
+
continue
|
| 172 |
+
repo = repo.rstrip("/").split("/")[0].removesuffix(".git")
|
| 173 |
+
if repo.lower() in _GH_EXCLUDED_REPOS:
|
| 174 |
+
continue
|
| 175 |
+
key = f"{owner}/{repo}"
|
| 176 |
+
if key not in seen:
|
| 177 |
+
seen.add(key)
|
| 178 |
+
return f"https://github.com/{owner}/{repo}"
|
| 179 |
+
|
| 180 |
+
return None
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _is_doc_link(url: str) -> bool:
|
| 184 |
+
"""Filter out non-documentation links."""
|
| 185 |
+
bad_exts = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico",
|
| 186 |
+
".css", ".js", ".woff", ".woff2", ".ttf", ".map")
|
| 187 |
+
bad_paths = ("/api/", "/cdn-cgi/", "/_next/", "/__vite", "/assets/")
|
| 188 |
+
path = urlparse(url).path.lower()
|
| 189 |
+
return (
|
| 190 |
+
not any(path.endswith(ext) for ext in bad_exts)
|
| 191 |
+
and not any(bad in path for bad in bad_paths)
|
| 192 |
+
)
|
backend/services/structurer.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Structuring engine: Gemini 2.0 Flash (primary) β Groq LLaMA (fallback).
|
| 3 |
+
|
| 4 |
+
Takes raw Markdown documentation and produces a structured JSON representation
|
| 5 |
+
with components, props, gotchas, and version-specific details.
|
| 6 |
+
|
| 7 |
+
The prompt is engineered to hunt specifically for AI codegen failure patterns
|
| 8 |
+
(gotchas) that training data tends to get wrong.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import logging
|
| 13 |
+
import os
|
| 14 |
+
import re
|
| 15 |
+
import time
|
| 16 |
+
from datetime import date
|
| 17 |
+
|
| 18 |
+
import asyncio
|
| 19 |
+
|
| 20 |
+
import httpx
|
| 21 |
+
|
| 22 |
+
logger = logging.getLogger(__name__)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class _QuotaError(Exception):
|
| 26 |
+
"""Raised internally when an LLM provider returns 429."""
|
| 27 |
+
|
| 28 |
+
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
|
| 29 |
+
GEMINI_MODEL = "gemini-2.0-flash"
|
| 30 |
+
GEMINI_ENDPOINT = (
|
| 31 |
+
f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent"
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
|
| 35 |
+
GROQ_MODEL = "llama-3.3-70b-versatile"
|
| 36 |
+
GROQ_ENDPOINT = "https://api.groq.com/openai/v1/chat/completions"
|
| 37 |
+
# Groq's context window is 128k tokens; keep well under to avoid truncation errors
|
| 38 |
+
GROQ_MAX_DOC_CHARS = 40_000
|
| 39 |
+
|
| 40 |
+
STRUCTURED_SCHEMA = """{
|
| 41 |
+
"library": "string",
|
| 42 |
+
"version": "string",
|
| 43 |
+
"description": "string",
|
| 44 |
+
"components": [
|
| 45 |
+
{
|
| 46 |
+
"name": "string",
|
| 47 |
+
"import_path": "string (exact import statement, e.g. import X from '@/components/X')",
|
| 48 |
+
"install_command": "string (exact command from docs, NOT generic 'npm install')",
|
| 49 |
+
"props": [
|
| 50 |
+
{
|
| 51 |
+
"name": "string",
|
| 52 |
+
"type": "string",
|
| 53 |
+
"required": "boolean",
|
| 54 |
+
"default": "string or null"
|
| 55 |
+
}
|
| 56 |
+
],
|
| 57 |
+
"gotchas": ["string (specific AI codegen failure patterns)"],
|
| 58 |
+
"peer_dependencies": ["string"],
|
| 59 |
+
"usage_example": "string (complete, runnable code snippet)",
|
| 60 |
+
"last_verified": "string (ISO date)"
|
| 61 |
+
}
|
| 62 |
+
]
|
| 63 |
+
}"""
|
| 64 |
+
|
| 65 |
+
STRUCTURING_PROMPT = """You are a documentation structuring engine for DocForge. Your job is to extract \
|
| 66 |
+
precise, developer-critical information from raw library documentation.
|
| 67 |
+
|
| 68 |
+
Given the following raw documentation for {library}@{version}, extract structured information.
|
| 69 |
+
|
| 70 |
+
CRITICAL β You must specifically hunt for:
|
| 71 |
+
1. GOTCHAS: Things that commonly break AI code generation. Examples:
|
| 72 |
+
- Required directives ("use client", "use server" in Next.js App Router)
|
| 73 |
+
- Deprecated props or methods from previous versions
|
| 74 |
+
- Breaking changes between major versions (e.g. v4 β v5 API changes)
|
| 75 |
+
- Required peer dependencies that must be installed separately
|
| 76 |
+
- Common misuse patterns (wrong import path, wrong prop name)
|
| 77 |
+
- Framework-specific requirements (Next.js App Router vs Pages Router)
|
| 78 |
+
- CSS imports that must be added manually
|
| 79 |
+
- Configuration file changes required
|
| 80 |
+
|
| 81 |
+
2. EXACT install commands β NOT generic "npm install {library}". Look for npx, dlx, or custom CLI commands.
|
| 82 |
+
3. EXACT import paths β these change between major versions and are the #1 AI codegen failure point.
|
| 83 |
+
4. All props with their TypeScript types, whether they are required, and default values.
|
| 84 |
+
5. Version-specific API differences if mentioned in the docs.
|
| 85 |
+
|
| 86 |
+
Return ONLY valid JSON (no markdown code fences, no explanation, no extra whitespace) in EXACTLY this schema:
|
| 87 |
+
{schema}
|
| 88 |
+
|
| 89 |
+
If the documentation mentions multiple components or modules, include each as a separate entry in "components".
|
| 90 |
+
If there is only one main export, use a single component entry with the library name as the component name.
|
| 91 |
+
Set last_verified to today's date: {today}.
|
| 92 |
+
|
| 93 |
+
Raw documentation:
|
| 94 |
+
{raw_docs}"""
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
async def structure_with_gemini(
|
| 98 |
+
library: str,
|
| 99 |
+
version: str,
|
| 100 |
+
raw_docs: str,
|
| 101 |
+
gemini_key: str | None = None,
|
| 102 |
+
groq_key: str | None = None,
|
| 103 |
+
) -> dict:
|
| 104 |
+
"""
|
| 105 |
+
Call Gemini 2.0 Flash to structure raw documentation into our JSON schema.
|
| 106 |
+
|
| 107 |
+
gemini_key / groq_key β per-user keys that override environment variables.
|
| 108 |
+
Falls back to env vars when not provided.
|
| 109 |
+
"""
|
| 110 |
+
effective_gemini = gemini_key or GEMINI_API_KEY
|
| 111 |
+
effective_groq = groq_key or GROQ_API_KEY
|
| 112 |
+
|
| 113 |
+
if not effective_gemini:
|
| 114 |
+
logger.warning("GEMINI_API_KEY not set, using mock structurer")
|
| 115 |
+
return _mock_structure(library, version)
|
| 116 |
+
|
| 117 |
+
# Truncate to ~80k chars β keeps Gemini token usage around 20k per request,
|
| 118 |
+
# which is much friendlier to the free tier daily quota (1M tokens/day).
|
| 119 |
+
# Quality is maintained: the most relevant documentation is always first.
|
| 120 |
+
max_doc_chars = 80_000
|
| 121 |
+
truncated = raw_docs[:max_doc_chars]
|
| 122 |
+
if len(raw_docs) > max_doc_chars:
|
| 123 |
+
logger.info("Truncated docs from %d to %d chars for Gemini", len(raw_docs), max_doc_chars)
|
| 124 |
+
|
| 125 |
+
prompt = STRUCTURING_PROMPT.format(
|
| 126 |
+
library=library,
|
| 127 |
+
version=version,
|
| 128 |
+
schema=STRUCTURED_SCHEMA,
|
| 129 |
+
today=date.today().isoformat(),
|
| 130 |
+
raw_docs=truncated,
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
t0 = time.time()
|
| 134 |
+
payload = {
|
| 135 |
+
"contents": [{"parts": [{"text": prompt}]}],
|
| 136 |
+
"generationConfig": {
|
| 137 |
+
"temperature": 0.1,
|
| 138 |
+
"maxOutputTokens": 8192,
|
| 139 |
+
"responseMimeType": "application/json",
|
| 140 |
+
},
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
try:
|
| 144 |
+
async with httpx.AsyncClient(timeout=120) as client:
|
| 145 |
+
resp = await client.post(
|
| 146 |
+
GEMINI_ENDPOINT,
|
| 147 |
+
params={"key": effective_gemini},
|
| 148 |
+
json=payload,
|
| 149 |
+
)
|
| 150 |
+
if resp.status_code == 429:
|
| 151 |
+
raise _QuotaError("Gemini")
|
| 152 |
+
resp.raise_for_status()
|
| 153 |
+
data = resp.json()
|
| 154 |
+
|
| 155 |
+
elapsed = time.time() - t0
|
| 156 |
+
logger.info("Gemini structuring took %.1fs for %s@%s", elapsed, library, version)
|
| 157 |
+
raw_output = data["candidates"][0]["content"]["parts"][0]["text"]
|
| 158 |
+
return _parse_gemini_json(raw_output, library, version)
|
| 159 |
+
|
| 160 |
+
except _QuotaError:
|
| 161 |
+
logger.warning("Gemini quota hit for %s@%s β falling back to Groq", library, version)
|
| 162 |
+
except httpx.HTTPStatusError as exc:
|
| 163 |
+
if exc.response.status_code == 429:
|
| 164 |
+
logger.warning("Gemini quota hit for %s@%s β falling back to Groq", library, version)
|
| 165 |
+
else:
|
| 166 |
+
logger.error("Gemini failed for %s@%s: %s", library, version, exc)
|
| 167 |
+
raise
|
| 168 |
+
except Exception as exc:
|
| 169 |
+
logger.error("Gemini failed for %s@%s: %s", library, version, exc)
|
| 170 |
+
raise
|
| 171 |
+
|
| 172 |
+
# ββ Groq fallback ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 173 |
+
return await _structure_with_groq(library, version, raw_docs, t0, groq_key=effective_groq)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
async def _structure_with_groq(
|
| 177 |
+
library: str, version: str, raw_docs: str, t0: float, groq_key: str | None = None
|
| 178 |
+
) -> dict:
|
| 179 |
+
"""Call Groq (LLaMA 3.3 70B) as a fallback when Gemini quota is exhausted."""
|
| 180 |
+
effective_groq = groq_key or GROQ_API_KEY
|
| 181 |
+
if not effective_groq:
|
| 182 |
+
raise RuntimeError(
|
| 183 |
+
"Gemini quota exceeded and no Groq key available. "
|
| 184 |
+
"Add a Groq key in your DocForge settings (free at console.groq.com)."
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
truncated = raw_docs[:GROQ_MAX_DOC_CHARS]
|
| 188 |
+
if len(raw_docs) > GROQ_MAX_DOC_CHARS:
|
| 189 |
+
logger.info("Truncated docs from %d to %d chars for Groq", len(raw_docs), GROQ_MAX_DOC_CHARS)
|
| 190 |
+
|
| 191 |
+
prompt = STRUCTURING_PROMPT.format(
|
| 192 |
+
library=library,
|
| 193 |
+
version=version,
|
| 194 |
+
schema=STRUCTURED_SCHEMA,
|
| 195 |
+
today=date.today().isoformat(),
|
| 196 |
+
raw_docs=truncated,
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
payload = {
|
| 200 |
+
"model": GROQ_MODEL,
|
| 201 |
+
"messages": [{"role": "user", "content": prompt}],
|
| 202 |
+
"temperature": 0.1,
|
| 203 |
+
"max_tokens": 32768,
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
# Retry up to 3 times on 429 with exponential backoff
|
| 207 |
+
retry_delays = [10, 30, 60]
|
| 208 |
+
for attempt, delay in enumerate(retry_delays):
|
| 209 |
+
async with httpx.AsyncClient(timeout=120) as client:
|
| 210 |
+
resp = await client.post(
|
| 211 |
+
GROQ_ENDPOINT,
|
| 212 |
+
headers={"Authorization": f"Bearer {effective_groq}"},
|
| 213 |
+
json=payload,
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
if resp.status_code == 429:
|
| 217 |
+
if attempt < len(retry_delays) - 1:
|
| 218 |
+
logger.warning(
|
| 219 |
+
"Groq rate limited for %s@%s β waiting %ds before retry %d/3",
|
| 220 |
+
library, version, delay, attempt + 1,
|
| 221 |
+
)
|
| 222 |
+
await asyncio.sleep(delay)
|
| 223 |
+
continue
|
| 224 |
+
raise RuntimeError(
|
| 225 |
+
f"Groq rate limit exceeded for {library}@{version}. "
|
| 226 |
+
"Both Gemini and Groq are rate-limited. Wait a minute and try again, "
|
| 227 |
+
"or add a fresh API key in DocForge settings."
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
resp.raise_for_status()
|
| 231 |
+
data = resp.json()
|
| 232 |
+
break
|
| 233 |
+
|
| 234 |
+
elapsed = time.time() - t0
|
| 235 |
+
logger.info("Groq structuring took %.1fs for %s@%s", elapsed, library, version)
|
| 236 |
+
raw_output = data["choices"][0]["message"]["content"]
|
| 237 |
+
return _parse_gemini_json(raw_output, library, version)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _parse_gemini_json(raw: str, library: str, version: str) -> dict:
|
| 241 |
+
"""
|
| 242 |
+
Parse Gemini's response, handling common formatting issues.
|
| 243 |
+
Gemini sometimes wraps JSON in markdown code fences even when asked not to.
|
| 244 |
+
"""
|
| 245 |
+
# Strip markdown code fences if present
|
| 246 |
+
raw = re.sub(r"^```(?:json)?\s*", "", raw.strip(), flags=re.MULTILINE)
|
| 247 |
+
raw = re.sub(r"\s*```$", "", raw.strip(), flags=re.MULTILINE)
|
| 248 |
+
raw = raw.strip()
|
| 249 |
+
|
| 250 |
+
try:
|
| 251 |
+
result = json.loads(raw)
|
| 252 |
+
except json.JSONDecodeError as exc:
|
| 253 |
+
logger.error("Failed to parse Gemini JSON output: %s\nRaw: %s...", exc, raw[:500])
|
| 254 |
+
# Return a minimal valid structure rather than crashing
|
| 255 |
+
return _mock_structure(library, version)
|
| 256 |
+
|
| 257 |
+
# Ensure required top-level fields
|
| 258 |
+
result.setdefault("library", library)
|
| 259 |
+
result.setdefault("version", version)
|
| 260 |
+
result.setdefault("components", [])
|
| 261 |
+
|
| 262 |
+
# Normalize component fields
|
| 263 |
+
today = date.today().isoformat()
|
| 264 |
+
for comp in result.get("components", []):
|
| 265 |
+
comp.setdefault("gotchas", [])
|
| 266 |
+
comp.setdefault("props", [])
|
| 267 |
+
comp.setdefault("peer_dependencies", [])
|
| 268 |
+
comp.setdefault("last_verified", today)
|
| 269 |
+
|
| 270 |
+
return result
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _mock_structure(library: str, version: str) -> dict:
|
| 274 |
+
"""
|
| 275 |
+
Fallback when Gemini API key is not configured.
|
| 276 |
+
Returns a minimal but valid structure for local testing.
|
| 277 |
+
"""
|
| 278 |
+
return {
|
| 279 |
+
"library": library,
|
| 280 |
+
"version": version,
|
| 281 |
+
"description": f"Documentation for {library}@{version}",
|
| 282 |
+
"components": [
|
| 283 |
+
{
|
| 284 |
+
"name": library,
|
| 285 |
+
"import_path": f"import {{ {library} }} from '{library}'",
|
| 286 |
+
"install_command": f"npm install {library}",
|
| 287 |
+
"props": [],
|
| 288 |
+
"gotchas": [
|
| 289 |
+
"β οΈ Gemini API key not configured β run with GEMINI_API_KEY set for real extraction"
|
| 290 |
+
],
|
| 291 |
+
"peer_dependencies": [],
|
| 292 |
+
"usage_example": f"<{library} />",
|
| 293 |
+
"last_verified": date.today().isoformat(),
|
| 294 |
+
}
|
| 295 |
+
],
|
| 296 |
+
}
|