Spaces:
Sleeping
Sleeping
Download backend/routers/context.py from solvorlabs/docforge-api: direct link, hf CLI and curl.
- Browser
- Download file 22.5 kB
-
https://huggingface.co/spaces/solvorlabs/docforge-api/resolve/main/backend/routers/context.py
- Command line
-
hf download hf://spaces/solvorlabs/docforge-api/backend/routers/context.py
-
curl -L -o context.py https://huggingface.co/spaces/solvorlabs/docforge-api/resolve/main/backend/routers/context.py
22.5 kB
| """ | |
| /api/context routes β the core DocForge pipeline endpoint. | |
| POST /api/context β queues a job, returns job_id immediately | |
| GET /api/context/{job_id} β returns job status and result when done | |
| """ | |
| import logging | |
| import time | |
| import uuid | |
| from datetime import date | |
| from urllib.parse import urlparse | |
| from fastapi import APIRouter, BackgroundTasks, HTTPException, Request | |
| from ..models.schemas import ( | |
| ContextJobResult, | |
| ContextRequest, | |
| InputType, | |
| JobCreatedResponse, | |
| JobStatus, | |
| OutputFormat, | |
| ) | |
| from ..services import cache | |
| from ..services.formatter import format_context_md | |
| from ..services.ingestion.npm_ingester import ( | |
| resolve_npm_package, | |
| resolve_pypi_package, | |
| ) | |
| from ..services.ingestion.paste_ingester import ingest_paste | |
| from ..services.ingestion.registries_ingester import ( | |
| resolve_crates, | |
| resolve_cran, | |
| resolve_hex, | |
| resolve_maven, | |
| resolve_nuget, | |
| resolve_pubdev, | |
| resolve_rubygems, | |
| ) | |
| from ..services.ingestion.search_ingester import find_docs_url | |
| from ..services.ingestion.url_ingester import crawl_url | |
| from ..services.structurer import structure_with_gemini | |
| logger = logging.getLogger(__name__) | |
| router = APIRouter(prefix="/api/context", tags=["context"]) | |
| async def create_context_job( | |
| request: ContextRequest, | |
| background_tasks: BackgroundTasks, | |
| http_request: Request, | |
| ) -> JobCreatedResponse: | |
| """ | |
| Queue a documentation context generation job. | |
| Returns immediately with a job_id for polling. | |
| If a valid Bearer token is provided the user's stored API keys are used for | |
| the AI step; otherwise falls back to env-var keys (dev / open access). | |
| """ | |
| # ββ Auth: prefer stored keys from Bearer token, fall back to request body β | |
| gemini_key: str | None = None | |
| groq_key: str | None = None | |
| auth_header = http_request.headers.get("authorization", "") | |
| if auth_header.startswith("Bearer "): | |
| from ..services.auth_service import decode_token, get_api_keys | |
| payload = decode_token(auth_header.split(" ", 1)[1]) | |
| if payload: | |
| keys = await get_api_keys(payload["sub"]) | |
| gemini_key = keys.get("gemini_key") | |
| groq_key = keys.get("groq_key") | |
| # CLI local-key mode: keys sent directly in the request body | |
| if not gemini_key and not groq_key: | |
| gemini_key = request.gemini_key or None | |
| groq_key = request.groq_key or None | |
| # Enforce user-supplied key β DocForge does not maintain a server-level key. | |
| # DEV_MODE bypasses this so local dev works without a key configured. | |
| import os as _os | |
| if not gemini_key and not _os.getenv("DEV_MODE", "false").lower() == "true": | |
| raise HTTPException( | |
| status_code=400, | |
| detail=( | |
| "Gemini API key required. " | |
| "Get a free key at https://aistudio.google.com/app/apikey " | |
| "then add it in DocForge Settings." | |
| ), | |
| ) | |
| job_id = str(uuid.uuid4()) | |
| # Store initial job state | |
| await cache.set_job(job_id, { | |
| "status": JobStatus.queued, | |
| "job_id": job_id, | |
| "input": request.input, | |
| "input_type": request.input_type, | |
| }) | |
| # Run the heavy pipeline work in the background so the HTTP response | |
| # returns instantly (crawling takes 30β90s) | |
| background_tasks.add_task( | |
| _run_pipeline, | |
| job_id=job_id, | |
| request=request, | |
| gemini_key=gemini_key, | |
| groq_key=groq_key, | |
| ) | |
| logger.info("Queued job %s for input=%s type=%s", job_id, request.input, request.input_type) | |
| return JobCreatedResponse(job_id=job_id, status=JobStatus.queued) | |
| async def get_context_job(job_id: str) -> ContextJobResult: | |
| """Poll job status. Returns output when status == 'complete'.""" | |
| job = await cache.get_job(job_id) | |
| if job is None: | |
| raise HTTPException(status_code=404, detail=f"Job {job_id} not found") | |
| return ContextJobResult(**job) | |
| # --------------------------------------------------------------------------- | |
| # Background pipeline | |
| # --------------------------------------------------------------------------- | |
| async def _run_pipeline( | |
| job_id: str, | |
| request: ContextRequest, | |
| gemini_key: str | None = None, | |
| groq_key: str | None = None, | |
| ) -> None: | |
| """ | |
| Full DocForge pipeline: | |
| 1. Resolve package / fetch URL / clone repo | |
| 2. Crawl documentation | |
| 3. Structure with Gemini (using user's stored key if authenticated) | |
| 4. Format as .context.md | |
| 5. Cache result | |
| """ | |
| t_start = time.time() | |
| async def update_status(status: str, **extra: object) -> None: | |
| job = await cache.get_job(job_id) or {} | |
| job.update({"status": status, "job_id": job_id, **extra}) | |
| await cache.set_job(job_id, job) | |
| try: | |
| await update_status(JobStatus.processing) | |
| # ββ Step 1: Resolve metadata ββββββββββββββββββββββββββββββββββββββ | |
| library = request.input | |
| version = "latest" | |
| docs_url: str | None = None | |
| version_tag: str | None = None | |
| copypaste_docs: str | None = None # pre-fetched docs for copy-paste libs | |
| pre_structured: dict | None = None # pre-built structured dict (skips LLM) | |
| if request.input_type == InputType.npm: | |
| # If the user somehow passed a URL as an npm package, treat it as a URL crawl | |
| if request.input.startswith(("http://", "https://")): | |
| logger.warning("npm input looks like a URL β switching to URL crawl: %s", request.input) | |
| raw_docs = await crawl_url(request.input) | |
| structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key) | |
| output = format_context_md(structured) if request.output_format == OutputFormat.context_md else __import__("json").dumps(structured, indent=2) | |
| structured["cached_at"] = date.today().isoformat() | |
| await cache.set_cached_context(library, version, structured) | |
| await update_status(JobStatus.complete, library=library, version=version, output=output, components=structured.get("components", [])) | |
| return | |
| meta = await resolve_npm_package(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| version_tag = meta.get("_version_tag") # set when version not on npm but GitHub exists | |
| # Copy-paste library: docs already fetched during resolution | |
| if meta.get("_is_copypaste"): | |
| copypaste_docs = meta.get("_raw_docs", "") | |
| pre_structured = meta.get("_structured") or None | |
| logger.info( | |
| "Copy-paste library '%s' detected via %s β skipping crawl (%d chars pre-fetched)", | |
| library, meta.get("_copypaste_method"), len(copypaste_docs), | |
| ) | |
| else: | |
| logger.info("Resolved npm: %s@%s docs=%s tag=%s", library, version, docs_url, version_tag) | |
| elif request.input_type == InputType.pypi: | |
| meta = await resolve_pypi_package(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved PyPI: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.crates: | |
| meta = await resolve_crates(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved crates.io: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.rubygems: | |
| meta = await resolve_rubygems(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved RubyGems: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.pubdev: | |
| meta = await resolve_pubdev(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved pub.dev: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.nuget: | |
| meta = await resolve_nuget(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved NuGet: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.maven: | |
| meta = await resolve_maven(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved Maven: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.hex: | |
| meta = await resolve_hex(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved Hex: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.cran: | |
| meta = await resolve_cran(request.input) | |
| library = meta["name"] | |
| version = meta["version"] | |
| docs_url = meta.get("homepage") | |
| logger.info("Resolved CRAN: %s@%s docs=%s", library, version, docs_url) | |
| elif request.input_type == InputType.url: | |
| # Derive a clean library name from the domain so the cache key is | |
| # human-readable ("reactbits") rather than the raw URL. | |
| # GitHub URLs keep their repo name; plain URLs use the domain stem. | |
| if "github.com" in request.input: | |
| # e.g. https://github.com/DavidHDev/react-bits β react-bits | |
| parts = [p for p in urlparse(request.input).path.split("/") if p] | |
| library = parts[1] if len(parts) >= 2 else request.input | |
| else: | |
| domain = urlparse(request.input).netloc.removeprefix("www.") | |
| library = domain.split(".")[0] # "reactbits" from "reactbits.dev" | |
| logger.info("URL input '%s' β library name '%s' for cache/structuring", request.input, library) | |
| # ββ Step 2: Check cache (skip expensive crawl if already done) ββββ | |
| cached = await cache.get_cached_context(library, version) | |
| if cached: | |
| output = format_context_md(cached) if request.output_format == OutputFormat.context_md else str(cached) | |
| await update_status( | |
| JobStatus.complete, | |
| library=library, | |
| version=version, | |
| output=output, | |
| components=cached.get("components", []), | |
| ) | |
| logger.info("Cache hit for %s@%s, job %s done in %.1fs", library, version, job_id, time.time() - t_start) | |
| return | |
| # ββ Step 3: Crawl documentation βββββββββββββββββββββββββββββββββββ | |
| raw_docs = "" | |
| # Copy-paste library β docs were fetched during resolution, skip crawl | |
| if copypaste_docs is not None: | |
| raw_docs = copypaste_docs | |
| elif request.input_type == InputType.paste: | |
| raw_docs = await ingest_paste(request.content or request.input) | |
| elif request.input_type == InputType.github: | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| raw_docs = await ingest_github_repo(request.input) | |
| elif request.input_type == InputType.url: | |
| if "github.com" in request.input: | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| raw_docs = await ingest_github_repo(request.input) | |
| else: | |
| import asyncio as _asyncio | |
| from ..services.ingestion.url_ingester import extract_github_url | |
| from ..services.ingestion.copypaste_ingester import ( | |
| check_jsrepo, check_shadcn_registry, detect_and_ingest, | |
| ) | |
| # library is already set to the clean name_hint in Step 1 | |
| # ββ Parallel: all three lightweight checks at once βββββββββββββ | |
| # check_shadcn_registry hits /r/registry.json etc. β it does NOT | |
| # need a GitHub URL, so it runs alongside the other two. | |
| github_url, jsrepo_name_docs, shadcn_result = await _asyncio.gather( | |
| extract_github_url(request.input), | |
| check_jsrepo(library, None), | |
| check_shadcn_registry(request.input, library), | |
| ) | |
| # shadcn registry returns a pre-built structured dict β skip LLM entirely | |
| if shadcn_result is not None: | |
| pre_structured = shadcn_result | |
| logger.info( | |
| "URL '%s' β shadcn registry match for '%s' (%d components, skipping LLM)", | |
| request.input, library, len(pre_structured.get("components", [])), | |
| ) | |
| else: | |
| # Prefer jsrepo registry hits for the raw-docs path | |
| quick_docs = jsrepo_name_docs | |
| # If we also have a GitHub URL, do a targeted jsrepo check | |
| # (raw GitHub paths + jsrepo/{owner}/{repo}). | |
| jsrepo_targeted: str | None = None | |
| if github_url and not quick_docs: | |
| jsrepo_targeted = await check_jsrepo(library, github_url) | |
| cp_docs = jsrepo_targeted or quick_docs | |
| if cp_docs: | |
| raw_docs = cp_docs | |
| logger.info("URL '%s' β registry match for '%s' (%d chars)", request.input, library, len(raw_docs)) | |
| elif github_url: | |
| logger.info("URL '%s' β GitHub repo found: %s", request.input, github_url) | |
| # copy-paste detection with full github_url context | |
| cp_result = await detect_and_ingest( | |
| name=library, | |
| github_url=github_url, | |
| homepage=request.input, | |
| ) | |
| if cp_result: | |
| raw_docs = cp_result.get("_raw_docs", "") | |
| logger.info( | |
| "URL '%s' β copy-paste lib (method=%s) %d chars", | |
| request.input, cp_result.get("_copypaste_method"), len(raw_docs), | |
| ) | |
| else: | |
| # Not copy-paste β GitHub ingester still beats Playwright | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| raw_docs = await ingest_github_repo(github_url) | |
| logger.info("URL '%s' β GitHub ingestion (%d chars)", request.input, len(raw_docs)) | |
| else: | |
| # No GitHub found β fall back to Playwright | |
| raw_docs = await crawl_url(request.input) | |
| logger.info("URL '%s' β Playwright crawl (%d chars)", request.input, len(raw_docs)) | |
| # Sparse content β try GitHub search by domain name. | |
| # Only accept a fallback repo whose homepage points back to the | |
| # same domain we started from (prevents picking up unrelated | |
| # packages like MCP wrappers that happen to mention the name). | |
| if len(raw_docs) < 20_000: | |
| import httpx | |
| input_domain = urlparse(request.input).netloc.removeprefix("www.") | |
| async with httpx.AsyncClient(timeout=10) as _client: | |
| from ..services.ingestion.npm_ingester import _github_search | |
| fallback = await _github_search(_client, library) | |
| if fallback: | |
| # Quick check: does the GitHub repo's homepage match? | |
| try: | |
| meta_resp = await _client.get( | |
| f"https://api.github.com/repos/{fallback.removeprefix('https://github.com/')}", | |
| headers={"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"}, | |
| timeout=5, | |
| ) | |
| if meta_resp.is_success: | |
| repo_homepage = meta_resp.json().get("homepage") or "" | |
| if input_domain not in repo_homepage: | |
| logger.info( | |
| "GitHub fallback '%s' homepage '%s' doesn't match domain '%s' β skipping", | |
| fallback, repo_homepage, input_domain, | |
| ) | |
| fallback = None | |
| except Exception: | |
| fallback = None # if we can't verify, skip it | |
| if fallback: | |
| logger.info( | |
| "Sparse Playwright content (%d chars) for '%s' β trying GitHub: %s", | |
| len(raw_docs), request.input, fallback, | |
| ) | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| github_docs = await ingest_github_repo(fallback) | |
| if len(github_docs) > len(raw_docs): | |
| raw_docs = github_docs | |
| else: | |
| # npm or pypi: crawl the resolved docs URL | |
| if version_tag and not docs_url: | |
| # Version not on npm but GitHub repo known β use git tag directly | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| repo_url = meta.get("repository", "") | |
| import re as _re | |
| repo_url = _re.sub(r"^git\+", "", repo_url) | |
| logger.info("Using GitHub tag %s for %s", version_tag, repo_url) | |
| raw_docs = await ingest_github_repo(repo_url, version_tag=version_tag) | |
| elif docs_url: | |
| if "github.com" in docs_url: | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| raw_docs = await ingest_github_repo(docs_url, version_tag=version_tag) | |
| else: | |
| raw_docs = await crawl_url(docs_url) | |
| else: | |
| # Web search fallback: query DuckDuckGo for docs | |
| found_url = await find_docs_url(f"{library} {version}") | |
| if found_url: | |
| logger.info("Web search found docs for %s: %s", library, found_url) | |
| if "github.com" in found_url: | |
| from ..services.ingestion.github_ingester import ingest_github_repo | |
| raw_docs = await ingest_github_repo(found_url) | |
| else: | |
| raw_docs = await crawl_url(found_url) | |
| else: | |
| logger.warning("No docs URL found for %s, sending minimal stub", library) | |
| raw_docs = f"Library: {library}\nVersion: {version}\nNo documentation URL found." | |
| logger.info("Crawl done for %s@%s: %d chars in %.1fs", library, version, len(raw_docs), time.time() - t_start) | |
| # ββ Step 4: Structure with Gemini βββββββββββββββββββββββββββββββββ | |
| # Skip LLM for copy-paste libraries with pre-built structured data | |
| if pre_structured is not None: | |
| structured = pre_structured | |
| structured.setdefault("library", library) | |
| structured.setdefault("version", version) | |
| logger.info("Using pre-built structure for %s@%s (%d components)", library, version, len(structured.get("components", []))) | |
| else: | |
| structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key) | |
| # Filter to requested components if specified | |
| if request.components: | |
| requested = {c.lower() for c in request.components} | |
| structured["components"] = [ | |
| c for c in structured.get("components", []) | |
| if c.get("name", "").lower() in requested | |
| ] | |
| # ββ Step 5: Format output βββββββββββββββββββββββββββββββββββββββββ | |
| if request.output_format == OutputFormat.context_md: | |
| output = format_context_md(structured) | |
| else: | |
| import json | |
| output = json.dumps(structured, indent=2) | |
| # ββ Step 6: Cache and complete ββββββββββββββββββββββββββββββββββββ | |
| structured["cached_at"] = date.today().isoformat() | |
| await cache.set_cached_context(library, version, structured) | |
| total = time.time() - t_start | |
| logger.info("Pipeline complete for %s@%s in %.1fs", library, version, total) | |
| await update_status( | |
| JobStatus.complete, | |
| library=library, | |
| version=version, | |
| output=output, | |
| components=structured.get("components", []), | |
| ) | |
| except Exception as exc: | |
| logger.exception("Pipeline failed for job %s: %s", job_id, exc) | |
| await update_status(JobStatus.failed, error=str(exc)) |