Spaces:
Sleeping
Sleeping
Download backend/services/structurer.py from solvorlabs/docforge-api: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/spaces/solvorlabs/docforge-api/resolve/main/backend/services/structurer.py
- Command line
-
hf download hf://spaces/solvorlabs/docforge-api/backend/services/structurer.py
-
curl -L -o structurer.py https://huggingface.co/spaces/solvorlabs/docforge-api/resolve/main/backend/services/structurer.py
10.9 kB
| """ | |
| Structuring engine: Gemini 2.0 Flash (primary) β Groq LLaMA (fallback). | |
| Takes raw Markdown documentation and produces a structured JSON representation | |
| with components, props, gotchas, and version-specific details. | |
| The prompt is engineered to hunt specifically for AI codegen failure patterns | |
| (gotchas) that training data tends to get wrong. | |
| """ | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import time | |
| from datetime import date | |
| import asyncio | |
| import httpx | |
| logger = logging.getLogger(__name__) | |
| class _QuotaError(Exception): | |
| """Raised internally when an LLM provider returns 429.""" | |
| GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") | |
| GEMINI_MODEL = "gemini-2.0-flash" | |
| GEMINI_ENDPOINT = ( | |
| f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent" | |
| ) | |
| GROQ_API_KEY = os.getenv("GROQ_API_KEY", "") | |
| GROQ_MODEL = "llama-3.3-70b-versatile" | |
| GROQ_ENDPOINT = "https://api.groq.com/openai/v1/chat/completions" | |
| # Groq's context window is 128k tokens; keep well under to avoid truncation errors | |
| GROQ_MAX_DOC_CHARS = 40_000 | |
| STRUCTURED_SCHEMA = """{ | |
| "library": "string", | |
| "version": "string", | |
| "description": "string", | |
| "components": [ | |
| { | |
| "name": "string", | |
| "import_path": "string (exact import statement, e.g. import X from '@/components/X')", | |
| "install_command": "string (exact command from docs, NOT generic 'npm install')", | |
| "props": [ | |
| { | |
| "name": "string", | |
| "type": "string", | |
| "required": "boolean", | |
| "default": "string or null" | |
| } | |
| ], | |
| "gotchas": ["string (specific AI codegen failure patterns)"], | |
| "peer_dependencies": ["string"], | |
| "usage_example": "string (complete, runnable code snippet)", | |
| "last_verified": "string (ISO date)" | |
| } | |
| ] | |
| }""" | |
| STRUCTURING_PROMPT = """You are a documentation structuring engine for DocForge. Your job is to extract \ | |
| precise, developer-critical information from raw library documentation. | |
| Given the following raw documentation for {library}@{version}, extract structured information. | |
| CRITICAL β You must specifically hunt for: | |
| 1. GOTCHAS: Things that commonly break AI code generation. Examples: | |
| - Required directives ("use client", "use server" in Next.js App Router) | |
| - Deprecated props or methods from previous versions | |
| - Breaking changes between major versions (e.g. v4 β v5 API changes) | |
| - Required peer dependencies that must be installed separately | |
| - Common misuse patterns (wrong import path, wrong prop name) | |
| - Framework-specific requirements (Next.js App Router vs Pages Router) | |
| - CSS imports that must be added manually | |
| - Configuration file changes required | |
| 2. EXACT install commands β NOT generic "npm install {library}". Look for npx, dlx, or custom CLI commands. | |
| 3. EXACT import paths β these change between major versions and are the #1 AI codegen failure point. | |
| 4. All props with their TypeScript types, whether they are required, and default values. | |
| 5. Version-specific API differences if mentioned in the docs. | |
| Return ONLY valid JSON (no markdown code fences, no explanation, no extra whitespace) in EXACTLY this schema: | |
| {schema} | |
| If the documentation mentions multiple components or modules, include each as a separate entry in "components". | |
| If there is only one main export, use a single component entry with the library name as the component name. | |
| Set last_verified to today's date: {today}. | |
| Raw documentation: | |
| {raw_docs}""" | |
| async def structure_with_gemini( | |
| library: str, | |
| version: str, | |
| raw_docs: str, | |
| gemini_key: str | None = None, | |
| groq_key: str | None = None, | |
| ) -> dict: | |
| """ | |
| Call Gemini 2.0 Flash to structure raw documentation into our JSON schema. | |
| gemini_key / groq_key β per-user keys that override environment variables. | |
| Falls back to env vars when not provided. | |
| """ | |
| effective_gemini = gemini_key or GEMINI_API_KEY | |
| effective_groq = groq_key or GROQ_API_KEY | |
| if not effective_gemini: | |
| logger.warning("GEMINI_API_KEY not set, using mock structurer") | |
| return _mock_structure(library, version) | |
| # Truncate to ~80k chars β keeps Gemini token usage around 20k per request, | |
| # which is much friendlier to the free tier daily quota (1M tokens/day). | |
| # Quality is maintained: the most relevant documentation is always first. | |
| max_doc_chars = 80_000 | |
| truncated = raw_docs[:max_doc_chars] | |
| if len(raw_docs) > max_doc_chars: | |
| logger.info("Truncated docs from %d to %d chars for Gemini", len(raw_docs), max_doc_chars) | |
| prompt = STRUCTURING_PROMPT.format( | |
| library=library, | |
| version=version, | |
| schema=STRUCTURED_SCHEMA, | |
| today=date.today().isoformat(), | |
| raw_docs=truncated, | |
| ) | |
| t0 = time.time() | |
| payload = { | |
| "contents": [{"parts": [{"text": prompt}]}], | |
| "generationConfig": { | |
| "temperature": 0.1, | |
| "maxOutputTokens": 8192, | |
| "responseMimeType": "application/json", | |
| }, | |
| } | |
| try: | |
| async with httpx.AsyncClient(timeout=120) as client: | |
| resp = await client.post( | |
| GEMINI_ENDPOINT, | |
| params={"key": effective_gemini}, | |
| json=payload, | |
| ) | |
| if resp.status_code == 429: | |
| raise _QuotaError("Gemini") | |
| resp.raise_for_status() | |
| data = resp.json() | |
| elapsed = time.time() - t0 | |
| logger.info("Gemini structuring took %.1fs for %s@%s", elapsed, library, version) | |
| raw_output = data["candidates"][0]["content"]["parts"][0]["text"] | |
| return _parse_gemini_json(raw_output, library, version) | |
| except _QuotaError: | |
| logger.warning("Gemini quota hit for %s@%s β falling back to Groq", library, version) | |
| except httpx.HTTPStatusError as exc: | |
| if exc.response.status_code == 429: | |
| logger.warning("Gemini quota hit for %s@%s β falling back to Groq", library, version) | |
| else: | |
| logger.error("Gemini failed for %s@%s: %s", library, version, exc) | |
| raise | |
| except Exception as exc: | |
| logger.error("Gemini failed for %s@%s: %s", library, version, exc) | |
| raise | |
| # ββ Groq fallback ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| return await _structure_with_groq(library, version, raw_docs, t0, groq_key=effective_groq) | |
| async def _structure_with_groq( | |
| library: str, version: str, raw_docs: str, t0: float, groq_key: str | None = None | |
| ) -> dict: | |
| """Call Groq (LLaMA 3.3 70B) as a fallback when Gemini quota is exhausted.""" | |
| effective_groq = groq_key or GROQ_API_KEY | |
| if not effective_groq: | |
| raise RuntimeError( | |
| "Gemini quota exceeded and no Groq key available. " | |
| "Add a Groq key in your DocForge settings (free at console.groq.com)." | |
| ) | |
| truncated = raw_docs[:GROQ_MAX_DOC_CHARS] | |
| if len(raw_docs) > GROQ_MAX_DOC_CHARS: | |
| logger.info("Truncated docs from %d to %d chars for Groq", len(raw_docs), GROQ_MAX_DOC_CHARS) | |
| prompt = STRUCTURING_PROMPT.format( | |
| library=library, | |
| version=version, | |
| schema=STRUCTURED_SCHEMA, | |
| today=date.today().isoformat(), | |
| raw_docs=truncated, | |
| ) | |
| payload = { | |
| "model": GROQ_MODEL, | |
| "messages": [{"role": "user", "content": prompt}], | |
| "temperature": 0.1, | |
| "max_tokens": 32768, | |
| } | |
| # Retry up to 3 times on 429 with exponential backoff | |
| retry_delays = [10, 30, 60] | |
| for attempt, delay in enumerate(retry_delays): | |
| async with httpx.AsyncClient(timeout=120) as client: | |
| resp = await client.post( | |
| GROQ_ENDPOINT, | |
| headers={"Authorization": f"Bearer {effective_groq}"}, | |
| json=payload, | |
| ) | |
| if resp.status_code == 429: | |
| if attempt < len(retry_delays) - 1: | |
| logger.warning( | |
| "Groq rate limited for %s@%s β waiting %ds before retry %d/3", | |
| library, version, delay, attempt + 1, | |
| ) | |
| await asyncio.sleep(delay) | |
| continue | |
| raise RuntimeError( | |
| f"Groq rate limit exceeded for {library}@{version}. " | |
| "Both Gemini and Groq are rate-limited. Wait a minute and try again, " | |
| "or add a fresh API key in DocForge settings." | |
| ) | |
| resp.raise_for_status() | |
| data = resp.json() | |
| break | |
| elapsed = time.time() - t0 | |
| logger.info("Groq structuring took %.1fs for %s@%s", elapsed, library, version) | |
| raw_output = data["choices"][0]["message"]["content"] | |
| return _parse_gemini_json(raw_output, library, version) | |
| def _parse_gemini_json(raw: str, library: str, version: str) -> dict: | |
| """ | |
| Parse Gemini's response, handling common formatting issues. | |
| Gemini sometimes wraps JSON in markdown code fences even when asked not to. | |
| """ | |
| # Strip markdown code fences if present | |
| raw = re.sub(r"^```(?:json)?\s*", "", raw.strip(), flags=re.MULTILINE) | |
| raw = re.sub(r"\s*```$", "", raw.strip(), flags=re.MULTILINE) | |
| raw = raw.strip() | |
| try: | |
| result = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| logger.error("Failed to parse Gemini JSON output: %s\nRaw: %s...", exc, raw[:500]) | |
| # Return a minimal valid structure rather than crashing | |
| return _mock_structure(library, version) | |
| # Ensure required top-level fields | |
| result.setdefault("library", library) | |
| result.setdefault("version", version) | |
| result.setdefault("components", []) | |
| # Normalize component fields | |
| today = date.today().isoformat() | |
| for comp in result.get("components", []): | |
| comp.setdefault("gotchas", []) | |
| comp.setdefault("props", []) | |
| comp.setdefault("peer_dependencies", []) | |
| comp.setdefault("last_verified", today) | |
| return result | |
| def _mock_structure(library: str, version: str) -> dict: | |
| """ | |
| Fallback when Gemini API key is not configured. | |
| Returns a minimal but valid structure for local testing. | |
| """ | |
| return { | |
| "library": library, | |
| "version": version, | |
| "description": f"Documentation for {library}@{version}", | |
| "components": [ | |
| { | |
| "name": library, | |
| "import_path": f"import {{ {library} }} from '{library}'", | |
| "install_command": f"npm install {library}", | |
| "props": [], | |
| "gotchas": [ | |
| "β οΈ Gemini API key not configured β run with GEMINI_API_KEY set for real extraction" | |
| ], | |
| "peer_dependencies": [], | |
| "usage_example": f"<{library} />", | |
| "last_verified": date.today().isoformat(), | |
| } | |
| ], | |
| } |