Spaces:
Sleeping
Sleeping
File size: 10,926 Bytes
6ca8e48 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 | """
Structuring engine: Gemini 2.0 Flash (primary) β Groq LLaMA (fallback).
Takes raw Markdown documentation and produces a structured JSON representation
with components, props, gotchas, and version-specific details.
The prompt is engineered to hunt specifically for AI codegen failure patterns
(gotchas) that training data tends to get wrong.
"""
import json
import logging
import os
import re
import time
from datetime import date
import asyncio
import httpx
logger = logging.getLogger(__name__)
class _QuotaError(Exception):
"""Raised internally when an LLM provider returns 429."""
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
GEMINI_MODEL = "gemini-2.0-flash"
GEMINI_ENDPOINT = (
f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent"
)
GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
GROQ_MODEL = "llama-3.3-70b-versatile"
GROQ_ENDPOINT = "https://api.groq.com/openai/v1/chat/completions"
# Groq's context window is 128k tokens; keep well under to avoid truncation errors
GROQ_MAX_DOC_CHARS = 40_000
STRUCTURED_SCHEMA = """{
"library": "string",
"version": "string",
"description": "string",
"components": [
{
"name": "string",
"import_path": "string (exact import statement, e.g. import X from '@/components/X')",
"install_command": "string (exact command from docs, NOT generic 'npm install')",
"props": [
{
"name": "string",
"type": "string",
"required": "boolean",
"default": "string or null"
}
],
"gotchas": ["string (specific AI codegen failure patterns)"],
"peer_dependencies": ["string"],
"usage_example": "string (complete, runnable code snippet)",
"last_verified": "string (ISO date)"
}
]
}"""
STRUCTURING_PROMPT = """You are a documentation structuring engine for DocForge. Your job is to extract \
precise, developer-critical information from raw library documentation.
Given the following raw documentation for {library}@{version}, extract structured information.
CRITICAL β You must specifically hunt for:
1. GOTCHAS: Things that commonly break AI code generation. Examples:
- Required directives ("use client", "use server" in Next.js App Router)
- Deprecated props or methods from previous versions
- Breaking changes between major versions (e.g. v4 β v5 API changes)
- Required peer dependencies that must be installed separately
- Common misuse patterns (wrong import path, wrong prop name)
- Framework-specific requirements (Next.js App Router vs Pages Router)
- CSS imports that must be added manually
- Configuration file changes required
2. EXACT install commands β NOT generic "npm install {library}". Look for npx, dlx, or custom CLI commands.
3. EXACT import paths β these change between major versions and are the #1 AI codegen failure point.
4. All props with their TypeScript types, whether they are required, and default values.
5. Version-specific API differences if mentioned in the docs.
Return ONLY valid JSON (no markdown code fences, no explanation, no extra whitespace) in EXACTLY this schema:
{schema}
If the documentation mentions multiple components or modules, include each as a separate entry in "components".
If there is only one main export, use a single component entry with the library name as the component name.
Set last_verified to today's date: {today}.
Raw documentation:
{raw_docs}"""
async def structure_with_gemini(
library: str,
version: str,
raw_docs: str,
gemini_key: str | None = None,
groq_key: str | None = None,
) -> dict:
"""
Call Gemini 2.0 Flash to structure raw documentation into our JSON schema.
gemini_key / groq_key β per-user keys that override environment variables.
Falls back to env vars when not provided.
"""
effective_gemini = gemini_key or GEMINI_API_KEY
effective_groq = groq_key or GROQ_API_KEY
if not effective_gemini:
logger.warning("GEMINI_API_KEY not set, using mock structurer")
return _mock_structure(library, version)
# Truncate to ~80k chars β keeps Gemini token usage around 20k per request,
# which is much friendlier to the free tier daily quota (1M tokens/day).
# Quality is maintained: the most relevant documentation is always first.
max_doc_chars = 80_000
truncated = raw_docs[:max_doc_chars]
if len(raw_docs) > max_doc_chars:
logger.info("Truncated docs from %d to %d chars for Gemini", len(raw_docs), max_doc_chars)
prompt = STRUCTURING_PROMPT.format(
library=library,
version=version,
schema=STRUCTURED_SCHEMA,
today=date.today().isoformat(),
raw_docs=truncated,
)
t0 = time.time()
payload = {
"contents": [{"parts": [{"text": prompt}]}],
"generationConfig": {
"temperature": 0.1,
"maxOutputTokens": 8192,
"responseMimeType": "application/json",
},
}
try:
async with httpx.AsyncClient(timeout=120) as client:
resp = await client.post(
GEMINI_ENDPOINT,
params={"key": effective_gemini},
json=payload,
)
if resp.status_code == 429:
raise _QuotaError("Gemini")
resp.raise_for_status()
data = resp.json()
elapsed = time.time() - t0
logger.info("Gemini structuring took %.1fs for %s@%s", elapsed, library, version)
raw_output = data["candidates"][0]["content"]["parts"][0]["text"]
return _parse_gemini_json(raw_output, library, version)
except _QuotaError:
logger.warning("Gemini quota hit for %s@%s β falling back to Groq", library, version)
except httpx.HTTPStatusError as exc:
if exc.response.status_code == 429:
logger.warning("Gemini quota hit for %s@%s β falling back to Groq", library, version)
else:
logger.error("Gemini failed for %s@%s: %s", library, version, exc)
raise
except Exception as exc:
logger.error("Gemini failed for %s@%s: %s", library, version, exc)
raise
# ββ Groq fallback ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
return await _structure_with_groq(library, version, raw_docs, t0, groq_key=effective_groq)
async def _structure_with_groq(
library: str, version: str, raw_docs: str, t0: float, groq_key: str | None = None
) -> dict:
"""Call Groq (LLaMA 3.3 70B) as a fallback when Gemini quota is exhausted."""
effective_groq = groq_key or GROQ_API_KEY
if not effective_groq:
raise RuntimeError(
"Gemini quota exceeded and no Groq key available. "
"Add a Groq key in your DocForge settings (free at console.groq.com)."
)
truncated = raw_docs[:GROQ_MAX_DOC_CHARS]
if len(raw_docs) > GROQ_MAX_DOC_CHARS:
logger.info("Truncated docs from %d to %d chars for Groq", len(raw_docs), GROQ_MAX_DOC_CHARS)
prompt = STRUCTURING_PROMPT.format(
library=library,
version=version,
schema=STRUCTURED_SCHEMA,
today=date.today().isoformat(),
raw_docs=truncated,
)
payload = {
"model": GROQ_MODEL,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.1,
"max_tokens": 32768,
}
# Retry up to 3 times on 429 with exponential backoff
retry_delays = [10, 30, 60]
for attempt, delay in enumerate(retry_delays):
async with httpx.AsyncClient(timeout=120) as client:
resp = await client.post(
GROQ_ENDPOINT,
headers={"Authorization": f"Bearer {effective_groq}"},
json=payload,
)
if resp.status_code == 429:
if attempt < len(retry_delays) - 1:
logger.warning(
"Groq rate limited for %s@%s β waiting %ds before retry %d/3",
library, version, delay, attempt + 1,
)
await asyncio.sleep(delay)
continue
raise RuntimeError(
f"Groq rate limit exceeded for {library}@{version}. "
"Both Gemini and Groq are rate-limited. Wait a minute and try again, "
"or add a fresh API key in DocForge settings."
)
resp.raise_for_status()
data = resp.json()
break
elapsed = time.time() - t0
logger.info("Groq structuring took %.1fs for %s@%s", elapsed, library, version)
raw_output = data["choices"][0]["message"]["content"]
return _parse_gemini_json(raw_output, library, version)
def _parse_gemini_json(raw: str, library: str, version: str) -> dict:
"""
Parse Gemini's response, handling common formatting issues.
Gemini sometimes wraps JSON in markdown code fences even when asked not to.
"""
# Strip markdown code fences if present
raw = re.sub(r"^```(?:json)?\s*", "", raw.strip(), flags=re.MULTILINE)
raw = re.sub(r"\s*```$", "", raw.strip(), flags=re.MULTILINE)
raw = raw.strip()
try:
result = json.loads(raw)
except json.JSONDecodeError as exc:
logger.error("Failed to parse Gemini JSON output: %s\nRaw: %s...", exc, raw[:500])
# Return a minimal valid structure rather than crashing
return _mock_structure(library, version)
# Ensure required top-level fields
result.setdefault("library", library)
result.setdefault("version", version)
result.setdefault("components", [])
# Normalize component fields
today = date.today().isoformat()
for comp in result.get("components", []):
comp.setdefault("gotchas", [])
comp.setdefault("props", [])
comp.setdefault("peer_dependencies", [])
comp.setdefault("last_verified", today)
return result
def _mock_structure(library: str, version: str) -> dict:
"""
Fallback when Gemini API key is not configured.
Returns a minimal but valid structure for local testing.
"""
return {
"library": library,
"version": version,
"description": f"Documentation for {library}@{version}",
"components": [
{
"name": library,
"import_path": f"import {{ {library} }} from '{library}'",
"install_command": f"npm install {library}",
"props": [],
"gotchas": [
"β οΈ Gemini API key not configured β run with GEMINI_API_KEY set for real extraction"
],
"peer_dependencies": [],
"usage_example": f"<{library} />",
"last_verified": date.today().isoformat(),
}
],
} |