File size: 10,926 Bytes
6ca8e48
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
"""
Structuring engine: Gemini 2.0 Flash (primary) β†’ Groq LLaMA (fallback).

Takes raw Markdown documentation and produces a structured JSON representation
with components, props, gotchas, and version-specific details.

The prompt is engineered to hunt specifically for AI codegen failure patterns
(gotchas) that training data tends to get wrong.
"""

import json
import logging
import os
import re
import time
from datetime import date

import asyncio

import httpx

logger = logging.getLogger(__name__)


class _QuotaError(Exception):
    """Raised internally when an LLM provider returns 429."""

GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
GEMINI_MODEL = "gemini-2.0-flash"
GEMINI_ENDPOINT = (
    f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent"
)

GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
GROQ_MODEL = "llama-3.3-70b-versatile"
GROQ_ENDPOINT = "https://api.groq.com/openai/v1/chat/completions"
# Groq's context window is 128k tokens; keep well under to avoid truncation errors
GROQ_MAX_DOC_CHARS = 40_000

STRUCTURED_SCHEMA = """{
  "library": "string",
  "version": "string",
  "description": "string",
  "components": [
    {
      "name": "string",
      "import_path": "string (exact import statement, e.g. import X from '@/components/X')",
      "install_command": "string (exact command from docs, NOT generic 'npm install')",
      "props": [
        {
          "name": "string",
          "type": "string",
          "required": "boolean",
          "default": "string or null"
        }
      ],
      "gotchas": ["string (specific AI codegen failure patterns)"],
      "peer_dependencies": ["string"],
      "usage_example": "string (complete, runnable code snippet)",
      "last_verified": "string (ISO date)"
    }
  ]
}"""

STRUCTURING_PROMPT = """You are a documentation structuring engine for DocForge. Your job is to extract \
precise, developer-critical information from raw library documentation.

Given the following raw documentation for {library}@{version}, extract structured information.

CRITICAL β€” You must specifically hunt for:
1. GOTCHAS: Things that commonly break AI code generation. Examples:
   - Required directives ("use client", "use server" in Next.js App Router)
   - Deprecated props or methods from previous versions
   - Breaking changes between major versions (e.g. v4 β†’ v5 API changes)
   - Required peer dependencies that must be installed separately
   - Common misuse patterns (wrong import path, wrong prop name)
   - Framework-specific requirements (Next.js App Router vs Pages Router)
   - CSS imports that must be added manually
   - Configuration file changes required

2. EXACT install commands β€” NOT generic "npm install {library}". Look for npx, dlx, or custom CLI commands.
3. EXACT import paths β€” these change between major versions and are the #1 AI codegen failure point.
4. All props with their TypeScript types, whether they are required, and default values.
5. Version-specific API differences if mentioned in the docs.

Return ONLY valid JSON (no markdown code fences, no explanation, no extra whitespace) in EXACTLY this schema:
{schema}

If the documentation mentions multiple components or modules, include each as a separate entry in "components".
If there is only one main export, use a single component entry with the library name as the component name.
Set last_verified to today's date: {today}.

Raw documentation:
{raw_docs}"""


async def structure_with_gemini(
    library: str,
    version: str,
    raw_docs: str,
    gemini_key: str | None = None,
    groq_key: str | None = None,
) -> dict:
    """
    Call Gemini 2.0 Flash to structure raw documentation into our JSON schema.

    gemini_key / groq_key β€” per-user keys that override environment variables.
    Falls back to env vars when not provided.
    """
    effective_gemini = gemini_key or GEMINI_API_KEY
    effective_groq   = groq_key   or GROQ_API_KEY

    if not effective_gemini:
        logger.warning("GEMINI_API_KEY not set, using mock structurer")
        return _mock_structure(library, version)

    # Truncate to ~80k chars β€” keeps Gemini token usage around 20k per request,
    # which is much friendlier to the free tier daily quota (1M tokens/day).
    # Quality is maintained: the most relevant documentation is always first.
    max_doc_chars = 80_000
    truncated = raw_docs[:max_doc_chars]
    if len(raw_docs) > max_doc_chars:
        logger.info("Truncated docs from %d to %d chars for Gemini", len(raw_docs), max_doc_chars)

    prompt = STRUCTURING_PROMPT.format(
        library=library,
        version=version,
        schema=STRUCTURED_SCHEMA,
        today=date.today().isoformat(),
        raw_docs=truncated,
    )

    t0 = time.time()
    payload = {
        "contents": [{"parts": [{"text": prompt}]}],
        "generationConfig": {
            "temperature": 0.1,
            "maxOutputTokens": 8192,
            "responseMimeType": "application/json",
        },
    }

    try:
        async with httpx.AsyncClient(timeout=120) as client:
            resp = await client.post(
                GEMINI_ENDPOINT,
                params={"key": effective_gemini},
                json=payload,
            )
            if resp.status_code == 429:
                raise _QuotaError("Gemini")
            resp.raise_for_status()
            data = resp.json()

        elapsed = time.time() - t0
        logger.info("Gemini structuring took %.1fs for %s@%s", elapsed, library, version)
        raw_output = data["candidates"][0]["content"]["parts"][0]["text"]
        return _parse_gemini_json(raw_output, library, version)

    except _QuotaError:
        logger.warning("Gemini quota hit for %s@%s β€” falling back to Groq", library, version)
    except httpx.HTTPStatusError as exc:
        if exc.response.status_code == 429:
            logger.warning("Gemini quota hit for %s@%s β€” falling back to Groq", library, version)
        else:
            logger.error("Gemini failed for %s@%s: %s", library, version, exc)
            raise
    except Exception as exc:
        logger.error("Gemini failed for %s@%s: %s", library, version, exc)
        raise

    # ── Groq fallback ────────────────────────────────────────────────────────
    return await _structure_with_groq(library, version, raw_docs, t0, groq_key=effective_groq)


async def _structure_with_groq(
    library: str, version: str, raw_docs: str, t0: float, groq_key: str | None = None
) -> dict:
    """Call Groq (LLaMA 3.3 70B) as a fallback when Gemini quota is exhausted."""
    effective_groq = groq_key or GROQ_API_KEY
    if not effective_groq:
        raise RuntimeError(
            "Gemini quota exceeded and no Groq key available. "
            "Add a Groq key in your DocForge settings (free at console.groq.com)."
        )

    truncated = raw_docs[:GROQ_MAX_DOC_CHARS]
    if len(raw_docs) > GROQ_MAX_DOC_CHARS:
        logger.info("Truncated docs from %d to %d chars for Groq", len(raw_docs), GROQ_MAX_DOC_CHARS)

    prompt = STRUCTURING_PROMPT.format(
        library=library,
        version=version,
        schema=STRUCTURED_SCHEMA,
        today=date.today().isoformat(),
        raw_docs=truncated,
    )

    payload = {
        "model": GROQ_MODEL,
        "messages": [{"role": "user", "content": prompt}],
        "temperature": 0.1,
        "max_tokens": 32768,
    }

    # Retry up to 3 times on 429 with exponential backoff
    retry_delays = [10, 30, 60]
    for attempt, delay in enumerate(retry_delays):
        async with httpx.AsyncClient(timeout=120) as client:
            resp = await client.post(
                GROQ_ENDPOINT,
                headers={"Authorization": f"Bearer {effective_groq}"},
                json=payload,
            )

        if resp.status_code == 429:
            if attempt < len(retry_delays) - 1:
                logger.warning(
                    "Groq rate limited for %s@%s β€” waiting %ds before retry %d/3",
                    library, version, delay, attempt + 1,
                )
                await asyncio.sleep(delay)
                continue
            raise RuntimeError(
                f"Groq rate limit exceeded for {library}@{version}. "
                "Both Gemini and Groq are rate-limited. Wait a minute and try again, "
                "or add a fresh API key in DocForge settings."
            )

        resp.raise_for_status()
        data = resp.json()
        break

    elapsed = time.time() - t0
    logger.info("Groq structuring took %.1fs for %s@%s", elapsed, library, version)
    raw_output = data["choices"][0]["message"]["content"]
    return _parse_gemini_json(raw_output, library, version)


def _parse_gemini_json(raw: str, library: str, version: str) -> dict:
    """
    Parse Gemini's response, handling common formatting issues.
    Gemini sometimes wraps JSON in markdown code fences even when asked not to.
    """
    # Strip markdown code fences if present
    raw = re.sub(r"^```(?:json)?\s*", "", raw.strip(), flags=re.MULTILINE)
    raw = re.sub(r"\s*```$", "", raw.strip(), flags=re.MULTILINE)
    raw = raw.strip()

    try:
        result = json.loads(raw)
    except json.JSONDecodeError as exc:
        logger.error("Failed to parse Gemini JSON output: %s\nRaw: %s...", exc, raw[:500])
        # Return a minimal valid structure rather than crashing
        return _mock_structure(library, version)

    # Ensure required top-level fields
    result.setdefault("library", library)
    result.setdefault("version", version)
    result.setdefault("components", [])

    # Normalize component fields
    today = date.today().isoformat()
    for comp in result.get("components", []):
        comp.setdefault("gotchas", [])
        comp.setdefault("props", [])
        comp.setdefault("peer_dependencies", [])
        comp.setdefault("last_verified", today)

    return result


def _mock_structure(library: str, version: str) -> dict:
    """
    Fallback when Gemini API key is not configured.
    Returns a minimal but valid structure for local testing.
    """
    return {
        "library": library,
        "version": version,
        "description": f"Documentation for {library}@{version}",
        "components": [
            {
                "name": library,
                "import_path": f"import {{ {library} }} from '{library}'",
                "install_command": f"npm install {library}",
                "props": [],
                "gotchas": [
                    "⚠️  Gemini API key not configured β€” run with GEMINI_API_KEY set for real extraction"
                ],
                "peer_dependencies": [],
                "usage_example": f"<{library} />",
                "last_verified": date.today().isoformat(),
            }
        ],
    }