File size: 22,513 Bytes
6ca8e48
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
"""
/api/context routes β€” the core DocForge pipeline endpoint.

POST /api/context   β†’ queues a job, returns job_id immediately
GET  /api/context/{job_id} β†’ returns job status and result when done
"""

import logging
import time
import uuid
from datetime import date
from urllib.parse import urlparse

from fastapi import APIRouter, BackgroundTasks, HTTPException, Request

from ..models.schemas import (
    ContextJobResult,
    ContextRequest,
    InputType,
    JobCreatedResponse,
    JobStatus,
    OutputFormat,
)
from ..services import cache
from ..services.formatter import format_context_md
from ..services.ingestion.npm_ingester import (
    resolve_npm_package,
    resolve_pypi_package,
)
from ..services.ingestion.paste_ingester import ingest_paste
from ..services.ingestion.registries_ingester import (
    resolve_crates,
    resolve_cran,
    resolve_hex,
    resolve_maven,
    resolve_nuget,
    resolve_pubdev,
    resolve_rubygems,
)
from ..services.ingestion.search_ingester import find_docs_url
from ..services.ingestion.url_ingester import crawl_url
from ..services.structurer import structure_with_gemini

logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/context", tags=["context"])


@router.post("", response_model=JobCreatedResponse)
async def create_context_job(
    request: ContextRequest,
    background_tasks: BackgroundTasks,
    http_request: Request,
) -> JobCreatedResponse:
    """
    Queue a documentation context generation job.
    Returns immediately with a job_id for polling.

    If a valid Bearer token is provided the user's stored API keys are used for
    the AI step; otherwise falls back to env-var keys (dev / open access).
    """
    # ── Auth: prefer stored keys from Bearer token, fall back to request body ─
    gemini_key: str | None = None
    groq_key:   str | None = None

    auth_header = http_request.headers.get("authorization", "")
    if auth_header.startswith("Bearer "):
        from ..services.auth_service import decode_token, get_api_keys
        payload = decode_token(auth_header.split(" ", 1)[1])
        if payload:
            keys = await get_api_keys(payload["sub"])
            gemini_key = keys.get("gemini_key")
            groq_key   = keys.get("groq_key")

    # CLI local-key mode: keys sent directly in the request body
    if not gemini_key and not groq_key:
        gemini_key = request.gemini_key or None
        groq_key   = request.groq_key or None

    # Enforce user-supplied key β€” DocForge does not maintain a server-level key.
    # DEV_MODE bypasses this so local dev works without a key configured.
    import os as _os
    if not gemini_key and not _os.getenv("DEV_MODE", "false").lower() == "true":
        raise HTTPException(
            status_code=400,
            detail=(
                "Gemini API key required. "
                "Get a free key at https://aistudio.google.com/app/apikey "
                "then add it in DocForge Settings."
            ),
        )

    job_id = str(uuid.uuid4())

    # Store initial job state
    await cache.set_job(job_id, {
        "status": JobStatus.queued,
        "job_id": job_id,
        "input": request.input,
        "input_type": request.input_type,
    })

    # Run the heavy pipeline work in the background so the HTTP response
    # returns instantly (crawling takes 30–90s)
    background_tasks.add_task(
        _run_pipeline,
        job_id=job_id,
        request=request,
        gemini_key=gemini_key,
        groq_key=groq_key,
    )

    logger.info("Queued job %s for input=%s type=%s", job_id, request.input, request.input_type)
    return JobCreatedResponse(job_id=job_id, status=JobStatus.queued)


@router.get("/{job_id}", response_model=ContextJobResult)
async def get_context_job(job_id: str) -> ContextJobResult:
    """Poll job status. Returns output when status == 'complete'."""
    job = await cache.get_job(job_id)
    if job is None:
        raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
    return ContextJobResult(**job)


# ---------------------------------------------------------------------------
# Background pipeline
# ---------------------------------------------------------------------------

async def _run_pipeline(
    job_id: str,
    request: ContextRequest,
    gemini_key: str | None = None,
    groq_key: str | None = None,
) -> None:
    """
    Full DocForge pipeline:
    1. Resolve package / fetch URL / clone repo
    2. Crawl documentation
    3. Structure with Gemini (using user's stored key if authenticated)
    4. Format as .context.md
    5. Cache result
    """
    t_start = time.time()

    async def update_status(status: str, **extra: object) -> None:
        job = await cache.get_job(job_id) or {}
        job.update({"status": status, "job_id": job_id, **extra})
        await cache.set_job(job_id, job)

    try:
        await update_status(JobStatus.processing)

        # ── Step 1: Resolve metadata ──────────────────────────────────────
        library = request.input
        version = "latest"
        docs_url: str | None = None
        version_tag: str | None = None
        copypaste_docs: str | None = None  # pre-fetched docs for copy-paste libs
        pre_structured: dict | None = None  # pre-built structured dict (skips LLM)

        if request.input_type == InputType.npm:
            # If the user somehow passed a URL as an npm package, treat it as a URL crawl
            if request.input.startswith(("http://", "https://")):
                logger.warning("npm input looks like a URL β€” switching to URL crawl: %s", request.input)
                raw_docs = await crawl_url(request.input)
                structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key)
                output = format_context_md(structured) if request.output_format == OutputFormat.context_md else __import__("json").dumps(structured, indent=2)
                structured["cached_at"] = date.today().isoformat()
                await cache.set_cached_context(library, version, structured)
                await update_status(JobStatus.complete, library=library, version=version, output=output, components=structured.get("components", []))
                return
            meta = await resolve_npm_package(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            version_tag = meta.get("_version_tag")  # set when version not on npm but GitHub exists
            # Copy-paste library: docs already fetched during resolution
            if meta.get("_is_copypaste"):
                copypaste_docs = meta.get("_raw_docs", "")
                pre_structured = meta.get("_structured") or None
                logger.info(
                    "Copy-paste library '%s' detected via %s β€” skipping crawl (%d chars pre-fetched)",
                    library, meta.get("_copypaste_method"), len(copypaste_docs),
                )
            else:
                logger.info("Resolved npm: %s@%s docs=%s tag=%s", library, version, docs_url, version_tag)

        elif request.input_type == InputType.pypi:
            meta = await resolve_pypi_package(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved PyPI: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.crates:
            meta = await resolve_crates(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved crates.io: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.rubygems:
            meta = await resolve_rubygems(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved RubyGems: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.pubdev:
            meta = await resolve_pubdev(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved pub.dev: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.nuget:
            meta = await resolve_nuget(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved NuGet: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.maven:
            meta = await resolve_maven(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved Maven: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.hex:
            meta = await resolve_hex(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved Hex: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.cran:
            meta = await resolve_cran(request.input)
            library = meta["name"]
            version = meta["version"]
            docs_url = meta.get("homepage")
            logger.info("Resolved CRAN: %s@%s docs=%s", library, version, docs_url)

        elif request.input_type == InputType.url:
            # Derive a clean library name from the domain so the cache key is
            # human-readable ("reactbits") rather than the raw URL.
            # GitHub URLs keep their repo name; plain URLs use the domain stem.
            if "github.com" in request.input:
                # e.g. https://github.com/DavidHDev/react-bits β†’ react-bits
                parts = [p for p in urlparse(request.input).path.split("/") if p]
                library = parts[1] if len(parts) >= 2 else request.input
            else:
                domain = urlparse(request.input).netloc.removeprefix("www.")
                library = domain.split(".")[0]  # "reactbits" from "reactbits.dev"
            logger.info("URL input '%s' β†’ library name '%s' for cache/structuring", request.input, library)

        # ── Step 2: Check cache (skip expensive crawl if already done) ────
        cached = await cache.get_cached_context(library, version)
        if cached:
            output = format_context_md(cached) if request.output_format == OutputFormat.context_md else str(cached)
            await update_status(
                JobStatus.complete,
                library=library,
                version=version,
                output=output,
                components=cached.get("components", []),
            )
            logger.info("Cache hit for %s@%s, job %s done in %.1fs", library, version, job_id, time.time() - t_start)
            return

        # ── Step 3: Crawl documentation ───────────────────────────────────
        raw_docs = ""

        # Copy-paste library β€” docs were fetched during resolution, skip crawl
        if copypaste_docs is not None:
            raw_docs = copypaste_docs

        elif request.input_type == InputType.paste:
            raw_docs = await ingest_paste(request.content or request.input)

        elif request.input_type == InputType.github:
            from ..services.ingestion.github_ingester import ingest_github_repo
            raw_docs = await ingest_github_repo(request.input)

        elif request.input_type == InputType.url:
            if "github.com" in request.input:
                from ..services.ingestion.github_ingester import ingest_github_repo
                raw_docs = await ingest_github_repo(request.input)
            else:
                import asyncio as _asyncio
                from ..services.ingestion.url_ingester import extract_github_url
                from ..services.ingestion.copypaste_ingester import (
                    check_jsrepo, check_shadcn_registry, detect_and_ingest,
                )

                # library is already set to the clean name_hint in Step 1
                # ── Parallel: all three lightweight checks at once ─────────────
                # check_shadcn_registry hits /r/registry.json etc. β€” it does NOT
                # need a GitHub URL, so it runs alongside the other two.
                github_url, jsrepo_name_docs, shadcn_result = await _asyncio.gather(
                    extract_github_url(request.input),
                    check_jsrepo(library, None),
                    check_shadcn_registry(request.input, library),
                )

                # shadcn registry returns a pre-built structured dict β€” skip LLM entirely
                if shadcn_result is not None:
                    pre_structured = shadcn_result
                    logger.info(
                        "URL '%s' β†’ shadcn registry match for '%s' (%d components, skipping LLM)",
                        request.input, library, len(pre_structured.get("components", [])),
                    )

                else:
                    # Prefer jsrepo registry hits for the raw-docs path
                    quick_docs = jsrepo_name_docs

                    # If we also have a GitHub URL, do a targeted jsrepo check
                    # (raw GitHub paths + jsrepo/{owner}/{repo}).
                    jsrepo_targeted: str | None = None
                    if github_url and not quick_docs:
                        jsrepo_targeted = await check_jsrepo(library, github_url)

                    cp_docs = jsrepo_targeted or quick_docs

                    if cp_docs:
                        raw_docs = cp_docs
                        logger.info("URL '%s' β†’ registry match for '%s' (%d chars)", request.input, library, len(raw_docs))

                    elif github_url:
                        logger.info("URL '%s' β†’ GitHub repo found: %s", request.input, github_url)
                        # copy-paste detection with full github_url context
                        cp_result = await detect_and_ingest(
                            name=library,
                            github_url=github_url,
                            homepage=request.input,
                        )
                        if cp_result:
                            raw_docs = cp_result.get("_raw_docs", "")
                            logger.info(
                                "URL '%s' β†’ copy-paste lib (method=%s) %d chars",
                                request.input, cp_result.get("_copypaste_method"), len(raw_docs),
                            )
                        else:
                            # Not copy-paste β€” GitHub ingester still beats Playwright
                            from ..services.ingestion.github_ingester import ingest_github_repo
                            raw_docs = await ingest_github_repo(github_url)
                            logger.info("URL '%s' β†’ GitHub ingestion (%d chars)", request.input, len(raw_docs))

                    else:
                        # No GitHub found β€” fall back to Playwright
                        raw_docs = await crawl_url(request.input)
                        logger.info("URL '%s' β†’ Playwright crawl (%d chars)", request.input, len(raw_docs))

                        # Sparse content β€” try GitHub search by domain name.
                        # Only accept a fallback repo whose homepage points back to the
                        # same domain we started from (prevents picking up unrelated
                        # packages like MCP wrappers that happen to mention the name).
                        if len(raw_docs) < 20_000:
                            import httpx
                            input_domain = urlparse(request.input).netloc.removeprefix("www.")
                            async with httpx.AsyncClient(timeout=10) as _client:
                                from ..services.ingestion.npm_ingester import _github_search
                                fallback = await _github_search(_client, library)
                                if fallback:
                                    # Quick check: does the GitHub repo's homepage match?
                                    try:
                                        meta_resp = await _client.get(
                                            f"https://api.github.com/repos/{fallback.removeprefix('https://github.com/')}",
                                            headers={"User-Agent": "docforge/1.0", "Accept": "application/vnd.github+json"},
                                            timeout=5,
                                        )
                                        if meta_resp.is_success:
                                            repo_homepage = meta_resp.json().get("homepage") or ""
                                            if input_domain not in repo_homepage:
                                                logger.info(
                                                    "GitHub fallback '%s' homepage '%s' doesn't match domain '%s' β€” skipping",
                                                    fallback, repo_homepage, input_domain,
                                                )
                                                fallback = None
                                    except Exception:
                                        fallback = None  # if we can't verify, skip it

                            if fallback:
                                logger.info(
                                    "Sparse Playwright content (%d chars) for '%s' β€” trying GitHub: %s",
                                    len(raw_docs), request.input, fallback,
                                )
                                from ..services.ingestion.github_ingester import ingest_github_repo
                                github_docs = await ingest_github_repo(fallback)
                                if len(github_docs) > len(raw_docs):
                                    raw_docs = github_docs

        else:
            # npm or pypi: crawl the resolved docs URL
            if version_tag and not docs_url:
                # Version not on npm but GitHub repo known β€” use git tag directly
                from ..services.ingestion.github_ingester import ingest_github_repo
                repo_url = meta.get("repository", "")
                import re as _re
                repo_url = _re.sub(r"^git\+", "", repo_url)
                logger.info("Using GitHub tag %s for %s", version_tag, repo_url)
                raw_docs = await ingest_github_repo(repo_url, version_tag=version_tag)
            elif docs_url:
                if "github.com" in docs_url:
                    from ..services.ingestion.github_ingester import ingest_github_repo
                    raw_docs = await ingest_github_repo(docs_url, version_tag=version_tag)
                else:
                    raw_docs = await crawl_url(docs_url)
            else:
                # Web search fallback: query DuckDuckGo for docs
                found_url = await find_docs_url(f"{library} {version}")
                if found_url:
                    logger.info("Web search found docs for %s: %s", library, found_url)
                    if "github.com" in found_url:
                        from ..services.ingestion.github_ingester import ingest_github_repo
                        raw_docs = await ingest_github_repo(found_url)
                    else:
                        raw_docs = await crawl_url(found_url)
                else:
                    logger.warning("No docs URL found for %s, sending minimal stub", library)
                    raw_docs = f"Library: {library}\nVersion: {version}\nNo documentation URL found."

        logger.info("Crawl done for %s@%s: %d chars in %.1fs", library, version, len(raw_docs), time.time() - t_start)

        # ── Step 4: Structure with Gemini ─────────────────────────────────
        # Skip LLM for copy-paste libraries with pre-built structured data
        if pre_structured is not None:
            structured = pre_structured
            structured.setdefault("library", library)
            structured.setdefault("version", version)
            logger.info("Using pre-built structure for %s@%s (%d components)", library, version, len(structured.get("components", [])))
        else:
            structured = await structure_with_gemini(library, version, raw_docs, gemini_key=gemini_key, groq_key=groq_key)

        # Filter to requested components if specified
        if request.components:
            requested = {c.lower() for c in request.components}
            structured["components"] = [
                c for c in structured.get("components", [])
                if c.get("name", "").lower() in requested
            ]

        # ── Step 5: Format output ─────────────────────────────────────────
        if request.output_format == OutputFormat.context_md:
            output = format_context_md(structured)
        else:
            import json
            output = json.dumps(structured, indent=2)

        # ── Step 6: Cache and complete ────────────────────────────────────
        structured["cached_at"] = date.today().isoformat()
        await cache.set_cached_context(library, version, structured)

        total = time.time() - t_start
        logger.info("Pipeline complete for %s@%s in %.1fs", library, version, total)

        await update_status(
            JobStatus.complete,
            library=library,
            version=version,
            output=output,
            components=structured.get("components", []),
        )

    except Exception as exc:
        logger.exception("Pipeline failed for job %s: %s", job_id, exc)
        await update_status(JobStatus.failed, error=str(exc))