services/agent_loop.py

"""
Agent loop — Orchestrates the autonomous job application pipeline.

For each site in user allowlist:
  1. Send instruction to extension via WebSocket
  2. Extension navigates site, returns job listing DOM
  3. For each discovered job:
     a. Hybrid search: job desc ↔ knowledge indices → match_score
     b. If score ≥ threshold: tailor resume, generate answers, apply
  4. Sleep gap_seconds between applications

All intelligence comes from call_es_agent() — the extension is just hands.
"""

import asyncio
import json
import uuid
import logging
import time
from datetime import datetime
from urllib.parse import urlparse

logger = logging.getLogger("cereal.services.agent_loop")

MAX_STEPS_PER_APPLICATION = 60


async def start_agent_loop(user_id: str, store):
    """Start the agent loop as a background task."""
    asyncio.create_task(_run_agent_loop(user_id, store))


async def _run_agent_loop(user_id: str, store):
    """Main agent loop — iterates over allowlist sites and applies to jobs."""
    from services.elasticsearch import es_client, call_es_agent, bulk_index
    from services.response_sync import extract_employer_domain

    logger.info(f"[LOOP] ========== Agent loop starting | user={user_id} ==========")
    loop_start = time.monotonic()

    client = es_client()

    try:
        # --- Load preferences ---
        try:
            prefs_doc = await client.get(
                index="user_preferences", id=f"{user_id}_prefs"
            )
            prefs = prefs_doc["_source"].get("preferences", {})
            logger.info(f"[LOOP] Preferences loaded: {prefs}")
        except Exception as e:
            logger.warning(f"[LOOP] Could not load preferences, using defaults: {e}")
            prefs = {}

        threshold    = prefs.get("match_score_threshold", 60.0)
        gap_seconds  = prefs.get("gap_seconds", 30)
        daily_limit  = prefs.get("daily_application_limit", 50)
        auto_apply   = prefs.get("auto_apply", True)

        logger.info(
            f"[LOOP] Config | threshold={threshold} | gap={gap_seconds}s | "
            f"daily_limit={daily_limit} | auto_apply={auto_apply}"
        )

        # --- Load allowlist ---
        try:
            allowlist_doc = await client.get(
                index="user_preferences", id=f"{user_id}_allowlist"
            )
            sites = allowlist_doc["_source"].get("sites", [])
            logger.info(f"[LOOP] Allowlist loaded: {len(sites)} site(s)")
            for s in sites:
                logger.debug(f"[LOOP]   site: {s.get('url')} | enabled={s.get('enabled', True)}")
        except Exception as e:
            logger.warning(f"[LOOP] Could not load allowlist: {e}")
            sites = []

        if not sites:
            logger.warning("[LOOP] No sites in allowlist — stopping")
            await store.update(user_id, state="stopped", current_action="No sites in allowlist")
            return

        applications_count = 0

        # --- Site loop ---
        for site in sites:
            current = await store.get(user_id)
            if current.get("state") != "running":
                logger.info(f"[LOOP] Agent state changed to {current.get('state')!r} — exiting site loop")
                break

            if not site.get("enabled", True):
                logger.debug(f"[LOOP] Skipping disabled site: {site.get('url')}")
                continue

            site_url = site["url"]
            logger.info(f"[LOOP] ── Scanning site: {site_url}")
            await store.update(user_id, current_site=site_url, current_action=f"Scanning {site_url}")

            if not _is_url_allowed(site_url, sites):
                logger.warning(f"[LOOP] URL failed allowlist check, skipping: {site_url}")
                continue

            t_site = time.monotonic()
            jobs = await _discover_jobs(user_id, site_url)
            logger.info(
                f"[LOOP] Discovered {len(jobs)} job(s) from {site_url} "
                f"in {time.monotonic() - t_site:.1f}s"
            )

            # --- Job loop ---
            for job in jobs:
                current = await store.get(user_id)
                if current.get("state") != "running":
                    logger.info("[LOOP] Agent paused/stopped mid-job-loop — breaking")
                    break

                if applications_count >= daily_limit:
                    logger.info(f"[LOOP] Daily limit reached ({daily_limit}) — stopping")
                    await store.update(user_id, state="stopped", current_action="Daily limit reached")
                    return

                title   = job.get("title", "Unknown")
                company = job.get("company", "Unknown")
                logger.info(f"[LOOP] ── Evaluating: {title!r} at {company!r}")
                await store.update(
                    user_id,
                    current_action=f"Evaluating: {title} at {company}"
                )

                # Score
                t_score = time.monotonic()
                match_score = await _score_job(user_id, job)
                logger.info(
                    f"[LOOP] Score: {match_score:.1f} (threshold={threshold}) | "
                    f"took={time.monotonic() - t_score:.1f}s"
                )

                if match_score < threshold:
                    logger.info(f"[LOOP] Below threshold — skipping {title!r}")
                    continue

                if not auto_apply:
                    logger.info(f"[LOOP] auto_apply=False — storing {title!r} for review")
                    job["status"]      = "needs_review"
                    job["match_score"] = match_score
                    await _index_job(user_id, job)
                    continue

                # Apply
                logger.info(f"[LOOP] Applying to: {title!r} at {company!r}")
                await store.update(
                    user_id,
                    current_action=f"Applying: {title} at {company}"
                )

                t_apply = time.monotonic()
                try:
                    await _process_application(user_id, job, match_score)
                    applications_count += 1
                    await store.update(user_id, applications_this_session=applications_count)
                    logger.info(
                        f"[LOOP] ✓ Application complete | {title!r} at {company!r} | "
                        f"took={time.monotonic() - t_apply:.1f}s | "
                        f"total_this_session={applications_count}"
                    )
                except Exception as e:
                    logger.error(
                        f"[LOOP] ✗ Application failed | {title!r} at {company!r} | "
                        f"error={e}", exc_info=True
                    )
                    await _flag_for_review(user_id, job, str(e))

                logger.debug(f"[LOOP] Sleeping {gap_seconds}s before next application")
                await asyncio.sleep(gap_seconds)

        total_elapsed = time.monotonic() - loop_start
        logger.info(
            f"[LOOP] ========== Agent loop complete | user={user_id} | "
            f"applications={applications_count} | elapsed={total_elapsed:.1f}s =========="
        )
        await store.update(
            user_id,
            state="stopped",
            current_action=f"Complete — {applications_count} applications sent"
        )

    except Exception as e:
        elapsed = time.monotonic() - loop_start
        logger.error(
            f"[LOOP] ✗ Agent loop crashed | user={user_id} | "
            f"elapsed={elapsed:.1f}s | error={e}", exc_info=True
        )
        await store.update(user_id, state="stopped", current_action=f"Error: {str(e)[:100]}")


def _is_url_allowed(url: str, allowlist: list[dict]) -> bool:
    """Check if a URL's netloc matches any enabled allowlist entry."""
    parsed = urlparse(url)
    for site in allowlist:
        if not site.get("enabled", True):
            continue
        allowed = urlparse(site["url"])
        if parsed.netloc == allowed.netloc:
            return True
    return False


async def _discover_jobs(user_id: str, site_url: str) -> list[dict]:
    """
    Navigate the allowlisted site via the extension, run the content
    script job detector, fetch each job description page, return job dicts.
    """
    import json as _json
    import services.ws_client as ws

    jobs = []

    try:
        logger.debug(f"[DISCOVER] Navigating to {site_url}")
        t0 = time.monotonic()
        await ws.navigate(site_url)
        await ws.wait_ms(2500)
        logger.debug(f"[DISCOVER] Page loaded in {time.monotonic() - t0:.1f}s")

        # Discover job listing links
        result = await ws.evaluate("""
            (() => {
                const links = Array.from(document.querySelectorAll("a[href]"));
                const JOB_PATTERNS = [
                    /\/job/, /\/position/, /\/opening/, /\/role/,
                    /\/apply/, /\/careers\//, /\/jobs\//,
                ];
                return links
                    .filter(a => JOB_PATTERNS.some(p => p.test(a.href)))
                    .slice(0, 30)
                    .map(a => ({
                        title: a.textContent.trim().slice(0, 200),
                        apply_url: a.href,
                        site_url: window.location.origin,
                        company: document.title.split(/[|\-–]/)[1]?.trim()
                                 || window.location.hostname
                                      .replace(/^www\./, "").split(".")[0],
                    }));
            })()
        """)

        raw_jobs = result.get("value", [])
        if not isinstance(raw_jobs, list):
            raw_jobs = []

        logger.debug(f"[DISCOVER] Found {len(raw_jobs)} candidate links on {site_url}")

        # Fetch full description for each listing (cap 10)
        for i, job_meta in enumerate(raw_jobs[:10]):
            apply_url = job_meta.get("apply_url", "")
            logger.debug(f"[DISCOVER] Fetching job [{i+1}/{min(len(raw_jobs), 10)}]: {apply_url}")

            try:
                t_job = time.monotonic()
                await ws.navigate(apply_url)
                await ws.wait_ms(1500)

                desc = await ws.evaluate("""
                    (() => {
                        const SELECTORS = [
                            "[class*=\"job-description\"]",
                            "[class*=\"jobDescription\"]",
                            "[class*=\"description\"]",
                            "[data-testid*=\"description\"]",
                            "article", "main",
                        ];
                        for (const s of SELECTORS) {
                            const el = document.querySelector(s);
                            if (el && el.innerText.length > 200)
                                return el.innerText.slice(0, 3000);
                        }
                        return document.body.innerText.slice(0, 3000);
                    })()
                """)

                q_raw = await ws.evaluate(
                    "window.__cereal"
                    "  ? JSON.stringify(window.__cereal.detectCustomQuestions())"
                    "  : \"[]\"",
                )
                questions = []
                try:
                    questions = _json.loads(q_raw.get("value", "[]"))
                except Exception:
                    pass

                desc_text = desc.get("value", "")
                logger.debug(
                    f"[DISCOVER]   ✓ {job_meta['title']!r} | "
                    f"desc_len={len(desc_text)} | "
                    f"custom_questions={len(questions)} | "
                    f"took={time.monotonic() - t_job:.1f}s"
                )

                jobs.append({
                    "job_id":           str(uuid.uuid4()),
                    "title":            job_meta["title"],
                    "company":          job_meta["company"].title(),
                    "apply_url":        apply_url,
                    "site_url":         job_meta["site_url"],
                    "description":      desc_text,
                    "custom_questions": [q["label"] for q in questions],
                })

            except Exception as e:
                logger.warning(f"[DISCOVER] Failed to fetch {apply_url}: {e}")
                continue

    except Exception as e:
        logger.error(f"[DISCOVER] Job discovery failed for {site_url}: {e}", exc_info=True)

    return jobs


async def _score_job(user_id: str, job: dict) -> float:
    """Score job fit via ES Agent Builder."""
    from services.elasticsearch import call_es_agent

    title   = job.get("title", "")
    company = job.get("company", "")
    desc    = job.get("description", "")[:1500]

    prompt = (
        f"Job: {title} at {company}\n"
        f"Description: {desc}\n\n"
        f"Task: Rate how well this candidate fits this job. "
        f"Search the knowledge base for their relevant skills, experience, and projects. "
        f'Return ONLY a JSON object: {{"score": <0-100>, "reason": "<one sentence>"}}'
    )

    try:
        response = await call_es_agent(user_id=user_id, message=prompt)
        raw = response["message"]
        logger.debug(f"[SCORE] Raw agent response: {raw[:300]}")

        result = json.loads(raw)
        score  = float(result.get("score", 0))
        reason = result.get("reason", "")
        logger.info(f"[SCORE] {title!r} at {company!r} → score={score} | reason={reason!r}")
        return score

    except json.JSONDecodeError as e:
        logger.warning(f"[SCORE] Failed to parse agent JSON response: {e} | raw={response.get('message','')[:200]}")
        return 0.0
    except Exception as e:
        logger.warning(f"[SCORE] Scoring failed, defaulting to 0: {e}")
        return 0.0


async def _fill_application_form(
    user_id: str,
    job: dict,
    tailored_resume: dict,
    resume_pdf_path: str,
    custom_answers: dict,
) -> bool:
    """Fill form fields, upload resume PDF, submit. Returns True if submitted."""
    import json as _json
    import services.ws_client as ws

    title = job.get("title", "")
    logger.info(f"[FORM] Starting form fill for {title!r} | url={job.get('apply_url')}")

    await ws.navigate(job["apply_url"])
    await ws.wait_ms(2000)

    # CAPTCHA check
    captcha = await ws.evaluate(
        "window.__cereal ? window.__cereal.detectCaptcha() : false"
    )
    if captcha.get("value", False):
        logger.warning(f"[FORM] CAPTCHA detected at {job['apply_url']} — flagging for review")
        return False

    # Detect fields
    f_raw = await ws.evaluate(
        "JSON.stringify(window.__cereal ? window.__cereal.detectFormFields() : [])"
    )
    fields = []
    try:
        fields = _json.loads(f_raw.get("value", "[]"))
    except Exception:
        pass

    logger.debug(f"[FORM] Detected {len(fields)} form fields")

    name_parts   = tailored_resume.get("name", "").split()
    field_values = {
        "first":    name_parts[0] if name_parts else "",
        "last":     name_parts[-1] if len(name_parts) > 1 else "",
        "name":     tailored_resume.get("name", ""),
        "email":    tailored_resume.get("email", ""),
        "phone":    tailored_resume.get("phone", ""),
        "linkedin": tailored_resume.get("linkedin", ""),
        "github":   tailored_resume.get("github", ""),
        "website":  tailored_resume.get("github", ""),
    }

    filled = 0
    for field in fields:
        if field.get("type") == "file":
            continue
        label    = field.get("label", "").lower()
        selector = field.get("selector", "")
        if not selector:
            continue

        value = None
        for key, val in field_values.items():
            if key in label and val:
                value = val
                break

        if value is None:
            for question, answer in custom_answers.items():
                if question.lower()[:40] in label or label[:40] in question.lower():
                    value = answer
                    break

        if value:
            try:
                await ws.type_text(selector, str(value))
                await ws.wait_ms(250)
                filled += 1
                logger.debug(f"[FORM] Filled field {selector!r} (label={label!r})")
            except Exception as e:
                logger.warning(f"[FORM] Could not fill field {selector!r}: {e}")

    logger.debug(f"[FORM] Filled {filled}/{len(fields)} fields")

    # Upload resume
    u_raw = await ws.evaluate(
        "JSON.stringify(window.__cereal ? window.__cereal.detectFileUpload() : {exists: false})"
    )
    try:
        upload_info = _json.loads(u_raw.get("value", "{}") or "{}")
        if upload_info.get("exists") and upload_info.get("selector"):
            logger.debug(f"[FORM] Uploading resume PDF to {upload_info['selector']!r}")
            await ws.upload_file(upload_info["selector"], resume_pdf_path)
            await ws.wait_ms(1000)
            logger.debug("[FORM] Resume PDF uploaded")
        else:
            logger.debug("[FORM] No file upload field detected")
    except Exception as e:
        logger.warning(f"[FORM] Resume upload failed: {e}")

    # Submit
    sub = await ws.evaluate("""
        (() => {
            const btn = document.querySelector(
                'button[type="submit"], input[type="submit"], button:not([type])'
            );
            if (btn) { btn.click(); return { submitted: true }; }
            const form = document.querySelector("form");
            if (form) { form.submit(); return { submitted: true }; }
            return { submitted: false };
        })()
    """)
    await ws.wait_ms(2000)

    submitted = sub.get("value", {}).get("submitted", False)
    logger.info(f"[FORM] Submit result: submitted={submitted}")
    return submitted


async def _process_application(user_id: str, job: dict, match_score: float):
    """Full application pipeline: tailor resume → generate answers → fill form → index."""
    from services.elasticsearch import call_es_agent, bulk_index
    from services.response_sync import extract_employer_domain

    title   = job.get("title", "")
    company = job.get("company", "")

    # 1. Tailor resume
    logger.info(f"[APPLY] Tailoring resume for {title!r} at {company!r}")
    t0 = time.monotonic()

    resume_prompt = (
        f"Job Title: {title}\n"
        f"Company: {company}\n"
        f"Job Description: {job.get('description', '')[:2000]}\n\n"
        f"Task: Generate a tailored resume for this candidate for this specific job. "
        f"Retrieve the most relevant sections of their background from the knowledge base. "
        f"Output JSON matching this schema exactly:\n"
        f'{{"name":"","email":"","phone":"","linkedin":"","github":"",'
        f'"summary":"(2-3 sentences, tailored to THIS job)",'
        f'"experience":[{{"company":"","title":"","dates":"","bullets":[""]}}],'
        f'"projects":[{{"name":"","description":"","url":""}}],'
        f'"skills":[""],'
        f'"education":[{{"institution":"","degree":"","year":""}}]}}\n'
        f"Output ONLY the JSON. No markdown. No explanation."
    )

    resume_response = await call_es_agent(user_id=user_id, message=resume_prompt)
    logger.debug(f"[APPLY] Resume agent response (first 300): {resume_response['message'][:300]}")

    try:
        tailored_resume = json.loads(resume_response["message"])
        logger.info(
            f"[APPLY] Resume tailored in {time.monotonic() - t0:.1f}s | "
            f"name={tailored_resume.get('name')!r} | "
            f"skills={tailored_resume.get('skills', [])[:5]}"
        )
    except json.JSONDecodeError as e:
        logger.error(f"[APPLY] Failed to parse tailored resume JSON: {e}")
        raise

    # 2. Custom question answers
    custom_answers = {}
    questions = job.get("custom_questions", [])
    logger.info(f"[APPLY] Generating answers for {len(questions)} custom question(s)")

    for question in questions:
        logger.debug(f"[APPLY] Answering: {question!r}")
        t_q = time.monotonic()
        answer_prompt = (
            f"Job: {title} at {company}\n"
            f"Application Question: {question}\n\n"
            f"Task: Write the best possible answer to this question using this candidate's "
            f"real background. Search the knowledge base for relevant context. "
            f"Answer in first person. Be specific. Use real examples. "
            f"Length: 100-250 words unless it's a yes/no or short answer. "
            f"Output only the answer text — no preamble."
        )
        answer_response = await call_es_agent(user_id=user_id, message=answer_prompt)
        answer = answer_response["message"]
        custom_answers[question] = answer
        logger.debug(
            f"[APPLY] Answer for {question[:60]!r}: "
            f"{answer[:100]!r}... (took {time.monotonic() - t_q:.1f}s)"
        )

    # 3. Render resume PDF
    logger.debug("[APPLY] Rendering resume PDF")
    from services.resume_renderer import render_resume_pdf, cleanup_resume
    app_id = str(uuid.uuid4())
    t_pdf = time.monotonic()
    resume_pdf_path = await render_resume_pdf(user_id, tailored_resume, app_id)
    logger.debug(f"[APPLY] PDF rendered in {time.monotonic() - t_pdf:.1f}s → {resume_pdf_path}")

    # 4. Fill form and submit
    submitted = await _fill_application_form(
        user_id=user_id,
        job=job,
        tailored_resume=tailored_resume,
        resume_pdf_path=resume_pdf_path,
        custom_answers=custom_answers,
    )

    # 5. Cleanup PDF
    cleanup_resume(resume_pdf_path)

    # 6. Index application
    now = datetime.utcnow().isoformat()
    status = "applied" if submitted else "needs_review"

    application_doc = {
        "user_id":         user_id,
        "application_id":  app_id,
        "job_id":          job.get("job_id", ""),
        "company":         company,
        "role":            title,
        "site_url":        job.get("site_url", ""),
        "apply_url":       job.get("apply_url", ""),
        "employer_domain": extract_employer_domain(job.get("apply_url", "")),
        "status":          status,
        "match_score":     match_score,
        "custom_answers":  custom_answers,
        "agent_log": {
            "resume_citations": resume_response.get("citations", []),
            "steps":            [],
            "form_submitted":   submitted,
        },
        "applied_at": now,
    }

    await bulk_index("applications", [application_doc])
    logger.info(
        f"[APPLY] Application indexed | id={app_id} | status={status} | "
        f"score={match_score} | submitted={submitted}"
    )


async def _index_job(user_id: str, job: dict):
    """Index a discovered job for manual review."""
    from services.elasticsearch import bulk_index

    job["user_id"]      = user_id
    job["job_id"]       = job.get("job_id", str(uuid.uuid4()))
    job["discovered_at"] = datetime.utcnow().isoformat()
    await bulk_index("jobs", [job])
    logger.debug(f"[LOOP] Indexed job for review: {job.get('title')!r}")


async def _flag_for_review(user_id: str, job: dict, error: str):
    """Flag a failed application for manual review."""
    from services.elasticsearch import bulk_index
    from services.response_sync import extract_employer_domain

    app_id = str(uuid.uuid4())
    application_doc = {
        "user_id":         user_id,
        "application_id":  app_id,
        "job_id":          job.get("job_id", ""),
        "company":         job.get("company", ""),
        "role":            job.get("title", ""),
        "site_url":        job.get("site_url", ""),
        "apply_url":       job.get("apply_url", ""),
        "employer_domain": extract_employer_domain(job.get("apply_url", "")),
        "status":          "needs_review",
        "match_score":     job.get("match_score", 0),
        "agent_log":       {"error": error},
        "applied_at":      datetime.utcnow().isoformat(),
    }

    await bulk_index("applications", [application_doc])
    logger.info(f"[LOOP] Flagged for review | id={app_id} | error={error[:100]!r}")