scripts/pull_handoff_bundle.py

#!/usr/bin/env python3
"""Pull live Habit Journal state into a context-free handoff markdown bundle.

Reads env (or repo-root .env): HF_SPACE_URL, AGENT_TOKEN, APP_PASSWORD.
Writes handoff/HANDOFF_BUNDLE_{TODAY}.md and handoff/raw/*.json (gitignored).

Never prints APP_PASSWORD or bearer tokens into the bundle.
"""

from __future__ import annotations

import json
import os
import re
import sys
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, build_opener, HTTPCookieProcessor
from http.cookiejar import CookieJar

try:
    from zoneinfo import ZoneInfo
except ImportError:  # pragma: no cover
    ZoneInfo = None  # type: ignore[misc, assignment]

REPO_ROOT = Path(__file__).resolve().parents[1]
HANDOFF_DIR = REPO_ROOT / "handoff"
RAW_DIR = HANDOFF_DIR / "raw"
TZ_NAME = "Africa/Dar_es_Salaam"
DEFAULT_SPACE = "https://Mbonea-Fastwhisper.hf.space"

PHONE_RE = re.compile(r"\b\d{8,}\b")
EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
SECRET_KEYS = frozenset(
    {
        "password",
        "old_password",
        "new_password",
        "app_password",
        "authorization",
        "cookie",
        "token",
        "agent_token",
        "api_key",
        "openrouter_api_key",
        "secret",
        "secret_key",
    }
)


def load_dotenv(path: Path) -> None:
    if not path.is_file():
        return
    for line in path.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        key, value = line.split("=", 1)
        key = key.strip()
        value = value.strip().strip("'").strip('"')
        if key and key not in os.environ:
            os.environ[key] = value


def today_in_tz() -> date:
    """Africa/Dar_es_Salaam is UTC+3 year-round (no DST)."""

    try:
        if ZoneInfo is not None:
            return datetime.now(ZoneInfo(TZ_NAME)).date()
    except Exception:  # noqa: BLE001 — Windows often lacks tzdata
        pass
    return (datetime.now(timezone.utc) + timedelta(hours=3)).date()


def redact_text(value: str) -> str:
    value = EMAIL_RE.sub("[email]", value)
    value = PHONE_RE.sub("[digits]", value)
    return value


def redact_obj(value: Any) -> Any:
    if isinstance(value, dict):
        out: dict[str, Any] = {}
        for k, v in value.items():
            if k.lower() in SECRET_KEYS or any(s in k.lower() for s in ("password", "token", "secret", "cookie")):
                out[k] = "[redacted]"
            else:
                out[k] = redact_obj(v)
        return out
    if isinstance(value, list):
        return [redact_obj(v) for v in value]
    if isinstance(value, str):
        return redact_text(value)
    return value


def trunc(text: Any, n: int = 200) -> str:
    s = redact_text(str(text or "")).replace("\n", " ").strip()
    return s if len(s) <= n else s[: n - 1] + "…"


def pretty(data: Any) -> str:
    return json.dumps(redact_obj(data), indent=2, ensure_ascii=False, default=str)


class SpaceClient:
    def __init__(self, base_url: str, agent_token: str | None, password: str | None) -> None:
        self.base = base_url.rstrip("/")
        self.agent_token = agent_token or None
        self.password = password or None
        self.jar = CookieJar()
        self.opener = build_opener(HTTPCookieProcessor(self.jar))
        self.session_ok = False

    def _request(
        self,
        method: str,
        path: str,
        *,
        query: dict[str, Any] | None = None,
        body: dict[str, Any] | None = None,
        use_agent: bool = False,
        use_session: bool = False,
    ) -> tuple[int, Any]:
        url = f"{self.base}{path}"
        if query:
            url = f"{url}?{urlencode({k: v for k, v in query.items() if v is not None})}"
        headers = {
            "Accept": "application/json",
            "User-Agent": "habit-journal-handoff-pull/1.0",
        }
        data = None
        if body is not None:
            data = json.dumps(body).encode("utf-8")
            headers["Content-Type"] = "application/json"
        if use_agent and self.agent_token:
            headers["Authorization"] = f"Bearer {self.agent_token}"
        req = Request(url, data=data, headers=headers, method=method)
        try:
            with self.opener.open(req, timeout=60) as resp:
                raw = resp.read().decode("utf-8", errors="replace")
                status = getattr(resp, "status", 200)
                try:
                    return status, json.loads(raw) if raw else None
                except json.JSONDecodeError:
                    return status, {"raw_text": trunc(raw, 500)}
        except HTTPError as exc:
            raw = exc.read().decode("utf-8", errors="replace")
            try:
                payload = json.loads(raw) if raw else {"error": {"message": str(exc)}}
            except json.JSONDecodeError:
                payload = {"error": {"message": trunc(raw, 400)}}
            return exc.code, payload
        except URLError as exc:
            return 0, {"error": {"message": str(exc.reason)}}

    def get(
        self,
        path: str,
        *,
        query: dict[str, Any] | None = None,
        prefer_agent: bool = False,
    ) -> tuple[int, Any]:
        if prefer_agent and self.agent_token:
            status, data = self._request("GET", path, query=query, use_agent=True)
            if status in (200, 201):
                return status, data
            if status not in (401, 403, 404):
                return status, data
        if self.session_ok or self.ensure_session():
            return self._request("GET", path, query=query, use_session=True)
        return 401, {"error": {"message": "No AGENT_TOKEN and session login failed"}}

    def ensure_session(self) -> bool:
        if self.session_ok:
            return True
        if not self.password:
            return False
        status, data = self._request(
            "POST",
            "/api/auth/login",
            body={"password": self.password},
        )
        ok = status == 200 and isinstance(data, dict) and data.get("ok") is True
        self.session_ok = ok
        return ok


def unwrap(data: Any) -> Any:
    if isinstance(data, dict) and "data" in data:
        return data.get("data")
    return data


def write_raw(name: str, payload: Any) -> Path:
    RAW_DIR.mkdir(parents=True, exist_ok=True)
    path = RAW_DIR / name
    path.write_text(pretty(payload) + "\n", encoding="utf-8")
    return path


def checklist_line(flag: bool, label: str) -> str:
    mark = "x" if flag else " "
    return f"- [{mark}] {label}"


def build_bundle(
    *,
    pulled_at: str,
    space_url: str,
    today: date,
    yday: date,
    health: Any,
    ready: Any,
    agent_ctx: Any,
    agent_status: int,
    pending: list[dict[str, Any]],
    recent: list[dict[str, Any]],
    daily: dict[str, Any] | None,
    plan_today: dict[str, Any] | None,
    plan_yday: dict[str, Any] | None,
    priors: Any,
    stats: Any,
    settings: Any,
    raw_names: list[str],
) -> str:
    ctx = agent_ctx if isinstance(agent_ctx, dict) else {}
    daily_items = list((daily or {}).get("items") or [])
    week_points = (daily or {}).get("week_points")
    risk = ctx.get("risk_1h") or (plan_today or {}).get("risk_1h") or {}
    triggers = ctx.get("triggers_yesterday") or (plan_today or {}).get("triggers_yesterday") or []
    capacity = ctx.get("capacity_hint")
    if capacity is None:
        capacity = (plan_today or {}).get("capacity_hint")

    now_utc = datetime.now(timezone.utc)
    cutoff_24h = now_utc - timedelta(hours=24)
    entries_24h = 0
    pending_old = 0
    for e in recent:
        ts = e.get("ts")
        try:
            dt = datetime.fromisoformat(str(ts).replace("Z", "+00:00"))
        except ValueError:
            continue
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        if dt >= cutoff_24h:
            entries_24h += 1
        if e.get("result") == "pending" and dt < cutoff_24h:
            pending_old += 1

    blocks_today = list((plan_today or {}).get("blocks") or [])
    with_fb = sum(1 for b in blocks_today if b.get("feedback"))
    strong = 0
    for b in blocks_today:
        fb = b.get("feedback") or {}
        if fb.get("strong") or (
            fb.get("did") in ("done", "partial")
            and fb.get("actual_min") is not None
            and fb.get("quality") is not None
            and fb.get("fun") is not None
        ):
            strong += 1
    strong_rate = (strong / len(blocks_today)) if blocks_today else None
    has_explore = any(
        b.get("intent") in ("explore", "restore_fun") for b in blocks_today
    )

    high_pending = any(
        (e.get("intensity") or 0) >= 7 and e.get("result") == "pending" for e in pending + recent
    )
    court_rerun_flag = False
    for row in daily_items:
        if str(row.get("date")) not in (today.isoformat(), yday.isoformat()):
            continue
        if row.get("court") == "court":
            court_rerun_flag = True
        rerun = str(row.get("rerun") or "none").lower()
        if rerun not in ("none", "", "false", "0", "closed"):
            court_rerun_flag = True

    overload = isinstance(capacity, (int, float)) and capacity < 0.55 and len(blocks_today) >= 5
    prior_kinds = []
    if isinstance(priors, dict):
        prior_kinds = list(priors.get("kinds") or [])
    empty_priors = not prior_kinds or all(int(k.get("n") or 0) == 0 for k in prior_kinds if isinstance(k, dict))

    lines: list[str] = []
    a = lines.append
    a("# Habit Journal pull — context-free bundle")
    a(f"pulled_at_iso: {pulled_at}")
    a(f"space_url: {space_url}")
    a(f"timezone: {TZ_NAME}")
    a(f"date_today: {today.isoformat()}")
    a("")
    a("## How to read this (for the external AI)")
    a("You have NO prior chat unless the user adds it. Use only this bundle + what the user types.")
    a(
        "User goals often include: FSE/avoidance awareness, Spain GO (don’t cancel; don’t pay flights pre-visa), "
        "Daily vs Log, schedule feedback, boundaries — but prefer DATA here over assumptions."
    )
    a("Do not ask for passwords.")
    a("")
    a("## Health")
    a("### /api/health")
    a("```json")
    a(pretty(health))
    a("```")
    a("### /api/ready")
    a("```json")
    a(pretty(ready))
    a("```")
    if settings is not None:
        a("### /api/settings/status")
        a("```json")
        a(pretty(settings))
        a("```")
    a("")
    a("## Agent context (today)")
    a(f"http_status: {agent_status}")
    if ctx:
        a("```json")
        a(pretty(ctx))
        a("```")
    else:
        a("_Agent context unavailable — used human-route fallbacks._")
    a("")
    a("## Pending entries")
    a(f"count: {len(pending)}")
    if not pending:
        a("_none_")
    else:
        for e in pending:
            a(
                f"- `{e.get('id')}` · {e.get('ts')} · intensity={e.get('intensity')} · "
                f"emotions={e.get('emotions')} · remedy={e.get('remedy')} · result={e.get('result')}"
            )
            a(f"  happened: {trunc(e.get('happened'), 200)}")
    a("")
    a("## Recent entries (newest first, max 50)")
    if not recent:
        a("_none_")
    else:
        a("| ts | tags | emotions | intensity | result | remedy | happened |")
        a("|---|---|---|---|---|---|---|")
        for e in recent[:50]:
            a(
                f"| {e.get('ts')} | {','.join(e.get('tags') or [])} | "
                f"{','.join(e.get('emotions') or [])} | {e.get('intensity')} | "
                f"{e.get('result')} | {trunc(e.get('remedy'), 40)} | {trunc(e.get('happened'), 200)} |"
            )
    a("")
    a("## Daily rows (last 7 days)")
    if week_points is not None:
        a(f"week_points: {week_points} · band: {(daily or {}).get('band')}")
    if not daily_items:
        a("_none_")
    else:
        a(
            "| date | primary_brick | brick_done | corn_sessions | delay_ok | "
            "daydream | rerun | court | points | note |"
        )
        a("|---|---|---|---|---|---|---|---|---|---|")
        for row in daily_items:
            a(
                f"| {row.get('date')} | {row.get('primary_brick')} | {row.get('brick_done')} | "
                f"{row.get('corn_sessions')} | {row.get('delay_ok')} | {row.get('daydream')} | "
                f"{row.get('rerun')} | {row.get('court')} | {row.get('points')} | "
                f"{trunc(row.get('note'), 80)} |"
            )
    a("")
    a("## Plan today")
    _append_plan(a, plan_today)
    a("## Plan yesterday")
    _append_plan(a, plan_yday)
    a("")
    a("## Feedback summary (if in context or derivable)")
    a(f"blocks_today: {len(blocks_today)}")
    a(f"blocks_with_feedback: {with_fb}")
    a(f"strong_feedback_count: {strong}")
    a(f"strong_feedback_rate: {None if strong_rate is None else round(strong_rate, 3)}")
    a(f"has_explore_or_restore_block: {has_explore}")
    for b in blocks_today:
        fb = b.get("feedback")
        if not fb:
            continue
        a(
            f"- {b.get('start')}-{b.get('end')} {b.get('title')}: did={fb.get('did')} "
            f"min={fb.get('actual_min')} q={fb.get('quality')} fun={fb.get('fun')} "
            f"energy={fb.get('energy_after')} repeat={fb.get('would_repeat')}"
        )
    a("")
    a("## Priors (schedule)")
    if prior_kinds:
        a("| kind | n | d_hat | p_done | mean_fun | repeat_score |")
        a("|---|---|---|---|---|---|")
        for k in prior_kinds:
            if not isinstance(k, dict):
                continue
            a(
                f"| {k.get('kind')} | {k.get('n')} | {k.get('d_hat')} | {k.get('p_done')} | "
                f"{k.get('mean_fun')} | {k.get('repeat_score')} |"
            )
        md = (priors or {}).get("markdown") if isinstance(priors, dict) else None
        if md:
            a("")
            a("```")
            a(str(md))
            a("```")
    else:
        a("_none / unavailable_")
    a("")
    a("## Stats snapshot (if any)")
    if stats is None:
        a("_unavailable_")
    else:
        a("```json")
        a(pretty(stats))
        a("```")
    a("")
    a("## Risk / triggers (from context)")
    a(f"risk_1h: {pretty(risk)}")
    a(f"triggers_yesterday: {pretty(triggers)}")
    a(f"capacity_hint: {capacity}")
    ops = ctx.get("operators") or (plan_today or {}).get("operators")
    if ops:
        a(f"operators: {pretty(ops)}")
    a("")
    a("## Derived (compute in script if easy)")
    a(f"- pending_count: {len(pending)}")
    a(f"- week_points_sum: {week_points}")
    a(f"- entries_last_24h_count: {entries_24h}")
    a(f"- pending_older_than_24h: {pending_old}")
    a(f"- blocks_planned_today: {len(blocks_today)}")
    a(f"- blocks_with_feedback: {with_fb}")
    a(f"- strong_feedback_rate_today: {None if strong_rate is None else round(strong_rate, 3)}")
    a("")
    a("## Red flags (heuristic checklist for external AI)")
    a(checklist_line(high_pending, "high intensity + pending unresolved"))
    a(checklist_line(court_rerun_flag, "court/rerun open on daily"))
    a(checklist_line(bool(overload), "plan overload vs capacity_hint"))
    a(checklist_line(bool(blocks_today) and not has_explore, "no explore/restore_fun block"))
    a(checklist_line(empty_priors, "DATA_THIN / empty priors"))
    a("")
    a("## Raw pointers")
    a("local files: handoff/raw/...")
    for name in raw_names:
        a(f"- handoff/raw/{name}")
    a("")
    return "\n".join(lines)


def _append_plan(a, plan: dict[str, Any] | None) -> None:
    if not plan:
        a("_none / unavailable_")
        a("")
        return
    a(f"version: {plan.get('version')} · source: {plan.get('source')} · capacity_hint: {plan.get('capacity_hint')}")
    blocks = plan.get("blocks") or []
    if not blocks:
        a("_no blocks_")
    else:
        for b in blocks:
            a(
                f"- {b.get('start')}-{b.get('end')} · {b.get('title')} · kind={b.get('kind')} · "
                f"priority={b.get('priority')} · status={b.get('status')} · locked={b.get('locked')}"
            )
    a("")


def main() -> int:
    load_dotenv(REPO_ROOT / ".env")
    space = (os.environ.get("HF_SPACE_URL") or DEFAULT_SPACE).rstrip("/")
    agent_token = os.environ.get("AGENT_TOKEN") or ""
    password = os.environ.get("APP_PASSWORD") or ""

    today = today_in_tz()
    yday = today - timedelta(days=1)
    week_start = today - timedelta(days=6)
    pulled_at = datetime.now(timezone.utc).isoformat()

    client = SpaceClient(space, agent_token or None, password or None)
    HANDOFF_DIR.mkdir(parents=True, exist_ok=True)
    RAW_DIR.mkdir(parents=True, exist_ok=True)

    raw_names: list[str] = []

    def save(name: str, status: int, payload: Any) -> Any:
        envelope = {"http_status": status, "body": payload}
        write_raw(name, envelope)
        raw_names.append(name)
        return unwrap(payload) if status == 200 else None

    st_h, health_body = client.get("/api/health")
    health = save(f"health_{today.isoformat()}.json", st_h, health_body)

    st_r, ready_body = client.get("/api/ready")
    ready = save(f"ready_{today.isoformat()}.json", st_r, ready_body)

    # Agent API uses ?day= (not date=)
    st_a, agent_body = (0, None)
    if agent_token:
        st_a, agent_body = client._request(
            "GET",
            "/api/agent/context",
            query={"day": today.isoformat()},
            use_agent=True,
        )
    if st_a not in (200, 201):
        # session fallback to same agent route (supports session) or skip
        st_a2, agent_body2 = client.get(
            "/api/agent/context",
            query={"day": today.isoformat()},
            prefer_agent=False,
        )
        if st_a2 in (200, 201):
            st_a, agent_body = st_a2, agent_body2
    agent_ctx = save(f"agent_context_{today.isoformat()}.json", st_a, agent_body)

    # Human extras (session)
    if not client.session_ok:
        client.ensure_session()

    st_p, pending_body = client.get("/api/entries", query={"result": "pending", "limit": 50})
    pending_data = save(f"entries_pending_{today.isoformat()}.json", st_p, pending_body)
    pending = list((pending_data or {}).get("items") or []) if isinstance(pending_data, dict) else []

    st_e, recent_body = client.get(
        "/api/entries",
        query={"start": week_start.isoformat(), "end": today.isoformat(), "limit": 50},
    )
    recent_data = save(f"entries_recent_{today.isoformat()}.json", st_e, recent_body)
    recent = list((recent_data or {}).get("items") or []) if isinstance(recent_data, dict) else []
    # API may return oldest-first; prefer newest first
    recent = sorted(recent, key=lambda e: str(e.get("ts") or ""), reverse=True)

    st_d, daily_body = client.get(
        "/api/daily",
        query={"start": week_start.isoformat(), "end": today.isoformat()},
    )
    daily = save(f"daily_{week_start.isoformat()}_{today.isoformat()}.json", st_d, daily_body)

    st_pt, plan_t_body = client.get(f"/api/plan/{today.isoformat()}")
    plan_today = save(f"plan_{today.isoformat()}.json", st_pt, plan_t_body)

    st_py, plan_y_body = client.get(f"/api/plan/{yday.isoformat()}")
    plan_yday = save(f"plan_{yday.isoformat()}.json", st_py, plan_y_body)

    st_pr, priors_body = client.get("/api/schedule/priors")
    priors = save(f"priors_{today.isoformat()}.json", st_pr, priors_body)

    st_s, stats_body = client.get(
        "/api/stats",
        query={"start": week_start.isoformat(), "end": today.isoformat()},
    )
    stats = save(f"stats_{today.isoformat()}.json", st_s, stats_body)

    st_set, settings_body = client.get("/api/settings/status")
    settings = save(f"settings_status_{today.isoformat()}.json", st_set, settings_body)

    # Prefer agent-context plan/priors when human routes empty
    if isinstance(agent_ctx, dict):
        if not plan_today and isinstance(agent_ctx.get("current_plan"), dict):
            plan_today = agent_ctx["current_plan"]
        if not priors and agent_ctx.get("priors") is not None:
            priors = {
                "kinds": agent_ctx.get("priors"),
                "markdown": agent_ctx.get("server_priors_markdown"),
            }
        if not pending and agent_ctx.get("pending_entries_count"):
            # keep pending list from entries; count still in derived
            pass

    md = build_bundle(
        pulled_at=pulled_at,
        space_url=space,
        today=today,
        yday=yday,
        health=health if health is not None else health_body,
        ready=ready if ready is not None else ready_body,
        agent_ctx=agent_ctx if isinstance(agent_ctx, dict) else None,
        agent_status=st_a,
        pending=pending,
        recent=recent,
        daily=daily if isinstance(daily, dict) else None,
        plan_today=plan_today if isinstance(plan_today, dict) else None,
        plan_yday=plan_yday if isinstance(plan_yday, dict) else None,
        priors=priors,
        stats=stats,
        settings=settings,
        raw_names=sorted(set(raw_names)),
    )

    out_path = HANDOFF_DIR / f"HANDOFF_BUNDLE_{today.isoformat()}.md"
    out_path.write_text(md, encoding="utf-8")
    print(f"Wrote {out_path.relative_to(REPO_ROOT)}")
    print(f"Raw JSON under {RAW_DIR.relative_to(REPO_ROOT)}/")
    if not agent_token and not password:
        print("WARN: set AGENT_TOKEN and/or APP_PASSWORD in .env for authenticated pulls", file=sys.stderr)
        return 2
    if st_a not in (200, 201) and st_e not in (200, 201):
        print(f"WARN: limited data (agent={st_a}, entries={st_e})", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())