Spaces:
Sleeping
Sleeping
| """Unified weekly markdown report assembled from analytics functions.""" | |
| from __future__ import annotations | |
| from datetime import datetime, timezone | |
| import json as _json | |
| from pathlib import Path | |
| from scripts.lib.chat_eval.analytics import ( | |
| coverage, | |
| doc_utilization, | |
| misconception_frequency, | |
| voice_failures, | |
| timing_density, | |
| query_frequency, | |
| score_summary, | |
| cost_summary, | |
| cache_summary, | |
| cache_recommendation, | |
| topics_by_session, | |
| chat_sessions, | |
| ) | |
| def build_report( | |
| path: Path, | |
| *, | |
| since: str | None = None, | |
| session_core_path: Path | None = None, | |
| catalog_path: Path | None = None, | |
| title: str = "Chat Feedback Report", | |
| heading_level: int = 1, | |
| ) -> str: | |
| """Concatenate every analytics view into one markdown document.""" | |
| parts: list[str] = [] | |
| parts.append(f"{'#' * heading_level} {title}") | |
| parts.append("") | |
| parts.append(f"Source log: `{path}`") | |
| if since: | |
| parts.append(f"Since: `{since}`") | |
| parts.append("") | |
| # 1. Summary | |
| parts.append("## Summary") | |
| summary = score_summary(path, since=since) | |
| if summary: | |
| parts.append("Mean score per dimension across judged turns:") | |
| parts.append("") | |
| parts.append("| Dimension | Mean |") | |
| parts.append("|---|---|") | |
| for dim, mean in summary.items(): | |
| parts.append(f"| {dim} | {mean} |") | |
| else: | |
| parts.append("(No judged turns yet — run `python -m scripts.lib.chat_eval.cli judge`.)") | |
| parts.append("") | |
| # 1.5 Cost | |
| parts.append("## Cost") | |
| cost = cost_summary(path, since=since) | |
| if cost["n_turns"] > 0: | |
| parts.append(f"Total spend (chat tutor only, est): **${cost['total_cost_usd']:.2f}** across {cost['n_turns']} turns.") | |
| parts.append("") | |
| parts.append(f"Mean per turn: **${cost['mean_cost_usd']:.3f}**. Most expensive turn: ${cost['max_cost_usd']:.3f} (`{cost['max_turn_id']}`).") | |
| parts.append("") | |
| if cost["by_intent"]: | |
| parts.append("By intent:") | |
| parts.append("") | |
| parts.append("| Intent | n | Total | Mean |") | |
| parts.append("|---|---|---|---|") | |
| for intent, stats in sorted(cost["by_intent"].items(), key=lambda kv: -kv[1]["total"]): | |
| parts.append(f"| {intent} | {stats['n']} | ${stats['total']:.3f} | ${stats['mean']:.3f} |") | |
| parts.append("") | |
| parts.append("_Judge + planner + embedder costs not included; estimated via `len(text)//4` tokens (±20%)._") | |
| else: | |
| parts.append("(No turns logged — no cost to report.)") | |
| parts.append("") | |
| # 1.6 Caching (cache hit rate + savings + cache strategy candidates) | |
| parts.append("## Caching") | |
| cache = cache_summary(path, since=since) | |
| if cache["n_turns_with_data"] > 0: | |
| parts.append( | |
| f"Cache hit rate: **{cache['cache_hit_rate']*100:.1f}%** across " | |
| f"{cache['n_turns_with_data']} turns with usage data." | |
| ) | |
| parts.append("") | |
| parts.append( | |
| f"- Uncached input: **{cache['total_uncached_input']:,}** tokens " | |
| f"(full rate)" | |
| ) | |
| parts.append( | |
| f"- Cache writes: **{cache['total_cache_writes']:,}** tokens " | |
| f"(1.25x premium)" | |
| ) | |
| parts.append( | |
| f"- Cache reads: **{cache['total_cache_reads']:,}** tokens " | |
| f"(0.10x discount)" | |
| ) | |
| parts.append( | |
| f"- Estimated savings vs no-cache: " | |
| f"**${cache['estimated_savings_usd']:.2f}**" | |
| ) | |
| parts.append("") | |
| else: | |
| parts.append("(No cache-usage data yet — chat sessions before " | |
| "the caching commit lack `actual_input_tokens`. New " | |
| "sessions will populate this section.)") | |
| parts.append("") | |
| # 1.7 Permanent-cache recommendation: which retrieved docs are worth | |
| # pinning into a 1-hour cached prefix? Ranked by projected savings, | |
| # accounting for the write premium. | |
| parts.append("### Permanent cache: which docs to pin") | |
| rec = cache_recommendation(path, since=since, ttl="1h", top_n=15) | |
| if rec["recommended"]: | |
| parts.append( | |
| f"If you put these {len(rec['recommended'])} docs into a 1-hour " | |
| f"cached prefix (write at 2x input, read at 0.10x), projected " | |
| f"savings across the logged window: " | |
| f"**${rec['total_recommended_savings_usd']:.4f}**." | |
| ) | |
| parts.append("") | |
| parts.append("| doc_id | kind | topic | retrieved | body tok | savings |") | |
| parts.append("|---|---|---|---|---|---|") | |
| for r in rec["recommended"]: | |
| parts.append( | |
| f"| `{r['doc_id']}` | {r['kind']} | {r['topic']} | " | |
| f"{r['retrieval_count']} | {r['body_tokens']:,} | " | |
| f"${r['savings_per_term_usd']:.4f} |" | |
| ) | |
| parts.append("") | |
| parts.append( | |
| f"_Break-even at N={rec['recommended'][0]['break_even_at']} " | |
| f"retrievals for 1-hour TTL. Docs retrieved fewer times don't " | |
| f"pay back the write premium._" | |
| ) | |
| elif rec["no_data_warning"]: | |
| parts.append(f"({rec['no_data_warning']})") | |
| else: | |
| parts.append( | |
| "(No docs were retrieved 2+ times in the logged window — " | |
| "permanent caching has no docs to recommend yet.)" | |
| ) | |
| parts.append("") | |
| # 1.8 Session core (if file exists) | |
| parts.append("### Session core") | |
| core_meta_path = session_core_path or ( | |
| path.parent / "session_core.json" | |
| ) | |
| if Path(core_meta_path).is_file(): | |
| try: | |
| meta = _json.loads(Path(core_meta_path).read_text(encoding="utf-8")) | |
| except (OSError, _json.JSONDecodeError): | |
| meta = None | |
| if meta: | |
| ts = meta.get("build_timestamp", "") | |
| try: | |
| built = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace( | |
| tzinfo=timezone.utc) | |
| age_days = (datetime.now(timezone.utc) - built).days | |
| except ValueError: | |
| age_days = -1 | |
| n = len(meta.get("doc_ids", [])) | |
| est = meta.get("concurrent_sessions_estimate", "?") | |
| proj = meta.get("projected_savings_usd", 0.0) | |
| parts.append( | |
| f"Built: `{ts}` ({age_days} days ago). " | |
| f"Docs: **{n}**. " | |
| f"Concurrent-sessions estimate: **{est}**. " | |
| f"Projected savings/window: **${proj:.4f}**." | |
| ) | |
| if age_days >= 14: | |
| parts.append("") | |
| parts.append( | |
| "_Session core is **stale** (>= 14 days old). " | |
| "Re-run `python -m scripts.build_session_core ...`._" | |
| ) | |
| else: | |
| parts.append("(session_core.json present but unreadable.)") | |
| else: | |
| parts.append("(No session_core.json — run `build_session_core` to " | |
| "generate one.)") | |
| parts.append("") | |
| # 2. Concept coverage | |
| parts.append("## Concept coverage") | |
| cov = coverage(path, since=since) | |
| if cov: | |
| top = sorted(cov.items(), key=lambda kv: -kv[1])[:15] | |
| parts.append("Top canonical terms by retrieval frequency:") | |
| parts.append("") | |
| parts.append("| Term | Retrievals |") | |
| parts.append("|---|---|") | |
| for term, count in top: | |
| parts.append(f"| `{term}` | {count} |") | |
| else: | |
| parts.append("(No retrievals logged.)") | |
| parts.append("") | |
| # 3. Doc utilization | |
| parts.append("## Doc utilization") | |
| util = doc_utilization(path, since=since) | |
| if util: | |
| # Sort by lowest util_pct first (the failures) | |
| ranked = sorted(util.items(), key=lambda kv: (kv[1]["util_pct"], -kv[1]["retrieved_count"]))[:20] | |
| parts.append("Lowest-utilization docs (retrieved often, used rarely):") | |
| parts.append("") | |
| parts.append("| doc_id | retrieved | used | util % |") | |
| parts.append("|---|---|---|---|") | |
| for doc_id, stats in ranked: | |
| parts.append(f"| `{doc_id}` | {stats['retrieved_count']} | {stats['used_count']} | {stats['util_pct']} |") | |
| else: | |
| parts.append("(No retrievals logged.)") | |
| parts.append("") | |
| # 4. Misconceptions | |
| parts.append("## Misconceptions") | |
| mc = misconception_frequency(path, since=since) | |
| if mc: | |
| parts.append("`count` = total flags. `in_query` = student exhibited the " | |
| "misconception. `handled` = response addressed it. " | |
| "`missed` = student exhibited it AND response didn't address " | |
| "it (the actionable failure mode).") | |
| parts.append("") | |
| parts.append("| id | count | in_query | handled | missed |") | |
| parts.append("|---|---|---|---|---|") | |
| # Sort by missed desc, then count desc | |
| for mid, stats in sorted(mc.items(), | |
| key=lambda kv: (-kv[1]["missed"], -kv[1]["count"])): | |
| parts.append( | |
| f"| `{mid}` | {stats['count']} | {stats['in_query']} | " | |
| f"{stats['handled']} | {stats['missed']} |" | |
| ) | |
| else: | |
| parts.append("(No misconceptions detected, or no judged turns.)") | |
| parts.append("") | |
| # 5. Voice failures | |
| parts.append("## Voice failures") | |
| vf = voice_failures(path, since=since) | |
| if vf: | |
| parts.append(f"({len(vf)} turns scored <= 2 on voice.)") | |
| parts.append("") | |
| for v in vf[:10]: | |
| parts.append(f"### {v['turn_id']} (voice {v['voice_score']})") | |
| parts.append(f"- **Query:** {v['user_message']}") | |
| parts.append(f"- **Response excerpt:** {v['response'][:200]}{'...' if len(v['response']) > 200 else ''}") | |
| parts.append(f"- **Diagnosis:** {v['diagnosis']}") | |
| parts.append("") | |
| else: | |
| parts.append("(No voice failures detected.)") | |
| parts.append("") | |
| # 6. Timing patterns | |
| parts.append("## Timing patterns") | |
| timing = timing_density(path, since=since) | |
| if timing: | |
| parts.append("Turn count by weekday:") | |
| parts.append("") | |
| parts.append("| Weekday | Total |") | |
| parts.append("|---|---|") | |
| for wd in ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]: | |
| total = timing.get(wd, {}).get("total", 0) | |
| parts.append(f"| {wd} | {total} |") | |
| else: | |
| parts.append("(No turns logged.)") | |
| parts.append("") | |
| # 7. Topics by class session | |
| parts.append("## Topics by class session") | |
| catalog_p = catalog_path or (path.parent / "catalog.jsonl") | |
| tbs = topics_by_session(path, catalog_path=catalog_p, since=since) | |
| sessions = tbs.get("sessions", {}) | |
| unmapped = tbs.get("unmapped_turns", 0) | |
| if sessions: | |
| parts.append( | |
| "Each turn is associated with the lecture session(s) its " | |
| "retrieved docs trace to via `lecture_problem` doc_ids. " | |
| "Top topics for a date are taken from the broader retrieval " | |
| "pack (syntopicon entries surfaced alongside that lecture's " | |
| "problems). One turn can land under multiple sessions if it " | |
| "pulled from several lectures." | |
| ) | |
| parts.append("") | |
| parts.append("| Lecture date | Turns | Top topics | Intent breakdown | Example query |") | |
| parts.append("|---|---|---|---|---|") | |
| # Sort by date descending so most recent lecture appears first | |
| for date in sorted(sessions.keys(), reverse=True): | |
| stats = sessions[date] | |
| top_topics = ", ".join( | |
| f"{t} ({n})" for t, n in | |
| sorted(stats["topics"].items(), key=lambda kv: -kv[1])[:3] | |
| ) or "—" | |
| intents_breakdown = ", ".join( | |
| f"{i} {n}" for i, n in | |
| sorted(stats["intents"].items(), key=lambda kv: -kv[1]) | |
| ) | |
| sample = stats["sample_query"].replace("|", "\\|") | |
| parts.append( | |
| f"| {date} | {stats['turn_count']} | {top_topics} | " | |
| f"{intents_breakdown} | {sample} |" | |
| ) | |
| if unmapped: | |
| parts.append("") | |
| parts.append( | |
| f"_{unmapped} turn(s) had no retrieval that linked to a " | |
| "dated lecture session — typically pure-syntopicon hits without " | |
| "lecture references, or generic slide retrieval._" | |
| ) | |
| elif unmapped: | |
| parts.append( | |
| f"(No turns mapped to lecture sessions. {unmapped} turn(s) had " | |
| "retrieval that didn't link to any dated lecture.)" | |
| ) | |
| else: | |
| parts.append("(No turns logged.)") | |
| parts.append("") | |
| # 7.5 Chat sessions (one row per conversation) | |
| parts.append("## Chat sessions") | |
| chats = chat_sessions(path, since=since) | |
| if chats: | |
| parts.append( | |
| "One row per student chat. The opener is the first message — " | |
| "usually what the student walked in with. Topics are aggregated " | |
| "syntopicon hits across the whole chat." | |
| ) | |
| parts.append("") | |
| parts.append("| Started | Turns | Opener | Top topics | Intents | Final message |") | |
| parts.append("|---|---|---|---|---|---|") | |
| for chat in chats: | |
| opener = chat["opener"][:80].replace("|", "\\|") | |
| last = chat["last_user_message"][:60].replace("|", "\\|") | |
| top_topics = ", ".join( | |
| t for t, _ in | |
| sorted(chat["topics"].items(), key=lambda kv: -kv[1])[:3] | |
| ) or "—" | |
| intents = ", ".join( | |
| f"{i} {n}" for i, n in | |
| sorted(chat["intents"].items(), key=lambda kv: -kv[1]) | |
| ) | |
| parts.append( | |
| f"| {chat['started_at']} | {chat['turn_count']} | " | |
| f"{opener} | {top_topics} | {intents} | {last} |" | |
| ) | |
| else: | |
| parts.append("(No chat sessions in this period.)") | |
| parts.append("") | |
| # 8. Suggested actions | |
| parts.append("## Suggested actions") | |
| suggestions: list[str] = [] | |
| if summary and summary.get("voice", 5) < 3: | |
| suggestions.append("- **Voice scores are low.** Review prompt fragments for AI-tutor tells.") | |
| if summary and summary.get("substance", 5) < 3: | |
| suggestions.append("- **Substance scores are low.** Tighten Socratic-default; engage on `explain` intents.") | |
| if util: | |
| unused = [d for d, s in util.items() if s["retrieved_count"] >= 3 and s["util_pct"] < 20] | |
| if unused: | |
| suggestions.append(f"- **{len(unused)} docs are retrieved often but rarely used.** Consider tuning quotas or pruning the catalog: {unused[:5]}") | |
| if mc: | |
| missed = [(mid, s) for mid, s in mc.items() if s["missed"] > 0] | |
| if missed: | |
| suggestions.append( | |
| f"- **{len(missed)} misconception(s) appeared in student " | |
| f"queries and were NOT addressed in the response.** " | |
| f"Faculty may want to tighten the prompt's displacement " | |
| f"language: {[m[0] for m in missed[:5]]}" | |
| ) | |
| if Path(core_meta_path).is_file(): | |
| try: | |
| meta = _json.loads(Path(core_meta_path).read_text(encoding="utf-8")) | |
| ts = meta.get("build_timestamp", "") | |
| built = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace( | |
| tzinfo=timezone.utc) | |
| age_days = (datetime.now(timezone.utc) - built).days | |
| if age_days >= 14: | |
| suggestions.append( | |
| f"- **Session core is stale ({age_days} days old).** " | |
| "Re-run the nightly builder." | |
| ) | |
| except (OSError, ValueError, _json.JSONDecodeError): | |
| pass | |
| if not suggestions: | |
| suggestions.append("- (No actionable patterns detected in this period.)") | |
| parts.extend(suggestions) | |
| parts.append("") | |
| return "\n".join(parts) | |