"""Per-turn JSONL telemetry: append (live), iter (read), update_judge (atomic). The log is a single append-only JSONL file. Each line is one turn record. The `judge` field on each record is initially null; `update_judge` rewrites the entire file via stage-and-rename to fill it in. Concurrency: append + update_judge + snapshot are serialized via _LOG_LOCK so a reader (e.g. the history pusher) never observes a torn append or a half-replaced file. """ from __future__ import annotations import json import logging import os import shutil import threading from pathlib import Path from typing import Iterator logger = logging.getLogger(__name__) # In-process lock. Chat handler thread (calling append) and the history # pusher thread (calling snapshot) share one Python process on the HF # Space, so a threading.Lock is sufficient. _LOG_LOCK = threading.Lock() def append(record: dict, path: Path) -> None: """Append one JSON-encoded record as a new line in path. Creates the parent directory if needed.""" path.parent.mkdir(parents=True, exist_ok=True) line = json.dumps(record, ensure_ascii=False) with _LOG_LOCK: with open(path, "a", encoding="utf-8") as f: f.write(line + "\n") # Notify history_pusher (no-op if not active). Imported here, after # the write completes, so a faulty pusher import can never block the # write itself. try: from scripts.lib.chat import history_pusher history_pusher.notify() except Exception as e: logger.warning("history_pusher.notify failed: %s", e) def snapshot(src: Path, dest: Path) -> Path | None: """Copy `src` to `dest` under _LOG_LOCK. Returns dest on success, or None if `src` does not exist (cold-start cases). The pusher uploads from the snapshot, not from the live log, so concurrent appends cannot tear the upload's view.""" with _LOG_LOCK: if not src.is_file(): return None shutil.copyfile(src, dest) return dest def load_all(path: Path) -> list[dict]: """Load every JSONL turn into memory as a list of dicts. Skips blank lines and malformed JSON (with a WARNING log). Returns an empty list if the path does not exist. Use this to amortize many reads of the same turn log within a single operation (e.g., a tuning step that calls multiple analytics helpers); pass the resulting list to helpers that accept a pre-loaded list. """ return list(iter_turns(path)) def iter_turns(path: Path, *, since: str | None = None) -> Iterator[dict]: """Yield each record in the log. Skips blank lines and malformed JSON (with a WARNING log). Optional `since` filter on `ts` field (lexicographic comparison; ISO-8601 timestamps sort correctly). A record whose `ts` equals `since` exactly is included (the comparison is strict less-than for exclusion).""" if not path.is_file(): return with open(path, "r", encoding="utf-8") as f: for line_no, raw in enumerate(f, 1): stripped = raw.strip() if not stripped: continue try: rec = json.loads(stripped) except json.JSONDecodeError: logger.warning( "turn_log.iter_turns: malformed JSON at %s:%d — skipping", path, line_no, ) continue if since is not None and rec.get("ts", "") < since: continue yield rec def is_student_mode_turn(record: dict) -> bool: """True for turns that belong in student-learning analytics (judge, tune, etc.). Faculty `mode="review"` turns are deliberate searches and are excluded. Records missing the `mode` field predate the feature and are treated as tutor-mode for backward compatibility.""" return record.get("mode", "tutor") != "review" def iter_turns_compat(path_or_turns, *, since: str | None = None) -> Iterator[dict]: """Yield turns from either a Path (via iter_turns) or a pre-loaded list. For lists, applies the same `since` lower-bound filter that iter_turns(since=...) applies. Lets callers accept either shape with no behavior change. """ if isinstance(path_or_turns, list): for t in path_or_turns: if since is not None and t.get("ts", "") < since: continue yield t else: yield from iter_turns(path_or_turns, since=since) def update_judge(turn_id: str, judge: dict, path: Path) -> None: """Find the record with matching turn_id, set its judge field to `judge`, and rewrite the log atomically via stage-and-rename. Raises KeyError if turn_id is not in the log.""" if not path.is_file(): raise KeyError(turn_id) staging = path.with_name(path.name + ".tmp") with _LOG_LOCK: try: found = False with open(path, "r", encoding="utf-8") as src, \ open(staging, "w", encoding="utf-8") as dst: for raw in src: stripped = raw.strip() if not stripped: continue try: rec = json.loads(stripped) except json.JSONDecodeError: # Preserve malformed lines verbatim — don't silently drop. dst.write(raw if raw.endswith("\n") else raw + "\n") continue if rec.get("turn_id") == turn_id: rec["judge"] = judge found = True dst.write(json.dumps(rec, ensure_ascii=False) + "\n") if not found: raise KeyError(turn_id) # Atomic-ish replace; on Windows os.replace handles the rename. os.replace(staging, path) except BaseException: # On any failure (KeyError, OSError, KeyboardInterrupt, etc.), clean up # the staging file so it doesn't shadow future updates or confuse operators. if staging.exists(): try: os.unlink(staging) except OSError: pass raise