Spaces:
Running
Running
Download scripts/lib/chat_eval/turn_log.py from defenestrate/asa: direct link, hf CLI and curl.
- Browser
- Download file 6.26 kB
-
https://huggingface.co/spaces/defenestrate/asa/resolve/main/scripts/lib/chat_eval/turn_log.py
- Command line
-
hf download hf://spaces/defenestrate/asa/scripts/lib/chat_eval/turn_log.py
-
curl -L -o turn_log.py https://huggingface.co/spaces/defenestrate/asa/resolve/main/scripts/lib/chat_eval/turn_log.py
6.26 kB
| """Per-turn JSONL telemetry: append (live), iter (read), update_judge (atomic). | |
| The log is a single append-only JSONL file. Each line is one turn record. | |
| The `judge` field on each record is initially null; `update_judge` rewrites | |
| the entire file via stage-and-rename to fill it in. | |
| Concurrency: append + update_judge + snapshot are serialized via _LOG_LOCK | |
| so a reader (e.g. the history pusher) never observes a torn append or a | |
| half-replaced file. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import logging | |
| import os | |
| import shutil | |
| import threading | |
| from pathlib import Path | |
| from typing import Iterator | |
| logger = logging.getLogger(__name__) | |
| # In-process lock. Chat handler thread (calling append) and the history | |
| # pusher thread (calling snapshot) share one Python process on the HF | |
| # Space, so a threading.Lock is sufficient. | |
| _LOG_LOCK = threading.Lock() | |
| def append(record: dict, path: Path) -> None: | |
| """Append one JSON-encoded record as a new line in path. Creates the | |
| parent directory if needed.""" | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| line = json.dumps(record, ensure_ascii=False) | |
| with _LOG_LOCK: | |
| with open(path, "a", encoding="utf-8") as f: | |
| f.write(line + "\n") | |
| # Notify history_pusher (no-op if not active). Imported here, after | |
| # the write completes, so a faulty pusher import can never block the | |
| # write itself. | |
| try: | |
| from scripts.lib.chat import history_pusher | |
| history_pusher.notify() | |
| except Exception as e: | |
| logger.warning("history_pusher.notify failed: %s", e) | |
| def snapshot(src: Path, dest: Path) -> Path | None: | |
| """Copy `src` to `dest` under _LOG_LOCK. Returns dest on success, or | |
| None if `src` does not exist (cold-start cases). The pusher uploads | |
| from the snapshot, not from the live log, so concurrent appends | |
| cannot tear the upload's view.""" | |
| with _LOG_LOCK: | |
| if not src.is_file(): | |
| return None | |
| shutil.copyfile(src, dest) | |
| return dest | |
| def load_all(path: Path) -> list[dict]: | |
| """Load every JSONL turn into memory as a list of dicts. | |
| Skips blank lines and malformed JSON (with a WARNING log). Returns an | |
| empty list if the path does not exist. Use this to amortize many reads | |
| of the same turn log within a single operation (e.g., a tuning step | |
| that calls multiple analytics helpers); pass the resulting list to | |
| helpers that accept a pre-loaded list. | |
| """ | |
| return list(iter_turns(path)) | |
| def iter_turns(path: Path, *, since: str | None = None) -> Iterator[dict]: | |
| """Yield each record in the log. Skips blank lines and malformed JSON | |
| (with a WARNING log). Optional `since` filter on `ts` field | |
| (lexicographic comparison; ISO-8601 timestamps sort correctly). | |
| A record whose `ts` equals `since` exactly is included (the comparison | |
| is strict less-than for exclusion).""" | |
| if not path.is_file(): | |
| return | |
| with open(path, "r", encoding="utf-8") as f: | |
| for line_no, raw in enumerate(f, 1): | |
| stripped = raw.strip() | |
| if not stripped: | |
| continue | |
| try: | |
| rec = json.loads(stripped) | |
| except json.JSONDecodeError: | |
| logger.warning( | |
| "turn_log.iter_turns: malformed JSON at %s:%d — skipping", | |
| path, line_no, | |
| ) | |
| continue | |
| if since is not None and rec.get("ts", "") < since: | |
| continue | |
| yield rec | |
| def is_student_mode_turn(record: dict) -> bool: | |
| """True for turns that belong in student-learning analytics (judge, | |
| tune, etc.). Faculty `mode="review"` turns are deliberate searches and | |
| are excluded. Records missing the `mode` field predate the feature and | |
| are treated as tutor-mode for backward compatibility.""" | |
| return record.get("mode", "tutor") != "review" | |
| def iter_turns_compat(path_or_turns, *, since: str | None = None) -> Iterator[dict]: | |
| """Yield turns from either a Path (via iter_turns) or a pre-loaded list. | |
| For lists, applies the same `since` lower-bound filter that | |
| iter_turns(since=...) applies. Lets callers accept either shape with | |
| no behavior change. | |
| """ | |
| if isinstance(path_or_turns, list): | |
| for t in path_or_turns: | |
| if since is not None and t.get("ts", "") < since: | |
| continue | |
| yield t | |
| else: | |
| yield from iter_turns(path_or_turns, since=since) | |
| def update_judge(turn_id: str, judge: dict, path: Path) -> None: | |
| """Find the record with matching turn_id, set its judge field to `judge`, | |
| and rewrite the log atomically via stage-and-rename. Raises KeyError if | |
| turn_id is not in the log.""" | |
| if not path.is_file(): | |
| raise KeyError(turn_id) | |
| staging = path.with_name(path.name + ".tmp") | |
| with _LOG_LOCK: | |
| try: | |
| found = False | |
| with open(path, "r", encoding="utf-8") as src, \ | |
| open(staging, "w", encoding="utf-8") as dst: | |
| for raw in src: | |
| stripped = raw.strip() | |
| if not stripped: | |
| continue | |
| try: | |
| rec = json.loads(stripped) | |
| except json.JSONDecodeError: | |
| # Preserve malformed lines verbatim — don't silently drop. | |
| dst.write(raw if raw.endswith("\n") else raw + "\n") | |
| continue | |
| if rec.get("turn_id") == turn_id: | |
| rec["judge"] = judge | |
| found = True | |
| dst.write(json.dumps(rec, ensure_ascii=False) + "\n") | |
| if not found: | |
| raise KeyError(turn_id) | |
| # Atomic-ish replace; on Windows os.replace handles the rename. | |
| os.replace(staging, path) | |
| except BaseException: | |
| # On any failure (KeyError, OSError, KeyboardInterrupt, etc.), clean up | |
| # the staging file so it doesn't shadow future updates or confuse operators. | |
| if staging.exists(): | |
| try: | |
| os.unlink(staging) | |
| except OSError: | |
| pass | |
| raise | |