asa / scripts /lib /chat_eval /turn_log.py
defenestrate's picture
ASA Space: squashed history (instructions and settings live in the private pack)
526950e
Raw History Blame Contribute Delete
6.26 kB
"""Per-turn JSONL telemetry: append (live), iter (read), update_judge (atomic).
The log is a single append-only JSONL file. Each line is one turn record.
The `judge` field on each record is initially null; `update_judge` rewrites
the entire file via stage-and-rename to fill it in.
Concurrency: append + update_judge + snapshot are serialized via _LOG_LOCK
so a reader (e.g. the history pusher) never observes a torn append or a
half-replaced file.
"""
from __future__ import annotations
import json
import logging
import os
import shutil
import threading
from pathlib import Path
from typing import Iterator
logger = logging.getLogger(__name__)
# In-process lock. Chat handler thread (calling append) and the history
# pusher thread (calling snapshot) share one Python process on the HF
# Space, so a threading.Lock is sufficient.
_LOG_LOCK = threading.Lock()
def append(record: dict, path: Path) -> None:
"""Append one JSON-encoded record as a new line in path. Creates the
parent directory if needed."""
path.parent.mkdir(parents=True, exist_ok=True)
line = json.dumps(record, ensure_ascii=False)
with _LOG_LOCK:
with open(path, "a", encoding="utf-8") as f:
f.write(line + "\n")
# Notify history_pusher (no-op if not active). Imported here, after
# the write completes, so a faulty pusher import can never block the
# write itself.
try:
from scripts.lib.chat import history_pusher
history_pusher.notify()
except Exception as e:
logger.warning("history_pusher.notify failed: %s", e)
def snapshot(src: Path, dest: Path) -> Path | None:
"""Copy `src` to `dest` under _LOG_LOCK. Returns dest on success, or
None if `src` does not exist (cold-start cases). The pusher uploads
from the snapshot, not from the live log, so concurrent appends
cannot tear the upload's view."""
with _LOG_LOCK:
if not src.is_file():
return None
shutil.copyfile(src, dest)
return dest
def load_all(path: Path) -> list[dict]:
"""Load every JSONL turn into memory as a list of dicts.
Skips blank lines and malformed JSON (with a WARNING log). Returns an
empty list if the path does not exist. Use this to amortize many reads
of the same turn log within a single operation (e.g., a tuning step
that calls multiple analytics helpers); pass the resulting list to
helpers that accept a pre-loaded list.
"""
return list(iter_turns(path))
def iter_turns(path: Path, *, since: str | None = None) -> Iterator[dict]:
"""Yield each record in the log. Skips blank lines and malformed JSON
(with a WARNING log). Optional `since` filter on `ts` field
(lexicographic comparison; ISO-8601 timestamps sort correctly).
A record whose `ts` equals `since` exactly is included (the comparison
is strict less-than for exclusion)."""
if not path.is_file():
return
with open(path, "r", encoding="utf-8") as f:
for line_no, raw in enumerate(f, 1):
stripped = raw.strip()
if not stripped:
continue
try:
rec = json.loads(stripped)
except json.JSONDecodeError:
logger.warning(
"turn_log.iter_turns: malformed JSON at %s:%d — skipping",
path, line_no,
)
continue
if since is not None and rec.get("ts", "") < since:
continue
yield rec
def is_student_mode_turn(record: dict) -> bool:
"""True for turns that belong in student-learning analytics (judge,
tune, etc.). Faculty `mode="review"` turns are deliberate searches and
are excluded. Records missing the `mode` field predate the feature and
are treated as tutor-mode for backward compatibility."""
return record.get("mode", "tutor") != "review"
def iter_turns_compat(path_or_turns, *, since: str | None = None) -> Iterator[dict]:
"""Yield turns from either a Path (via iter_turns) or a pre-loaded list.
For lists, applies the same `since` lower-bound filter that
iter_turns(since=...) applies. Lets callers accept either shape with
no behavior change.
"""
if isinstance(path_or_turns, list):
for t in path_or_turns:
if since is not None and t.get("ts", "") < since:
continue
yield t
else:
yield from iter_turns(path_or_turns, since=since)
def update_judge(turn_id: str, judge: dict, path: Path) -> None:
"""Find the record with matching turn_id, set its judge field to `judge`,
and rewrite the log atomically via stage-and-rename. Raises KeyError if
turn_id is not in the log."""
if not path.is_file():
raise KeyError(turn_id)
staging = path.with_name(path.name + ".tmp")
with _LOG_LOCK:
try:
found = False
with open(path, "r", encoding="utf-8") as src, \
open(staging, "w", encoding="utf-8") as dst:
for raw in src:
stripped = raw.strip()
if not stripped:
continue
try:
rec = json.loads(stripped)
except json.JSONDecodeError:
# Preserve malformed lines verbatim — don't silently drop.
dst.write(raw if raw.endswith("\n") else raw + "\n")
continue
if rec.get("turn_id") == turn_id:
rec["judge"] = judge
found = True
dst.write(json.dumps(rec, ensure_ascii=False) + "\n")
if not found:
raise KeyError(turn_id)
# Atomic-ish replace; on Windows os.replace handles the rename.
os.replace(staging, path)
except BaseException:
# On any failure (KeyError, OSError, KeyboardInterrupt, etc.), clean up
# the staging file so it doesn't shadow future updates or confuse operators.
if staging.exists():
try:
os.unlink(staging)
except OSError:
pass
raise