#!/usr/bin/env python """Flask backend for the AVIO human evaluation (HF Docker Space). Serves static/index.html + items.json + media/, and collects one JSON record per rated screen: data/records//.jsonl (append-only, fsync'ed) A huggingface_hub.CommitScheduler pushes data/ to the private dataset repo DATASET_REPO every 2 minutes; on start the repo is restored into data/ so records survive restarts and rebuilds. Each container only ever appends to files named after its own boot_id, so the upload-only scheduler can never overwrite another container's records. env: ACCESS_CODE, ADMIN_CODE, HF_TOKEN (Space secrets); DATASET_REPO, PAGE_VERSION (variables); DATA_DIR (default ./data), NO_HF=1 (local test: no restore, no scheduler). """ import csv from collections import defaultdict import hashlib import hmac import io import json import os import re import secrets import signal import threading import time import uuid from pathlib import Path from flask import Flask, abort, jsonify, request, send_from_directory, Response HERE = Path(__file__).resolve().parent DATA_DIR = Path(os.environ.get("DATA_DIR", HERE / "data")) REC_DIR = DATA_DIR / "records" META_DIR = DATA_DIR / "meta" MEDIA_DIR = HERE / "media" STATIC_DIR = HERE / "static" ACCESS_CODE = os.environ.get("ACCESS_CODE", "") ADMIN_CODE = os.environ.get("ADMIN_CODE", "") DATASET_REPO = os.environ.get("DATASET_REPO", "WitneyWW/avio-human-eval-records") NO_HF = os.environ.get("NO_HF") == "1" BOOT_ID = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + "-" + secrets.token_hex(3) ITEMS = json.loads((HERE / "items.json").read_text()) PAGE_VERSION = os.environ.get("PAGE_VERSION") or ITEMS.get("page_version", "dev") ITEM_BY_ID = {it["item_id"]: it for it in ITEMS["items"]} SETS = ("test", "pilot", "train_audit", "sc_prepass", "qual", "profile") GATED = ("test", "pilot", "train_audit") # need profile + qualification (Q1) first VERSIONS = ITEMS.get("versions") or {} if "SC" not in VERSIONS and not ITEMS.get("sc_frozen"): # v2 (sc_frozen, no C question) has no pre-pass VERSIONS["SC"] = {"set": "sc_prepass", "items": [it["item_id"] for it in ITEMS["items"] if it["set"] == "test"]} VERSIONS.setdefault("PROFILE", {"set": "profile", "items": ["profile"]}) QUAL_VERSION = "Q1" if "Q1" in VERSIONS else None PROFILE_OPTS = {"experience": ("none", "some", "professional"), "headphones": ("Yes", "No"), "hc_side": ("Left", "Right", "Both", "Not sure")} ACCESS_REQUIRED = bool(ACCESS_CODE) RATER_RE = re.compile(r"^[a-z0-9_]{2,24}$") TRI = ("Pass", "Fail", "Uncertain") ART = ("None", "Minor", "Major") app = Flask(__name__, static_folder=None) app.config["MAX_CONTENT_LENGTH"] = 256 * 1024 _lock = threading.RLock() # re-entrant: record() holds it while append_record() takes it again _latest = {} # (rater_id, set, item_id, output_id) -> latest record (by submitted_at) _record_ids = set() _hf = {"restored": None, "last_push_at": None, "last_error": None, "enabled": not NO_HF} _scheduler = None # ----------------------------------------------------------------------------- helpers def hx(*parts): return hashlib.sha256("|".join(parts).encode()).hexdigest() def key_of(rec): return (rec["rater_id"], rec["set"], rec["item_id"], rec.get("output_id")) def iter_records(): for p in sorted(REC_DIR.rglob("*.jsonl")): with open(p, encoding="utf-8") as f: for line in f: line = line.strip() if line: try: yield json.loads(line) except json.JSONDecodeError: continue def load_index(): _latest.clear() _record_ids.clear() for rec in iter_records(): _record_ids.add(rec["record_id"]) k = key_of(rec) if k not in _latest or rec["submitted_at"] >= _latest[k]["submitted_at"]: _latest[k] = rec def screens_for(rater_id, version): """Deterministic hash-sorted order inside a version; the outputs of an item are consecutive.""" v = VERSIONS[version] # v2 versions have ordered "sections" (Part 1, Part 2): shuffle inside a section, keep the section order sections = v.get("sections") or [v["items"]] screens = [] for si, sec in enumerate(sections): items = [ITEM_BY_ID[i] for i in sec] if v["set"] != "qual": # the practice round keeps its fixed, didactic order items.sort(key=lambda it: hx(rater_id, it["item_id"])) for it in items: if v["set"] == "sc_prepass": screens.append({"item_id": it["item_id"], "output_id": None, "section": si}) continue outs = sorted(it["outputs"], key=lambda o: hx(rater_id, it["item_id"], o["output_id"])) for o in outs: screens.append({"item_id": it["item_id"], "output_id": o["output_id"], "section": si}) for i, s in enumerate(screens): s["screen_index"] = i return screens def n_screens(version): v = VERSIONS[version] if v["set"] in ("sc_prepass", "profile"): return len(v["items"]) return sum(len(ITEM_BY_ID[i]["outputs"]) for i in v["items"]) def version_stats(): """Per version: how many raters completed it / are in progress (from the latest-record index).""" done_by = defaultdict(lambda: defaultdict(int)) # version -> rater -> n distinct screens for (r, s, iid, oid), rec in _latest.items(): if r.startswith("dryrun"): continue done_by[rec.get("version", "?")][r] += 1 out = {} for vid, v in VERSIONS.items(): total = n_screens(vid) cnt = done_by.get(vid, {}) out[vid] = {"version": vid, "set": v["set"], "n_items": len(v["items"]), "n_screens": total, "raters_completed": sum(1 for n in cnt.values() if n >= total), "raters_in_progress": sum(1 for n in cnt.values() if 0 < n < total)} return out def pick_version(set_name, rater_id=None): """Auto-assign: the version of `set_name` with the fewest completed raters, then fewest in progress, then random; skip versions this rater has already finished.""" st = version_stats() cands = [v for v in st.values() if v["set"] == set_name and not VERSIONS[v["version"]].get("demo_only")] if rater_id: done = {k[2] for k in _latest if k[0] == rater_id} # item ids done by this rater (any version) cands = [v for v in cands if not all(i in done for i in VERSIONS[v["version"]]["items"])] or cands if not cands: return None best = min(cands, key=lambda v: (v["raters_completed"], v["raters_in_progress"])) pool = [v for v in cands if (v["raters_completed"], v["raters_in_progress"]) == (best["raters_completed"], best["raters_in_progress"])] return secrets.choice(pool)["version"] def rater_status(rid): """profile_done / qual_done for the gate in front of pilot/test questionnaires.""" prof = any(k[0] == rid and k[1] == "profile" for k in _latest) if QUAL_VERSION is None: return {"profile_done": prof, "qual_done": True, "qual_version": None} need = set(VERSIONS[QUAL_VERSION]["items"]) got = {k[2] for k in _latest if k[0] == rid and k[1] == "qual"} return {"profile_done": prof, "qual_done": need <= got, "qual_version": QUAL_VERSION} def gate_error(rid, set_name): if set_name not in GATED: return None st = rater_status(rid) if st["profile_done"] and st["qual_done"]: return None return jsonify(error="qual_required", detail="Please complete the short profile and the practice round first.", **st), 409 def check_code(header, code): given = request.headers.get(header, "") or (request.get_json(silent=True) or {}).get(header.lower().replace("x-", "").replace("-", "_"), "") return bool(code) and hmac.compare_digest(str(given), code) def require_access(): if ACCESS_REQUIRED and not check_code("X-Access-Code", ACCESS_CODE): abort(401) def require_admin(): if not check_code("X-Admin-Code", ADMIN_CODE): abort(401) def validate(rec): """Server-side mirror of the client validation. Returns a list of error strings.""" e = [] for f in ("record_id", "rater_id", "set", "item_id", "submitted_at", "page_version"): if not rec.get(f): e.append(f"missing {f}") if e: return e if not RATER_RE.match(rec["rater_id"]): e.append("bad rater_id") if rec["set"] not in SETS: e.append("bad set") if rec["set"] == "profile": if rec.get("item_id") != "profile" or rec.get("version") != "PROFILE": e.append("profile record must have item_id=profile, version=PROFILE") for f, allowed in PROFILE_OPTS.items(): if rec.get(f) not in allowed: e.append(f"{f} must be one of {'/'.join(allowed)}") if not isinstance(rec.get("hc_count"), int) or not 0 <= rec["hc_count"] <= 9: e.append("hc_count must be an integer 0-9") return e v = VERSIONS.get(rec.get("version") or "") if v is None: return e + ["unknown version"] if v.get("demo_only"): return e + ["this version is preview-only and cannot be rated"] if v["set"] != rec["set"]: e.append("version/set mismatch") it = ITEM_BY_ID.get(rec["item_id"]) if it is None: return e + ["unknown item_id"] if rec["item_id"] not in v["items"]: e.append("item not in version") if rec["set"] == "sc_prepass": if it["set"] != "test": e.append("sc_prepass only covers test items") if rec.get("output_id") is not None: e.append("sc_prepass has no output_id") if rec.get("consistency_eligible") not in ("Yes", "No"): e.append("consistency_eligible must be Yes/No") return e if it["set"] != rec["set"]: e.append("item does not belong to set") if rec.get("output_id") not in {o["output_id"] for o in it["outputs"]}: e.append("output_id not in item") for f in ("V", "A", "P_visual"): if rec.get(f) not in TRI: e.append(f"{f} must be Pass/Fail/Uncertain") if rec.get("P_audio") not in TRI + ("NA",): e.append("P_audio must be Pass/Fail/Uncertain/NA") for f in ("artifact_visual", "artifact_audio"): if rec.get(f) not in ART: e.append(f"{f} must be None/Minor/Major") if it["task"] == "removal": if rec.get("recon_plausible") not in TRI: e.append("recon_plausible required for removal") elif rec.get("recon_plausible") is not None: e.append("recon_plausible must be null for addition") if it.get("sc_eligible"): if rec.get("consistency_verifiable") not in ("Yes", "No"): e.append("consistency_verifiable must be Yes/No") if rec.get("consistency_verifiable") == "Yes": if rec.get("consistency_label") not in TRI: e.append("consistency_label required when verifiable") elif rec.get("consistency_label") is not None: e.append("consistency_label must be null when not verifiable") else: if rec.get("consistency_verifiable") is not None or rec.get("consistency_label") is not None: e.append("consistency fields must be null for non-eligible items") unc = any(rec.get(f) == "Uncertain" for f in ("V", "A", "P_visual", "P_audio", "recon_plausible", "consistency_label")) if unc and not (rec.get("uncertainty_reason") or "").strip(): e.append("uncertainty_reason required when any field is Uncertain") return e def append_record(rec): path = REC_DIR / rec["rater_id"] / f"{BOOT_ID}.jsonl" path.parent.mkdir(parents=True, exist_ok=True) line = json.dumps(rec, ensure_ascii=False) + "\n" lock = _scheduler.lock if _scheduler is not None else _lock with lock: with open(path, "a", encoding="utf-8") as f: f.write(line) f.flush() os.fsync(f.fileno()) # ----------------------------------------------------------------------------- HF sync def hf_setup(): global _scheduler REC_DIR.mkdir(parents=True, exist_ok=True) META_DIR.mkdir(parents=True, exist_ok=True) if NO_HF: load_index() return from huggingface_hub import CommitScheduler, HfApi, snapshot_download token = os.environ.get("HF_TOKEN") try: HfApi(token=token).create_repo(DATASET_REPO, repo_type="dataset", private=True, exist_ok=True) snapshot_download(DATASET_REPO, repo_type="dataset", local_dir=str(DATA_DIR), token=token, allow_patterns=["records/**", "meta/**"]) _hf["restored"] = True except Exception as ex: # keep serving; records accumulate locally and the client retries _hf["restored"] = False _hf["last_error"] = f"restore: {type(ex).__name__}: {str(ex)[:200]}" load_index() with open(META_DIR / "boots.jsonl", "a") as f: f.write(json.dumps({"boot_id": BOOT_ID, "page_version": PAGE_VERSION, "items_sha256": ITEMS.get("items_sha256"), "n_records_restored": len(_record_ids), "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}) + "\n") _scheduler = CommitScheduler(repo_id=DATASET_REPO, repo_type="dataset", folder_path=str(DATA_DIR), path_in_repo=None, every=2, private=True, token=token, allow_patterns=["records/**/*.jsonl", "meta/*.json", "meta/*.jsonl"], squash_history=False) # make the scheduler skip the files we just restored (identical content) on its first tick for p in DATA_DIR.rglob("*"): if p.is_file(): _scheduler.last_uploaded[p] = p.stat().st_mtime def _on_term(signum, frame): try: _scheduler.trigger().result(timeout=25) except Exception: pass raise SystemExit(0) signal.signal(signal.SIGTERM, _on_term) hf_setup() # ----------------------------------------------------------------------------- routes @app.get("/") def index(): return send_from_directory(STATIC_DIR, "index.html", max_age=0) @app.get("/demo") def demo(): return send_from_directory(STATIC_DIR, "demo.html", max_age=0) @app.get("/items.json") def items_json(): return send_from_directory(HERE, "items.json", max_age=0) @app.get("/static/") def static_file(name): return send_from_directory(STATIC_DIR, name, max_age=0) @app.get("/media/") def media(name): resp = send_from_directory(MEDIA_DIR, name, conditional=True) resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" return resp @app.get("/api/health") def health(): last = None if _scheduler is not None: last = getattr(_scheduler, "last_push_at", None) return jsonify(ok=True, boot_id=BOOT_ID, page_version=PAGE_VERSION, items_sha256=ITEMS.get("items_sha256"), sc_frozen=ITEMS.get("sc_frozen", False), n_records=len(_record_ids), access_required=ACCESS_REQUIRED, hf={**_hf, "last_push_at": last}) @app.get("/api/versions") def versions(): require_access() return jsonify(versions=list(version_stats().values()), sc_frozen=ITEMS.get("sc_frozen", False)) @app.get("/api/assign") def assign(): require_access() set_name = request.args.get("set", "test") rid = request.args.get("rater_id", "").strip().lower() or None if set_name not in SETS: return jsonify(error="bad set"), 400 if set_name == "test" and not ITEMS.get("sc_frozen", False): return jsonify(error="sc_not_frozen"), 409 if rid and (g := gate_error(rid, set_name)) is not None: return g v = pick_version(set_name, rid) return jsonify(version=v, set=set_name, n_screens=n_screens(v) if v else 0) @app.post("/api/login") def login(): body = request.get_json(silent=True) or {} if ACCESS_REQUIRED and not hmac.compare_digest(str(body.get("access_code", "")), ACCESS_CODE): return jsonify(ok=False, error="bad access code"), 401 rid = str(body.get("rater_id", "")).strip().lower() if not RATER_RE.match(rid): return jsonify(ok=False, error="rater id must be 2-24 chars of a-z 0-9 _"), 400 return jsonify(ok=True, rater_id=rid, page_version=PAGE_VERSION, sc_frozen=ITEMS.get("sc_frozen", False), **rater_status(rid)) @app.get("/api/session") def session(): require_access() rid = request.args.get("rater_id", "").strip().lower() version = request.args.get("version", "") if not RATER_RE.match(rid) or version not in VERSIONS: return jsonify(error="bad rater_id or version"), 400 if VERSIONS[version].get("demo_only"): return jsonify(error="demo_only", detail="This version is a preview only; open the main link to be assigned a questionnaire."), 409 set_name = VERSIONS[version]["set"] if set_name == "test" and not ITEMS.get("sc_frozen", False): return jsonify(error="sc_not_frozen", detail="The consistency pre-pass has not been frozen yet; " "the test versions open afterwards."), 409 if (g := gate_error(rid, set_name)) is not None: return g screens = screens_for(rid, version) latest = {} for (r, s_, iid, oid), rec in _latest.items(): if r == rid and s_ == set_name and rec.get("version") == version: latest[f"{iid}|{oid or ''}"] = rec next_index = next((s_["screen_index"] for s_ in screens if f"{s_['item_id']}|{s_['output_id'] or ''}" not in latest), len(screens)) return jsonify(rater_id=rid, set=set_name, version=version, screens=screens, latest=latest, next_index=next_index, n_total=len(screens), n_done=len(latest), block_size=ITEMS.get("block_size", 20), page_version=PAGE_VERSION, boot_id=BOOT_ID) @app.post("/api/record") def record(): require_access() rec = request.get_json(silent=True) if not isinstance(rec, dict): return jsonify(ok=False, errors=["body must be a JSON object"]), 400 rec["rater_id"] = str(rec.get("rater_id", "")).strip().lower() if rec.get("version") in VERSIONS: rec["set"] = VERSIONS[rec["version"]]["set"] errs = validate(rec) if errs: return jsonify(ok=False, errors=errs), 400 it = ITEM_BY_ID.get(rec["item_id"], {"task": None}) rec["task"] = it["task"] if rec["set"] == "profile": rec["output_id"] = None rec["hc_pass"] = rec["hc_count"] == ITEMS.get("headphone_check", {}).get("answer_count") and \ rec["hc_side"] == ITEMS.get("headphone_check", {}).get("answer_side") rec["phase"] = {"pilot": "pilot", "qual": "qual", "profile": "profile"}.get(rec["set"], "main") rec["server_received_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) rec["boot_id"] = BOOT_ID rec["user_agent"] = request.headers.get("User-Agent", "")[:200] rec.setdefault("mode", "api") with _lock: if rec["record_id"] in _record_ids: n = sum(1 for k in _latest if k[0] == rec["rater_id"] and k[1] == rec["set"]) return jsonify(ok=True, record_id=rec["record_id"], n_done=n, duplicate=True) append_record(rec) _record_ids.add(rec["record_id"]) k = key_of(rec) if k not in _latest or rec["submitted_at"] >= _latest[k]["submitted_at"]: _latest[k] = rec n = sum(1 for k in _latest if k[0] == rec["rater_id"] and k[1] == rec["set"]) return jsonify(ok=True, record_id=rec["record_id"], n_done=n, duplicate=False) @app.get("/api/progress") def progress(): require_admin() out = {} for (r, s_, iid, oid), rec in _latest.items(): key = f"{r}/{rec.get('version')}" d = out.setdefault(key, {"rater_id": r, "set": s_, "version": rec.get("version"), "done": 0, "last_submit": "", "revisions": 0}) d["done"] += 1 d["last_submit"] = max(d["last_submit"], rec["submitted_at"]) for rec in iter_records(): if rec.get("revision"): key = f"{rec['rater_id']}/{rec.get('version')}" if key in out: out[key]["revisions"] += 1 for d in out.values(): d["total"] = n_screens(d["version"]) if d["version"] in VERSIONS else None return jsonify(progress=sorted(out.values(), key=lambda d: (d["version"] or "", d["rater_id"])), versions=list(version_stats().values()), sc_frozen=ITEMS.get("sc_frozen", False), boot_id=BOOT_ID, n_records=len(_record_ids), hf=_hf) FIELDS = ["record_id", "rater_id", "set", "version", "phase", "item_id", "output_id", "task", "screen_index", "block", "revision", "revises_record_id", "V", "A", "P_visual", "P_audio", "artifact_visual", "artifact_audio", "recon_plausible", "consistency_verifiable", "consistency_label", "consistency_eligible", "uncertainty_reason", "comment", "experience", "headphones", "hc_count", "hc_side", "hc_pass", "plays_src", "plays_out", "plays_ref", "seconds_on_screen", "submitted_at", "server_received_at", "boot_id", "page_version", "mode", "user_agent", "is_latest"] @app.get("/api/export.") def export(fmt): require_admin() recs = list(iter_records()) for rec in recs: rec["is_latest"] = _latest.get(key_of(rec), {}).get("record_id") == rec["record_id"] if fmt == "jsonl": return Response("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in recs), mimetype="application/x-ndjson") buf = io.StringIO() w = csv.DictWriter(buf, fieldnames=FIELDS, extrasaction="ignore") w.writeheader() for r in recs: w.writerow(r) return Response(buf.getvalue(), mimetype="text/csv", headers={"Content-Disposition": "attachment; filename=avio_human_eval_records.csv"}) if __name__ == "__main__": app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 7860)), threaded=True)