WitneyWW's picture
rater Space: TEST + TEST2 only (items 052c78793bc12e67, page 2026-11-09v2guide)
b92bed6 verified
Raw History Blame Contribute Delete
22.1 kB
#!/usr/bin/env python
"""Flask backend for the AVIO human evaluation (HF Docker Space).
Serves static/index.html + items.json + media/, and collects one JSON record per rated screen:
data/records/<rater_id>/<boot_id>.jsonl (append-only, fsync'ed)
A huggingface_hub.CommitScheduler pushes data/ to the private dataset repo DATASET_REPO every
2 minutes; on start the repo is restored into data/ so records survive restarts and rebuilds.
Each container only ever appends to files named after its own boot_id, so the upload-only
scheduler can never overwrite another container's records.
env: ACCESS_CODE, ADMIN_CODE, HF_TOKEN (Space secrets); DATASET_REPO, PAGE_VERSION (variables);
DATA_DIR (default ./data), NO_HF=1 (local test: no restore, no scheduler).
"""
import csv
from collections import defaultdict
import hashlib
import hmac
import io
import json
import os
import re
import secrets
import signal
import threading
import time
import uuid
from pathlib import Path
from flask import Flask, abort, jsonify, request, send_from_directory, Response
HERE = Path(__file__).resolve().parent
DATA_DIR = Path(os.environ.get("DATA_DIR", HERE / "data"))
REC_DIR = DATA_DIR / "records"
META_DIR = DATA_DIR / "meta"
MEDIA_DIR = HERE / "media"
STATIC_DIR = HERE / "static"
ACCESS_CODE = os.environ.get("ACCESS_CODE", "")
ADMIN_CODE = os.environ.get("ADMIN_CODE", "")
DATASET_REPO = os.environ.get("DATASET_REPO", "WitneyWW/avio-human-eval-records")
NO_HF = os.environ.get("NO_HF") == "1"
BOOT_ID = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + "-" + secrets.token_hex(3)
ITEMS = json.loads((HERE / "items.json").read_text())
PAGE_VERSION = os.environ.get("PAGE_VERSION") or ITEMS.get("page_version", "dev")
ITEM_BY_ID = {it["item_id"]: it for it in ITEMS["items"]}
SETS = ("test", "pilot", "train_audit", "sc_prepass", "qual", "profile")
GATED = ("test", "pilot", "train_audit") # need profile + qualification (Q1) first
VERSIONS = ITEMS.get("versions") or {}
if "SC" not in VERSIONS and not ITEMS.get("sc_frozen"): # v2 (sc_frozen, no C question) has no pre-pass
VERSIONS["SC"] = {"set": "sc_prepass", "items": [it["item_id"] for it in ITEMS["items"] if it["set"] == "test"]}
VERSIONS.setdefault("PROFILE", {"set": "profile", "items": ["profile"]})
QUAL_VERSION = "Q1" if "Q1" in VERSIONS else None
PROFILE_OPTS = {"experience": ("none", "some", "professional"),
"headphones": ("Yes", "No"), "hc_side": ("Left", "Right", "Both", "Not sure")}
ACCESS_REQUIRED = bool(ACCESS_CODE)
RATER_RE = re.compile(r"^[a-z0-9_]{2,24}$")
TRI = ("Pass", "Fail", "Uncertain")
ART = ("None", "Minor", "Major")
app = Flask(__name__, static_folder=None)
app.config["MAX_CONTENT_LENGTH"] = 256 * 1024
_lock = threading.RLock() # re-entrant: record() holds it while append_record() takes it again
_latest = {} # (rater_id, set, item_id, output_id) -> latest record (by submitted_at)
_record_ids = set()
_hf = {"restored": None, "last_push_at": None, "last_error": None, "enabled": not NO_HF}
_scheduler = None
# ----------------------------------------------------------------------------- helpers
def hx(*parts):
return hashlib.sha256("|".join(parts).encode()).hexdigest()
def key_of(rec):
return (rec["rater_id"], rec["set"], rec["item_id"], rec.get("output_id"))
def iter_records():
for p in sorted(REC_DIR.rglob("*.jsonl")):
with open(p, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
try:
yield json.loads(line)
except json.JSONDecodeError:
continue
def load_index():
_latest.clear()
_record_ids.clear()
for rec in iter_records():
_record_ids.add(rec["record_id"])
k = key_of(rec)
if k not in _latest or rec["submitted_at"] >= _latest[k]["submitted_at"]:
_latest[k] = rec
def screens_for(rater_id, version):
"""Deterministic hash-sorted order inside a version; the outputs of an item are consecutive."""
v = VERSIONS[version]
# v2 versions have ordered "sections" (Part 1, Part 2): shuffle inside a section, keep the section order
sections = v.get("sections") or [v["items"]]
screens = []
for si, sec in enumerate(sections):
items = [ITEM_BY_ID[i] for i in sec]
if v["set"] != "qual": # the practice round keeps its fixed, didactic order
items.sort(key=lambda it: hx(rater_id, it["item_id"]))
for it in items:
if v["set"] == "sc_prepass":
screens.append({"item_id": it["item_id"], "output_id": None, "section": si})
continue
outs = sorted(it["outputs"], key=lambda o: hx(rater_id, it["item_id"], o["output_id"]))
for o in outs:
screens.append({"item_id": it["item_id"], "output_id": o["output_id"], "section": si})
for i, s in enumerate(screens):
s["screen_index"] = i
return screens
def n_screens(version):
v = VERSIONS[version]
if v["set"] in ("sc_prepass", "profile"):
return len(v["items"])
return sum(len(ITEM_BY_ID[i]["outputs"]) for i in v["items"])
def version_stats():
"""Per version: how many raters completed it / are in progress (from the latest-record index)."""
done_by = defaultdict(lambda: defaultdict(int)) # version -> rater -> n distinct screens
for (r, s, iid, oid), rec in _latest.items():
if r.startswith("dryrun"):
continue
done_by[rec.get("version", "?")][r] += 1
out = {}
for vid, v in VERSIONS.items():
total = n_screens(vid)
cnt = done_by.get(vid, {})
out[vid] = {"version": vid, "set": v["set"], "n_items": len(v["items"]), "n_screens": total,
"raters_completed": sum(1 for n in cnt.values() if n >= total),
"raters_in_progress": sum(1 for n in cnt.values() if 0 < n < total)}
return out
def pick_version(set_name, rater_id=None):
"""Auto-assign: the version of `set_name` with the fewest completed raters, then fewest in progress,
then random; skip versions this rater has already finished."""
st = version_stats()
cands = [v for v in st.values() if v["set"] == set_name and not VERSIONS[v["version"]].get("demo_only")]
if rater_id:
done = {k[2] for k in _latest if k[0] == rater_id} # item ids done by this rater (any version)
cands = [v for v in cands if not all(i in done for i in VERSIONS[v["version"]]["items"])] or cands
if not cands:
return None
best = min(cands, key=lambda v: (v["raters_completed"], v["raters_in_progress"]))
pool = [v for v in cands if (v["raters_completed"], v["raters_in_progress"]) == (best["raters_completed"], best["raters_in_progress"])]
return secrets.choice(pool)["version"]
def rater_status(rid):
"""profile_done / qual_done for the gate in front of pilot/test questionnaires."""
prof = any(k[0] == rid and k[1] == "profile" for k in _latest)
if QUAL_VERSION is None:
return {"profile_done": prof, "qual_done": True, "qual_version": None}
need = set(VERSIONS[QUAL_VERSION]["items"])
got = {k[2] for k in _latest if k[0] == rid and k[1] == "qual"}
return {"profile_done": prof, "qual_done": need <= got, "qual_version": QUAL_VERSION}
def gate_error(rid, set_name):
if set_name not in GATED:
return None
st = rater_status(rid)
if st["profile_done"] and st["qual_done"]:
return None
return jsonify(error="qual_required", detail="Please complete the short profile and the practice round first.", **st), 409
def check_code(header, code):
given = request.headers.get(header, "") or (request.get_json(silent=True) or {}).get(header.lower().replace("x-", "").replace("-", "_"), "")
return bool(code) and hmac.compare_digest(str(given), code)
def require_access():
if ACCESS_REQUIRED and not check_code("X-Access-Code", ACCESS_CODE):
abort(401)
def require_admin():
if not check_code("X-Admin-Code", ADMIN_CODE):
abort(401)
def validate(rec):
"""Server-side mirror of the client validation. Returns a list of error strings."""
e = []
for f in ("record_id", "rater_id", "set", "item_id", "submitted_at", "page_version"):
if not rec.get(f):
e.append(f"missing {f}")
if e:
return e
if not RATER_RE.match(rec["rater_id"]):
e.append("bad rater_id")
if rec["set"] not in SETS:
e.append("bad set")
if rec["set"] == "profile":
if rec.get("item_id") != "profile" or rec.get("version") != "PROFILE":
e.append("profile record must have item_id=profile, version=PROFILE")
for f, allowed in PROFILE_OPTS.items():
if rec.get(f) not in allowed:
e.append(f"{f} must be one of {'/'.join(allowed)}")
if not isinstance(rec.get("hc_count"), int) or not 0 <= rec["hc_count"] <= 9:
e.append("hc_count must be an integer 0-9")
return e
v = VERSIONS.get(rec.get("version") or "")
if v is None:
return e + ["unknown version"]
if v.get("demo_only"):
return e + ["this version is preview-only and cannot be rated"]
if v["set"] != rec["set"]:
e.append("version/set mismatch")
it = ITEM_BY_ID.get(rec["item_id"])
if it is None:
return e + ["unknown item_id"]
if rec["item_id"] not in v["items"]:
e.append("item not in version")
if rec["set"] == "sc_prepass":
if it["set"] != "test":
e.append("sc_prepass only covers test items")
if rec.get("output_id") is not None:
e.append("sc_prepass has no output_id")
if rec.get("consistency_eligible") not in ("Yes", "No"):
e.append("consistency_eligible must be Yes/No")
return e
if it["set"] != rec["set"]:
e.append("item does not belong to set")
if rec.get("output_id") not in {o["output_id"] for o in it["outputs"]}:
e.append("output_id not in item")
for f in ("V", "A", "P_visual"):
if rec.get(f) not in TRI:
e.append(f"{f} must be Pass/Fail/Uncertain")
if rec.get("P_audio") not in TRI + ("NA",):
e.append("P_audio must be Pass/Fail/Uncertain/NA")
for f in ("artifact_visual", "artifact_audio"):
if rec.get(f) not in ART:
e.append(f"{f} must be None/Minor/Major")
if it["task"] == "removal":
if rec.get("recon_plausible") not in TRI:
e.append("recon_plausible required for removal")
elif rec.get("recon_plausible") is not None:
e.append("recon_plausible must be null for addition")
if it.get("sc_eligible"):
if rec.get("consistency_verifiable") not in ("Yes", "No"):
e.append("consistency_verifiable must be Yes/No")
if rec.get("consistency_verifiable") == "Yes":
if rec.get("consistency_label") not in TRI:
e.append("consistency_label required when verifiable")
elif rec.get("consistency_label") is not None:
e.append("consistency_label must be null when not verifiable")
else:
if rec.get("consistency_verifiable") is not None or rec.get("consistency_label") is not None:
e.append("consistency fields must be null for non-eligible items")
unc = any(rec.get(f) == "Uncertain" for f in ("V", "A", "P_visual", "P_audio", "recon_plausible", "consistency_label"))
if unc and not (rec.get("uncertainty_reason") or "").strip():
e.append("uncertainty_reason required when any field is Uncertain")
return e
def append_record(rec):
path = REC_DIR / rec["rater_id"] / f"{BOOT_ID}.jsonl"
path.parent.mkdir(parents=True, exist_ok=True)
line = json.dumps(rec, ensure_ascii=False) + "\n"
lock = _scheduler.lock if _scheduler is not None else _lock
with lock:
with open(path, "a", encoding="utf-8") as f:
f.write(line)
f.flush()
os.fsync(f.fileno())
# ----------------------------------------------------------------------------- HF sync
def hf_setup():
global _scheduler
REC_DIR.mkdir(parents=True, exist_ok=True)
META_DIR.mkdir(parents=True, exist_ok=True)
if NO_HF:
load_index()
return
from huggingface_hub import CommitScheduler, HfApi, snapshot_download
token = os.environ.get("HF_TOKEN")
try:
HfApi(token=token).create_repo(DATASET_REPO, repo_type="dataset", private=True, exist_ok=True)
snapshot_download(DATASET_REPO, repo_type="dataset", local_dir=str(DATA_DIR), token=token,
allow_patterns=["records/**", "meta/**"])
_hf["restored"] = True
except Exception as ex: # keep serving; records accumulate locally and the client retries
_hf["restored"] = False
_hf["last_error"] = f"restore: {type(ex).__name__}: {str(ex)[:200]}"
load_index()
with open(META_DIR / "boots.jsonl", "a") as f:
f.write(json.dumps({"boot_id": BOOT_ID, "page_version": PAGE_VERSION,
"items_sha256": ITEMS.get("items_sha256"), "n_records_restored": len(_record_ids),
"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}) + "\n")
_scheduler = CommitScheduler(repo_id=DATASET_REPO, repo_type="dataset", folder_path=str(DATA_DIR),
path_in_repo=None, every=2, private=True, token=token,
allow_patterns=["records/**/*.jsonl", "meta/*.json", "meta/*.jsonl"],
squash_history=False)
# make the scheduler skip the files we just restored (identical content) on its first tick
for p in DATA_DIR.rglob("*"):
if p.is_file():
_scheduler.last_uploaded[p] = p.stat().st_mtime
def _on_term(signum, frame):
try:
_scheduler.trigger().result(timeout=25)
except Exception:
pass
raise SystemExit(0)
signal.signal(signal.SIGTERM, _on_term)
hf_setup()
# ----------------------------------------------------------------------------- routes
@app.get("/")
def index():
return send_from_directory(STATIC_DIR, "index.html", max_age=0)
@app.get("/demo")
def demo():
return send_from_directory(STATIC_DIR, "demo.html", max_age=0)
@app.get("/items.json")
def items_json():
return send_from_directory(HERE, "items.json", max_age=0)
@app.get("/static/<path:name>")
def static_file(name):
return send_from_directory(STATIC_DIR, name, max_age=0)
@app.get("/media/<path:name>")
def media(name):
resp = send_from_directory(MEDIA_DIR, name, conditional=True)
resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
return resp
@app.get("/api/health")
def health():
last = None
if _scheduler is not None:
last = getattr(_scheduler, "last_push_at", None)
return jsonify(ok=True, boot_id=BOOT_ID, page_version=PAGE_VERSION, items_sha256=ITEMS.get("items_sha256"),
sc_frozen=ITEMS.get("sc_frozen", False), n_records=len(_record_ids), access_required=ACCESS_REQUIRED,
hf={**_hf, "last_push_at": last})
@app.get("/api/versions")
def versions():
require_access()
return jsonify(versions=list(version_stats().values()), sc_frozen=ITEMS.get("sc_frozen", False))
@app.get("/api/assign")
def assign():
require_access()
set_name = request.args.get("set", "test")
rid = request.args.get("rater_id", "").strip().lower() or None
if set_name not in SETS:
return jsonify(error="bad set"), 400
if set_name == "test" and not ITEMS.get("sc_frozen", False):
return jsonify(error="sc_not_frozen"), 409
if rid and (g := gate_error(rid, set_name)) is not None:
return g
v = pick_version(set_name, rid)
return jsonify(version=v, set=set_name, n_screens=n_screens(v) if v else 0)
@app.post("/api/login")
def login():
body = request.get_json(silent=True) or {}
if ACCESS_REQUIRED and not hmac.compare_digest(str(body.get("access_code", "")), ACCESS_CODE):
return jsonify(ok=False, error="bad access code"), 401
rid = str(body.get("rater_id", "")).strip().lower()
if not RATER_RE.match(rid):
return jsonify(ok=False, error="rater id must be 2-24 chars of a-z 0-9 _"), 400
return jsonify(ok=True, rater_id=rid, page_version=PAGE_VERSION, sc_frozen=ITEMS.get("sc_frozen", False), **rater_status(rid))
@app.get("/api/session")
def session():
require_access()
rid = request.args.get("rater_id", "").strip().lower()
version = request.args.get("version", "")
if not RATER_RE.match(rid) or version not in VERSIONS:
return jsonify(error="bad rater_id or version"), 400
if VERSIONS[version].get("demo_only"):
return jsonify(error="demo_only", detail="This version is a preview only; open the main link to be assigned a questionnaire."), 409
set_name = VERSIONS[version]["set"]
if set_name == "test" and not ITEMS.get("sc_frozen", False):
return jsonify(error="sc_not_frozen", detail="The consistency pre-pass has not been frozen yet; "
"the test versions open afterwards."), 409
if (g := gate_error(rid, set_name)) is not None:
return g
screens = screens_for(rid, version)
latest = {}
for (r, s_, iid, oid), rec in _latest.items():
if r == rid and s_ == set_name and rec.get("version") == version:
latest[f"{iid}|{oid or ''}"] = rec
next_index = next((s_["screen_index"] for s_ in screens if f"{s_['item_id']}|{s_['output_id'] or ''}" not in latest),
len(screens))
return jsonify(rater_id=rid, set=set_name, version=version, screens=screens, latest=latest, next_index=next_index,
n_total=len(screens), n_done=len(latest), block_size=ITEMS.get("block_size", 20),
page_version=PAGE_VERSION, boot_id=BOOT_ID)
@app.post("/api/record")
def record():
require_access()
rec = request.get_json(silent=True)
if not isinstance(rec, dict):
return jsonify(ok=False, errors=["body must be a JSON object"]), 400
rec["rater_id"] = str(rec.get("rater_id", "")).strip().lower()
if rec.get("version") in VERSIONS:
rec["set"] = VERSIONS[rec["version"]]["set"]
errs = validate(rec)
if errs:
return jsonify(ok=False, errors=errs), 400
it = ITEM_BY_ID.get(rec["item_id"], {"task": None})
rec["task"] = it["task"]
if rec["set"] == "profile":
rec["output_id"] = None
rec["hc_pass"] = rec["hc_count"] == ITEMS.get("headphone_check", {}).get("answer_count") and \
rec["hc_side"] == ITEMS.get("headphone_check", {}).get("answer_side")
rec["phase"] = {"pilot": "pilot", "qual": "qual", "profile": "profile"}.get(rec["set"], "main")
rec["server_received_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
rec["boot_id"] = BOOT_ID
rec["user_agent"] = request.headers.get("User-Agent", "")[:200]
rec.setdefault("mode", "api")
with _lock:
if rec["record_id"] in _record_ids:
n = sum(1 for k in _latest if k[0] == rec["rater_id"] and k[1] == rec["set"])
return jsonify(ok=True, record_id=rec["record_id"], n_done=n, duplicate=True)
append_record(rec)
_record_ids.add(rec["record_id"])
k = key_of(rec)
if k not in _latest or rec["submitted_at"] >= _latest[k]["submitted_at"]:
_latest[k] = rec
n = sum(1 for k in _latest if k[0] == rec["rater_id"] and k[1] == rec["set"])
return jsonify(ok=True, record_id=rec["record_id"], n_done=n, duplicate=False)
@app.get("/api/progress")
def progress():
require_admin()
out = {}
for (r, s_, iid, oid), rec in _latest.items():
key = f"{r}/{rec.get('version')}"
d = out.setdefault(key, {"rater_id": r, "set": s_, "version": rec.get("version"), "done": 0, "last_submit": "", "revisions": 0})
d["done"] += 1
d["last_submit"] = max(d["last_submit"], rec["submitted_at"])
for rec in iter_records():
if rec.get("revision"):
key = f"{rec['rater_id']}/{rec.get('version')}"
if key in out:
out[key]["revisions"] += 1
for d in out.values():
d["total"] = n_screens(d["version"]) if d["version"] in VERSIONS else None
return jsonify(progress=sorted(out.values(), key=lambda d: (d["version"] or "", d["rater_id"])),
versions=list(version_stats().values()), sc_frozen=ITEMS.get("sc_frozen", False),
boot_id=BOOT_ID, n_records=len(_record_ids), hf=_hf)
FIELDS = ["record_id", "rater_id", "set", "version", "phase", "item_id", "output_id", "task", "screen_index", "block",
"revision", "revises_record_id", "V", "A", "P_visual", "P_audio", "artifact_visual", "artifact_audio",
"recon_plausible", "consistency_verifiable", "consistency_label", "consistency_eligible",
"uncertainty_reason", "comment", "experience", "headphones", "hc_count", "hc_side", "hc_pass", "plays_src", "plays_out", "plays_ref", "seconds_on_screen",
"submitted_at", "server_received_at", "boot_id", "page_version", "mode", "user_agent", "is_latest"]
@app.get("/api/export.<fmt>")
def export(fmt):
require_admin()
recs = list(iter_records())
for rec in recs:
rec["is_latest"] = _latest.get(key_of(rec), {}).get("record_id") == rec["record_id"]
if fmt == "jsonl":
return Response("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in recs),
mimetype="application/x-ndjson")
buf = io.StringIO()
w = csv.DictWriter(buf, fieldnames=FIELDS, extrasaction="ignore")
w.writeheader()
for r in recs:
w.writerow(r)
return Response(buf.getvalue(), mimetype="text/csv",
headers={"Content-Disposition": "attachment; filename=avio_human_eval_records.csv"})
if __name__ == "__main__":
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 7860)), threaded=True)