File size: 2,294 Bytes
abba2fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | """Parity: the released fp16 package must reproduce the frozen pointer_v2 strict outcomes on AndroidControl test steps.
Streams the dataset and keeps only the steps it checks (the 8 GB Mac cannot hold the whole file next to the model).
Usage (repo root): .venv/bin/python release/milo-pointer/parity_check.py [n]"""
import gzip, json, sys, time
from collections import Counter
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(ROOT / "bench"), str(Path(__file__).parent)]
from ac_frontier import DATA, strict_ok
def wanted_test_rows(want): # same history/consequence string as ac_frontier.test_rows
def sig(r):
return Counter((e["text"] or e["desc"] or e["rid"], e["cls"]) for e in r["els"])
out, prev, t = {}, None, -1
for l in gzip.open(DATA, "rt"):
r = json.loads(l)
s = sig(r)
if prev and prev[0] == r["ep"] and prev[1] == r["instr"].strip():
j = sum((s & prev[3]).values()) / max(sum((s | prev[3]).values()), 1)
r["hist"] = (
f"repeat of last step (last action {prev[2]}), screen {int(round(j * 10)) * 10}% unchanged"
)
else:
r["hist"] = "new step" + (f" (last action {prev[2]})" if prev and prev[0] == r["ep"] else "")
prev = (r["ep"], r["instr"].strip(), r["action"]["action_type"], s)
if r["split"] == "test":
t += 1
if t in want:
out[t] = r
return out
if __name__ == "__main__":
n = int(sys.argv[1]) if len(sys.argv) > 1 else 200
ref = [r["strict"] for r in json.load(open(ROOT / "training/ac/pointer_v2/local_eval.json"))]
idx = json.load(open(ROOT / "bench/ac_sample1000.json"))[:n]
T = wanted_test_rows(set(idx))
from milo_pointer import MiloPointer
m = MiloPointer.load(Path(__file__).parent)
same = ok = 0
ts = []
for i in idx:
r = T[i]
t0 = time.time()
p = m.predict(r["goal"], r["instr"], r["els"], r["w"], r["h"], r["hist"])
ts.append(time.time() - t0)
s = strict_ok(r, p)
ok += s
same += s == bool(ref[i])
ts.sort()
print(
f"n={n} released {ok} vs frozen {sum(ref[i] for i in idx)}; same outcome {same}/{n}; median {1000*ts[n//2]:.0f} ms"
)
|