File size: 1,435 Bytes
ee3e28a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | import json, statistics as st
def load(p):
d={}
for line in open(p):
r=json.loads(line); d[r["pid"]]=r
return d
def analyze(tag, spec_f, van1_f, van2_f):
sp, v1, v2 = load(spec_f), load(van1_f), load(van2_f)
pids = sorted(set(sp)&set(v1)&set(v2))
n=len(pids)
sp_m = st.mean([sp[p]["tok_s"] for p in pids])
v1_m = st.mean([v1[p]["tok_s"] for p in pids])
v2_m = st.mean([v2[p]["tok_s"] for p in pids])
ratios_paired = sorted(sp[p]["tok_s"]/((v1[p]["tok_s"]+v2[p]["tok_s"])/2) for p in pids)
med = st.median(ratios_paired)
q1 = ratios_paired[n//4]; q3 = ratios_paired[(3*n)//4]
a = sp_m/v1_m
b = st.mean([sp[p]["tok_s"]/v1[p]["tok_s"] for p in pids])
c = (sum(sp[p]["gen_len"] for p in pids)/sum(sp[p]["t"] for p in pids)) / (sum(v1[p]["gen_len"] for p in pids)/sum(v1[p]["t"] for p in pids))
d = sp_m/min(v1_m, v2_m)
print(f"{tag}: n={n} ours_median={med:.3f} [IQR {q1:.3f}-{q3:.3f}] | litA_meanratio={a:.3f} litB_meanofratios={b:.3f} litC_totaltok={c:.3f} litD_vs_slower_van={d:.3f}")
print(f" abs tok/s: spec {sp_m:.1f} van1 {v1_m:.1f} van2 {v2_m:.1f}")
B="/testessfs10/users/zeyu.zhang/wangyu_ssd/stage1/"
for g in (4,6,8):
analyze(f"strict g{g}", B+f"nb_spec_g{g}.jsonl", B+f"nb_van1_g{g}.jsonl", B+f"nb_van2_g{g}.jsonl")
for g in (6,8):
analyze(f"relax g{g}", B+f"rxb_spec_g{g}.jsonl", B+f"rxb_van1_g{g}.jsonl", B+f"rxb_van2_g{g}.jsonl")
|