Buckets:
| import sys, numpy as np, pandas as pd | |
| sys.path.insert(0, "/home/user/RestoreKV") | |
| from prgf.evaluate import load_ruler | |
| exec(open("pertask_lb.py").read().split("v4 = ")[0]) | |
| split = sys.argv[1]; runs = sys.argv[2:] # name=path | |
| df = load_ruler(split) | |
| cols = {} | |
| for r in runs: | |
| name, path = r.split("=") | |
| p = pd.read_json(path, lines=True).set_index("index").predicted_answer | |
| cols[name] = p.loc[df.index].values | |
| for m in ["RestoreKV", "RestoreKV_plus", "no_press"]: | |
| cols[m] = pd.read_csv(f"lb/{m}_predictions.csv").predicted_answer.iloc[df.index].values | |
| S = pd.DataFrame({k: per_sample(v, df.answer.values, df.task.values) for k, v in cols.items()}); S["task"] = df.task.values | |
| t = S.groupby("task").mean().round(1); t.loc["AVG"] = t.mean().round(2); print(t.to_string()) | |
| rng = np.random.default_rng(0); names = list(cols) | |
| idx = {k: np.where(S.task.values == k)[0] for k in S.task.unique()} | |
| def avg(s, ii): return np.mean([s[ii[k]].mean() for k in ii]) | |
| a = names[-4] # last run given | |
| for b in names: | |
| if b == a: continue | |
| d = S[a].values - S[b].values | |
| boots = [np.mean([d[rng.choice(v, len(v))].mean() for v in idx.values()]) for _ in range(2000)] | |
| print(f"{a} - {b}: {avg(d, idx):+.2f} [{np.percentile(boots, 2.5):+.2f}, {np.percentile(boots, 97.5):+.2f}]") | |
Xet Storage Details
- Size:
- 1.28 kB
- Xet hash:
- 4901a41668d8071756b9aadddc756e90cf9d0637f841c20b7290335f6f80da15
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.