Buckets:
| import glob, json, sys, numpy as np, pandas as pd | |
| sys.path.insert(0, "/home/user/RestoreKV") | |
| from prgf.evaluate import load_ruler | |
| exec(open("pertask_lb.py").read().split("v4 = ")[0]) | |
| df = load_ruler("all") | |
| pred = {} | |
| for f in glob.glob("res/shards/**/*.json", recursive=True): | |
| for k, v in json.load(open(f))["preds"].items(): pred[int(k)] = v | |
| n_shard = len(pred) | |
| for f in ["res/v4plus_test50_preds.jsonl", "res/results/full_v4plus/dev/0.9375_preds.jsonl"]: | |
| d = pd.read_json(f, lines=True) | |
| pred.update(dict(zip(d["index"], d["predicted_answer"]))) | |
| idx = sorted(pred) | |
| sub = df.loc[idx] | |
| S = {"PRGF_v4plus": per_sample([pred[i] for i in idx], sub.answer.values, sub.task.values)} | |
| for m in ["RestoreKV", "RestoreKV_plus", "no_press"]: | |
| S[m] = per_sample(pd.read_csv(f"lb/{m}_predictions.csv").predicted_answer.iloc[idx].values, sub.answer.values, sub.task.values) | |
| S = pd.DataFrame(S); S["task"] = sub.task.values | |
| t = S.groupby("task").mean().round(1); t.loc["AVG"] = t.mean().round(2) | |
| print(f"rows: {len(idx)} / 6500 (shards {n_shard})"); print(t.to_string()) | |
| ids = [np.where(S.task.values == k)[0] for k in S.task.unique()]; rng = np.random.default_rng(0) | |
| for b in ["RestoreKV", "RestoreKV_plus"]: | |
| d = S["PRGF_v4plus"].values - S[b].values | |
| bo = [np.mean([d[rng.choice(v, len(v))].mean() for v in ids]) for _ in range(3000)] | |
| print(f"PRGF_v4plus - {b}: {np.mean([d[v].mean() for v in ids]):+.2f} [{np.percentile(bo,2.5):+.2f}, {np.percentile(bo,97.5):+.2f}]") | |
Xet Storage Details
- Size:
- 1.48 kB
- Xet hash:
- d89398b5d4e6e7a049ddf5fdf5d2b422e875e2c69eddf5a5da3e1d8dc073c36a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.