| { |
| "stages": { |
| "stable:queen-v4-r1-stable": { |
| "wall_seconds": 11067.888490438461, |
| "invocations": 1 |
| }, |
| "branch:queen-v4-r1-x1": { |
| "wall_seconds": 1519.7983586788177, |
| "invocations": 1 |
| }, |
| "branch:queen-v4-r1-x1.5": { |
| "wall_seconds": 2177.17232131958, |
| "invocations": 1 |
| }, |
| "branch:queen-v4-r1-x2": { |
| "wall_seconds": 2837.695327758789, |
| "invocations": 1 |
| }, |
| "sft": { |
| "wall_seconds": 4540.518196582794, |
| "invocations": 1 |
| } |
| }, |
| "gpu": { |
| "count": 1 |
| }, |
| "wall_hours": 6.151, |
| "gpu_hours": 6.151, |
| "usd_per_gpu_hour": 0.34, |
| "cost_usd": 2.09, |
| "runs": { |
| "queen-v4-r1-stable": { |
| "last_step": 3846, |
| "last_val_bpb": 0.9321817332330166, |
| "best_step": 3072, |
| "best_val_bpb": 0.9221891606398556, |
| "sft_from_step": 3072 |
| }, |
| "queen-v4-r1-x1": { |
| "last_step": 2404, |
| "last_val_bpb": 1.2690717874147677, |
| "best_step": 1980, |
| "best_val_bpb": 1.004812328168178, |
| "sft_from_step": 1980 |
| }, |
| "queen-v4-r1-x1.5": { |
| "last_step": 3605, |
| "last_val_bpb": 1.4380317966698843, |
| "best_step": 2970, |
| "best_val_bpb": 0.9712996006066769, |
| "sft_from_step": 2970 |
| }, |
| "queen-v4-r1-x2": { |
| "last_step": 4808, |
| "last_val_bpb": 1.6860155733525417, |
| "best_step": 3960, |
| "best_val_bpb": 0.9732352409739348, |
| "sft_from_step": 3960 |
| } |
| }, |
| "base": { |
| "last_step": 3605, |
| "last_val_bpb": 1.4380317966698843, |
| "best_step": 3605, |
| "best_val_bpb": 1.4380317966698843, |
| "sft_from_step": 3605, |
| "promoted_from": "queen-v4-r1-x1.5" |
| }, |
| "sft": { |
| "method": "synthetic+openbook", |
| "files": [ |
| "sft/checked/mix.train.jsonl" |
| ], |
| "conversations": 1811163, |
| "epochs": 1, |
| "base_step": 3605, |
| "init_lr_frac": 0.5, |
| "mix": { |
| "weights": { |
| "synthetic": 0.55, |
| "openbook": 0.25, |
| "headings": 0.2 |
| }, |
| "anchor": "synthetic", |
| "seed": 0, |
| "sources": { |
| "synthetic": { |
| "available": 996393, |
| "used": 996393 |
| }, |
| "openbook": { |
| "available": 919976, |
| "used": 452905 |
| }, |
| "headings": { |
| "available": 521532, |
| "used": 362324 |
| } |
| }, |
| "shares": { |
| "synthetic": 0.55, |
| "openbook": 0.25, |
| "headings": 0.2 |
| }, |
| "conversations": 1811622 |
| }, |
| "contamination": { |
| "bench_files": { |
| "arc_easy_test.json": { |
| "sha256": "76e55faa15bce1b6ef26212c0c67a242a39952088d3b9e1a4d6071ea33c15190", |
| "items": 2376 |
| }, |
| "cryptomc.json": { |
| "sha256": "c81b3367d654f1e207d2dcbb48f490ff5feb496e2d08a049d39fb8aa8de35f87", |
| "items": 150 |
| }, |
| "cryptoterms.json": { |
| "sha256": "918b257c12deac8818cebf271187eedd88f026f875908b5a2e6987434b991f90", |
| "items": 150 |
| }, |
| "ctslang.json": { |
| "sha256": "d967e9196888027a4055592381892fca6185ee3a41b70be6a7351fa06d0bfc70", |
| "items": 150 |
| }, |
| "projectid.json": { |
| "sha256": "6feec798ea7269d2931fba111c48b33af2030a885e4c589235072ca6e488f51e", |
| "items": 200 |
| }, |
| "projectid_xl.json": { |
| "sha256": "859b5fc0d6a197bdf38e09e980d4577bd32f321e4fb3aec674725b6e4ed9a709", |
| "items": 2593 |
| }, |
| "qa.json": { |
| "sha256": "d5f699bdec9dca94e355332380adf317ef497784a56d3d9277ca6c0f993af9c1", |
| "items": 100 |
| } |
| }, |
| "skipped_files": [ |
| "._arc_easy_test.json", |
| "._cryptomc.json", |
| "._cryptoterms.json", |
| "._ctslang.json", |
| "._fresh_pages.json", |
| "._fresh_pages_log.json", |
| "._grade-0.json", |
| "._grade-1.json", |
| "._grade-2.json", |
| "._grade-3.json", |
| "._grade-4.json", |
| "._grade-key.json", |
| "._grade2-a.json", |
| "._grade2-b.json", |
| "._grade2-c.json", |
| "._grade2-grades.json", |
| "._grade2-key.json", |
| "._grade3-a.json", |
| "._grade3-b.json", |
| "._grade3-c.json", |
| "._grade3-grades.json", |
| "._grade3-key.json", |
| "._models.json", |
| "._projectid.json", |
| "._projectid_bench_protocols.json", |
| "._projectid_defillama_raw.json", |
| "._projectid_meta.json", |
| "._projectid_results.json", |
| "._projectid_results_detail.json", |
| "._projectid_split.json", |
| "._projectid_xl.json", |
| "._projectid_xl_meta.json", |
| "._qa.json", |
| "._qa_answers.json", |
| "._report_q31.json", |
| "._results.json", |
| "._results_ctslang.json", |
| "._results_ctslang_detail.json", |
| "._results_detail.json", |
| "._results_domain.json", |
| "._results_domain_detail.json", |
| "._smoke.json", |
| "._web_snapshot.json", |
| "fresh_pages.json", |
| "fresh_pages_log.json", |
| "grade-0.json", |
| "grade-1.json", |
| "grade-2.json", |
| "grade-3.json", |
| "grade-4.json", |
| "grade-key.json", |
| "grade2-a.json", |
| "grade2-b.json", |
| "grade2-c.json", |
| "grade2-grades.json", |
| "grade2-key.json", |
| "grade3-a.json", |
| "grade3-b.json", |
| "grade3-c.json", |
| "grade3-grades.json", |
| "grade3-key.json", |
| "models.json", |
| "projectid_bench_protocols.json", |
| "projectid_defillama_raw.json", |
| "projectid_meta.json", |
| "projectid_results.json", |
| "projectid_results_detail.json", |
| "projectid_split.json", |
| "projectid_xl_meta.json", |
| "qa_answers.json", |
| "report_q31.json", |
| "results.json", |
| "results_ctslang.json", |
| "results_ctslang_detail.json", |
| "results_detail.json", |
| "results_domain.json", |
| "results_domain_detail.json", |
| "smoke.json", |
| "web_snapshot.json" |
| ], |
| "criteria": { |
| "ngram13": 13, |
| "ngram8": 8, |
| "ngram8_share": 0.7, |
| "question_min_words_in_text": 5, |
| "answer_within_words": 100 |
| }, |
| "dropped": { |
| "mix.train.jsonl": 459, |
| "synthetic.val.jsonl": 5, |
| "openbook.val.jsonl": 5, |
| "headings.val.jsonl": 0 |
| }, |
| "checked": { |
| "mix.train.jsonl": 1811622, |
| "synthetic.val.jsonl": 20337, |
| "openbook.val.jsonl": 18826, |
| "headings.val.jsonl": 12012 |
| }, |
| "by_criterion": { |
| "mix.train.jsonl": { |
| "question": 451, |
| "ngram8_70": 20, |
| "question_answer": 19, |
| "ngram13": 2 |
| }, |
| "synthetic.val.jsonl": { |
| "question": 5 |
| }, |
| "openbook.val.jsonl": { |
| "question": 5 |
| } |
| }, |
| "by_bench": { |
| "mix.train.jsonl": { |
| "qa.json": 402, |
| "cryptomc.json": 67, |
| "projectid_xl.json": 10 |
| }, |
| "synthetic.val.jsonl": { |
| "qa.json": 5 |
| }, |
| "openbook.val.jsonl": { |
| "qa.json": 5 |
| } |
| }, |
| "bench_fingerprint": "656b4d7aa50c7d85e43d3915aae7dce568d54be7842219a97db529bbc349c125" |
| }, |
| "synth": { |
| "coverage": { |
| "eligible": 188828, |
| "answered": 146686, |
| "with_pairs": 139073, |
| "answered_share": 0.7768, |
| "pairs_share": 0.7365 |
| }, |
| "gate": 0.75, |
| "teacher": "a general-purpose LLM" |
| } |
| } |
| } |
