Add files using upload-large-folder tool
Browse files- ablations/README.md +103 -0
- ablations/flame1e18_overlap.csv +16 -0
- ablations/olmoe_adapt_lmeval_impose.csv +7 -0
- ablations/olmoe_cal2.csv +13 -0
- ablations/olmoe_minflow_bpb_full.csv +12 -0
- ablations/serving_benchmarks.csv +59 -0
- ablations/specialization_summary.csv +8 -0
- ablations/stability_fakequant.csv +29 -0
- merged_ce_model/tokenizer.json +0 -0
- olmoe_adapt/logs/.gitignore +8 -0
- olmoe_adapt/logs/bake_bake_B.log +205 -0
- olmoe_adapt/logs/bake_bake_C.log +205 -0
- olmoe_adapt/logs/bake_bake_D.log +777 -0
- olmoe_adapt/logs/bake_bake_E.log +205 -0
- olmoe_adapt/logs/bake_bake_Er64.log +53 -0
- olmoe_adapt/logs/bake_bake_Er8.log +53 -0
- olmoe_adapt/logs/bake_bake_Fprime.log +633 -0
- olmoe_adapt/logs/bake_bake_G.log +470 -0
- olmoe_adapt/logs/bake_bake_H.log +205 -0
- olmoe_adapt/logs/bakeoff_run.log +11 -0
- olmoe_adapt/logs/c4val.log +32 -0
- olmoe_adapt/logs/cal0.log +11 -0
- olmoe_adapt/logs/cal2.log +59 -0
- olmoe_adapt/logs/ce_gate.log +46 -0
- olmoe_adapt/logs/corpus_build.log +9 -0
- olmoe_adapt/logs/corpus_smoke.log +9 -0
- olmoe_adapt/logs/dense_bracket.log +0 -0
- olmoe_adapt/logs/hf_delex.log +9 -0
- olmoe_adapt/logs/impose.log +8 -0
- olmoe_adapt/logs/live_greedy.log +5 -0
- olmoe_adapt/logs/lmeval_downstream.log +0 -0
- olmoe_adapt/logs/lmeval_impose.log +0 -0
- olmoe_adapt/logs/merge_ce.log +11 -0
- olmoe_adapt/logs/model_download.log +8 -0
- olmoe_adapt/logs/oseries_full.log +16 -0
- olmoe_adapt/logs/oseries_ladder.log +10 -0
- olmoe_adapt/logs/oseries_msweep.log +10 -0
- olmoe_adapt/logs/oseries_o2.log +28 -0
- olmoe_adapt/logs/oseries_o2full.log +17 -0
- olmoe_adapt/logs/oseries_paired.log +44 -0
- olmoe_adapt/logs/probe1.log +12 -0
- olmoe_adapt/logs/rsweep.log +6 -0
- olmoe_adapt/logs/run_h.out +3 -0
- olmoe_adapt/logs/run_post_c.out +6 -0
- olmoe_adapt/logs/sweep.log +7 -0
- olmoe_adapt/logs/sweep_sweep_lr1e-4.log +205 -0
- olmoe_adapt/logs/sweep_sweep_lr3e-4.log +205 -0
- olmoe_adapt/logs/sweep_sweep_lr3e-5.log +205 -0
- olmoe_adapt/oseries/meta.json +47 -0
- olmoe_adapt/oseries_ce/meta.json +12 -0
ablations/README.md
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Ablation Results — CSV index
|
| 2 |
+
|
| 3 |
+
Prose narrative of the Phase-0 findings (baselines, dense floor, acceptance, temporal, fine-graining, 1e18): [FINDINGS.md](FINDINGS.md).
|
| 4 |
+
|
| 5 |
+
Consolidated result tables from the temporal-MoE ablation program, gathered from the `a6000` / `h100` orchestration branches (comms + code excluded). One row per file: a one-line description and the branch commit(s) whose result-distillation produced it.
|
| 6 |
+
|
| 7 |
+
\* Quality numbers: end-of-training **test-set** evals (`ce_test_final`) are the canonical series (protocol-consistent across all runs); earlier write-ups sometimes quoted during-training val evals ~0.004 CE higher — both series preserved in `phase0_isoflop_points.csv`.
|
| 8 |
+
|
| 9 |
+
**Provenance** — `a6000@<sha>` / `h100@<sha>` is the commit on that branch that wrote the source data (read that commit's message for the exact run); files split across both boxes cite both. Pre-consolidation and pre-combine versions of every file live in this branch's earlier commits (`7e89690` consolidate → `2934c9d` combine → `abc63c3` fold/rename → `56758d4` seed-merge).
|
| 10 |
+
|
| 11 |
+
| CSV | What it is | Generated by |
|
| 12 |
+
|---|---|---|
|
| 13 |
+
| `alignment_cells.csv` | ΔBPB-vs-dense-baseline and A3 routing-overlap per ablation cell across paradigms. | a6000@5e3614ac |
|
| 14 |
+
| `alignment_frontier.csv` | Test-BPB vs A3-cacheability frontier per temporal cell (the diversity↔alignment tradeoff). | h100@d9de0a30 |
|
| 15 |
+
| `anomaly_pred.csv` | Predictability of swap 'anomalies' (positive frac, logistic AUC, ridge R²) at several prefetch horizons. | a6000@09530444 |
|
| 16 |
+
| `b1_eval_bpb.csv` | B1 eval: test-BPB vs baseline (Δ) across τ_ρ / EMA-β router configs. | a6000@85c6f4e7 |
|
| 17 |
+
| `block_replay.csv` | Block-wise (chunked) routing replay — retained mass, coverage, union, turnover per block boundary. | a6000@97de0d04 |
|
| 18 |
+
| `downstream_evals.csv` | Downstream lm-eval task accuracies per budget/checkpoint/model/task. | a6000@10a8bf6a |
|
| 19 |
+
| `e1_swap_rate_by_layer.csv` | Mean rolling-residency swap rate + p95 swap-burst length per layer. | a6000@cd588fd6, h100@cd588fd6 |
|
| 20 |
+
| `e1_victim_cache_hitrate.csv` | Expert-reload hit-rate vs victim-cache size. | a6000@cd588fd6, h100@cd588fd6 |
|
| 21 |
+
| `e2_streamed_diversity.csv` | Streamed expert diversity — union / effective-experts per model. | a6000@cd588fd6, h100@cd588fd6 |
|
| 22 |
+
| `e3_mass_vs_set_consistency.csv` | Mass- vs set-consistency of routing across positions. | a6000@cd588fd6, h100@cd588fd6 |
|
| 23 |
+
| `e4_swap_vs_retained_mass.csv` | Swap rate vs retained routing mass across τ thresholds. | a6000@cd588fd6, h100@cd588fd6 |
|
| 24 |
+
| `e5_eviction_policy_headroom.csv` | Set/mass coverage across eviction policies (headroom). | a6000@cd588fd6, h100@cd588fd6 |
|
| 25 |
+
| `e6_per_layer_ranking.csv` | Per-layer hit-rate / swap-rate / expert-lifetime ranking. | a6000@cd588fd6, h100@cd588fd6 |
|
| 26 |
+
| `e7_demand_smoothing.csv` | EMA-β demand smoothing — swap rate vs set/mass coverage. | a6000@cd588fd6, h100@cd588fd6 |
|
| 27 |
+
| `e8_document_boundary.csv` | Routing churn at document (EOD) boundaries. | a6000@cd588fd6, h100@cd588fd6 |
|
| 28 |
+
| `expert_selection_per_token.csv` | Per-token expert selections (full-MoE vs temporal) for the 8/15/38M models (model in active_params_M col). | a6000@53aa24ba, h100@53aa24ba |
|
| 29 |
+
| `flame192_leftflank_1e18.csv` | 1e18 isoFLOP LEFT-flank (hidden-192/~26M) 5-cell final CE panel: dense + {MoE,temporal}×{coarse,fine}. | a6000@ee73c64d, h100@5ed5a240 |
|
| 30 |
+
| `flame38m_overnight_seeds.csv` | Overnight seed extension (h100 split): fine-pair seed-2 (temporal+MoE), local dense floor, coarse-MoE seed-3 — end-of-run full-val + test CE/BPB. | h100@5a30342a |
|
| 31 |
+
| `flame38m_overnight_seeds_a6000.csv` | Overnight coarse-temporal seed-3 on the a6000 split (same schema). | a6000@e0a98204 |
|
| 32 |
+
| `flame38m_1e18_cells.csv` | 1e18 38M-middle 5-cell panel finals (dense/MoE/temporal × coarse/fine); coarse per-seed curves in t18_1e18_curves.csv. | temporal-moe-impl@66f786b7, h100@60b0e351/e53596d6 |
|
| 33 |
+
| `flame512_1e18_rightflank.csv` | 1e18 isoFLOP RIGHT-flank (hidden-512/98M) 5-cell final CE panel. | h100@d4827a1e |
|
| 34 |
+
| `flame512_1e18_rightflank_curves.csv` | Per-iteration training curves for the 1e18 right-flank cells. | h100@d4827a1e |
|
| 35 |
+
| `karen_center_replay.csv` | 'Karen' (aux-free momentum) center-of-mass replay — A3/eff/union/swap per seed. | a6000@91855c19 |
|
| 36 |
+
| `karen_promotion_s2_1e17.csv` | Karen s2@1e17 promotion checkpoint — val/test BPB + residency stats. | h100@ee5b26db |
|
| 37 |
+
| `learned_locality_vs_scale.csv` | Temporal vs full-MoE routing overlap ('learned locality') vs model scale. | a6000@cd588fd6, h100@cd588fd6 |
|
| 38 |
+
| `mechinterp_floors.csv` | Null/floor AUC baselines for the locus features (incl. the softmax-routing baseline model). | a6000@3ce9b63e/5125733c |
|
| 39 |
+
| `mechinterp_freerider.csv` | 'Free-rider' expert stats — distinct experts per sequence, tokens per expert. | a6000@1f416ef5 |
|
| 40 |
+
| `mechinterp_lens.csv` | Logit-lens effective-vocab & semantic dispersion per (layer,expert), static+weighted variants, incl. softmax baseline. | a6000@3ce9b63e/6a4d7bf0 |
|
| 41 |
+
| `mechinterp_locus.csv` | Token-AUC vs context-AUC 'locus' of expert specialization per (layer,expert); variant col = base/kfull/kwin k-config. | a6000@29c500ab/3ce9b63e/41c8d18a/804e10b6/dd4e900e |
|
| 42 |
+
| `mechinterp_logitlens.csv` | Unweighted logit-lens effective-vocab + semantic dispersion per (layer,expert). | a6000@53c381bd |
|
| 43 |
+
| `mechinterp_structural.csv` | Structural routing stats (PR, generalist frac, router entropy, eff-rank, centroid dist) per model. | a6000@3ce9b63e |
|
| 44 |
+
| `mechinterp_structural_1e19.csv` | Structural stats replicated at 1e19 (3 cells, same schema + budget col). | h100@8aef6c4c |
|
| 45 |
+
| `mechinterp_locus_1e19.csv` | Token-vs-context locus probes at 1e19 (per layer/expert AUCs; iid null 0.500-0.502, shift null 0.502-0.505 = residency autocorrelation). | h100@696da366 |
|
| 46 |
+
| `mechinterp_demand_1e19.csv` | Causal history-only demand-prediction AUC at 1e19 (0.604 moe / 0.926 t-coarse / 0.979 t-fine). | h100@fea7d5fa |
|
| 47 |
+
| `mechinterp_lens_1e19.csv` | Data-weighted logit lens at 1e19 (per layer/expert eff-vocab; static no-signal ref ~30k of 50k at this scale). | h100@fc47dd0c |
|
| 48 |
+
| `unmask_eval_1e19.csv` | Constraint swap at 1e19, eval-only: unmask temporal (+0.21/+0.20 BPB) and impose on moe_coarse (+0.43 BPB). | h100@54c7d767 |
|
| 49 |
+
| `momr_replay.csv` | Log-ratio momentum ('momr') replay — set/mass coverage, swap rate, union across β. | h100@256ae85c |
|
| 50 |
+
| `oracle_a3.csv` | Oracle A3-cacheability (with induced swap rate) vs prefetch horizon per k. | a6000@74d1d62a |
|
| 51 |
+
| `oracle_horizon_map.csv` | Oracle A3 gain (Δ over baseline) vs prefetch horizon per k. | a6000@37ef31f8 |
|
| 52 |
+
| `p2_fair_1swap.csv` | P2 systems: fair single-swap B=1 decode timing breakdown (router/GEMM/DMA/overlap). | a6000@f6e740aa |
|
| 53 |
+
| `p2_swap_bench.csv` | P2 swap microbench — compute/swap/DMA timings, %DMA hidden, VRAM per resident-set R. | a6000@e4c1a1ff |
|
| 54 |
+
| `p2b_swaprate_emul.csv` | Emulated swap-rate vs decode tok/s. | a6000@4ffd0c11 |
|
| 55 |
+
| `p3_integrated.csv` | P3 integrated decode — stock vs overlap vs serial tok/s and exposed ms. | a6000@4bdea753 |
|
| 56 |
+
| `p3b_unified.csv` | P3b unified-kernel decode variants — tok/s, VRAM reduction, correctness. | a6000@37a74bf9 |
|
| 57 |
+
| `p3c_overlap.csv` | P3c copy-overlap decode variants — tok/s, VRAM, correctness. | a6000@edc6bb7c |
|
| 58 |
+
| `p3d_fair.csv` | P3d fair-comparison decode variants — tok/s, VRAM, correctness. | a6000@d6f9b879 |
|
| 59 |
+
| `perf_baseline.csv` | Training perf baseline — s/iter, TFLOPs/GPU, peak GB per shape/mode. | a6000@eba7d3f4 |
|
| 60 |
+
| `phase0_isoflop_points.csv` | Per-point final CE/BPB for the 1e16/1e17 isoFLOP program (dense/MoE/temporal × coarse/fine × shapes, incl. s=2 and eviction variants); records the end-of-training test-set eval AND the published during-training val value where they differ. | temporal-moe-impl@66f786b7 (log.md), main (plot_g3_curves.py) |
|
| 61 |
+
| `phase0_lr_tuning.csv` | LR-selection + cross-size stability runs behind the locked peak-LR 3e-3. | temporal-moe-impl@66f786b7 (log.md) |
|
| 62 |
+
| `rolling_coverage_lifetime_vs_K.csv` | Rolling-residency hit-rate & expert lifetime vs resident-cache size K/k. | a6000@cd588fd6, h100@cd588fd6 |
|
| 63 |
+
| `rsweep.csv` | Resident-set-size (R) sweep — BPB + A3/coverage/eff-experts/swap-rate per cell. | a6000@b65454d8 |
|
| 64 |
+
| `seed_replicates.csv` | Seed-replicate CE/BPB across temporal & full-MoE variants (a6000+h100 combined; box col marks source). | a6000@89028580, h100@0562e4a4 |
|
| 65 |
+
| `specialization_m3.csv` | Expert-specialization M3 — centroid distances & pairwise cosine per cell. | a6000@0ec187a2 |
|
| 66 |
+
| `specialization_probe.csv` | Per-(layer,expert) specialization probe: residency share, selectivity, gate-when-selected, centroid dist. | a6000@0ec187a2 |
|
| 67 |
+
| `specialization_summary.csv` | Per-cell specialization summary (PR percentiles, generalist frac, eff-rank, gate stats). | a6000@0ec187a2 |
|
| 68 |
+
| `serving_benchmarks.csv` | llama.cpp real-engine serving results: decode a/b/c/d kernel setups + prefill A/D/C decomposition + matched-ubatch 16k sweep + vanilla-offload floor curve (TEMPORAL_SWAP_N pinned misses/layer on the depth-re-pinned comparable model). Toolchain now fully committed (gen_random_qwen3moe.py, build_models.sh, run_serving_bench.sh, refreshed systems_bench.patch). | fork@6094183 (a6000, local, orig rows), a6000@50af4263 (floor) |
|
| 69 |
+
| `stability_weights.csv` | Per-expert weight statics (Frobenius/maxabs/excess-kurtosis per matrix; routed/shared/dense-FFN/router-row), 1e18 38M + 1e19 cells. | h100@9e07cf38 |
|
| 70 |
+
| `stability_residency.csv` | Per-expert resident/selected fractions on the fixed eval batch (temporal cells), joins 1:1 to stability_weights router rows. | h100@5a7908d1 |
|
| 71 |
+
| `stability_activations.csv` | Per-expert activation stats (output norms, FFN-intermediate max/kurtosis, router logits resident/non-resident, gates), 1e18 + 1e19. | h100@2634e09b (38M), h100@ec9007c8 (1e19) |
|
| 72 |
+
| `stability_trunk.csv` | Trunk stats per layer/head (max attention logit, block out/in L2 ratios, residual-stream dims), 1e18 + 1e19. | h100@2634e09b (38M), h100@ec9007c8 (1e19) |
|
| 73 |
+
| `stability_gradnorms.csv` | Grad-norm series (every 10 iters) from surviving train.logs, 38M + 1e19 runs, spike census. | h100@efc43768 |
|
| 74 |
+
| `stability_fakequant.csv` | Fake-quant (per-group-128 RTN, routed experts only) test CE/BPB at 16/8/4/3 bits, 1e18 + 1e19; divisor 2.9780. | h100@8f5064e6 (38M), h100@ec9007c8 (1e19) |
|
| 75 |
+
| `t18_1e18_curves.csv` | 1e18 (38M) isoFLOP training curves — coarse full-MoE + coarse temporal cells. | h100@60b0e351/e53596d6 |
|
| 76 |
+
| `t19_1e19_curves.csv` | 1e19 isoFLOP training curves for the 4 t19 cells (dense / moe_coarse / coarse_temporal / fine_temporal). | h100@03a8e488/6a69b7f7/d2b7cd7e/de80543f |
|
| 77 |
+
| `t19_lmeval_stderr.csv` | t19 lm-eval rerun with per-task stderr (pinned-venv harness; accs reproduce t19_lmeval within noise). | h100@9bec50ac |
|
| 78 |
+
| `t19_lmeval.csv` | t19 downstream lm-eval accuracies (dense / moe_coarse / temporal_coarse). | h100@a42340f1 |
|
| 79 |
+
| `t19_lmeval_finegrain.csv` | t19 fine-grained temporal downstream lm-eval accuracies. | h100@3b8d9da8 |
|
| 80 |
+
| `t19_moe_coarse_probe.csv` | t19 coarse full-MoE routing probe — load balance, gate PR, self-consistency. | h100@5df8095d |
|
| 81 |
+
| `t19_temporal_probe.csv` | t19 temporal residency probe (coarse 6/64 + fine 18/192; test-CE, h100 train.log-verified) — A3 mass/set coverage, union, eff-experts. Combines t19_probe_triple + t19_fine_probe_triple. | h100@6a69b7f7 (coarse), h100@d2b7cd7e (fine) |
|
| 82 |
+
| `temporal_router_momentum_sweep.csv` | Temporal-router momentum (γ_m×β_m) sweep — swap rate & set/mass coverage (audit-scored). | a6000@6f9e02bc |
|
| 83 |
+
| `temporal_router_momentum_sweep_selfscored.csv` | Same momentum sweep, self-scored coverage. | a6000@6f9e02bc |
|
| 84 |
+
| `temporal_router_tau_ema_sweep.csv` | Temporal-router τ (min-logit eviction threshold) × EMA-β sweep — swap rate & coverage (audit-scored). | a6000@6f9e02bc |
|
| 85 |
+
| `temporal_router_tau_ema_sweep_selfscored.csv` | Same τ×EMA sweep, self-scored coverage. | a6000@6f9e02bc |
|
| 86 |
+
| `unmask_eval.csv` | Unmask / cross-regime eval — native vs cross-regime metric per scale & paradigm. | a6000@d3fbc238 |
|
| 87 |
+
| `olmoe_adapt_impose.csv` | OLMoE-1B-7B-0125 Stage-0 zero-shot impose (R=k=8 of 64) + R-sweep coherence + audited-slice restatement (base 0.6727 / impose 2.7507 BPB, D=3.1089 byte-derived). | h100@29556cc6, restated h100@dbc9461a |
|
| 88 |
+
| `olmoe_adapt_corpus_audit.md` | Adaptation program provenance: no-unconsumed-tail finding, parent-pool corpus recipe (1B tok, dedup counts), ReMoE/MELINOE budget priors, throughput root cause (HF per-expert python dispatch loop). | h100@92ccb2f2 + updates |
|
| 89 |
+
| `olmoe_adapt_sweep.csv` | Stage-2 router-only LR sweep under R=8 (3e-5/1e-4/3e-4 × 0.25B, full 50M eval series; winner 3e-4, 70.7% recovery, late crossover). | h100@358818ab |
|
| 90 |
+
| `olmoe_adapt_bakeoff.csv` | Stage-2b escalation bake-off, all 11 arms + curves + eval-noise sigma (A/B/D routing 70%; C norms 91.4% == E LoRA; CE winner 93.2%; F' full-FT 93.4% = constraint price; G distill hurts; Er8/Er64 rank saturation; H zone-anneal null). | h100 per-arm commits 0065–0101 |
|
| 91 |
+
| `olmoe_adapt_lmeval_impose.csv` | Stage-0 3-task downstream impose (arc_easy/piqa/obqa collapse under cold R=8 mask). | h100@dbc9461a |
|
| 92 |
+
| `olmoe_adapt_downstream.csv` | Stage-3 downstream 10-task lm-eval, 5 cells: base-free / impose-R8 / CE-adapted-R8 / OLMo-1B-0724 / OLMo-7B-0724 (74.4% accuracy recovery; CE-adapt 0.589 vs dense-1B 0.601; base 0.682 vs dense-7B 0.677). | h100@15200735 + dense bracket h100@01170b04 |
|
| 93 |
+
| `olmoe_adapt_forensics.csv` | Stage-3 de-lex probe battery, CE-adapt vs base-impose (locus flips token->context: ctx-tok -0.004 -> +0.049; generalists -> 0; demand AUC ~0.96 both = non-discriminator). | h100@0124 result commit |
|
| 94 |
+
| `olmoe_adapt_telemetry.csv` | Stage-3 routing telemetry of CE-adapted @R=8 (swap 0.9997 ~= ceiling, dwell 13.4 tok, usage entropy 0.9988). | h100@0125 result commit |
|
| 95 |
+
| `olmoe_scratch_ladder.csv` | Baseline-equivalence: c4val harness validation (ours 2.4730 vs OLMoE-0924 wandb 2.4807) + from-scratch 0924 checkpoint ladder on our slice (20B..524B) -> crossings ~28B (C) / ~46B (CE) tokens, agrees with wandb train-curve 25.7/42.8B. | h100@61db892b + ladder commits |
|
| 96 |
+
| `olmoe_minflow_calib.csv` | O-series captured-mass ladder per layer (windowed + full-4096: static 0.213 / greedy 0.277 / MinFlow-m1 0.327 / top-8 bound 0.405; flat-router coverage finding top-24=0.684). | h100@2aa82d49 + full h100@0137 |
|
| 97 |
+
| `olmoe_minflow_msweep.csv` | O-1 swap-budget m-sweep {1,2,4} (m>1 diagnostic-only, non-deployable: m× fetch bandwidth). | h100@f54f3fb3 |
|
| 98 |
+
| `olmoe_minflow_bpb.csv` | O-2/O-close reward->BPB transfer + the three decisive pairs: replay headroom 0.0231±0.0076; free-vs-drift 0.4034±0.0178; CE winner-gate scan-oracle -0.0113±0.0021 (live scan BEATS forced free-logit oracle -> offline scheduling dead). | h100@0110–0138 commits, final 0130a03a |
|
| 99 |
+
| `olmoe_minflow_capture_meta.json` | O-0 reward-field capture metadata (rank-mass histogram, coverage). | h100@2aa82d49 |
|
| 100 |
+
| `olmoe_cal0.csv` | Cal-0 closed-form moment-matching probe: REJECTED (raw 1.1% / clipped 31.5% recovery; not benign under free routing; cos~0.01 vs learned gains). | h100@e6795ec0 |
|
| 101 |
+
| `olmoe_cal2.csv` | Cal-2 calibrated-init training screen: NULL, undone-to-single-basin (final 0.9262 vs C@50M 0.8791; cos-to-init 1.0->0.17, cos-to-C -0.03->0.44; init axis closed). | h100@9f0ca8ae |
|
| 102 |
+
| `olmoe_adapt_RESULTS.md` | Full adaptation-study writeup (11-arm table, four-part mechanism story, recommendations). | h100@0101 closeout |
|
| 103 |
+
| `adapt_ckpts/*.safetensors` | Router-only checkpoints (~4MB each): LR-sweep arms, bake-off router arms (B/C/D), Cal-2. LoRA-bearing ckpts (474MB) remain pod-local by policy. | h100 arm commits |
|
ablations/flame1e18_overlap.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Overlap-arch 1e18-g3 promotion cells (orch 0150/0160). V1 early-router only (V2 failed the 1e17 gate).
|
| 2 |
+
# Exact flame38m 1e18 config: GRAIN=3 (192 exp/top-18), H256/L9 (moe-layer-freq [0]*1+[1]*8 = 8 MoE layers),
|
| 3 |
+
# gb1024/mb32/2121it/seed1234/WSD, pythia-50k. BPB = test_CE / 2.9780, LOWER better.
|
| 4 |
+
# Standard-arch 1e18-g3 comparators (seed1234): temporal 1.3354 (CE 3.976762), moe 1.3461 (CE 4.008667).
|
| 5 |
+
# FLAG: the 1e17-s2 (L=6, 5 MoE layers) screen predicted early-router tax +0.0036 (temporal); the 1e18-flame38m
|
| 6 |
+
# (L=9, 8 MoE layers) MEASUREMENT is far larger -> the screen did NOT transfer across depth. moe cell pending.
|
| 7 |
+
variant,cell,method,test_ce_nats,test_bpb,std_bpb,delta,screen_predicted
|
| 8 |
+
v1_early_router,flame38m_g3_temporal_ovlEarly,temporal,4.108881,1.3798,1.3354,+0.0444,+0.0036
|
| 9 |
+
v1_early_router,flame38m_g3_moe_ovlEarly,moe,4.456250,1.4964,1.3461,+0.1503,+0.0004
|
| 10 |
+
# VERDICT: V1 early-router FAILS at 1e18 depth on BOTH legs, far beyond the 1e17 screen's prediction.
|
| 11 |
+
# temporal +0.0444 (screen said +0.0036, 12x) moe +0.1503 (screen said +0.0004, 375x)
|
| 12 |
+
# The MoE leg is now WORSE THAN THE DENSE FLOOR (1.4964 vs 1.3911). Clean converged runs (no nan/skips).
|
| 13 |
+
# => this is a DEPTH effect, not temporal-specific: the 1e17 screen was s2 (L=6, 5 MoE layers), the 1e18
|
| 14 |
+
# cell is flame38m (L=9, 8 MoE layers); pre-attention routing degrades compound per MoE layer, and the
|
| 15 |
+
# free-MoE leg (which re-routes every layer with no residency smoothing) is hurt worst.
|
| 16 |
+
# => the screen->promotion protocol must screen AT THE DEPLOYMENT DEPTH. V1 should not have promoted.
|
ablations/olmoe_adapt_lmeval_impose.csv
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
task,metric,base,impose_mask_R8,delta
|
| 2 |
+
arc_easy,acc,0.7715,0.2689,-0.5025
|
| 3 |
+
arc_easy,acc_norm,0.7677,0.2593,-0.5084
|
| 4 |
+
piqa,acc,0.7856,0.5299,-0.2557
|
| 5 |
+
piqa,acc_norm,0.796,0.5136,-0.2824
|
| 6 |
+
openbookqa,acc,0.328,0.142,-0.186
|
| 7 |
+
openbookqa,acc_norm,0.454,0.278,-0.176
|
ablations/olmoe_cal2.csv
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Cal-2 (orch 0132): INIT-AXIS screen. Arm-C surface (router+65 norm gains both trainable), lr 3e-04, 50M tok,
|
| 2 |
+
# norm gains INITIALIZED from clipped closed-form g' (Cal-0 2.0965-BPB start); router from base. Eval @R=8, D=3.1089.
|
| 3 |
+
# Bar: beat arm C like-for-like C@50M=0.8791 by >2sigma(0.012). Ladder: base 0.6727 / arm C 0.8505 / impose 2.7507.
|
| 4 |
+
# cos_prime=cos(log(g_t/g_base),log(g'/g_base)) [retain g'?]; cos_C=cos(log(g_t/g_base),log(g_C/g_base)) [undo toward arm C?]
|
| 5 |
+
tok_M,BPB,recovery,swap_rate,usage_entropy,cos_prime,cos_C
|
| 6 |
+
0,2.0965,0.3148,0.9997,0.9941,1.0000,-0.0342
|
| 7 |
+
10,1.0241,0.8309,0.9997,0.9963,0.2822,0.3087
|
| 8 |
+
20,0.9730,0.8555,0.9997,0.9964,0.2337,0.3570
|
| 9 |
+
30,0.9502,0.8665,0.9997,0.9968,0.2040,0.3839
|
| 10 |
+
40,0.9362,0.8732,0.9998,0.9971,0.1842,0.4152
|
| 11 |
+
50,0.9262,0.8780,0.9998,0.9975,0.1702,0.4397
|
| 12 |
+
# FINAL @ 50M: BPB=0.9262 rec=0.878 cos_prime=0.1702 cos_C=0.4397 (UNDOES g' toward arm C (g_C))
|
| 13 |
+
# VERDICT: NULL: g'-init WORSE by 0.0471 -> init axis CLOSED (arm C's from-base start is not improved)
|
ablations/olmoe_minflow_bpb_full.csv
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# O-series closing: UNWINDOWED (full-4096, single realistic cold-fill) BPB pair, C=32 exact solver, 8 packs, D=3.1089.
|
| 2 |
+
# Removes the windowing asterisk. NOTE: 8 packs (noisier than the 256-pack evals); greedy vs MinFlow is PAIRED (same packs).
|
| 3 |
+
# Compare to the WINDOWED pair (olmoe_minflow_bpb.csv, 16 cold-fills/seq): greedy 2.2953 / MinFlow_m1 2.2881, gap -0.0072.
|
| 4 |
+
schedule,BPB,delta_vs_greedy
|
| 5 |
+
free_top8,0.6850,
|
| 6 |
+
greedy,2.2872,+0.0000
|
| 7 |
+
minflow_m1,2.2522,-0.0350
|
| 8 |
+
# FINDING: at FULL sequence (1 cold-fill), MinFlow m=1 beats greedy by 0.0350 BPB — ~5x the windowed 0.0072.
|
| 9 |
+
# The windowing (frequent free cold-fills) reset both policies and UNDERSTATED the scheduling headroom.
|
| 10 |
+
# So the paper claim needs care: greedy is NOT within ~1sigma of the hindsight-optimal <=1-swap schedule at full
|
| 11 |
+
# sequence; the oracle is ~0.035 BPB better. Still tiny vs the weight-side recovery (CE closes 2.078->0.14),
|
| 12 |
+
# so scheduling is a minor lever (~1.7% of the impose gap), but not the ~0 the windowed check implied.
|
ablations/serving_benchmarks.csv
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Serving benchmarks for temporal-MoE (llama.cpp CUDA fork, RTX A6000, Q4_K_M random-weight Qwen3-MoE).
|
| 2 |
+
# Fork commits (LOCAL on a6000, NOT pushed): 2cb4175 v1, 13cc828 v2/16k, fb0e979 configD, 5c2f7b2 skipseed, 1896afb wavemmid(NO-GO), 8fa7937 ub-counter, 6094183 matched-ub.
|
| 3 |
+
# tiers: ceiling=all experts VRAM-resident (stock mul_mat_id); deploy=streamed (-ncmoe 48 TEMPORAL_UNIFIED=1, R=k slots + CPU pool); resident_control=expert-major kernel with all experts resident (no upload, config D).
|
| 4 |
+
# prefill_ms lower better; decode_tok_s higher better; peak_vram_mib = nvidia-smi global max. 4 trials, warmup discarded, mean of last 3. correctness: streamed vs ceiling PPL delta 0.02-0.04% (FP-reorder, not a logic error).
|
| 5 |
+
phase,model,tier,setup,ubatch,context,prefill_ms,decode_tok_s,peak_vram_mib,note
|
| 6 |
+
decode,fine,ceiling,a_all_resident,,1024,,200,7672,bit-identical baseline
|
| 7 |
+
decode,fine,deploy,b_our_kernel_no_swap,,1024,,176,7672,p=0 code overhead only
|
| 8 |
+
decode,fine,deploy,c_active_only_router_after_attn_DEPLOY,,1024,,165,1502,PPL 4dp match; 5.1x total / 10.6x expert-tier VRAM cut
|
| 9 |
+
decode,fine,deploy,d_router_early,,1024,,173,1502,router-before-attn overlap variant
|
| 10 |
+
decode,coarse,ceiling,a_all_resident,,1024,,252,,
|
| 11 |
+
decode,coarse,deploy,b_our_kernel_no_swap,,1024,,217,,
|
| 12 |
+
decode,coarse,deploy,c_active_only_DEPLOY,,1024,,128,,2.5MiB swaps dominate at coarse granularity
|
| 13 |
+
decode,coarse,deploy,d_router_early,,1024,,143,,
|
| 14 |
+
prefill,fine,ceiling,A_mul_mat_id_resident,1024,1024,151,,8174,config-D control; kernel baseline
|
| 15 |
+
prefill,fine,resident_control,D_expert_major_resident,1024,1024,366,,8262,config-D control; no upload
|
| 16 |
+
prefill,fine,deploy,C_expert_major_streamed,1024,1024,358,,2172,C/D~1.00x streaming free; kernel-bound at 1k
|
| 17 |
+
prefill,fine,ceiling,A_mul_mat_id_resident,4096,4096,665,,10170,
|
| 18 |
+
prefill,fine,resident_control,D_expert_major_resident,4096,4096,978,,10500,
|
| 19 |
+
prefill,fine,deploy,C_expert_major_streamed,4096,4096,1029,,4410,C/D 1.05x
|
| 20 |
+
prefill,fine,ceiling,A_mul_mat_id_resident,16384,16384,4715,,18162,single-ubatch
|
| 21 |
+
prefill,fine,resident_control,D_expert_major_resident,16384,16384,4219,,19448,
|
| 22 |
+
prefill,fine,deploy,C_expert_major_streamed,16384,16384,4297,,13358,C/D 1.02x
|
| 23 |
+
prefill,fine,ceiling,A_matched_ub,16384,16384,4723,,18162,matched-ub sweep @16k
|
| 24 |
+
prefill,fine,deploy,C_matched_ub,16384,16384,4247,,13358,matched-ub sweep @16k
|
| 25 |
+
prefill,fine,ceiling,A_matched_ub,4096,16384,2922,,10746,ceiling optimum ub
|
| 26 |
+
prefill,fine,deploy,C_matched_ub,4096,16384,4132,,4986,deploy optimum ub; C/A 1.41x
|
| 27 |
+
prefill,fine,ceiling,A_matched_ub,2048,16384,2942,,9512,CHOSEN operating point (base near-optimal)
|
| 28 |
+
prefill,fine,deploy,C_matched_ub,2048,16384,4499,,3590,CHOSEN operating point; C/A 1.53x A/C-VRAM 2.65x
|
| 29 |
+
prefill,fine,ceiling,A_matched_ub,1024,16384,2975,,8894,ceiling VRAM floor ~KV+all-experts
|
| 30 |
+
prefill,fine,deploy,C_matched_ub,1024,16384,5527,,3159,deploy VRAM floor ~KV+ring
|
| 31 |
+
# --- context-length sweep (fine model, -ub 2048 both tiers, 4 trials discard-first) ---
|
| 32 |
+
# CORRECTED: deploy prefill is now TEMPORAL residency-masked (matches decode+training), not full-MoE. fork f43b81a.
|
| 33 |
+
# schema differs (context sweep): phase=context_sweep; prefill_ms + prefill/decode tok/s + vram per (context,tier). A=ceiling C=deploy.
|
| 34 |
+
context_sweep,fine,ceiling,ub2048,2048,1024,152,203.4,8174,pp 6756 tok/s
|
| 35 |
+
context_sweep,fine,deploy,ub2048,2048,1024,352,161.7,2182,pp 2906 tok/s (temporal residency-masked)
|
| 36 |
+
context_sweep,fine,ceiling,ub2048,2048,2048,309,203.0,8840,pp 6625 tok/s
|
| 37 |
+
context_sweep,fine,deploy,ub2048,2048,2048,574,159.6,2926,pp 3570 tok/s (temporal residency-masked)
|
| 38 |
+
context_sweep,fine,ceiling,ub2048,2048,4096,624,201.9,8936,pp 6568 tok/s
|
| 39 |
+
context_sweep,fine,deploy,ub2048,2048,4096,1103,159.2,3030,pp 3715 tok/s (temporal residency-masked)
|
| 40 |
+
context_sweep,fine,ceiling,ub2048,2048,8192,1304,201.1,9128,pp 6284 tok/s
|
| 41 |
+
context_sweep,fine,deploy,ub2048,2048,8192,2264,159.5,3224,pp 3618 tok/s (temporal residency-masked)
|
| 42 |
+
context_sweep,fine,ceiling,ub2048,2048,16384,2922,202.4,9512,pp 5608 tok/s
|
| 43 |
+
context_sweep,fine,deploy,ub2048,2048,16384,4592,159.3,3614,pp 3568 tok/s (temporal residency-masked)
|
| 44 |
+
# --- vanilla-offload FLOOR curve (a6000 2026-07-19): default MoE = only R=k expert slots VRAM-resident + free top-k every token + fetch-on-miss. ---
|
| 45 |
+
# Comparable model: depth re-pinned to the original's KV-cache slope (L=45 kv_heads=4 head_dim=128 H=1024) -> fine ceiling 200.8=orig 200; coarse 251.0=orig 252; VRAM 7152/7128~=7672; KV ~90 KiB/tok=orig 89. Regenerated by llamacpp-bench/gen_random_qwen3moe.py; systems_bench.patch adds TEMPORAL_SWAP_N (pin N cold swap-copies/layer; source experts cycled to defeat L2 cache). setup=multi_swap_active_only_n<N>: R=k active-only slots; N cold misses/layer; evict lowest-index non-selected slot (NOT LRU; eviction policy is byte-cost-neutral). N=1==deploy rate (1 swap/layer); N=0==our-kernel no-swap. Low-N (compute/launch-bound) rows carry GPU-boost variance; TARGET floor rows (high N) are bandwidth-bound + stable.
|
| 46 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n0,,1024,,186.6,1382,0 swaps/layer our-kernel resident-set baseline
|
| 47 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n1,,1024,,121.3,1382,1 swap/layer == deploy rate (boost-variance; orig deploy row=165)
|
| 48 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n2,,1024,,135.5,1382,2 swaps/layer
|
| 49 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n4,,1024,,99.5,1382,4 swaps/layer
|
| 50 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n8,,1024,,65.5,1382,8 swaps/layer
|
| 51 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n14,,1024,,43.1,1382,TARGET 14 swaps/layer=round(0.8k=14.4) real-MoE ~20pct consecutive overlap; 0.22x ceiling 0.26x deploy
|
| 52 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n16,,1024,,38.7,1382,TARGET 16 swaps/layer=round(k*(1-k/E)=16.3) uniform-random overlap; 0.19x ceiling(200) 0.23x deploy(165)
|
| 53 |
+
decode,fine,vanilla_offload,multi_swap_active_only_n18,,1024,,35.1,1382,18 swaps/layer all-miss (k=18)
|
| 54 |
+
decode,coarse,vanilla_offload,multi_swap_active_only_n0,,1024,,232.4,1356,0 swaps/layer baseline
|
| 55 |
+
decode,coarse,vanilla_offload,multi_swap_active_only_n1,,1024,,127.5,1356,1 swap/layer == deploy rate; reproduces orig deploy row=128
|
| 56 |
+
decode,coarse,vanilla_offload,multi_swap_active_only_n2,,1024,,82.9,1356,2 swaps/layer
|
| 57 |
+
decode,coarse,vanilla_offload,multi_swap_active_only_n4,,1024,,50.6,1356,4 swaps/layer
|
| 58 |
+
decode,coarse,vanilla_offload,multi_swap_active_only_n5,,1024,,42.0,1356,TARGET 5 swaps/layer; k*(1-k/E)=5.44 and 0.8k=4.8 both round to 5 here; 0.17x ceiling(252) 0.33x deploy(128)
|
| 59 |
+
decode,coarse,vanilla_offload,multi_swap_active_only_n6,,1024,,36.1,1356,6 swaps/layer all-miss (k=6)
|
ablations/specialization_summary.csv
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cell,run,E,k,PR_median,PR_p10,PR_p90,generalist_frac_gt0.5,gate_when_sel_median,eff_rank_mean,strong_corr_pairs_mean
|
| 2 |
+
L0_s1,g3_tmoe_s0_1e16,192,18,0.6572,0.2378,0.9232,0.6458,0.0095,11.47,68
|
| 3 |
+
L0_s3,g3_tmoe_s0_1e16_seed3,192,18,0.6502,0.2735,0.9226,0.6667,0.00948,12.14,36
|
| 4 |
+
A2_auxfree,g3_tmoe_s0_1e16_auxfree,192,18,0.6004,0.1343,0.9344,0.5816,0.01095,11.6,116
|
| 5 |
+
karen,g3_tmoe_s0_1e16_karen,192,18,0.6143,0.1658,0.9117,0.6337,0.01122,13.66,10
|
| 6 |
+
karen_soft_s1,g3_tmoe_s0_1e16_karen_soft,192,18,0.579,0.1499,0.9203,0.5781,0.01167,13.62,61
|
| 7 |
+
karen_soft_s2,g3_tmoe_s0_1e16_karen_soft_seed2,192,18,0.5925,0.1362,0.9027,0.5903,0.0118,12.68,62
|
| 8 |
+
LG1_momr_s2,g3_tmoe_s0_1e16_momr_seed2,192,18,0.6851,0.2268,0.9186,0.6979,0.00941,11.13,54
|
ablations/stability_fakequant.csv
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run,bits,test_ce,test_bpb
|
| 2 |
+
g1_moe,16,3.918474,1.315807
|
| 3 |
+
g1_moe,8,3.918539,1.315829
|
| 4 |
+
g1_moe,4,3.936437,1.321839
|
| 5 |
+
g1_moe,3,4.026195,1.351980
|
| 6 |
+
g1_temporal,16,3.909472,1.312784
|
| 7 |
+
g1_temporal,8,3.909512,1.312798
|
| 8 |
+
g1_temporal,4,3.923908,1.317632
|
| 9 |
+
g1_temporal,3,3.996078,1.341866
|
| 10 |
+
g3_moe,16,4.009125,1.346247
|
| 11 |
+
g3_moe,8,4.009167,1.346262
|
| 12 |
+
g3_moe,4,4.021676,1.350462
|
| 13 |
+
g3_moe,3,4.081373,1.370508
|
| 14 |
+
g3_temporal,16,3.976890,1.335423
|
| 15 |
+
g3_temporal,8,3.976963,1.335448
|
| 16 |
+
g3_temporal,4,3.987511,1.338990
|
| 17 |
+
g3_temporal,3,4.038702,1.356179
|
| 18 |
+
moe_coarse_1e19,16,3.129897,1.051006
|
| 19 |
+
moe_coarse_1e19,8,3.129929,1.051017
|
| 20 |
+
moe_coarse_1e19,4,3.137491,1.053556
|
| 21 |
+
moe_coarse_1e19,3,3.175067,1.066174
|
| 22 |
+
temporal_coarse_1e19,16,3.178994,1.067493
|
| 23 |
+
temporal_coarse_1e19,8,3.179076,1.067520
|
| 24 |
+
temporal_coarse_1e19,4,3.185608,1.069714
|
| 25 |
+
temporal_coarse_1e19,3,3.217265,1.080344
|
| 26 |
+
temporal_fine_1e19,16,3.171383,1.064937
|
| 27 |
+
temporal_fine_1e19,8,3.171479,1.064969
|
| 28 |
+
temporal_fine_1e19,4,3.177317,1.066930
|
| 29 |
+
temporal_fine_1e19,3,3.205341,1.076340
|
merged_ce_model/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
olmoe_adapt/logs/.gitignore
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
venv/
|
| 2 |
+
model/
|
| 3 |
+
data/
|
| 4 |
+
__pycache__/
|
| 5 |
+
*.pt
|
| 6 |
+
*.safetensors
|
| 7 |
+
|
| 8 |
+
*.log
|
olmoe_adapt/logs/bake_bake_B.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=B tag=bake_B lr=0.0003 trainable=2097152 (router=2097152 extra=0) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:43: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:45: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M R=64 lm=2.0987 19.3k tok/s
|
| 11 |
+
[step 40] tok=2.6M R=64 lm=2.2252 19.5k tok/s
|
| 12 |
+
[step 60] tok=3.9M R=63 lm=2.1470 19.6k tok/s
|
| 13 |
+
[step 80] tok=5.2M R=63 lm=2.0423 19.6k tok/s
|
| 14 |
+
[step 100] tok=6.6M R=62 lm=2.1752 19.6k tok/s
|
| 15 |
+
[step 120] tok=7.9M R=62 lm=2.1442 19.7k tok/s
|
| 16 |
+
[step 140] tok=9.2M R=61 lm=2.1745 19.7k tok/s
|
| 17 |
+
[step 160] tok=10.5M R=61 lm=2.2132 19.7k tok/s
|
| 18 |
+
[step 180] tok=11.8M R=60 lm=2.1269 19.7k tok/s
|
| 19 |
+
[step 200] tok=13.1M R=60 lm=2.2061 19.7k tok/s
|
| 20 |
+
[step 220] tok=14.4M R=59 lm=2.0731 19.7k tok/s
|
| 21 |
+
[step 240] tok=15.7M R=59 lm=1.9612 19.7k tok/s
|
| 22 |
+
[step 260] tok=17.0M R=58 lm=2.1708 19.7k tok/s
|
| 23 |
+
[step 280] tok=18.4M R=58 lm=2.2011 19.7k tok/s
|
| 24 |
+
[step 300] tok=19.7M R=57 lm=2.1055 19.7k tok/s
|
| 25 |
+
[step 320] tok=21.0M R=57 lm=2.1007 19.7k tok/s
|
| 26 |
+
[step 340] tok=22.3M R=56 lm=2.2233 19.7k tok/s
|
| 27 |
+
[step 360] tok=23.6M R=56 lm=2.2414 19.7k tok/s
|
| 28 |
+
[step 380] tok=24.9M R=55 lm=2.2380 19.7k tok/s
|
| 29 |
+
[step 400] tok=26.2M R=55 lm=2.1980 19.7k tok/s
|
| 30 |
+
[step 420] tok=27.5M R=54 lm=2.2394 19.7k tok/s
|
| 31 |
+
[step 440] tok=28.8M R=54 lm=2.1341 19.7k tok/s
|
| 32 |
+
[step 460] tok=30.1M R=53 lm=2.1742 19.7k tok/s
|
| 33 |
+
[step 480] tok=31.5M R=53 lm=2.2473 19.7k tok/s
|
| 34 |
+
[step 500] tok=32.8M R=52 lm=2.1159 19.7k tok/s
|
| 35 |
+
[step 520] tok=34.1M R=52 lm=2.2894 19.7k tok/s
|
| 36 |
+
[step 540] tok=35.4M R=51 lm=2.1763 19.7k tok/s
|
| 37 |
+
[step 560] tok=36.7M R=51 lm=2.2155 19.7k tok/s
|
| 38 |
+
[step 580] tok=38.0M R=50 lm=2.1895 19.7k tok/s
|
| 39 |
+
[step 600] tok=39.3M R=50 lm=2.3668 19.7k tok/s
|
| 40 |
+
[step 620] tok=40.6M R=49 lm=2.2144 19.7k tok/s
|
| 41 |
+
[step 640] tok=41.9M R=49 lm=2.0801 19.7k tok/s
|
| 42 |
+
[step 660] tok=43.3M R=48 lm=2.2699 19.7k tok/s
|
| 43 |
+
[step 680] tok=44.6M R=48 lm=2.1756 19.7k tok/s
|
| 44 |
+
[step 700] tok=45.9M R=47 lm=2.2155 19.7k tok/s
|
| 45 |
+
[step 720] tok=47.2M R=47 lm=2.2462 19.7k tok/s
|
| 46 |
+
[step 740] tok=48.5M R=46 lm=2.4119 19.7k tok/s
|
| 47 |
+
[step 760] tok=49.8M R=46 lm=2.3786 19.7k tok/s
|
| 48 |
+
[eval] bake_B arm=B tok=50M trainR=46 BPB=2.7264 swap=0.9992 ent=0.9878
|
| 49 |
+
[step 780] tok=51.1M R=45 lm=2.3171 19.5k tok/s
|
| 50 |
+
[step 800] tok=52.4M R=45 lm=2.1599 19.5k tok/s
|
| 51 |
+
[step 820] tok=53.7M R=44 lm=2.2998 19.5k tok/s
|
| 52 |
+
[step 840] tok=55.1M R=44 lm=2.1948 19.5k tok/s
|
| 53 |
+
[step 860] tok=56.4M R=43 lm=2.2849 19.5k tok/s
|
| 54 |
+
[step 880] tok=57.7M R=43 lm=2.3046 19.5k tok/s
|
| 55 |
+
[step 900] tok=59.0M R=43 lm=2.2421 19.5k tok/s
|
| 56 |
+
[step 920] tok=60.3M R=42 lm=2.2709 19.5k tok/s
|
| 57 |
+
[step 940] tok=61.6M R=42 lm=2.3135 19.6k tok/s
|
| 58 |
+
[step 960] tok=62.9M R=41 lm=2.2023 19.6k tok/s
|
| 59 |
+
[step 980] tok=64.2M R=41 lm=2.3974 19.6k tok/s
|
| 60 |
+
[step 1000] tok=65.5M R=40 lm=2.2026 19.6k tok/s
|
| 61 |
+
[step 1020] tok=66.8M R=40 lm=2.0962 19.6k tok/s
|
| 62 |
+
[step 1040] tok=68.2M R=39 lm=2.2051 19.6k tok/s
|
| 63 |
+
[step 1060] tok=69.5M R=39 lm=2.2462 19.6k tok/s
|
| 64 |
+
[step 1080] tok=70.8M R=38 lm=2.1954 19.6k tok/s
|
| 65 |
+
[step 1100] tok=72.1M R=38 lm=2.2615 19.6k tok/s
|
| 66 |
+
[step 1120] tok=73.4M R=37 lm=2.4683 19.6k tok/s
|
| 67 |
+
[step 1140] tok=74.7M R=37 lm=2.4157 19.6k tok/s
|
| 68 |
+
[step 1160] tok=76.0M R=36 lm=2.2574 19.6k tok/s
|
| 69 |
+
[step 1180] tok=77.3M R=36 lm=2.2718 19.6k tok/s
|
| 70 |
+
[step 1200] tok=78.6M R=35 lm=2.2224 19.6k tok/s
|
| 71 |
+
[step 1220] tok=80.0M R=35 lm=2.2565 19.6k tok/s
|
| 72 |
+
[step 1240] tok=81.3M R=34 lm=2.3976 19.6k tok/s
|
| 73 |
+
[step 1260] tok=82.6M R=34 lm=2.3440 19.6k tok/s
|
| 74 |
+
[step 1280] tok=83.9M R=33 lm=2.3858 19.6k tok/s
|
| 75 |
+
[step 1300] tok=85.2M R=33 lm=2.3377 19.6k tok/s
|
| 76 |
+
[step 1320] tok=86.5M R=32 lm=2.3479 19.6k tok/s
|
| 77 |
+
[step 1340] tok=87.8M R=32 lm=2.3583 19.6k tok/s
|
| 78 |
+
[step 1360] tok=89.1M R=31 lm=2.4302 19.6k tok/s
|
| 79 |
+
[step 1380] tok=90.4M R=31 lm=2.3159 19.6k tok/s
|
| 80 |
+
[step 1400] tok=91.8M R=30 lm=2.2853 19.6k tok/s
|
| 81 |
+
[step 1420] tok=93.1M R=30 lm=2.3583 19.6k tok/s
|
| 82 |
+
[step 1440] tok=94.4M R=29 lm=2.3446 19.6k tok/s
|
| 83 |
+
[step 1460] tok=95.7M R=29 lm=2.4297 19.6k tok/s
|
| 84 |
+
[step 1480] tok=97.0M R=28 lm=2.4223 19.6k tok/s
|
| 85 |
+
[step 1500] tok=98.3M R=28 lm=2.4586 19.6k tok/s
|
| 86 |
+
[step 1520] tok=99.6M R=27 lm=2.3323 19.6k tok/s
|
| 87 |
+
[eval] bake_B arm=B tok=100M trainR=27 BPB=2.6831 swap=0.9994 ent=0.9878
|
| 88 |
+
[step 1540] tok=100.9M R=27 lm=2.3342 19.5k tok/s
|
| 89 |
+
[step 1560] tok=102.2M R=26 lm=2.4722 19.5k tok/s
|
| 90 |
+
[step 1580] tok=103.5M R=26 lm=2.4445 19.5k tok/s
|
| 91 |
+
[step 1600] tok=104.9M R=25 lm=2.3412 19.5k tok/s
|
| 92 |
+
[step 1620] tok=106.2M R=25 lm=2.3519 19.5k tok/s
|
| 93 |
+
[step 1640] tok=107.5M R=24 lm=2.4504 19.5k tok/s
|
| 94 |
+
[step 1660] tok=108.8M R=24 lm=2.5108 19.5k tok/s
|
| 95 |
+
[step 1680] tok=110.1M R=23 lm=2.4768 19.5k tok/s
|
| 96 |
+
[step 1700] tok=111.4M R=23 lm=2.4387 19.5k tok/s
|
| 97 |
+
[step 1720] tok=112.7M R=22 lm=2.5928 19.5k tok/s
|
| 98 |
+
[step 1740] tok=114.0M R=22 lm=2.4976 19.5k tok/s
|
| 99 |
+
[step 1760] tok=115.3M R=21 lm=2.5815 19.5k tok/s
|
| 100 |
+
[step 1780] tok=116.7M R=21 lm=2.4928 19.5k tok/s
|
| 101 |
+
[step 1800] tok=118.0M R=20 lm=2.6618 19.5k tok/s
|
| 102 |
+
[step 1820] tok=119.3M R=20 lm=2.6313 19.6k tok/s
|
| 103 |
+
[step 1840] tok=120.6M R=20 lm=2.5968 19.6k tok/s
|
| 104 |
+
[step 1860] tok=121.9M R=19 lm=2.7466 19.6k tok/s
|
| 105 |
+
[step 1880] tok=123.2M R=19 lm=2.6852 19.6k tok/s
|
| 106 |
+
[step 1900] tok=124.5M R=18 lm=2.7558 19.6k tok/s
|
| 107 |
+
[step 1920] tok=125.8M R=18 lm=2.6827 19.6k tok/s
|
| 108 |
+
[step 1940] tok=127.1M R=17 lm=2.8658 19.6k tok/s
|
| 109 |
+
[step 1960] tok=128.5M R=17 lm=2.7450 19.6k tok/s
|
| 110 |
+
[step 1980] tok=129.8M R=16 lm=2.8324 19.6k tok/s
|
| 111 |
+
[step 2000] tok=131.1M R=16 lm=2.8189 19.6k tok/s
|
| 112 |
+
[step 2020] tok=132.4M R=15 lm=2.9286 19.6k tok/s
|
| 113 |
+
[step 2040] tok=133.7M R=15 lm=2.8378 19.6k tok/s
|
| 114 |
+
[step 2060] tok=135.0M R=14 lm=2.9801 19.6k tok/s
|
| 115 |
+
[step 2080] tok=136.3M R=14 lm=3.1011 19.6k tok/s
|
| 116 |
+
[step 2100] tok=137.6M R=13 lm=3.2752 19.6k tok/s
|
| 117 |
+
[step 2120] tok=138.9M R=13 lm=3.3033 19.6k tok/s
|
| 118 |
+
[step 2140] tok=140.2M R=12 lm=3.3180 19.6k tok/s
|
| 119 |
+
[step 2160] tok=141.6M R=12 lm=3.4398 19.6k tok/s
|
| 120 |
+
[step 2180] tok=142.9M R=11 lm=3.6111 19.6k tok/s
|
| 121 |
+
[step 2200] tok=144.2M R=11 lm=3.6040 19.6k tok/s
|
| 122 |
+
[step 2220] tok=145.5M R=10 lm=3.9649 19.6k tok/s
|
| 123 |
+
[step 2240] tok=146.8M R=10 lm=3.8094 19.6k tok/s
|
| 124 |
+
[step 2260] tok=148.1M R=9 lm=4.1075 19.6k tok/s
|
| 125 |
+
[step 2280] tok=149.4M R=9 lm=4.1917 19.6k tok/s
|
| 126 |
+
[eval] bake_B arm=B tok=150M trainR=8 BPB=1.4830 swap=0.9999 ent=0.9916
|
| 127 |
+
[step 2300] tok=150.7M R=8 lm=4.6120 19.5k tok/s
|
| 128 |
+
[step 2320] tok=152.0M R=8 lm=4.3906 19.5k tok/s
|
| 129 |
+
[step 2340] tok=153.4M R=8 lm=4.4153 19.5k tok/s
|
| 130 |
+
[step 2360] tok=154.7M R=8 lm=4.3065 19.5k tok/s
|
| 131 |
+
[step 2380] tok=156.0M R=8 lm=4.4565 19.5k tok/s
|
| 132 |
+
[step 2400] tok=157.3M R=8 lm=4.3428 19.5k tok/s
|
| 133 |
+
[step 2420] tok=158.6M R=8 lm=4.3049 19.5k tok/s
|
| 134 |
+
[step 2440] tok=159.9M R=8 lm=4.2387 19.5k tok/s
|
| 135 |
+
[step 2460] tok=161.2M R=8 lm=4.3002 19.5k tok/s
|
| 136 |
+
[step 2480] tok=162.5M R=8 lm=4.1037 19.5k tok/s
|
| 137 |
+
[step 2500] tok=163.8M R=8 lm=4.1680 19.5k tok/s
|
| 138 |
+
[step 2520] tok=165.2M R=8 lm=4.1394 19.5k tok/s
|
| 139 |
+
[step 2540] tok=166.5M R=8 lm=4.3114 19.5k tok/s
|
| 140 |
+
[step 2560] tok=167.8M R=8 lm=4.1563 19.6k tok/s
|
| 141 |
+
[step 2580] tok=169.1M R=8 lm=4.2261 19.6k tok/s
|
| 142 |
+
[step 2600] tok=170.4M R=8 lm=4.0132 19.6k tok/s
|
| 143 |
+
[step 2620] tok=171.7M R=8 lm=4.2112 19.6k tok/s
|
| 144 |
+
[step 2640] tok=173.0M R=8 lm=4.1323 19.6k tok/s
|
| 145 |
+
[step 2660] tok=174.3M R=8 lm=4.3646 19.6k tok/s
|
| 146 |
+
[step 2680] tok=175.6M R=8 lm=4.2462 19.6k tok/s
|
| 147 |
+
[step 2700] tok=176.9M R=8 lm=4.0994 19.6k tok/s
|
| 148 |
+
[step 2720] tok=178.3M R=8 lm=4.1714 19.6k tok/s
|
| 149 |
+
[step 2740] tok=179.6M R=8 lm=4.1147 19.6k tok/s
|
| 150 |
+
[step 2760] tok=180.9M R=8 lm=4.1921 19.6k tok/s
|
| 151 |
+
[step 2780] tok=182.2M R=8 lm=4.1272 19.6k tok/s
|
| 152 |
+
[step 2800] tok=183.5M R=8 lm=4.1416 19.6k tok/s
|
| 153 |
+
[step 2820] tok=184.8M R=8 lm=4.1661 19.6k tok/s
|
| 154 |
+
[step 2840] tok=186.1M R=8 lm=4.0360 19.6k tok/s
|
| 155 |
+
[step 2860] tok=187.4M R=8 lm=4.1205 19.6k tok/s
|
| 156 |
+
[step 2880] tok=188.7M R=8 lm=4.2419 19.6k tok/s
|
| 157 |
+
[step 2900] tok=190.1M R=8 lm=4.1887 19.6k tok/s
|
| 158 |
+
[step 2920] tok=191.4M R=8 lm=4.1100 19.6k tok/s
|
| 159 |
+
[step 2940] tok=192.7M R=8 lm=4.2353 19.6k tok/s
|
| 160 |
+
[step 2960] tok=194.0M R=8 lm=3.9857 19.6k tok/s
|
| 161 |
+
[step 2980] tok=195.3M R=8 lm=4.0978 19.6k tok/s
|
| 162 |
+
[step 3000] tok=196.6M R=8 lm=4.0624 19.6k tok/s
|
| 163 |
+
[step 3020] tok=197.9M R=8 lm=4.0828 19.6k tok/s
|
| 164 |
+
[step 3040] tok=199.2M R=8 lm=4.1774 19.6k tok/s
|
| 165 |
+
[eval] bake_B arm=B tok=200M trainR=8 BPB=1.2829 swap=0.9999 ent=0.9915
|
| 166 |
+
[step 3060] tok=200.5M R=8 lm=4.1373 19.5k tok/s
|
| 167 |
+
[step 3080] tok=201.9M R=8 lm=4.1524 19.5k tok/s
|
| 168 |
+
[step 3100] tok=203.2M R=8 lm=4.1770 19.5k tok/s
|
| 169 |
+
[step 3120] tok=204.5M R=8 lm=4.0530 19.5k tok/s
|
| 170 |
+
[step 3140] tok=205.8M R=8 lm=4.0915 19.6k tok/s
|
| 171 |
+
[step 3160] tok=207.1M R=8 lm=4.0947 19.6k tok/s
|
| 172 |
+
[step 3180] tok=208.4M R=8 lm=4.0568 19.6k tok/s
|
| 173 |
+
[step 3200] tok=209.7M R=8 lm=4.0451 19.6k tok/s
|
| 174 |
+
[step 3220] tok=211.0M R=8 lm=4.0109 19.6k tok/s
|
| 175 |
+
[step 3240] tok=212.3M R=8 lm=4.0837 19.6k tok/s
|
| 176 |
+
[step 3260] tok=213.6M R=8 lm=4.1350 19.6k tok/s
|
| 177 |
+
[step 3280] tok=215.0M R=8 lm=4.0931 19.6k tok/s
|
| 178 |
+
[step 3300] tok=216.3M R=8 lm=4.1614 19.6k tok/s
|
| 179 |
+
[step 3320] tok=217.6M R=8 lm=4.1405 19.6k tok/s
|
| 180 |
+
[step 3340] tok=218.9M R=8 lm=4.0433 19.6k tok/s
|
| 181 |
+
[step 3360] tok=220.2M R=8 lm=4.0909 19.6k tok/s
|
| 182 |
+
[step 3380] tok=221.5M R=8 lm=4.2159 19.6k tok/s
|
| 183 |
+
[step 3400] tok=222.8M R=8 lm=4.1443 19.6k tok/s
|
| 184 |
+
[step 3420] tok=224.1M R=8 lm=4.1206 19.6k tok/s
|
| 185 |
+
[step 3440] tok=225.4M R=8 lm=4.1010 19.6k tok/s
|
| 186 |
+
[step 3460] tok=226.8M R=8 lm=4.1388 19.6k tok/s
|
| 187 |
+
[step 3480] tok=228.1M R=8 lm=4.1162 19.6k tok/s
|
| 188 |
+
[step 3500] tok=229.4M R=8 lm=4.0706 19.6k tok/s
|
| 189 |
+
[step 3520] tok=230.7M R=8 lm=4.1209 19.6k tok/s
|
| 190 |
+
[step 3540] tok=232.0M R=8 lm=4.0758 19.6k tok/s
|
| 191 |
+
[step 3560] tok=233.3M R=8 lm=4.1009 19.6k tok/s
|
| 192 |
+
[step 3580] tok=234.6M R=8 lm=4.2391 19.6k tok/s
|
| 193 |
+
[step 3600] tok=235.9M R=8 lm=4.1356 19.6k tok/s
|
| 194 |
+
[step 3620] tok=237.2M R=8 lm=4.3200 19.6k tok/s
|
| 195 |
+
[step 3640] tok=238.6M R=8 lm=4.0635 19.6k tok/s
|
| 196 |
+
[step 3660] tok=239.9M R=8 lm=4.1947 19.6k tok/s
|
| 197 |
+
[step 3680] tok=241.2M R=8 lm=4.1784 19.6k tok/s
|
| 198 |
+
[step 3700] tok=242.5M R=8 lm=4.0311 19.6k tok/s
|
| 199 |
+
[step 3720] tok=243.8M R=8 lm=4.0900 19.6k tok/s
|
| 200 |
+
[step 3740] tok=245.1M R=8 lm=4.1343 19.6k tok/s
|
| 201 |
+
[step 3760] tok=246.4M R=8 lm=4.1831 19.6k tok/s
|
| 202 |
+
[step 3780] tok=247.7M R=8 lm=4.0765 19.6k tok/s
|
| 203 |
+
[step 3800] tok=249.0M R=8 lm=4.1704 19.6k tok/s
|
| 204 |
+
[eval] bake_B arm=B tok=250M trainR=8 BPB=1.2788 swap=0.9999 ent=0.9918
|
| 205 |
+
[DONE] arm=B final BPB=1.2788 swap=0.9999 ent=0.9918
|
olmoe_adapt/logs/bake_bake_C.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=C tag=bake_C lr=0.0003 trainable=2230272 (router=2097152 extra=133120) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:44: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:46: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M R=8 lm=5.4091 19.2k tok/s
|
| 11 |
+
[step 40] tok=2.6M R=8 lm=4.4112 19.4k tok/s
|
| 12 |
+
[step 60] tok=3.9M R=8 lm=3.7670 19.5k tok/s
|
| 13 |
+
[step 80] tok=5.2M R=8 lm=3.4740 19.5k tok/s
|
| 14 |
+
[step 100] tok=6.6M R=8 lm=3.4024 19.6k tok/s
|
| 15 |
+
[step 120] tok=7.9M R=8 lm=3.3032 19.6k tok/s
|
| 16 |
+
[step 140] tok=9.2M R=8 lm=3.1795 19.6k tok/s
|
| 17 |
+
[step 160] tok=10.5M R=8 lm=3.2217 19.6k tok/s
|
| 18 |
+
[step 180] tok=11.8M R=8 lm=3.1266 19.6k tok/s
|
| 19 |
+
[step 200] tok=13.1M R=8 lm=3.1323 19.6k tok/s
|
| 20 |
+
[step 220] tok=14.4M R=8 lm=2.9949 19.6k tok/s
|
| 21 |
+
[step 240] tok=15.7M R=8 lm=2.9062 19.6k tok/s
|
| 22 |
+
[step 260] tok=17.0M R=8 lm=2.9954 19.6k tok/s
|
| 23 |
+
[step 280] tok=18.4M R=8 lm=3.0044 19.6k tok/s
|
| 24 |
+
[step 300] tok=19.7M R=8 lm=2.9086 19.6k tok/s
|
| 25 |
+
[step 320] tok=21.0M R=8 lm=2.8822 19.6k tok/s
|
| 26 |
+
[step 340] tok=22.3M R=8 lm=2.9147 19.6k tok/s
|
| 27 |
+
[step 360] tok=23.6M R=8 lm=2.9106 19.6k tok/s
|
| 28 |
+
[step 380] tok=24.9M R=8 lm=2.9570 19.6k tok/s
|
| 29 |
+
[step 400] tok=26.2M R=8 lm=2.9146 19.6k tok/s
|
| 30 |
+
[step 420] tok=27.5M R=8 lm=2.9281 19.6k tok/s
|
| 31 |
+
[step 440] tok=28.8M R=8 lm=2.8367 19.6k tok/s
|
| 32 |
+
[step 460] tok=30.1M R=8 lm=2.8175 19.6k tok/s
|
| 33 |
+
[step 480] tok=31.5M R=8 lm=2.8998 19.6k tok/s
|
| 34 |
+
[step 500] tok=32.8M R=8 lm=2.8467 19.6k tok/s
|
| 35 |
+
[step 520] tok=34.1M R=8 lm=2.9359 19.6k tok/s
|
| 36 |
+
[step 540] tok=35.4M R=8 lm=2.8447 19.6k tok/s
|
| 37 |
+
[step 560] tok=36.7M R=8 lm=2.9027 19.6k tok/s
|
| 38 |
+
[step 580] tok=38.0M R=8 lm=2.8638 19.6k tok/s
|
| 39 |
+
[step 600] tok=39.3M R=8 lm=2.9616 19.6k tok/s
|
| 40 |
+
[step 620] tok=40.6M R=8 lm=2.8530 19.6k tok/s
|
| 41 |
+
[step 640] tok=41.9M R=8 lm=2.6619 19.6k tok/s
|
| 42 |
+
[step 660] tok=43.3M R=8 lm=2.8902 19.6k tok/s
|
| 43 |
+
[step 680] tok=44.6M R=8 lm=2.7440 19.6k tok/s
|
| 44 |
+
[step 700] tok=45.9M R=8 lm=2.8805 19.6k tok/s
|
| 45 |
+
[step 720] tok=47.2M R=8 lm=2.8308 19.6k tok/s
|
| 46 |
+
[step 740] tok=48.5M R=8 lm=3.0188 19.6k tok/s
|
| 47 |
+
[step 760] tok=49.8M R=8 lm=2.9329 19.6k tok/s
|
| 48 |
+
[eval] bake_C arm=C tok=50M trainR=8 BPB=0.8791 swap=0.9998 ent=0.9978
|
| 49 |
+
[step 780] tok=51.1M R=8 lm=2.9026 19.4k tok/s
|
| 50 |
+
[step 800] tok=52.4M R=8 lm=2.7695 19.4k tok/s
|
| 51 |
+
[step 820] tok=53.7M R=8 lm=2.8775 19.4k tok/s
|
| 52 |
+
[step 840] tok=55.1M R=8 lm=2.7520 19.4k tok/s
|
| 53 |
+
[step 860] tok=56.4M R=8 lm=2.8754 19.4k tok/s
|
| 54 |
+
[step 880] tok=57.7M R=8 lm=2.8497 19.4k tok/s
|
| 55 |
+
[step 900] tok=59.0M R=8 lm=2.8102 19.5k tok/s
|
| 56 |
+
[step 920] tok=60.3M R=8 lm=2.8091 19.5k tok/s
|
| 57 |
+
[step 940] tok=61.6M R=8 lm=2.8490 19.5k tok/s
|
| 58 |
+
[step 960] tok=62.9M R=8 lm=2.7195 19.5k tok/s
|
| 59 |
+
[step 980] tok=64.2M R=8 lm=2.9644 19.5k tok/s
|
| 60 |
+
[step 1000] tok=65.5M R=8 lm=2.7437 19.5k tok/s
|
| 61 |
+
[step 1020] tok=66.8M R=8 lm=2.6509 19.5k tok/s
|
| 62 |
+
[step 1040] tok=68.2M R=8 lm=2.7375 19.5k tok/s
|
| 63 |
+
[step 1060] tok=69.5M R=8 lm=2.8336 19.5k tok/s
|
| 64 |
+
[step 1080] tok=70.8M R=8 lm=2.7306 19.5k tok/s
|
| 65 |
+
[step 1100] tok=72.1M R=8 lm=2.7387 19.5k tok/s
|
| 66 |
+
[step 1120] tok=73.4M R=8 lm=2.9884 19.5k tok/s
|
| 67 |
+
[step 1140] tok=74.7M R=8 lm=2.9189 19.5k tok/s
|
| 68 |
+
[step 1160] tok=76.0M R=8 lm=2.7568 19.5k tok/s
|
| 69 |
+
[step 1180] tok=77.3M R=8 lm=2.8010 19.5k tok/s
|
| 70 |
+
[step 1200] tok=78.6M R=8 lm=2.7314 19.5k tok/s
|
| 71 |
+
[step 1220] tok=80.0M R=8 lm=2.7804 19.5k tok/s
|
| 72 |
+
[step 1240] tok=81.3M R=8 lm=2.9041 19.5k tok/s
|
| 73 |
+
[step 1260] tok=82.6M R=8 lm=2.8518 19.5k tok/s
|
| 74 |
+
[step 1280] tok=83.9M R=8 lm=2.8211 19.5k tok/s
|
| 75 |
+
[step 1300] tok=85.2M R=8 lm=2.7886 19.5k tok/s
|
| 76 |
+
[step 1320] tok=86.5M R=8 lm=2.7575 19.5k tok/s
|
| 77 |
+
[step 1340] tok=87.8M R=8 lm=2.7927 19.5k tok/s
|
| 78 |
+
[step 1360] tok=89.1M R=8 lm=2.8807 19.5k tok/s
|
| 79 |
+
[step 1380] tok=90.4M R=8 lm=2.7432 19.5k tok/s
|
| 80 |
+
[step 1400] tok=91.8M R=8 lm=2.7047 19.5k tok/s
|
| 81 |
+
[step 1420] tok=93.1M R=8 lm=2.8183 19.5k tok/s
|
| 82 |
+
[step 1440] tok=94.4M R=8 lm=2.7484 19.5k tok/s
|
| 83 |
+
[step 1460] tok=95.7M R=8 lm=2.8166 19.5k tok/s
|
| 84 |
+
[step 1480] tok=97.0M R=8 lm=2.8138 19.5k tok/s
|
| 85 |
+
[step 1500] tok=98.3M R=8 lm=2.8596 19.5k tok/s
|
| 86 |
+
[step 1520] tok=99.6M R=8 lm=2.6906 19.5k tok/s
|
| 87 |
+
[eval] bake_C arm=C tok=100M trainR=8 BPB=0.8627 swap=0.9998 ent=0.9982
|
| 88 |
+
[step 1540] tok=100.9M R=8 lm=2.6919 19.4k tok/s
|
| 89 |
+
[step 1560] tok=102.2M R=8 lm=2.8467 19.4k tok/s
|
| 90 |
+
[step 1580] tok=103.5M R=8 lm=2.8072 19.4k tok/s
|
| 91 |
+
[step 1600] tok=104.9M R=8 lm=2.6951 19.4k tok/s
|
| 92 |
+
[step 1620] tok=106.2M R=8 lm=2.6766 19.4k tok/s
|
| 93 |
+
[step 1640] tok=107.5M R=8 lm=2.7529 19.4k tok/s
|
| 94 |
+
[step 1660] tok=108.8M R=8 lm=2.8291 19.4k tok/s
|
| 95 |
+
[step 1680] tok=110.1M R=8 lm=2.7736 19.4k tok/s
|
| 96 |
+
[step 1700] tok=111.4M R=8 lm=2.7321 19.4k tok/s
|
| 97 |
+
[step 1720] tok=112.7M R=8 lm=2.8369 19.4k tok/s
|
| 98 |
+
[step 1740] tok=114.0M R=8 lm=2.7426 19.5k tok/s
|
| 99 |
+
[step 1760] tok=115.3M R=8 lm=2.7950 19.5k tok/s
|
| 100 |
+
[step 1780] tok=116.7M R=8 lm=2.7322 19.5k tok/s
|
| 101 |
+
[step 1800] tok=118.0M R=8 lm=2.8645 19.5k tok/s
|
| 102 |
+
[step 1820] tok=119.3M R=8 lm=2.7994 19.5k tok/s
|
| 103 |
+
[step 1840] tok=120.6M R=8 lm=2.7673 19.5k tok/s
|
| 104 |
+
[step 1860] tok=121.9M R=8 lm=2.8559 19.5k tok/s
|
| 105 |
+
[step 1880] tok=123.2M R=8 lm=2.8051 19.5k tok/s
|
| 106 |
+
[step 1900] tok=124.5M R=8 lm=2.7903 19.5k tok/s
|
| 107 |
+
[step 1920] tok=125.8M R=8 lm=2.7410 19.5k tok/s
|
| 108 |
+
[step 1940] tok=127.1M R=8 lm=2.8625 19.5k tok/s
|
| 109 |
+
[step 1960] tok=128.5M R=8 lm=2.7429 19.5k tok/s
|
| 110 |
+
[step 1980] tok=129.8M R=8 lm=2.7561 19.5k tok/s
|
| 111 |
+
[step 2000] tok=131.1M R=8 lm=2.7434 19.5k tok/s
|
| 112 |
+
[step 2020] tok=132.4M R=8 lm=2.7541 19.5k tok/s
|
| 113 |
+
[step 2040] tok=133.7M R=8 lm=2.6585 19.5k tok/s
|
| 114 |
+
[step 2060] tok=135.0M R=8 lm=2.6719 19.5k tok/s
|
| 115 |
+
[step 2080] tok=136.3M R=8 lm=2.8152 19.5k tok/s
|
| 116 |
+
[step 2100] tok=137.6M R=8 lm=2.8390 19.5k tok/s
|
| 117 |
+
[step 2120] tok=138.9M R=8 lm=2.8375 19.5k tok/s
|
| 118 |
+
[step 2140] tok=140.2M R=8 lm=2.6940 19.5k tok/s
|
| 119 |
+
[step 2160] tok=141.6M R=8 lm=2.8541 19.5k tok/s
|
| 120 |
+
[step 2180] tok=142.9M R=8 lm=2.7872 19.5k tok/s
|
| 121 |
+
[step 2200] tok=144.2M R=8 lm=2.7895 19.5k tok/s
|
| 122 |
+
[step 2220] tok=145.5M R=8 lm=2.8623 19.5k tok/s
|
| 123 |
+
[step 2240] tok=146.8M R=8 lm=2.7037 19.5k tok/s
|
| 124 |
+
[step 2260] tok=148.1M R=8 lm=2.6587 19.5k tok/s
|
| 125 |
+
[step 2280] tok=149.4M R=8 lm=2.8693 19.5k tok/s
|
| 126 |
+
[eval] bake_C arm=C tok=150M trainR=8 BPB=0.8560 swap=0.9998 ent=0.9984
|
| 127 |
+
[step 2300] tok=150.7M R=8 lm=2.7997 19.4k tok/s
|
| 128 |
+
[step 2320] tok=152.0M R=8 lm=2.7785 19.4k tok/s
|
| 129 |
+
[step 2340] tok=153.4M R=8 lm=2.7930 19.4k tok/s
|
| 130 |
+
[step 2360] tok=154.7M R=8 lm=2.6715 19.4k tok/s
|
| 131 |
+
[step 2380] tok=156.0M R=8 lm=2.7958 19.4k tok/s
|
| 132 |
+
[step 2400] tok=157.3M R=8 lm=2.8033 19.4k tok/s
|
| 133 |
+
[step 2420] tok=158.6M R=8 lm=2.8401 19.4k tok/s
|
| 134 |
+
[step 2440] tok=159.9M R=8 lm=2.7286 19.4k tok/s
|
| 135 |
+
[step 2460] tok=161.2M R=8 lm=2.8398 19.4k tok/s
|
| 136 |
+
[step 2480] tok=162.5M R=8 lm=2.6188 19.4k tok/s
|
| 137 |
+
[step 2500] tok=163.8M R=8 lm=2.7132 19.4k tok/s
|
| 138 |
+
[step 2520] tok=165.2M R=8 lm=2.7317 19.4k tok/s
|
| 139 |
+
[step 2540] tok=166.5M R=8 lm=2.8314 19.4k tok/s
|
| 140 |
+
[step 2560] tok=167.8M R=8 lm=2.6820 19.4k tok/s
|
| 141 |
+
[step 2580] tok=169.1M R=8 lm=2.7771 19.4k tok/s
|
| 142 |
+
[step 2600] tok=170.4M R=8 lm=2.5800 19.4k tok/s
|
| 143 |
+
[step 2620] tok=171.7M R=8 lm=2.7985 19.5k tok/s
|
| 144 |
+
[step 2640] tok=173.0M R=8 lm=2.7248 19.5k tok/s
|
| 145 |
+
[step 2660] tok=174.3M R=8 lm=2.8869 19.5k tok/s
|
| 146 |
+
[step 2680] tok=175.6M R=8 lm=2.8129 19.5k tok/s
|
| 147 |
+
[step 2700] tok=176.9M R=8 lm=2.7171 19.5k tok/s
|
| 148 |
+
[step 2720] tok=178.3M R=8 lm=2.8000 19.5k tok/s
|
| 149 |
+
[step 2740] tok=179.6M R=8 lm=2.6883 19.5k tok/s
|
| 150 |
+
[step 2760] tok=180.9M R=8 lm=2.8190 19.5k tok/s
|
| 151 |
+
[step 2780] tok=182.2M R=8 lm=2.7064 19.5k tok/s
|
| 152 |
+
[step 2800] tok=183.5M R=8 lm=2.7631 19.5k tok/s
|
| 153 |
+
[step 2820] tok=184.8M R=8 lm=2.8433 19.5k tok/s
|
| 154 |
+
[step 2840] tok=186.1M R=8 lm=2.5773 19.5k tok/s
|
| 155 |
+
[step 2860] tok=187.4M R=8 lm=2.7461 19.5k tok/s
|
| 156 |
+
[step 2880] tok=188.7M R=8 lm=2.8480 19.5k tok/s
|
| 157 |
+
[step 2900] tok=190.1M R=8 lm=2.8375 19.5k tok/s
|
| 158 |
+
[step 2920] tok=191.4M R=8 lm=2.7448 19.5k tok/s
|
| 159 |
+
[step 2940] tok=192.7M R=8 lm=2.8780 19.5k tok/s
|
| 160 |
+
[step 2960] tok=194.0M R=8 lm=2.5910 19.5k tok/s
|
| 161 |
+
[step 2980] tok=195.3M R=8 lm=2.7229 19.5k tok/s
|
| 162 |
+
[step 3000] tok=196.6M R=8 lm=2.5609 19.5k tok/s
|
| 163 |
+
[step 3020] tok=197.9M R=8 lm=2.7162 19.5k tok/s
|
| 164 |
+
[step 3040] tok=199.2M R=8 lm=2.8356 19.5k tok/s
|
| 165 |
+
[eval] bake_C arm=C tok=200M trainR=8 BPB=0.8526 swap=0.9998 ent=0.9987
|
| 166 |
+
[step 3060] tok=200.5M R=8 lm=2.7535 19.4k tok/s
|
| 167 |
+
[step 3080] tok=201.9M R=8 lm=2.7917 19.4k tok/s
|
| 168 |
+
[step 3100] tok=203.2M R=8 lm=2.7991 19.4k tok/s
|
| 169 |
+
[step 3120] tok=204.5M R=8 lm=2.6732 19.4k tok/s
|
| 170 |
+
[step 3140] tok=205.8M R=8 lm=2.7529 19.4k tok/s
|
| 171 |
+
[step 3160] tok=207.1M R=8 lm=2.6947 19.4k tok/s
|
| 172 |
+
[step 3180] tok=208.4M R=8 lm=2.6985 19.4k tok/s
|
| 173 |
+
[step 3200] tok=209.7M R=8 lm=2.7057 19.4k tok/s
|
| 174 |
+
[step 3220] tok=211.0M R=8 lm=2.5845 19.4k tok/s
|
| 175 |
+
[step 3240] tok=212.3M R=8 lm=2.7460 19.4k tok/s
|
| 176 |
+
[step 3260] tok=213.6M R=8 lm=2.7857 19.4k tok/s
|
| 177 |
+
[step 3280] tok=215.0M R=8 lm=2.7168 19.4k tok/s
|
| 178 |
+
[step 3300] tok=216.3M R=8 lm=2.7403 19.4k tok/s
|
| 179 |
+
[step 3320] tok=217.6M R=8 lm=2.7609 19.4k tok/s
|
| 180 |
+
[step 3340] tok=218.9M R=8 lm=2.7314 19.4k tok/s
|
| 181 |
+
[step 3360] tok=220.2M R=8 lm=2.7400 19.4k tok/s
|
| 182 |
+
[step 3380] tok=221.5M R=8 lm=2.7531 19.4k tok/s
|
| 183 |
+
[step 3400] tok=222.8M R=8 lm=2.7597 19.4k tok/s
|
| 184 |
+
[step 3420] tok=224.1M R=8 lm=2.7231 19.4k tok/s
|
| 185 |
+
[step 3440] tok=225.4M R=8 lm=2.6899 19.4k tok/s
|
| 186 |
+
[step 3460] tok=226.8M R=8 lm=2.7914 19.5k tok/s
|
| 187 |
+
[step 3480] tok=228.1M R=8 lm=2.7881 19.5k tok/s
|
| 188 |
+
[step 3500] tok=229.4M R=8 lm=2.7487 19.5k tok/s
|
| 189 |
+
[step 3520] tok=230.7M R=8 lm=2.7341 19.5k tok/s
|
| 190 |
+
[step 3540] tok=232.0M R=8 lm=2.7671 19.5k tok/s
|
| 191 |
+
[step 3560] tok=233.3M R=8 lm=2.7702 19.5k tok/s
|
| 192 |
+
[step 3580] tok=234.6M R=8 lm=2.9027 19.5k tok/s
|
| 193 |
+
[step 3600] tok=235.9M R=8 lm=2.7871 19.5k tok/s
|
| 194 |
+
[step 3620] tok=237.2M R=8 lm=3.0031 19.5k tok/s
|
| 195 |
+
[step 3640] tok=238.6M R=8 lm=2.7499 19.5k tok/s
|
| 196 |
+
[step 3660] tok=239.9M R=8 lm=2.8261 19.5k tok/s
|
| 197 |
+
[step 3680] tok=241.2M R=8 lm=2.7833 19.5k tok/s
|
| 198 |
+
[step 3700] tok=242.5M R=8 lm=2.7131 19.5k tok/s
|
| 199 |
+
[step 3720] tok=243.8M R=8 lm=2.7223 19.5k tok/s
|
| 200 |
+
[step 3740] tok=245.1M R=8 lm=2.7564 19.5k tok/s
|
| 201 |
+
[step 3760] tok=246.4M R=8 lm=2.8194 19.5k tok/s
|
| 202 |
+
[step 3780] tok=247.7M R=8 lm=2.7131 19.5k tok/s
|
| 203 |
+
[step 3800] tok=249.0M R=8 lm=2.8514 19.5k tok/s
|
| 204 |
+
[eval] bake_C arm=C tok=250M trainR=8 BPB=0.8505 swap=0.9998 ent=0.9988
|
| 205 |
+
[DONE] arm=C final BPB=0.8505 swap=0.9998 ent=0.9988
|
olmoe_adapt/logs/bake_bake_D.log
ADDED
|
@@ -0,0 +1,777 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=D tag=bake_D lr=0.0003 trainable=2097152 (router=2097152 extra=0) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:49: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:51: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=0.3M R=8 lm=6.7850 12.5k tok/s
|
| 11 |
+
[step 40] tok=0.7M R=8 lm=6.2973 12.9k tok/s
|
| 12 |
+
[step 60] tok=1.0M R=8 lm=5.8679 13.0k tok/s
|
| 13 |
+
[step 80] tok=1.3M R=8 lm=5.6394 13.1k tok/s
|
| 14 |
+
[step 100] tok=1.6M R=8 lm=5.0610 13.1k tok/s
|
| 15 |
+
[step 120] tok=2.0M R=8 lm=5.5136 13.1k tok/s
|
| 16 |
+
[step 140] tok=2.3M R=8 lm=5.1454 13.2k tok/s
|
| 17 |
+
[step 160] tok=2.6M R=8 lm=5.1089 13.2k tok/s
|
| 18 |
+
[step 180] tok=2.9M R=8 lm=4.9896 13.2k tok/s
|
| 19 |
+
[step 200] tok=3.3M R=8 lm=5.0350 13.2k tok/s
|
| 20 |
+
[step 220] tok=3.6M R=8 lm=4.9458 13.2k tok/s
|
| 21 |
+
[step 240] tok=3.9M R=8 lm=5.0722 13.2k tok/s
|
| 22 |
+
[step 260] tok=4.3M R=8 lm=4.7805 13.2k tok/s
|
| 23 |
+
[step 280] tok=4.6M R=8 lm=4.8617 13.2k tok/s
|
| 24 |
+
[step 300] tok=4.9M R=8 lm=4.6075 13.2k tok/s
|
| 25 |
+
[step 320] tok=5.2M R=8 lm=4.8984 13.2k tok/s
|
| 26 |
+
[step 340] tok=5.6M R=8 lm=4.6752 13.2k tok/s
|
| 27 |
+
[step 360] tok=5.9M R=8 lm=4.5779 13.2k tok/s
|
| 28 |
+
[step 380] tok=6.2M R=8 lm=4.5195 13.2k tok/s
|
| 29 |
+
[step 400] tok=6.6M R=8 lm=4.4749 13.3k tok/s
|
| 30 |
+
[step 420] tok=6.9M R=8 lm=4.5312 13.3k tok/s
|
| 31 |
+
[step 440] tok=7.2M R=8 lm=4.6045 13.3k tok/s
|
| 32 |
+
[step 460] tok=7.5M R=8 lm=4.3441 13.3k tok/s
|
| 33 |
+
[step 480] tok=7.9M R=8 lm=4.5783 13.3k tok/s
|
| 34 |
+
[step 500] tok=8.2M R=8 lm=4.5314 13.3k tok/s
|
| 35 |
+
[step 520] tok=8.5M R=8 lm=4.5672 13.3k tok/s
|
| 36 |
+
[step 540] tok=8.8M R=8 lm=4.2938 13.3k tok/s
|
| 37 |
+
[step 560] tok=9.2M R=8 lm=4.4947 13.3k tok/s
|
| 38 |
+
[step 580] tok=9.5M R=8 lm=4.3953 13.3k tok/s
|
| 39 |
+
[step 600] tok=9.8M R=8 lm=4.1726 13.3k tok/s
|
| 40 |
+
[step 620] tok=10.2M R=8 lm=4.2976 13.3k tok/s
|
| 41 |
+
[step 640] tok=10.5M R=8 lm=4.5372 13.3k tok/s
|
| 42 |
+
[step 660] tok=10.8M R=8 lm=4.5683 13.3k tok/s
|
| 43 |
+
[step 680] tok=11.1M R=8 lm=4.3489 13.3k tok/s
|
| 44 |
+
[step 700] tok=11.5M R=8 lm=4.4052 13.3k tok/s
|
| 45 |
+
[step 720] tok=11.8M R=8 lm=4.4298 13.3k tok/s
|
| 46 |
+
[step 740] tok=12.1M R=8 lm=4.3185 13.3k tok/s
|
| 47 |
+
[step 760] tok=12.5M R=8 lm=4.4732 13.3k tok/s
|
| 48 |
+
[step 780] tok=12.8M R=8 lm=4.5745 13.3k tok/s
|
| 49 |
+
[step 800] tok=13.1M R=8 lm=4.4922 13.3k tok/s
|
| 50 |
+
[step 820] tok=13.4M R=8 lm=4.2537 13.3k tok/s
|
| 51 |
+
[step 840] tok=13.8M R=8 lm=4.4566 13.3k tok/s
|
| 52 |
+
[step 860] tok=14.1M R=8 lm=4.2292 13.3k tok/s
|
| 53 |
+
[step 880] tok=14.4M R=8 lm=4.3961 13.3k tok/s
|
| 54 |
+
[step 900] tok=14.7M R=8 lm=4.3333 13.3k tok/s
|
| 55 |
+
[step 920] tok=15.1M R=8 lm=4.2244 13.3k tok/s
|
| 56 |
+
[step 940] tok=15.4M R=8 lm=4.3333 13.3k tok/s
|
| 57 |
+
[step 960] tok=15.7M R=8 lm=4.3701 13.3k tok/s
|
| 58 |
+
[step 980] tok=16.1M R=8 lm=4.5740 13.3k tok/s
|
| 59 |
+
[step 1000] tok=16.4M R=8 lm=4.1214 13.3k tok/s
|
| 60 |
+
[step 1020] tok=16.7M R=8 lm=4.4298 13.3k tok/s
|
| 61 |
+
[step 1040] tok=17.0M R=8 lm=4.2061 13.3k tok/s
|
| 62 |
+
[step 1060] tok=17.4M R=8 lm=4.5118 13.3k tok/s
|
| 63 |
+
[step 1080] tok=17.7M R=8 lm=4.2956 13.3k tok/s
|
| 64 |
+
[step 1100] tok=18.0M R=8 lm=4.2545 13.3k tok/s
|
| 65 |
+
[step 1120] tok=18.4M R=8 lm=4.4901 13.3k tok/s
|
| 66 |
+
[step 1140] tok=18.7M R=8 lm=4.2991 13.3k tok/s
|
| 67 |
+
[step 1160] tok=19.0M R=8 lm=4.0482 13.3k tok/s
|
| 68 |
+
[step 1180] tok=19.3M R=8 lm=4.4636 13.3k tok/s
|
| 69 |
+
[step 1200] tok=19.7M R=8 lm=4.5425 13.3k tok/s
|
| 70 |
+
[step 1220] tok=20.0M R=8 lm=4.2878 13.3k tok/s
|
| 71 |
+
[step 1240] tok=20.3M R=8 lm=4.4453 13.3k tok/s
|
| 72 |
+
[step 1260] tok=20.6M R=8 lm=4.1279 13.3k tok/s
|
| 73 |
+
[step 1280] tok=21.0M R=8 lm=4.2373 13.3k tok/s
|
| 74 |
+
[step 1300] tok=21.3M R=8 lm=4.3831 13.3k tok/s
|
| 75 |
+
[step 1320] tok=21.6M R=8 lm=4.3351 13.3k tok/s
|
| 76 |
+
[step 1340] tok=22.0M R=8 lm=3.9852 13.3k tok/s
|
| 77 |
+
[step 1360] tok=22.3M R=8 lm=4.3477 13.3k tok/s
|
| 78 |
+
[step 1380] tok=22.6M R=8 lm=4.1556 13.3k tok/s
|
| 79 |
+
[step 1400] tok=22.9M R=8 lm=4.5547 13.3k tok/s
|
| 80 |
+
[step 1420] tok=23.3M R=8 lm=4.3412 13.3k tok/s
|
| 81 |
+
[step 1440] tok=23.6M R=8 lm=4.2219 13.3k tok/s
|
| 82 |
+
[step 1460] tok=23.9M R=8 lm=4.2242 13.3k tok/s
|
| 83 |
+
[step 1480] tok=24.2M R=8 lm=4.2185 13.3k tok/s
|
| 84 |
+
[step 1500] tok=24.6M R=8 lm=4.1413 13.3k tok/s
|
| 85 |
+
[step 1520] tok=24.9M R=8 lm=4.3574 13.3k tok/s
|
| 86 |
+
[step 1540] tok=25.2M R=8 lm=4.1859 13.3k tok/s
|
| 87 |
+
[step 1560] tok=25.6M R=8 lm=4.2907 13.3k tok/s
|
| 88 |
+
[step 1580] tok=25.9M R=8 lm=4.1431 13.3k tok/s
|
| 89 |
+
[step 1600] tok=26.2M R=8 lm=4.1926 13.3k tok/s
|
| 90 |
+
[step 1620] tok=26.5M R=8 lm=4.0803 13.3k tok/s
|
| 91 |
+
[step 1640] tok=26.9M R=8 lm=4.2269 13.3k tok/s
|
| 92 |
+
[step 1660] tok=27.2M R=8 lm=4.5433 13.3k tok/s
|
| 93 |
+
[step 1680] tok=27.5M R=8 lm=4.1813 13.3k tok/s
|
| 94 |
+
[step 1700] tok=27.9M R=8 lm=4.3357 13.3k tok/s
|
| 95 |
+
[step 1720] tok=28.2M R=8 lm=4.4608 13.3k tok/s
|
| 96 |
+
[step 1740] tok=28.5M R=8 lm=4.1876 13.3k tok/s
|
| 97 |
+
[step 1760] tok=28.8M R=8 lm=4.1818 13.3k tok/s
|
| 98 |
+
[step 1780] tok=29.2M R=8 lm=4.0743 13.3k tok/s
|
| 99 |
+
[step 1800] tok=29.5M R=8 lm=4.2992 13.3k tok/s
|
| 100 |
+
[step 1820] tok=29.8M R=8 lm=4.1345 13.3k tok/s
|
| 101 |
+
[step 1840] tok=30.1M R=8 lm=4.2845 13.3k tok/s
|
| 102 |
+
[step 1860] tok=30.5M R=8 lm=4.2187 13.3k tok/s
|
| 103 |
+
[step 1880] tok=30.8M R=8 lm=4.1795 13.3k tok/s
|
| 104 |
+
[step 1900] tok=31.1M R=8 lm=4.1619 13.3k tok/s
|
| 105 |
+
[step 1920] tok=31.5M R=8 lm=4.1463 13.3k tok/s
|
| 106 |
+
[step 1940] tok=31.8M R=8 lm=4.3935 13.3k tok/s
|
| 107 |
+
[step 1960] tok=32.1M R=8 lm=4.2407 13.3k tok/s
|
| 108 |
+
[step 1980] tok=32.4M R=8 lm=4.2801 13.3k tok/s
|
| 109 |
+
[step 2000] tok=32.8M R=8 lm=4.2504 13.3k tok/s
|
| 110 |
+
[step 2020] tok=33.1M R=8 lm=3.9742 13.3k tok/s
|
| 111 |
+
[step 2040] tok=33.4M R=8 lm=4.1906 13.3k tok/s
|
| 112 |
+
[step 2060] tok=33.8M R=8 lm=4.1547 13.3k tok/s
|
| 113 |
+
[step 2080] tok=34.1M R=8 lm=4.3151 13.3k tok/s
|
| 114 |
+
[step 2100] tok=34.4M R=8 lm=4.2081 13.3k tok/s
|
| 115 |
+
[step 2120] tok=34.7M R=8 lm=4.4290 13.3k tok/s
|
| 116 |
+
[step 2140] tok=35.1M R=8 lm=4.8459 13.3k tok/s
|
| 117 |
+
[step 2160] tok=35.4M R=8 lm=4.1571 13.3k tok/s
|
| 118 |
+
[step 2180] tok=35.7M R=8 lm=4.0291 13.3k tok/s
|
| 119 |
+
[step 2200] tok=36.0M R=8 lm=4.2568 13.3k tok/s
|
| 120 |
+
[step 2220] tok=36.4M R=8 lm=4.4834 13.3k tok/s
|
| 121 |
+
[step 2240] tok=36.7M R=8 lm=4.4240 13.3k tok/s
|
| 122 |
+
[step 2260] tok=37.0M R=8 lm=4.3961 13.3k tok/s
|
| 123 |
+
[step 2280] tok=37.4M R=8 lm=4.0409 13.3k tok/s
|
| 124 |
+
[step 2300] tok=37.7M R=8 lm=4.2425 13.3k tok/s
|
| 125 |
+
[step 2320] tok=38.0M R=8 lm=4.5712 13.3k tok/s
|
| 126 |
+
[step 2340] tok=38.3M R=8 lm=4.4584 13.3k tok/s
|
| 127 |
+
[step 2360] tok=38.7M R=8 lm=4.0467 13.3k tok/s
|
| 128 |
+
[step 2380] tok=39.0M R=8 lm=4.1729 13.3k tok/s
|
| 129 |
+
[step 2400] tok=39.3M R=8 lm=4.5751 13.3k tok/s
|
| 130 |
+
[step 2420] tok=39.6M R=8 lm=4.4346 13.3k tok/s
|
| 131 |
+
[step 2440] tok=40.0M R=8 lm=4.2463 13.3k tok/s
|
| 132 |
+
[step 2460] tok=40.3M R=8 lm=4.1653 13.3k tok/s
|
| 133 |
+
[step 2480] tok=40.6M R=8 lm=4.1707 13.3k tok/s
|
| 134 |
+
[step 2500] tok=41.0M R=8 lm=4.1370 13.3k tok/s
|
| 135 |
+
[step 2520] tok=41.3M R=8 lm=4.3402 13.3k tok/s
|
| 136 |
+
[step 2540] tok=41.6M R=8 lm=4.2687 13.3k tok/s
|
| 137 |
+
[step 2560] tok=41.9M R=8 lm=3.9075 13.3k tok/s
|
| 138 |
+
[step 2580] tok=42.3M R=8 lm=4.5187 13.3k tok/s
|
| 139 |
+
[step 2600] tok=42.6M R=8 lm=4.2202 13.3k tok/s
|
| 140 |
+
[step 2620] tok=42.9M R=8 lm=4.1673 13.3k tok/s
|
| 141 |
+
[step 2640] tok=43.3M R=8 lm=4.1933 13.3k tok/s
|
| 142 |
+
[step 2660] tok=43.6M R=8 lm=4.3625 13.3k tok/s
|
| 143 |
+
[step 2680] tok=43.9M R=8 lm=4.1576 13.3k tok/s
|
| 144 |
+
[step 2700] tok=44.2M R=8 lm=4.4168 13.3k tok/s
|
| 145 |
+
[step 2720] tok=44.6M R=8 lm=3.8026 13.3k tok/s
|
| 146 |
+
[step 2740] tok=44.9M R=8 lm=4.2522 13.3k tok/s
|
| 147 |
+
[step 2760] tok=45.2M R=8 lm=4.1548 13.3k tok/s
|
| 148 |
+
[step 2780] tok=45.5M R=8 lm=4.2310 13.3k tok/s
|
| 149 |
+
[step 2800] tok=45.9M R=8 lm=4.4026 13.3k tok/s
|
| 150 |
+
[step 2820] tok=46.2M R=8 lm=3.9929 13.3k tok/s
|
| 151 |
+
[step 2840] tok=46.5M R=8 lm=3.9254 13.3k tok/s
|
| 152 |
+
[step 2860] tok=46.9M R=8 lm=4.1902 13.3k tok/s
|
| 153 |
+
[step 2880] tok=47.2M R=8 lm=4.3201 13.3k tok/s
|
| 154 |
+
[step 2900] tok=47.5M R=8 lm=4.2609 13.3k tok/s
|
| 155 |
+
[step 2920] tok=47.8M R=8 lm=4.2640 13.3k tok/s
|
| 156 |
+
[step 2940] tok=48.2M R=8 lm=3.9803 13.3k tok/s
|
| 157 |
+
[step 2960] tok=48.5M R=8 lm=4.2759 13.3k tok/s
|
| 158 |
+
[step 2980] tok=48.8M R=8 lm=4.1219 13.3k tok/s
|
| 159 |
+
[step 3000] tok=49.2M R=8 lm=4.2844 13.3k tok/s
|
| 160 |
+
[step 3020] tok=49.5M R=8 lm=4.1523 13.3k tok/s
|
| 161 |
+
[step 3040] tok=49.8M R=8 lm=4.2250 13.3k tok/s
|
| 162 |
+
[eval] bake_D arm=D tok=50M trainR=8 BPB=1.3107 swap=0.9999 ent=0.9907
|
| 163 |
+
[step 3060] tok=50.1M R=8 lm=4.0157 13.2k tok/s
|
| 164 |
+
[step 3080] tok=50.5M R=8 lm=4.0854 13.2k tok/s
|
| 165 |
+
[step 3100] tok=50.8M R=8 lm=4.0174 13.2k tok/s
|
| 166 |
+
[step 3120] tok=51.1M R=8 lm=4.3092 13.2k tok/s
|
| 167 |
+
[step 3140] tok=51.4M R=8 lm=4.6425 13.2k tok/s
|
| 168 |
+
[step 3160] tok=51.8M R=8 lm=3.9373 13.2k tok/s
|
| 169 |
+
[step 3180] tok=52.1M R=8 lm=4.2967 13.2k tok/s
|
| 170 |
+
[step 3200] tok=52.4M R=8 lm=4.2293 13.2k tok/s
|
| 171 |
+
[step 3220] tok=52.8M R=8 lm=4.1194 13.2k tok/s
|
| 172 |
+
[step 3240] tok=53.1M R=8 lm=4.0218 13.2k tok/s
|
| 173 |
+
[step 3260] tok=53.4M R=8 lm=3.6839 13.2k tok/s
|
| 174 |
+
[step 3280] tok=53.7M R=8 lm=4.2832 13.2k tok/s
|
| 175 |
+
[step 3300] tok=54.1M R=8 lm=4.1578 13.2k tok/s
|
| 176 |
+
[step 3320] tok=54.4M R=8 lm=4.1589 13.2k tok/s
|
| 177 |
+
[step 3340] tok=54.7M R=8 lm=4.1771 13.2k tok/s
|
| 178 |
+
[step 3360] tok=55.1M R=8 lm=4.2066 13.2k tok/s
|
| 179 |
+
[step 3380] tok=55.4M R=8 lm=4.4062 13.2k tok/s
|
| 180 |
+
[step 3400] tok=55.7M R=8 lm=3.9921 13.2k tok/s
|
| 181 |
+
[step 3420] tok=56.0M R=8 lm=4.0612 13.2k tok/s
|
| 182 |
+
[step 3440] tok=56.4M R=8 lm=4.4169 13.2k tok/s
|
| 183 |
+
[step 3460] tok=56.7M R=8 lm=4.1235 13.2k tok/s
|
| 184 |
+
[step 3480] tok=57.0M R=8 lm=4.1146 13.2k tok/s
|
| 185 |
+
[step 3500] tok=57.3M R=8 lm=4.3709 13.2k tok/s
|
| 186 |
+
[step 3520] tok=57.7M R=8 lm=4.1815 13.2k tok/s
|
| 187 |
+
[step 3540] tok=58.0M R=8 lm=4.2681 13.2k tok/s
|
| 188 |
+
[step 3560] tok=58.3M R=8 lm=4.3057 13.2k tok/s
|
| 189 |
+
[step 3580] tok=58.7M R=8 lm=4.4898 13.2k tok/s
|
| 190 |
+
[step 3600] tok=59.0M R=8 lm=4.1025 13.2k tok/s
|
| 191 |
+
[step 3620] tok=59.3M R=8 lm=4.0490 13.2k tok/s
|
| 192 |
+
[step 3640] tok=59.6M R=8 lm=4.2191 13.2k tok/s
|
| 193 |
+
[step 3660] tok=60.0M R=8 lm=4.0280 13.2k tok/s
|
| 194 |
+
[step 3680] tok=60.3M R=8 lm=4.0714 13.2k tok/s
|
| 195 |
+
[step 3700] tok=60.6M R=8 lm=4.1552 13.2k tok/s
|
| 196 |
+
[step 3720] tok=60.9M R=8 lm=4.2463 13.2k tok/s
|
| 197 |
+
[step 3740] tok=61.3M R=8 lm=4.2768 13.2k tok/s
|
| 198 |
+
[step 3760] tok=61.6M R=8 lm=4.1007 13.2k tok/s
|
| 199 |
+
[step 3780] tok=61.9M R=8 lm=4.2205 13.2k tok/s
|
| 200 |
+
[step 3800] tok=62.3M R=8 lm=3.9779 13.2k tok/s
|
| 201 |
+
[step 3820] tok=62.6M R=8 lm=4.1023 13.2k tok/s
|
| 202 |
+
[step 3840] tok=62.9M R=8 lm=4.1278 13.2k tok/s
|
| 203 |
+
[step 3860] tok=63.2M R=8 lm=3.8440 13.2k tok/s
|
| 204 |
+
[step 3880] tok=63.6M R=8 lm=4.2601 13.2k tok/s
|
| 205 |
+
[step 3900] tok=63.9M R=8 lm=4.2262 13.2k tok/s
|
| 206 |
+
[step 3920] tok=64.2M R=8 lm=4.2320 13.2k tok/s
|
| 207 |
+
[step 3940] tok=64.6M R=8 lm=4.2545 13.2k tok/s
|
| 208 |
+
[step 3960] tok=64.9M R=8 lm=4.1170 13.2k tok/s
|
| 209 |
+
[step 3980] tok=65.2M R=8 lm=4.0547 13.2k tok/s
|
| 210 |
+
[step 4000] tok=65.5M R=8 lm=4.0600 13.2k tok/s
|
| 211 |
+
[step 4020] tok=65.9M R=8 lm=4.1684 13.2k tok/s
|
| 212 |
+
[step 4040] tok=66.2M R=8 lm=4.6116 13.2k tok/s
|
| 213 |
+
[step 4060] tok=66.5M R=8 lm=4.0236 13.2k tok/s
|
| 214 |
+
[step 4080] tok=66.8M R=8 lm=4.1103 13.2k tok/s
|
| 215 |
+
[step 4100] tok=67.2M R=8 lm=4.0586 13.2k tok/s
|
| 216 |
+
[step 4120] tok=67.5M R=8 lm=4.2027 13.2k tok/s
|
| 217 |
+
[step 4140] tok=67.8M R=8 lm=4.1366 13.2k tok/s
|
| 218 |
+
[step 4160] tok=68.2M R=8 lm=4.1645 13.2k tok/s
|
| 219 |
+
[step 4180] tok=68.5M R=8 lm=4.0567 13.2k tok/s
|
| 220 |
+
[step 4200] tok=68.8M R=8 lm=4.3338 13.2k tok/s
|
| 221 |
+
[step 4220] tok=69.1M R=8 lm=4.3180 13.2k tok/s
|
| 222 |
+
[step 4240] tok=69.5M R=8 lm=4.0450 13.2k tok/s
|
| 223 |
+
[step 4260] tok=69.8M R=8 lm=4.0741 13.2k tok/s
|
| 224 |
+
[step 4280] tok=70.1M R=8 lm=4.0433 13.2k tok/s
|
| 225 |
+
[step 4300] tok=70.5M R=8 lm=4.2003 13.2k tok/s
|
| 226 |
+
[step 4320] tok=70.8M R=8 lm=4.2848 13.3k tok/s
|
| 227 |
+
[step 4340] tok=71.1M R=8 lm=4.1618 13.3k tok/s
|
| 228 |
+
[step 4360] tok=71.4M R=8 lm=4.2896 13.3k tok/s
|
| 229 |
+
[step 4380] tok=71.8M R=8 lm=4.0699 13.3k tok/s
|
| 230 |
+
[step 4400] tok=72.1M R=8 lm=4.1074 13.3k tok/s
|
| 231 |
+
[step 4420] tok=72.4M R=8 lm=4.2278 13.3k tok/s
|
| 232 |
+
[step 4440] tok=72.7M R=8 lm=4.2055 13.3k tok/s
|
| 233 |
+
[step 4460] tok=73.1M R=8 lm=3.8970 13.3k tok/s
|
| 234 |
+
[step 4480] tok=73.4M R=8 lm=4.2186 13.3k tok/s
|
| 235 |
+
[step 4500] tok=73.7M R=8 lm=4.2684 13.3k tok/s
|
| 236 |
+
[step 4520] tok=74.1M R=8 lm=4.4149 13.3k tok/s
|
| 237 |
+
[step 4540] tok=74.4M R=8 lm=4.1262 13.3k tok/s
|
| 238 |
+
[step 4560] tok=74.7M R=8 lm=4.2830 13.3k tok/s
|
| 239 |
+
[step 4580] tok=75.0M R=8 lm=4.0012 13.3k tok/s
|
| 240 |
+
[step 4600] tok=75.4M R=8 lm=4.1652 13.3k tok/s
|
| 241 |
+
[step 4620] tok=75.7M R=8 lm=4.2341 13.3k tok/s
|
| 242 |
+
[step 4640] tok=76.0M R=8 lm=4.1731 13.3k tok/s
|
| 243 |
+
[step 4660] tok=76.3M R=8 lm=4.1117 13.3k tok/s
|
| 244 |
+
[step 4680] tok=76.7M R=8 lm=4.3940 13.3k tok/s
|
| 245 |
+
[step 4700] tok=77.0M R=8 lm=4.2149 13.3k tok/s
|
| 246 |
+
[step 4720] tok=77.3M R=8 lm=4.2570 13.3k tok/s
|
| 247 |
+
[step 4740] tok=77.7M R=8 lm=4.0190 13.3k tok/s
|
| 248 |
+
[step 4760] tok=78.0M R=8 lm=4.2636 13.3k tok/s
|
| 249 |
+
[step 4780] tok=78.3M R=8 lm=4.1530 13.3k tok/s
|
| 250 |
+
[step 4800] tok=78.6M R=8 lm=4.2322 13.3k tok/s
|
| 251 |
+
[step 4820] tok=79.0M R=8 lm=4.0896 13.3k tok/s
|
| 252 |
+
[step 4840] tok=79.3M R=8 lm=4.1676 13.3k tok/s
|
| 253 |
+
[step 4860] tok=79.6M R=8 lm=4.1995 13.3k tok/s
|
| 254 |
+
[step 4880] tok=80.0M R=8 lm=4.0474 13.3k tok/s
|
| 255 |
+
[step 4900] tok=80.3M R=8 lm=4.2666 13.3k tok/s
|
| 256 |
+
[step 4920] tok=80.6M R=8 lm=4.2494 13.3k tok/s
|
| 257 |
+
[step 4940] tok=80.9M R=8 lm=4.2096 13.3k tok/s
|
| 258 |
+
[step 4960] tok=81.3M R=8 lm=4.3511 13.3k tok/s
|
| 259 |
+
[step 4980] tok=81.6M R=8 lm=4.3448 13.3k tok/s
|
| 260 |
+
[step 5000] tok=81.9M R=8 lm=4.0467 13.3k tok/s
|
| 261 |
+
[step 5020] tok=82.2M R=8 lm=4.4560 13.3k tok/s
|
| 262 |
+
[step 5040] tok=82.6M R=8 lm=4.4135 13.3k tok/s
|
| 263 |
+
[step 5060] tok=82.9M R=8 lm=3.9732 13.3k tok/s
|
| 264 |
+
[step 5080] tok=83.2M R=8 lm=3.9842 13.3k tok/s
|
| 265 |
+
[step 5100] tok=83.6M R=8 lm=4.1941 13.3k tok/s
|
| 266 |
+
[step 5120] tok=83.9M R=8 lm=4.0500 13.3k tok/s
|
| 267 |
+
[step 5140] tok=84.2M R=8 lm=3.7773 13.3k tok/s
|
| 268 |
+
[step 5160] tok=84.5M R=8 lm=4.1919 13.3k tok/s
|
| 269 |
+
[step 5180] tok=84.9M R=8 lm=4.2908 13.3k tok/s
|
| 270 |
+
[step 5200] tok=85.2M R=8 lm=4.1682 13.3k tok/s
|
| 271 |
+
[step 5220] tok=85.5M R=8 lm=4.3041 13.3k tok/s
|
| 272 |
+
[step 5240] tok=85.9M R=8 lm=4.3278 13.3k tok/s
|
| 273 |
+
[step 5260] tok=86.2M R=8 lm=4.4030 13.3k tok/s
|
| 274 |
+
[step 5280] tok=86.5M R=8 lm=4.1310 13.3k tok/s
|
| 275 |
+
[step 5300] tok=86.8M R=8 lm=3.7058 13.3k tok/s
|
| 276 |
+
[step 5320] tok=87.2M R=8 lm=4.1210 13.3k tok/s
|
| 277 |
+
[step 5340] tok=87.5M R=8 lm=4.2719 13.3k tok/s
|
| 278 |
+
[step 5360] tok=87.8M R=8 lm=4.2261 13.3k tok/s
|
| 279 |
+
[step 5380] tok=88.1M R=8 lm=4.1281 13.3k tok/s
|
| 280 |
+
[step 5400] tok=88.5M R=8 lm=4.0463 13.3k tok/s
|
| 281 |
+
[step 5420] tok=88.8M R=8 lm=4.0513 13.3k tok/s
|
| 282 |
+
[step 5440] tok=89.1M R=8 lm=4.3740 13.3k tok/s
|
| 283 |
+
[step 5460] tok=89.5M R=8 lm=4.4185 13.3k tok/s
|
| 284 |
+
[step 5480] tok=89.8M R=8 lm=4.0256 13.3k tok/s
|
| 285 |
+
[step 5500] tok=90.1M R=8 lm=4.1151 13.3k tok/s
|
| 286 |
+
[step 5520] tok=90.4M R=8 lm=3.9232 13.3k tok/s
|
| 287 |
+
[step 5540] tok=90.8M R=8 lm=4.0703 13.3k tok/s
|
| 288 |
+
[step 5560] tok=91.1M R=8 lm=4.0384 13.3k tok/s
|
| 289 |
+
[step 5580] tok=91.4M R=8 lm=4.0981 13.3k tok/s
|
| 290 |
+
[step 5600] tok=91.8M R=8 lm=4.1083 13.3k tok/s
|
| 291 |
+
[step 5620] tok=92.1M R=8 lm=4.1907 13.3k tok/s
|
| 292 |
+
[step 5640] tok=92.4M R=8 lm=4.0522 13.3k tok/s
|
| 293 |
+
[step 5660] tok=92.7M R=8 lm=4.3376 13.3k tok/s
|
| 294 |
+
[step 5680] tok=93.1M R=8 lm=4.1709 13.3k tok/s
|
| 295 |
+
[step 5700] tok=93.4M R=8 lm=4.3449 13.3k tok/s
|
| 296 |
+
[step 5720] tok=93.7M R=8 lm=4.1193 13.3k tok/s
|
| 297 |
+
[step 5740] tok=94.0M R=8 lm=4.0742 13.3k tok/s
|
| 298 |
+
[step 5760] tok=94.4M R=8 lm=4.2426 13.3k tok/s
|
| 299 |
+
[step 5780] tok=94.7M R=8 lm=4.2188 13.3k tok/s
|
| 300 |
+
[step 5800] tok=95.0M R=8 lm=4.0929 13.3k tok/s
|
| 301 |
+
[step 5820] tok=95.4M R=8 lm=4.3391 13.3k tok/s
|
| 302 |
+
[step 5840] tok=95.7M R=8 lm=4.2657 13.3k tok/s
|
| 303 |
+
[step 5860] tok=96.0M R=8 lm=3.9710 13.3k tok/s
|
| 304 |
+
[step 5880] tok=96.3M R=8 lm=4.0930 13.3k tok/s
|
| 305 |
+
[step 5900] tok=96.7M R=8 lm=4.2920 13.3k tok/s
|
| 306 |
+
[step 5920] tok=97.0M R=8 lm=4.2685 13.3k tok/s
|
| 307 |
+
[step 5940] tok=97.3M R=8 lm=4.0311 13.3k tok/s
|
| 308 |
+
[step 5960] tok=97.6M R=8 lm=3.8900 13.3k tok/s
|
| 309 |
+
[step 5980] tok=98.0M R=8 lm=4.2683 13.3k tok/s
|
| 310 |
+
[step 6000] tok=98.3M R=8 lm=3.9889 13.3k tok/s
|
| 311 |
+
[step 6020] tok=98.6M R=8 lm=4.1683 13.3k tok/s
|
| 312 |
+
[step 6040] tok=99.0M R=8 lm=4.3639 13.3k tok/s
|
| 313 |
+
[step 6060] tok=99.3M R=8 lm=4.1673 13.3k tok/s
|
| 314 |
+
[step 6080] tok=99.6M R=8 lm=4.0621 13.3k tok/s
|
| 315 |
+
[step 6100] tok=99.9M R=8 lm=4.0734 13.3k tok/s
|
| 316 |
+
[eval] bake_D arm=D tok=100M trainR=8 BPB=1.2986 swap=0.9999 ent=0.9896
|
| 317 |
+
[step 6120] tok=100.3M R=8 lm=4.1302 13.2k tok/s
|
| 318 |
+
[step 6140] tok=100.6M R=8 lm=3.9925 13.2k tok/s
|
| 319 |
+
[step 6160] tok=100.9M R=8 lm=4.1721 13.2k tok/s
|
| 320 |
+
[step 6180] tok=101.3M R=8 lm=4.3195 13.2k tok/s
|
| 321 |
+
[step 6200] tok=101.6M R=8 lm=4.0136 13.2k tok/s
|
| 322 |
+
[step 6220] tok=101.9M R=8 lm=4.2353 13.2k tok/s
|
| 323 |
+
[step 6240] tok=102.2M R=8 lm=4.1986 13.2k tok/s
|
| 324 |
+
[step 6260] tok=102.6M R=8 lm=4.4264 13.2k tok/s
|
| 325 |
+
[step 6280] tok=102.9M R=8 lm=4.1727 13.2k tok/s
|
| 326 |
+
[step 6300] tok=103.2M R=8 lm=4.5947 13.2k tok/s
|
| 327 |
+
[step 6320] tok=103.5M R=8 lm=4.2704 13.2k tok/s
|
| 328 |
+
[step 6340] tok=103.9M R=8 lm=4.1422 13.2k tok/s
|
| 329 |
+
[step 6360] tok=104.2M R=8 lm=4.2438 13.2k tok/s
|
| 330 |
+
[step 6380] tok=104.5M R=8 lm=4.0660 13.2k tok/s
|
| 331 |
+
[step 6400] tok=104.9M R=8 lm=4.3582 13.2k tok/s
|
| 332 |
+
[step 6420] tok=105.2M R=8 lm=4.2335 13.2k tok/s
|
| 333 |
+
[step 6440] tok=105.5M R=8 lm=4.0802 13.2k tok/s
|
| 334 |
+
[step 6460] tok=105.8M R=8 lm=4.2736 13.2k tok/s
|
| 335 |
+
[step 6480] tok=106.2M R=8 lm=4.1754 13.2k tok/s
|
| 336 |
+
[step 6500] tok=106.5M R=8 lm=4.1788 13.2k tok/s
|
| 337 |
+
[step 6520] tok=106.8M R=8 lm=4.0640 13.2k tok/s
|
| 338 |
+
[step 6540] tok=107.2M R=8 lm=3.9721 13.2k tok/s
|
| 339 |
+
[step 6560] tok=107.5M R=8 lm=4.4736 13.2k tok/s
|
| 340 |
+
[step 6580] tok=107.8M R=8 lm=4.1506 13.2k tok/s
|
| 341 |
+
[step 6600] tok=108.1M R=8 lm=4.2157 13.2k tok/s
|
| 342 |
+
[step 6620] tok=108.5M R=8 lm=4.1459 13.2k tok/s
|
| 343 |
+
[step 6640] tok=108.8M R=8 lm=4.0589 13.2k tok/s
|
| 344 |
+
[step 6660] tok=109.1M R=8 lm=4.2030 13.2k tok/s
|
| 345 |
+
[step 6680] tok=109.4M R=8 lm=4.1952 13.2k tok/s
|
| 346 |
+
[step 6700] tok=109.8M R=8 lm=4.3551 13.2k tok/s
|
| 347 |
+
[step 6720] tok=110.1M R=8 lm=4.2156 13.2k tok/s
|
| 348 |
+
[step 6740] tok=110.4M R=8 lm=4.1901 13.2k tok/s
|
| 349 |
+
[step 6760] tok=110.8M R=8 lm=3.8247 13.2k tok/s
|
| 350 |
+
[step 6780] tok=111.1M R=8 lm=4.0494 13.2k tok/s
|
| 351 |
+
[step 6800] tok=111.4M R=8 lm=4.1285 13.2k tok/s
|
| 352 |
+
[step 6820] tok=111.7M R=8 lm=3.9621 13.2k tok/s
|
| 353 |
+
[step 6840] tok=112.1M R=8 lm=4.1342 13.2k tok/s
|
| 354 |
+
[step 6860] tok=112.4M R=8 lm=4.4542 13.2k tok/s
|
| 355 |
+
[step 6880] tok=112.7M R=8 lm=4.1255 13.2k tok/s
|
| 356 |
+
[step 6900] tok=113.0M R=8 lm=3.9207 13.2k tok/s
|
| 357 |
+
[step 6920] tok=113.4M R=8 lm=4.3228 13.2k tok/s
|
| 358 |
+
[step 6940] tok=113.7M R=8 lm=4.2222 13.2k tok/s
|
| 359 |
+
[step 6960] tok=114.0M R=8 lm=4.1936 13.2k tok/s
|
| 360 |
+
[step 6980] tok=114.4M R=8 lm=4.3641 13.2k tok/s
|
| 361 |
+
[step 7000] tok=114.7M R=8 lm=4.2177 13.2k tok/s
|
| 362 |
+
[step 7020] tok=115.0M R=8 lm=4.1109 13.2k tok/s
|
| 363 |
+
[step 7040] tok=115.3M R=8 lm=4.1601 13.2k tok/s
|
| 364 |
+
[step 7060] tok=115.7M R=8 lm=4.1467 13.2k tok/s
|
| 365 |
+
[step 7080] tok=116.0M R=8 lm=4.1185 13.2k tok/s
|
| 366 |
+
[step 7100] tok=116.3M R=8 lm=4.2546 13.2k tok/s
|
| 367 |
+
[step 7120] tok=116.7M R=8 lm=4.0821 13.2k tok/s
|
| 368 |
+
[step 7140] tok=117.0M R=8 lm=4.2319 13.2k tok/s
|
| 369 |
+
[step 7160] tok=117.3M R=8 lm=3.9671 13.2k tok/s
|
| 370 |
+
[step 7180] tok=117.6M R=8 lm=3.9331 13.2k tok/s
|
| 371 |
+
[step 7200] tok=118.0M R=8 lm=4.2256 13.2k tok/s
|
| 372 |
+
[step 7220] tok=118.3M R=8 lm=3.8907 13.2k tok/s
|
| 373 |
+
[step 7240] tok=118.6M R=8 lm=4.0976 13.2k tok/s
|
| 374 |
+
[step 7260] tok=118.9M R=8 lm=4.2230 13.2k tok/s
|
| 375 |
+
[step 7280] tok=119.3M R=8 lm=4.2110 13.2k tok/s
|
| 376 |
+
[step 7300] tok=119.6M R=8 lm=4.3674 13.2k tok/s
|
| 377 |
+
[step 7320] tok=119.9M R=8 lm=4.1057 13.2k tok/s
|
| 378 |
+
[step 7340] tok=120.3M R=8 lm=3.9630 13.2k tok/s
|
| 379 |
+
[step 7360] tok=120.6M R=8 lm=4.1539 13.2k tok/s
|
| 380 |
+
[step 7380] tok=120.9M R=8 lm=4.1580 13.2k tok/s
|
| 381 |
+
[step 7400] tok=121.2M R=8 lm=4.3848 13.2k tok/s
|
| 382 |
+
[step 7420] tok=121.6M R=8 lm=4.2835 13.2k tok/s
|
| 383 |
+
[step 7440] tok=121.9M R=8 lm=4.2980 13.2k tok/s
|
| 384 |
+
[step 7460] tok=122.2M R=8 lm=4.2432 13.2k tok/s
|
| 385 |
+
[step 7480] tok=122.6M R=8 lm=4.2790 13.2k tok/s
|
| 386 |
+
[step 7500] tok=122.9M R=8 lm=4.0997 13.2k tok/s
|
| 387 |
+
[step 7520] tok=123.2M R=8 lm=4.3052 13.2k tok/s
|
| 388 |
+
[step 7540] tok=123.5M R=8 lm=4.4240 13.2k tok/s
|
| 389 |
+
[step 7560] tok=123.9M R=8 lm=4.1847 13.2k tok/s
|
| 390 |
+
[step 7580] tok=124.2M R=8 lm=4.1143 13.2k tok/s
|
| 391 |
+
[step 7600] tok=124.5M R=8 lm=4.1882 13.2k tok/s
|
| 392 |
+
[step 7620] tok=124.8M R=8 lm=4.0577 13.2k tok/s
|
| 393 |
+
[step 7640] tok=125.2M R=8 lm=4.1126 13.2k tok/s
|
| 394 |
+
[step 7660] tok=125.5M R=8 lm=4.2597 13.2k tok/s
|
| 395 |
+
[step 7680] tok=125.8M R=8 lm=4.4429 13.2k tok/s
|
| 396 |
+
[step 7700] tok=126.2M R=8 lm=3.8772 13.2k tok/s
|
| 397 |
+
[step 7720] tok=126.5M R=8 lm=4.1576 13.2k tok/s
|
| 398 |
+
[step 7740] tok=126.8M R=8 lm=4.0195 13.2k tok/s
|
| 399 |
+
[step 7760] tok=127.1M R=8 lm=4.2405 13.2k tok/s
|
| 400 |
+
[step 7780] tok=127.5M R=8 lm=4.2171 13.2k tok/s
|
| 401 |
+
[step 7800] tok=127.8M R=8 lm=3.9190 13.2k tok/s
|
| 402 |
+
[step 7820] tok=128.1M R=8 lm=4.0346 13.2k tok/s
|
| 403 |
+
[step 7840] tok=128.5M R=8 lm=4.0974 13.2k tok/s
|
| 404 |
+
[step 7860] tok=128.8M R=8 lm=4.0511 13.2k tok/s
|
| 405 |
+
[step 7880] tok=129.1M R=8 lm=4.1028 13.2k tok/s
|
| 406 |
+
[step 7900] tok=129.4M R=8 lm=4.1322 13.2k tok/s
|
| 407 |
+
[step 7920] tok=129.8M R=8 lm=4.0662 13.2k tok/s
|
| 408 |
+
[step 7940] tok=130.1M R=8 lm=4.3994 13.2k tok/s
|
| 409 |
+
[step 7960] tok=130.4M R=8 lm=4.2310 13.2k tok/s
|
| 410 |
+
[step 7980] tok=130.7M R=8 lm=4.1311 13.2k tok/s
|
| 411 |
+
[step 8000] tok=131.1M R=8 lm=4.1891 13.2k tok/s
|
| 412 |
+
[step 8020] tok=131.4M R=8 lm=4.2235 13.2k tok/s
|
| 413 |
+
[step 8040] tok=131.7M R=8 lm=4.3665 13.2k tok/s
|
| 414 |
+
[step 8060] tok=132.1M R=8 lm=3.9766 13.2k tok/s
|
| 415 |
+
[step 8080] tok=132.4M R=8 lm=4.1767 13.2k tok/s
|
| 416 |
+
[step 8100] tok=132.7M R=8 lm=4.1185 13.2k tok/s
|
| 417 |
+
[step 8120] tok=133.0M R=8 lm=4.3954 13.2k tok/s
|
| 418 |
+
[step 8140] tok=133.4M R=8 lm=4.4583 13.2k tok/s
|
| 419 |
+
[step 8160] tok=133.7M R=8 lm=4.0624 13.2k tok/s
|
| 420 |
+
[step 8180] tok=134.0M R=8 lm=4.0646 13.2k tok/s
|
| 421 |
+
[step 8200] tok=134.3M R=8 lm=4.2296 13.2k tok/s
|
| 422 |
+
[step 8220] tok=134.7M R=8 lm=4.2260 13.2k tok/s
|
| 423 |
+
[step 8240] tok=135.0M R=8 lm=4.0718 13.2k tok/s
|
| 424 |
+
[step 8260] tok=135.3M R=8 lm=4.0469 13.2k tok/s
|
| 425 |
+
[step 8280] tok=135.7M R=8 lm=4.2486 13.2k tok/s
|
| 426 |
+
[step 8300] tok=136.0M R=8 lm=3.8261 13.2k tok/s
|
| 427 |
+
[step 8320] tok=136.3M R=8 lm=4.2498 13.2k tok/s
|
| 428 |
+
[step 8340] tok=136.6M R=8 lm=4.1918 13.2k tok/s
|
| 429 |
+
[step 8360] tok=137.0M R=8 lm=4.2357 13.2k tok/s
|
| 430 |
+
[step 8380] tok=137.3M R=8 lm=4.0128 13.2k tok/s
|
| 431 |
+
[step 8400] tok=137.6M R=8 lm=4.2433 13.2k tok/s
|
| 432 |
+
[step 8420] tok=138.0M R=8 lm=3.8824 13.2k tok/s
|
| 433 |
+
[step 8440] tok=138.3M R=8 lm=4.4281 13.2k tok/s
|
| 434 |
+
[step 8460] tok=138.6M R=8 lm=4.1728 13.2k tok/s
|
| 435 |
+
[step 8480] tok=138.9M R=8 lm=4.4768 13.2k tok/s
|
| 436 |
+
[step 8500] tok=139.3M R=8 lm=4.0181 13.2k tok/s
|
| 437 |
+
[step 8520] tok=139.6M R=8 lm=4.2452 13.2k tok/s
|
| 438 |
+
[step 8540] tok=139.9M R=8 lm=4.1472 13.2k tok/s
|
| 439 |
+
[step 8560] tok=140.2M R=8 lm=4.2799 13.2k tok/s
|
| 440 |
+
[step 8580] tok=140.6M R=8 lm=4.2514 13.2k tok/s
|
| 441 |
+
[step 8600] tok=140.9M R=8 lm=4.0303 13.2k tok/s
|
| 442 |
+
[step 8620] tok=141.2M R=8 lm=4.0755 13.2k tok/s
|
| 443 |
+
[step 8640] tok=141.6M R=8 lm=4.1158 13.2k tok/s
|
| 444 |
+
[step 8660] tok=141.9M R=8 lm=4.0658 13.2k tok/s
|
| 445 |
+
[step 8680] tok=142.2M R=8 lm=4.0836 13.2k tok/s
|
| 446 |
+
[step 8700] tok=142.5M R=8 lm=4.1491 13.2k tok/s
|
| 447 |
+
[step 8720] tok=142.9M R=8 lm=4.2384 13.2k tok/s
|
| 448 |
+
[step 8740] tok=143.2M R=8 lm=4.1371 13.2k tok/s
|
| 449 |
+
[step 8760] tok=143.5M R=8 lm=4.0053 13.2k tok/s
|
| 450 |
+
[step 8780] tok=143.9M R=8 lm=4.2097 13.2k tok/s
|
| 451 |
+
[step 8800] tok=144.2M R=8 lm=4.2855 13.2k tok/s
|
| 452 |
+
[step 8820] tok=144.5M R=8 lm=4.2205 13.2k tok/s
|
| 453 |
+
[step 8840] tok=144.8M R=8 lm=3.9257 13.2k tok/s
|
| 454 |
+
[step 8860] tok=145.2M R=8 lm=4.0429 13.2k tok/s
|
| 455 |
+
[step 8880] tok=145.5M R=8 lm=4.2112 13.2k tok/s
|
| 456 |
+
[step 8900] tok=145.8M R=8 lm=3.8896 13.2k tok/s
|
| 457 |
+
[step 8920] tok=146.1M R=8 lm=3.8358 13.2k tok/s
|
| 458 |
+
[step 8940] tok=146.5M R=8 lm=4.4740 13.2k tok/s
|
| 459 |
+
[step 8960] tok=146.8M R=8 lm=3.9902 13.2k tok/s
|
| 460 |
+
[step 8980] tok=147.1M R=8 lm=4.1265 13.2k tok/s
|
| 461 |
+
[step 9000] tok=147.5M R=8 lm=3.9345 13.2k tok/s
|
| 462 |
+
[step 9020] tok=147.8M R=8 lm=4.1137 13.2k tok/s
|
| 463 |
+
[step 9040] tok=148.1M R=8 lm=4.0125 13.2k tok/s
|
| 464 |
+
[step 9060] tok=148.4M R=8 lm=4.1526 13.2k tok/s
|
| 465 |
+
[step 9080] tok=148.8M R=8 lm=4.0689 13.2k tok/s
|
| 466 |
+
[step 9100] tok=149.1M R=8 lm=4.1902 13.2k tok/s
|
| 467 |
+
[step 9120] tok=149.4M R=8 lm=4.2528 13.2k tok/s
|
| 468 |
+
[step 9140] tok=149.7M R=8 lm=4.3082 13.2k tok/s
|
| 469 |
+
[eval] bake_D arm=D tok=150M trainR=8 BPB=1.2929 swap=0.9999 ent=0.9896
|
| 470 |
+
[step 9160] tok=150.1M R=8 lm=4.7017 13.2k tok/s
|
| 471 |
+
[step 9180] tok=150.4M R=8 lm=4.2335 13.2k tok/s
|
| 472 |
+
[step 9200] tok=150.7M R=8 lm=4.0653 13.2k tok/s
|
| 473 |
+
[step 9220] tok=151.1M R=8 lm=4.1575 13.2k tok/s
|
| 474 |
+
[step 9240] tok=151.4M R=8 lm=4.2037 13.2k tok/s
|
| 475 |
+
[step 9260] tok=151.7M R=8 lm=4.2195 13.2k tok/s
|
| 476 |
+
[step 9280] tok=152.0M R=8 lm=4.0164 13.2k tok/s
|
| 477 |
+
[step 9300] tok=152.4M R=8 lm=4.3092 13.2k tok/s
|
| 478 |
+
[step 9320] tok=152.7M R=8 lm=4.2841 13.2k tok/s
|
| 479 |
+
[step 9340] tok=153.0M R=8 lm=3.9476 13.2k tok/s
|
| 480 |
+
[step 9360] tok=153.4M R=8 lm=4.3876 13.2k tok/s
|
| 481 |
+
[step 9380] tok=153.7M R=8 lm=4.0285 13.2k tok/s
|
| 482 |
+
[step 9400] tok=154.0M R=8 lm=3.9751 13.2k tok/s
|
| 483 |
+
[step 9420] tok=154.3M R=8 lm=4.0773 13.2k tok/s
|
| 484 |
+
[step 9440] tok=154.7M R=8 lm=3.9715 13.2k tok/s
|
| 485 |
+
[step 9460] tok=155.0M R=8 lm=4.2306 13.2k tok/s
|
| 486 |
+
[step 9480] tok=155.3M R=8 lm=4.2109 13.2k tok/s
|
| 487 |
+
[step 9500] tok=155.6M R=8 lm=4.1018 13.2k tok/s
|
| 488 |
+
[step 9520] tok=156.0M R=8 lm=4.1841 13.2k tok/s
|
| 489 |
+
[step 9540] tok=156.3M R=8 lm=4.1820 13.2k tok/s
|
| 490 |
+
[step 9560] tok=156.6M R=8 lm=4.1027 13.2k tok/s
|
| 491 |
+
[step 9580] tok=157.0M R=8 lm=4.0868 13.2k tok/s
|
| 492 |
+
[step 9600] tok=157.3M R=8 lm=4.2807 13.2k tok/s
|
| 493 |
+
[step 9620] tok=157.6M R=8 lm=4.1132 13.2k tok/s
|
| 494 |
+
[step 9640] tok=157.9M R=8 lm=4.2642 13.2k tok/s
|
| 495 |
+
[step 9660] tok=158.3M R=8 lm=4.0579 13.2k tok/s
|
| 496 |
+
[step 9680] tok=158.6M R=8 lm=4.2276 13.2k tok/s
|
| 497 |
+
[step 9700] tok=158.9M R=8 lm=4.2100 13.2k tok/s
|
| 498 |
+
[step 9720] tok=159.3M R=8 lm=3.9921 13.2k tok/s
|
| 499 |
+
[step 9740] tok=159.6M R=8 lm=4.1428 13.2k tok/s
|
| 500 |
+
[step 9760] tok=159.9M R=8 lm=4.2245 13.2k tok/s
|
| 501 |
+
[step 9780] tok=160.2M R=8 lm=4.2684 13.2k tok/s
|
| 502 |
+
[step 9800] tok=160.6M R=8 lm=3.8722 13.2k tok/s
|
| 503 |
+
[step 9820] tok=160.9M R=8 lm=4.1552 13.2k tok/s
|
| 504 |
+
[step 9840] tok=161.2M R=8 lm=4.0439 13.2k tok/s
|
| 505 |
+
[step 9860] tok=161.5M R=8 lm=4.1812 13.2k tok/s
|
| 506 |
+
[step 9880] tok=161.9M R=8 lm=4.2982 13.2k tok/s
|
| 507 |
+
[step 9900] tok=162.2M R=8 lm=4.1435 13.2k tok/s
|
| 508 |
+
[step 9920] tok=162.5M R=8 lm=3.9581 13.2k tok/s
|
| 509 |
+
[step 9940] tok=162.9M R=8 lm=4.2771 13.2k tok/s
|
| 510 |
+
[step 9960] tok=163.2M R=8 lm=4.1276 13.2k tok/s
|
| 511 |
+
[step 9980] tok=163.5M R=8 lm=3.9848 13.2k tok/s
|
| 512 |
+
[step 10000] tok=163.8M R=8 lm=4.2144 13.2k tok/s
|
| 513 |
+
[step 10020] tok=164.2M R=8 lm=3.9911 13.2k tok/s
|
| 514 |
+
[step 10040] tok=164.5M R=8 lm=3.9740 13.2k tok/s
|
| 515 |
+
[step 10060] tok=164.8M R=8 lm=3.8899 13.2k tok/s
|
| 516 |
+
[step 10080] tok=165.2M R=8 lm=4.1452 13.2k tok/s
|
| 517 |
+
[step 10100] tok=165.5M R=8 lm=4.2708 13.2k tok/s
|
| 518 |
+
[step 10120] tok=165.8M R=8 lm=4.0719 13.2k tok/s
|
| 519 |
+
[step 10140] tok=166.1M R=8 lm=4.1245 13.2k tok/s
|
| 520 |
+
[step 10160] tok=166.5M R=8 lm=4.3255 13.2k tok/s
|
| 521 |
+
[step 10180] tok=166.8M R=8 lm=4.2574 13.2k tok/s
|
| 522 |
+
[step 10200] tok=167.1M R=8 lm=3.8192 13.2k tok/s
|
| 523 |
+
[step 10220] tok=167.4M R=8 lm=4.2185 13.2k tok/s
|
| 524 |
+
[step 10240] tok=167.8M R=8 lm=3.9593 13.2k tok/s
|
| 525 |
+
[step 10260] tok=168.1M R=8 lm=4.2249 13.2k tok/s
|
| 526 |
+
[step 10280] tok=168.4M R=8 lm=4.1738 13.2k tok/s
|
| 527 |
+
[step 10300] tok=168.8M R=8 lm=4.2721 13.2k tok/s
|
| 528 |
+
[step 10320] tok=169.1M R=8 lm=4.0702 13.2k tok/s
|
| 529 |
+
[step 10340] tok=169.4M R=8 lm=4.3212 13.2k tok/s
|
| 530 |
+
[step 10360] tok=169.7M R=8 lm=3.9578 13.2k tok/s
|
| 531 |
+
[step 10380] tok=170.1M R=8 lm=4.2050 13.2k tok/s
|
| 532 |
+
[step 10400] tok=170.4M R=8 lm=3.9025 13.2k tok/s
|
| 533 |
+
[step 10420] tok=170.7M R=8 lm=4.1644 13.2k tok/s
|
| 534 |
+
[step 10440] tok=171.0M R=8 lm=3.8716 13.2k tok/s
|
| 535 |
+
[step 10460] tok=171.4M R=8 lm=4.2199 13.2k tok/s
|
| 536 |
+
[step 10480] tok=171.7M R=8 lm=3.9889 13.2k tok/s
|
| 537 |
+
[step 10500] tok=172.0M R=8 lm=4.0982 13.2k tok/s
|
| 538 |
+
[step 10520] tok=172.4M R=8 lm=4.2012 13.2k tok/s
|
| 539 |
+
[step 10540] tok=172.7M R=8 lm=4.1352 13.2k tok/s
|
| 540 |
+
[step 10560] tok=173.0M R=8 lm=3.8171 13.2k tok/s
|
| 541 |
+
[step 10580] tok=173.3M R=8 lm=4.4530 13.2k tok/s
|
| 542 |
+
[step 10600] tok=173.7M R=8 lm=4.1874 13.2k tok/s
|
| 543 |
+
[step 10620] tok=174.0M R=8 lm=4.0211 13.2k tok/s
|
| 544 |
+
[step 10640] tok=174.3M R=8 lm=4.0000 13.2k tok/s
|
| 545 |
+
[step 10660] tok=174.7M R=8 lm=4.1714 13.2k tok/s
|
| 546 |
+
[step 10680] tok=175.0M R=8 lm=4.2054 13.2k tok/s
|
| 547 |
+
[step 10700] tok=175.3M R=8 lm=4.0694 13.2k tok/s
|
| 548 |
+
[step 10720] tok=175.6M R=8 lm=4.1540 13.2k tok/s
|
| 549 |
+
[step 10740] tok=176.0M R=8 lm=4.2999 13.2k tok/s
|
| 550 |
+
[step 10760] tok=176.3M R=8 lm=4.0691 13.2k tok/s
|
| 551 |
+
[step 10780] tok=176.6M R=8 lm=4.6959 13.2k tok/s
|
| 552 |
+
[step 10800] tok=176.9M R=8 lm=4.1187 13.2k tok/s
|
| 553 |
+
[step 10820] tok=177.3M R=8 lm=3.9954 13.2k tok/s
|
| 554 |
+
[step 10840] tok=177.6M R=8 lm=4.0305 13.2k tok/s
|
| 555 |
+
[step 10860] tok=177.9M R=8 lm=4.0577 13.2k tok/s
|
| 556 |
+
[step 10880] tok=178.3M R=8 lm=3.9230 13.2k tok/s
|
| 557 |
+
[step 10900] tok=178.6M R=8 lm=3.9013 13.2k tok/s
|
| 558 |
+
[step 10920] tok=178.9M R=8 lm=4.1641 13.2k tok/s
|
| 559 |
+
[step 10940] tok=179.2M R=8 lm=3.8450 13.2k tok/s
|
| 560 |
+
[step 10960] tok=179.6M R=8 lm=4.2541 13.2k tok/s
|
| 561 |
+
[step 10980] tok=179.9M R=8 lm=4.4125 13.2k tok/s
|
| 562 |
+
[step 11000] tok=180.2M R=8 lm=4.0791 13.2k tok/s
|
| 563 |
+
[step 11020] tok=180.6M R=8 lm=4.0066 13.2k tok/s
|
| 564 |
+
[step 11040] tok=180.9M R=8 lm=4.3199 13.2k tok/s
|
| 565 |
+
[step 11060] tok=181.2M R=8 lm=4.1152 13.2k tok/s
|
| 566 |
+
[step 11080] tok=181.5M R=8 lm=4.2721 13.2k tok/s
|
| 567 |
+
[step 11100] tok=181.9M R=8 lm=3.8228 13.2k tok/s
|
| 568 |
+
[step 11120] tok=182.2M R=8 lm=4.1469 13.2k tok/s
|
| 569 |
+
[step 11140] tok=182.5M R=8 lm=4.2908 13.2k tok/s
|
| 570 |
+
[step 11160] tok=182.8M R=8 lm=4.0229 13.2k tok/s
|
| 571 |
+
[step 11180] tok=183.2M R=8 lm=4.0459 13.2k tok/s
|
| 572 |
+
[step 11200] tok=183.5M R=8 lm=4.5036 13.2k tok/s
|
| 573 |
+
[step 11220] tok=183.8M R=8 lm=4.1366 13.2k tok/s
|
| 574 |
+
[step 11240] tok=184.2M R=8 lm=3.9627 13.2k tok/s
|
| 575 |
+
[step 11260] tok=184.5M R=8 lm=4.0577 13.2k tok/s
|
| 576 |
+
[step 11280] tok=184.8M R=8 lm=4.0412 13.2k tok/s
|
| 577 |
+
[step 11300] tok=185.1M R=8 lm=4.2451 13.2k tok/s
|
| 578 |
+
[step 11320] tok=185.5M R=8 lm=4.1156 13.2k tok/s
|
| 579 |
+
[step 11340] tok=185.8M R=8 lm=4.0404 13.2k tok/s
|
| 580 |
+
[step 11360] tok=186.1M R=8 lm=4.2200 13.2k tok/s
|
| 581 |
+
[step 11380] tok=186.4M R=8 lm=4.2817 13.2k tok/s
|
| 582 |
+
[step 11400] tok=186.8M R=8 lm=4.2393 13.2k tok/s
|
| 583 |
+
[step 11420] tok=187.1M R=8 lm=4.1056 13.2k tok/s
|
| 584 |
+
[step 11440] tok=187.4M R=8 lm=4.0482 13.2k tok/s
|
| 585 |
+
[step 11460] tok=187.8M R=8 lm=4.2517 13.2k tok/s
|
| 586 |
+
[step 11480] tok=188.1M R=8 lm=4.2398 13.2k tok/s
|
| 587 |
+
[step 11500] tok=188.4M R=8 lm=3.9516 13.2k tok/s
|
| 588 |
+
[step 11520] tok=188.7M R=8 lm=4.1534 13.2k tok/s
|
| 589 |
+
[step 11540] tok=189.1M R=8 lm=4.0670 13.2k tok/s
|
| 590 |
+
[step 11560] tok=189.4M R=8 lm=4.2531 13.2k tok/s
|
| 591 |
+
[step 11580] tok=189.7M R=8 lm=3.9285 13.2k tok/s
|
| 592 |
+
[step 11600] tok=190.1M R=8 lm=4.4857 13.2k tok/s
|
| 593 |
+
[step 11620] tok=190.4M R=8 lm=4.0461 13.2k tok/s
|
| 594 |
+
[step 11640] tok=190.7M R=8 lm=3.9772 13.2k tok/s
|
| 595 |
+
[step 11660] tok=191.0M R=8 lm=4.0647 13.2k tok/s
|
| 596 |
+
[step 11680] tok=191.4M R=8 lm=4.1573 13.2k tok/s
|
| 597 |
+
[step 11700] tok=191.7M R=8 lm=4.1723 13.2k tok/s
|
| 598 |
+
[step 11720] tok=192.0M R=8 lm=4.0049 13.2k tok/s
|
| 599 |
+
[step 11740] tok=192.3M R=8 lm=4.0230 13.2k tok/s
|
| 600 |
+
[step 11760] tok=192.7M R=8 lm=3.9139 13.2k tok/s
|
| 601 |
+
[step 11780] tok=193.0M R=8 lm=3.9330 13.2k tok/s
|
| 602 |
+
[step 11800] tok=193.3M R=8 lm=4.0240 13.2k tok/s
|
| 603 |
+
[step 11820] tok=193.7M R=8 lm=4.2512 13.2k tok/s
|
| 604 |
+
[step 11840] tok=194.0M R=8 lm=3.9637 13.2k tok/s
|
| 605 |
+
[step 11860] tok=194.3M R=8 lm=4.1580 13.2k tok/s
|
| 606 |
+
[step 11880] tok=194.6M R=8 lm=4.0793 13.2k tok/s
|
| 607 |
+
[step 11900] tok=195.0M R=8 lm=3.9327 13.2k tok/s
|
| 608 |
+
[step 11920] tok=195.3M R=8 lm=4.1769 13.2k tok/s
|
| 609 |
+
[step 11940] tok=195.6M R=8 lm=3.8464 13.2k tok/s
|
| 610 |
+
[step 11960] tok=196.0M R=8 lm=4.2491 13.2k tok/s
|
| 611 |
+
[step 11980] tok=196.3M R=8 lm=4.2413 13.2k tok/s
|
| 612 |
+
[step 12000] tok=196.6M R=8 lm=4.0352 13.2k tok/s
|
| 613 |
+
[step 12020] tok=196.9M R=8 lm=4.0574 13.2k tok/s
|
| 614 |
+
[step 12040] tok=197.3M R=8 lm=4.2998 13.2k tok/s
|
| 615 |
+
[step 12060] tok=197.6M R=8 lm=4.0333 13.2k tok/s
|
| 616 |
+
[step 12080] tok=197.9M R=8 lm=4.1873 13.2k tok/s
|
| 617 |
+
[step 12100] tok=198.2M R=8 lm=4.0895 13.2k tok/s
|
| 618 |
+
[step 12120] tok=198.6M R=8 lm=4.1718 13.2k tok/s
|
| 619 |
+
[step 12140] tok=198.9M R=8 lm=3.8363 13.2k tok/s
|
| 620 |
+
[step 12160] tok=199.2M R=8 lm=4.1484 13.2k tok/s
|
| 621 |
+
[step 12180] tok=199.6M R=8 lm=4.4557 13.2k tok/s
|
| 622 |
+
[step 12200] tok=199.9M R=8 lm=4.1612 13.2k tok/s
|
| 623 |
+
[eval] bake_D arm=D tok=200M trainR=8 BPB=1.2875 swap=0.9999 ent=0.9891
|
| 624 |
+
[step 12220] tok=200.2M R=8 lm=4.2242 13.2k tok/s
|
| 625 |
+
[step 12240] tok=200.5M R=8 lm=4.2128 13.2k tok/s
|
| 626 |
+
[step 12260] tok=200.9M R=8 lm=4.0557 13.2k tok/s
|
| 627 |
+
[step 12280] tok=201.2M R=8 lm=4.3620 13.2k tok/s
|
| 628 |
+
[step 12300] tok=201.5M R=8 lm=4.3044 13.2k tok/s
|
| 629 |
+
[step 12320] tok=201.9M R=8 lm=4.3242 13.2k tok/s
|
| 630 |
+
[step 12340] tok=202.2M R=8 lm=4.2431 13.2k tok/s
|
| 631 |
+
[step 12360] tok=202.5M R=8 lm=4.0615 13.2k tok/s
|
| 632 |
+
[step 12380] tok=202.8M R=8 lm=4.2532 13.2k tok/s
|
| 633 |
+
[step 12400] tok=203.2M R=8 lm=4.1241 13.2k tok/s
|
| 634 |
+
[step 12420] tok=203.5M R=8 lm=4.2005 13.2k tok/s
|
| 635 |
+
[step 12440] tok=203.8M R=8 lm=4.0536 13.2k tok/s
|
| 636 |
+
[step 12460] tok=204.1M R=8 lm=4.0354 13.2k tok/s
|
| 637 |
+
[step 12480] tok=204.5M R=8 lm=4.4010 13.2k tok/s
|
| 638 |
+
[step 12500] tok=204.8M R=8 lm=4.0832 13.2k tok/s
|
| 639 |
+
[step 12520] tok=205.1M R=8 lm=4.4379 13.2k tok/s
|
| 640 |
+
[step 12540] tok=205.5M R=8 lm=4.0627 13.2k tok/s
|
| 641 |
+
[step 12560] tok=205.8M R=8 lm=4.2985 13.2k tok/s
|
| 642 |
+
[step 12580] tok=206.1M R=8 lm=3.9764 13.2k tok/s
|
| 643 |
+
[step 12600] tok=206.4M R=8 lm=4.1038 13.2k tok/s
|
| 644 |
+
[step 12620] tok=206.8M R=8 lm=4.0975 13.2k tok/s
|
| 645 |
+
[step 12640] tok=207.1M R=8 lm=4.2658 13.2k tok/s
|
| 646 |
+
[step 12660] tok=207.4M R=8 lm=4.3749 13.2k tok/s
|
| 647 |
+
[step 12680] tok=207.7M R=8 lm=4.0976 13.2k tok/s
|
| 648 |
+
[step 12700] tok=208.1M R=8 lm=4.3479 13.2k tok/s
|
| 649 |
+
[step 12720] tok=208.4M R=8 lm=4.2716 13.2k tok/s
|
| 650 |
+
[step 12740] tok=208.7M R=8 lm=4.2036 13.2k tok/s
|
| 651 |
+
[step 12760] tok=209.1M R=8 lm=4.0057 13.2k tok/s
|
| 652 |
+
[step 12780] tok=209.4M R=8 lm=4.1345 13.2k tok/s
|
| 653 |
+
[step 12800] tok=209.7M R=8 lm=4.0341 13.2k tok/s
|
| 654 |
+
[step 12820] tok=210.0M R=8 lm=4.1355 13.2k tok/s
|
| 655 |
+
[step 12840] tok=210.4M R=8 lm=3.9798 13.2k tok/s
|
| 656 |
+
[step 12860] tok=210.7M R=8 lm=4.2632 13.2k tok/s
|
| 657 |
+
[step 12880] tok=211.0M R=8 lm=4.0347 13.2k tok/s
|
| 658 |
+
[step 12900] tok=211.4M R=8 lm=4.2602 13.2k tok/s
|
| 659 |
+
[step 12920] tok=211.7M R=8 lm=4.0383 13.2k tok/s
|
| 660 |
+
[step 12940] tok=212.0M R=8 lm=4.0644 13.2k tok/s
|
| 661 |
+
[step 12960] tok=212.3M R=8 lm=4.0723 13.2k tok/s
|
| 662 |
+
[step 12980] tok=212.7M R=8 lm=4.0018 13.2k tok/s
|
| 663 |
+
[step 13000] tok=213.0M R=8 lm=3.9897 13.2k tok/s
|
| 664 |
+
[step 13020] tok=213.3M R=8 lm=4.1106 13.2k tok/s
|
| 665 |
+
[step 13040] tok=213.6M R=8 lm=4.2279 13.2k tok/s
|
| 666 |
+
[step 13060] tok=214.0M R=8 lm=4.2771 13.2k tok/s
|
| 667 |
+
[step 13080] tok=214.3M R=8 lm=3.9030 13.2k tok/s
|
| 668 |
+
[step 13100] tok=214.6M R=8 lm=4.1088 13.2k tok/s
|
| 669 |
+
[step 13120] tok=215.0M R=8 lm=4.1869 13.2k tok/s
|
| 670 |
+
[step 13140] tok=215.3M R=8 lm=4.0992 13.2k tok/s
|
| 671 |
+
[step 13160] tok=215.6M R=8 lm=3.9215 13.2k tok/s
|
| 672 |
+
[step 13180] tok=215.9M R=8 lm=4.3489 13.2k tok/s
|
| 673 |
+
[step 13200] tok=216.3M R=8 lm=4.0498 13.2k tok/s
|
| 674 |
+
[step 13220] tok=216.6M R=8 lm=4.3063 13.2k tok/s
|
| 675 |
+
[step 13240] tok=216.9M R=8 lm=4.0656 13.2k tok/s
|
| 676 |
+
[step 13260] tok=217.3M R=8 lm=3.9415 13.2k tok/s
|
| 677 |
+
[step 13280] tok=217.6M R=8 lm=4.5619 13.2k tok/s
|
| 678 |
+
[step 13300] tok=217.9M R=8 lm=3.7576 13.2k tok/s
|
| 679 |
+
[step 13320] tok=218.2M R=8 lm=4.1520 13.2k tok/s
|
| 680 |
+
[step 13340] tok=218.6M R=8 lm=4.1772 13.2k tok/s
|
| 681 |
+
[step 13360] tok=218.9M R=8 lm=3.9426 13.2k tok/s
|
| 682 |
+
[step 13380] tok=219.2M R=8 lm=4.2703 13.2k tok/s
|
| 683 |
+
[step 13400] tok=219.5M R=8 lm=3.9484 13.2k tok/s
|
| 684 |
+
[step 13420] tok=219.9M R=8 lm=4.0652 13.2k tok/s
|
| 685 |
+
[step 13440] tok=220.2M R=8 lm=3.8666 13.2k tok/s
|
| 686 |
+
[step 13460] tok=220.5M R=8 lm=4.2661 13.2k tok/s
|
| 687 |
+
[step 13480] tok=220.9M R=8 lm=4.2462 13.2k tok/s
|
| 688 |
+
[step 13500] tok=221.2M R=8 lm=3.8091 13.2k tok/s
|
| 689 |
+
[step 13520] tok=221.5M R=8 lm=4.1526 13.2k tok/s
|
| 690 |
+
[step 13540] tok=221.8M R=8 lm=4.2889 13.2k tok/s
|
| 691 |
+
[step 13560] tok=222.2M R=8 lm=3.9628 13.2k tok/s
|
| 692 |
+
[step 13580] tok=222.5M R=8 lm=4.0972 13.2k tok/s
|
| 693 |
+
[step 13600] tok=222.8M R=8 lm=4.0260 13.2k tok/s
|
| 694 |
+
[step 13620] tok=223.2M R=8 lm=3.9418 13.2k tok/s
|
| 695 |
+
[step 13640] tok=223.5M R=8 lm=4.1996 13.2k tok/s
|
| 696 |
+
[step 13660] tok=223.8M R=8 lm=4.4134 13.2k tok/s
|
| 697 |
+
[step 13680] tok=224.1M R=8 lm=4.0219 13.2k tok/s
|
| 698 |
+
[step 13700] tok=224.5M R=8 lm=4.4214 13.2k tok/s
|
| 699 |
+
[step 13720] tok=224.8M R=8 lm=4.1423 13.2k tok/s
|
| 700 |
+
[step 13740] tok=225.1M R=8 lm=4.1586 13.2k tok/s
|
| 701 |
+
[step 13760] tok=225.4M R=8 lm=4.1083 13.2k tok/s
|
| 702 |
+
[step 13780] tok=225.8M R=8 lm=3.9549 13.2k tok/s
|
| 703 |
+
[step 13800] tok=226.1M R=8 lm=3.9127 13.2k tok/s
|
| 704 |
+
[step 13820] tok=226.4M R=8 lm=4.0724 13.2k tok/s
|
| 705 |
+
[step 13840] tok=226.8M R=8 lm=4.3405 13.2k tok/s
|
| 706 |
+
[step 13860] tok=227.1M R=8 lm=4.3959 13.2k tok/s
|
| 707 |
+
[step 13880] tok=227.4M R=8 lm=4.0238 13.2k tok/s
|
| 708 |
+
[step 13900] tok=227.7M R=8 lm=4.0965 13.2k tok/s
|
| 709 |
+
[step 13920] tok=228.1M R=8 lm=4.1780 13.2k tok/s
|
| 710 |
+
[step 13940] tok=228.4M R=8 lm=4.0472 13.2k tok/s
|
| 711 |
+
[step 13960] tok=228.7M R=8 lm=4.2570 13.2k tok/s
|
| 712 |
+
[step 13980] tok=229.0M R=8 lm=4.0400 13.2k tok/s
|
| 713 |
+
[step 14000] tok=229.4M R=8 lm=4.1341 13.2k tok/s
|
| 714 |
+
[step 14020] tok=229.7M R=8 lm=3.9901 13.2k tok/s
|
| 715 |
+
[step 14040] tok=230.0M R=8 lm=4.1411 13.2k tok/s
|
| 716 |
+
[step 14060] tok=230.4M R=8 lm=4.0246 13.2k tok/s
|
| 717 |
+
[step 14080] tok=230.7M R=8 lm=4.2587 13.2k tok/s
|
| 718 |
+
[step 14100] tok=231.0M R=8 lm=4.3362 13.2k tok/s
|
| 719 |
+
[step 14120] tok=231.3M R=8 lm=4.3849 13.2k tok/s
|
| 720 |
+
[step 14140] tok=231.7M R=8 lm=3.8971 13.2k tok/s
|
| 721 |
+
[step 14160] tok=232.0M R=8 lm=4.2106 13.2k tok/s
|
| 722 |
+
[step 14180] tok=232.3M R=8 lm=4.3919 13.2k tok/s
|
| 723 |
+
[step 14200] tok=232.7M R=8 lm=4.0475 13.2k tok/s
|
| 724 |
+
[step 14220] tok=233.0M R=8 lm=4.3718 13.2k tok/s
|
| 725 |
+
[step 14240] tok=233.3M R=8 lm=4.0766 13.2k tok/s
|
| 726 |
+
[step 14260] tok=233.6M R=8 lm=4.1246 13.2k tok/s
|
| 727 |
+
[step 14280] tok=234.0M R=8 lm=4.3372 13.2k tok/s
|
| 728 |
+
[step 14300] tok=234.3M R=8 lm=4.0901 13.2k tok/s
|
| 729 |
+
[step 14320] tok=234.6M R=8 lm=4.2145 13.2k tok/s
|
| 730 |
+
[step 14340] tok=234.9M R=8 lm=4.0310 13.2k tok/s
|
| 731 |
+
[step 14360] tok=235.3M R=8 lm=4.1091 13.2k tok/s
|
| 732 |
+
[step 14380] tok=235.6M R=8 lm=3.8782 13.2k tok/s
|
| 733 |
+
[step 14400] tok=235.9M R=8 lm=4.1878 13.2k tok/s
|
| 734 |
+
[step 14420] tok=236.3M R=8 lm=3.8672 13.2k tok/s
|
| 735 |
+
[step 14440] tok=236.6M R=8 lm=4.0077 13.2k tok/s
|
| 736 |
+
[step 14460] tok=236.9M R=8 lm=4.0181 13.2k tok/s
|
| 737 |
+
[step 14480] tok=237.2M R=8 lm=4.2384 13.2k tok/s
|
| 738 |
+
[step 14500] tok=237.6M R=8 lm=3.8301 13.2k tok/s
|
| 739 |
+
[step 14520] tok=237.9M R=8 lm=4.2639 13.2k tok/s
|
| 740 |
+
[step 14540] tok=238.2M R=8 lm=4.0865 13.2k tok/s
|
| 741 |
+
[step 14560] tok=238.6M R=8 lm=4.1787 13.2k tok/s
|
| 742 |
+
[step 14580] tok=238.9M R=8 lm=4.2418 13.2k tok/s
|
| 743 |
+
[step 14600] tok=239.2M R=8 lm=3.9771 13.2k tok/s
|
| 744 |
+
[step 14620] tok=239.5M R=8 lm=4.4096 13.2k tok/s
|
| 745 |
+
[step 14640] tok=239.9M R=8 lm=4.1691 13.2k tok/s
|
| 746 |
+
[step 14660] tok=240.2M R=8 lm=4.0147 13.2k tok/s
|
| 747 |
+
[step 14680] tok=240.5M R=8 lm=4.3445 13.2k tok/s
|
| 748 |
+
[step 14700] tok=240.8M R=8 lm=4.3034 13.2k tok/s
|
| 749 |
+
[step 14720] tok=241.2M R=8 lm=4.1159 13.2k tok/s
|
| 750 |
+
[step 14740] tok=241.5M R=8 lm=4.3670 13.2k tok/s
|
| 751 |
+
[step 14760] tok=241.8M R=8 lm=4.1731 13.2k tok/s
|
| 752 |
+
[step 14780] tok=242.2M R=8 lm=4.0126 13.2k tok/s
|
| 753 |
+
[step 14800] tok=242.5M R=8 lm=3.9965 13.2k tok/s
|
| 754 |
+
[step 14820] tok=242.8M R=8 lm=4.1615 13.2k tok/s
|
| 755 |
+
[step 14840] tok=243.1M R=8 lm=3.9905 13.2k tok/s
|
| 756 |
+
[step 14860] tok=243.5M R=8 lm=4.1726 13.2k tok/s
|
| 757 |
+
[step 14880] tok=243.8M R=8 lm=4.3358 13.2k tok/s
|
| 758 |
+
[step 14900] tok=244.1M R=8 lm=4.2530 13.2k tok/s
|
| 759 |
+
[step 14920] tok=244.4M R=8 lm=4.2352 13.2k tok/s
|
| 760 |
+
[step 14940] tok=244.8M R=8 lm=4.2025 13.2k tok/s
|
| 761 |
+
[step 14960] tok=245.1M R=8 lm=4.1536 13.2k tok/s
|
| 762 |
+
[step 14980] tok=245.4M R=8 lm=3.9844 13.2k tok/s
|
| 763 |
+
[step 15000] tok=245.8M R=8 lm=4.2799 13.2k tok/s
|
| 764 |
+
[step 15020] tok=246.1M R=8 lm=4.2515 13.2k tok/s
|
| 765 |
+
[step 15040] tok=246.4M R=8 lm=4.1880 13.2k tok/s
|
| 766 |
+
[step 15060] tok=246.7M R=8 lm=4.0734 13.2k tok/s
|
| 767 |
+
[step 15080] tok=247.1M R=8 lm=3.9584 13.2k tok/s
|
| 768 |
+
[step 15100] tok=247.4M R=8 lm=3.8955 13.2k tok/s
|
| 769 |
+
[step 15120] tok=247.7M R=8 lm=4.0868 13.2k tok/s
|
| 770 |
+
[step 15140] tok=248.1M R=8 lm=4.0800 13.2k tok/s
|
| 771 |
+
[step 15160] tok=248.4M R=8 lm=4.2114 13.2k tok/s
|
| 772 |
+
[step 15180] tok=248.7M R=8 lm=3.8351 13.2k tok/s
|
| 773 |
+
[step 15200] tok=249.0M R=8 lm=4.3013 13.2k tok/s
|
| 774 |
+
[step 15220] tok=249.4M R=8 lm=4.0596 13.2k tok/s
|
| 775 |
+
[step 15240] tok=249.7M R=8 lm=4.0671 13.2k tok/s
|
| 776 |
+
[eval] bake_D arm=D tok=250M trainR=8 BPB=1.2919 swap=0.9999 ent=0.9903
|
| 777 |
+
[DONE] arm=D final BPB=1.2919 swap=0.9999 ent=0.9903
|
olmoe_adapt/logs/bake_bake_E.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=E tag=bake_E lr=0.0003 trainable=236978176 (router=2097152 extra=234881024) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:43: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:45: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M R=8 lm=3.4997 12.5k tok/s
|
| 11 |
+
[step 40] tok=2.6M R=8 lm=3.2526 12.6k tok/s
|
| 12 |
+
[step 60] tok=3.9M R=8 lm=3.0169 12.6k tok/s
|
| 13 |
+
[step 80] tok=5.2M R=8 lm=2.9145 12.6k tok/s
|
| 14 |
+
[step 100] tok=6.6M R=8 lm=2.9657 12.6k tok/s
|
| 15 |
+
[step 120] tok=7.9M R=8 lm=2.9300 12.6k tok/s
|
| 16 |
+
[step 140] tok=9.2M R=8 lm=2.8611 12.7k tok/s
|
| 17 |
+
[step 160] tok=10.5M R=8 lm=2.9271 12.7k tok/s
|
| 18 |
+
[step 180] tok=11.8M R=8 lm=2.8716 12.7k tok/s
|
| 19 |
+
[step 200] tok=13.1M R=8 lm=2.9038 12.7k tok/s
|
| 20 |
+
[step 220] tok=14.4M R=8 lm=2.8007 12.7k tok/s
|
| 21 |
+
[step 240] tok=15.7M R=8 lm=2.7035 12.7k tok/s
|
| 22 |
+
[step 260] tok=17.0M R=8 lm=2.8169 12.7k tok/s
|
| 23 |
+
[step 280] tok=18.4M R=8 lm=2.8439 12.7k tok/s
|
| 24 |
+
[step 300] tok=19.7M R=8 lm=2.7478 12.7k tok/s
|
| 25 |
+
[step 320] tok=21.0M R=8 lm=2.7411 12.7k tok/s
|
| 26 |
+
[step 340] tok=22.3M R=8 lm=2.7949 12.7k tok/s
|
| 27 |
+
[step 360] tok=23.6M R=8 lm=2.7915 12.7k tok/s
|
| 28 |
+
[step 380] tok=24.9M R=8 lm=2.8438 12.7k tok/s
|
| 29 |
+
[step 400] tok=26.2M R=8 lm=2.7834 12.7k tok/s
|
| 30 |
+
[step 420] tok=27.5M R=8 lm=2.8417 12.7k tok/s
|
| 31 |
+
[step 440] tok=28.8M R=8 lm=2.7292 12.7k tok/s
|
| 32 |
+
[step 460] tok=30.1M R=8 lm=2.7248 12.7k tok/s
|
| 33 |
+
[step 480] tok=31.5M R=8 lm=2.7996 12.7k tok/s
|
| 34 |
+
[step 500] tok=32.8M R=8 lm=2.7546 12.7k tok/s
|
| 35 |
+
[step 520] tok=34.1M R=8 lm=2.8439 12.7k tok/s
|
| 36 |
+
[step 540] tok=35.4M R=8 lm=2.7754 12.7k tok/s
|
| 37 |
+
[step 560] tok=36.7M R=8 lm=2.8281 12.7k tok/s
|
| 38 |
+
[step 580] tok=38.0M R=8 lm=2.7873 12.7k tok/s
|
| 39 |
+
[step 600] tok=39.3M R=8 lm=2.9244 12.7k tok/s
|
| 40 |
+
[step 620] tok=40.6M R=8 lm=2.7813 12.7k tok/s
|
| 41 |
+
[step 640] tok=41.9M R=8 lm=2.6098 12.7k tok/s
|
| 42 |
+
[step 660] tok=43.3M R=8 lm=2.8320 12.7k tok/s
|
| 43 |
+
[step 680] tok=44.6M R=8 lm=2.6781 12.7k tok/s
|
| 44 |
+
[step 700] tok=45.9M R=8 lm=2.8299 12.7k tok/s
|
| 45 |
+
[step 720] tok=47.2M R=8 lm=2.7727 12.7k tok/s
|
| 46 |
+
[step 740] tok=48.5M R=8 lm=2.9567 12.7k tok/s
|
| 47 |
+
[step 760] tok=49.8M R=8 lm=2.8822 12.7k tok/s
|
| 48 |
+
[eval] bake_E arm=E tok=50M trainR=8 BPB=0.8642 swap=0.9997 ent=0.9975
|
| 49 |
+
[step 780] tok=51.1M R=8 lm=2.8399 12.4k tok/s
|
| 50 |
+
[step 800] tok=52.4M R=8 lm=2.7232 12.4k tok/s
|
| 51 |
+
[step 820] tok=53.7M R=8 lm=2.8306 12.4k tok/s
|
| 52 |
+
[step 840] tok=55.1M R=8 lm=2.7099 12.4k tok/s
|
| 53 |
+
[step 860] tok=56.4M R=8 lm=2.8355 12.4k tok/s
|
| 54 |
+
[step 880] tok=57.7M R=8 lm=2.8058 12.4k tok/s
|
| 55 |
+
[step 900] tok=59.0M R=8 lm=2.7680 12.4k tok/s
|
| 56 |
+
[step 920] tok=60.3M R=8 lm=2.7629 12.4k tok/s
|
| 57 |
+
[step 940] tok=61.6M R=8 lm=2.8075 12.4k tok/s
|
| 58 |
+
[step 960] tok=62.9M R=8 lm=2.7009 12.4k tok/s
|
| 59 |
+
[step 980] tok=64.2M R=8 lm=2.9320 12.4k tok/s
|
| 60 |
+
[step 1000] tok=65.5M R=8 lm=2.7092 12.4k tok/s
|
| 61 |
+
[step 1020] tok=66.8M R=8 lm=2.6144 12.4k tok/s
|
| 62 |
+
[step 1040] tok=68.2M R=8 lm=2.7086 12.5k tok/s
|
| 63 |
+
[step 1060] tok=69.5M R=8 lm=2.8094 12.5k tok/s
|
| 64 |
+
[step 1080] tok=70.8M R=8 lm=2.7073 12.5k tok/s
|
| 65 |
+
[step 1100] tok=72.1M R=8 lm=2.7115 12.5k tok/s
|
| 66 |
+
[step 1120] tok=73.4M R=8 lm=2.9526 12.5k tok/s
|
| 67 |
+
[step 1140] tok=74.7M R=8 lm=2.8854 12.5k tok/s
|
| 68 |
+
[step 1160] tok=76.0M R=8 lm=2.7320 12.5k tok/s
|
| 69 |
+
[step 1180] tok=77.3M R=8 lm=2.7656 12.5k tok/s
|
| 70 |
+
[step 1200] tok=78.6M R=8 lm=2.7083 12.5k tok/s
|
| 71 |
+
[step 1220] tok=80.0M R=8 lm=2.7353 12.5k tok/s
|
| 72 |
+
[step 1240] tok=81.3M R=8 lm=2.8784 12.5k tok/s
|
| 73 |
+
[step 1260] tok=82.6M R=8 lm=2.8200 12.5k tok/s
|
| 74 |
+
[step 1280] tok=83.9M R=8 lm=2.8099 12.5k tok/s
|
| 75 |
+
[step 1300] tok=85.2M R=8 lm=2.7634 12.5k tok/s
|
| 76 |
+
[step 1320] tok=86.5M R=8 lm=2.7347 12.5k tok/s
|
| 77 |
+
[step 1340] tok=87.8M R=8 lm=2.7661 12.5k tok/s
|
| 78 |
+
[step 1360] tok=89.1M R=8 lm=2.8573 12.5k tok/s
|
| 79 |
+
[step 1380] tok=90.4M R=8 lm=2.7180 12.5k tok/s
|
| 80 |
+
[step 1400] tok=91.8M R=8 lm=2.6929 12.5k tok/s
|
| 81 |
+
[step 1420] tok=93.1M R=8 lm=2.7862 12.5k tok/s
|
| 82 |
+
[step 1440] tok=94.4M R=8 lm=2.7165 12.5k tok/s
|
| 83 |
+
[step 1460] tok=95.7M R=8 lm=2.7895 12.5k tok/s
|
| 84 |
+
[step 1480] tok=97.0M R=8 lm=2.7894 12.5k tok/s
|
| 85 |
+
[step 1500] tok=98.3M R=8 lm=2.8344 12.5k tok/s
|
| 86 |
+
[step 1520] tok=99.6M R=8 lm=2.6693 12.5k tok/s
|
| 87 |
+
[eval] bake_E arm=E tok=100M trainR=8 BPB=0.8586 swap=0.9997 ent=0.9972
|
| 88 |
+
[step 1540] tok=100.9M R=8 lm=2.6722 12.4k tok/s
|
| 89 |
+
[step 1560] tok=102.2M R=8 lm=2.8134 12.4k tok/s
|
| 90 |
+
[step 1580] tok=103.5M R=8 lm=2.7843 12.4k tok/s
|
| 91 |
+
[step 1600] tok=104.9M R=8 lm=2.6668 12.4k tok/s
|
| 92 |
+
[step 1620] tok=106.2M R=8 lm=2.6655 12.4k tok/s
|
| 93 |
+
[step 1640] tok=107.5M R=8 lm=2.7521 12.4k tok/s
|
| 94 |
+
[step 1660] tok=108.8M R=8 lm=2.8206 12.4k tok/s
|
| 95 |
+
[step 1680] tok=110.1M R=8 lm=2.7519 12.4k tok/s
|
| 96 |
+
[step 1700] tok=111.4M R=8 lm=2.7194 12.4k tok/s
|
| 97 |
+
[step 1720] tok=112.7M R=8 lm=2.8120 12.4k tok/s
|
| 98 |
+
[step 1740] tok=114.0M R=8 lm=2.7209 12.4k tok/s
|
| 99 |
+
[step 1760] tok=115.3M R=8 lm=2.7596 12.4k tok/s
|
| 100 |
+
[step 1780] tok=116.7M R=8 lm=2.7663 12.4k tok/s
|
| 101 |
+
[step 1800] tok=118.0M R=8 lm=2.8604 12.4k tok/s
|
| 102 |
+
[step 1820] tok=119.3M R=8 lm=2.7781 12.4k tok/s
|
| 103 |
+
[step 1840] tok=120.6M R=8 lm=2.7715 12.4k tok/s
|
| 104 |
+
[step 1860] tok=121.9M R=8 lm=2.8404 12.4k tok/s
|
| 105 |
+
[step 1880] tok=123.2M R=8 lm=2.7916 12.5k tok/s
|
| 106 |
+
[step 1900] tok=124.5M R=8 lm=2.7846 12.5k tok/s
|
| 107 |
+
[step 1920] tok=125.8M R=8 lm=2.7733 12.5k tok/s
|
| 108 |
+
[step 1940] tok=127.1M R=8 lm=2.8467 12.5k tok/s
|
| 109 |
+
[step 1960] tok=128.5M R=8 lm=2.7333 12.5k tok/s
|
| 110 |
+
[step 1980] tok=129.8M R=8 lm=2.7321 12.5k tok/s
|
| 111 |
+
[step 2000] tok=131.1M R=8 lm=2.7324 12.5k tok/s
|
| 112 |
+
[step 2020] tok=132.4M R=8 lm=2.7341 12.5k tok/s
|
| 113 |
+
[step 2040] tok=133.7M R=8 lm=2.6455 12.5k tok/s
|
| 114 |
+
[step 2060] tok=135.0M R=8 lm=2.6623 12.5k tok/s
|
| 115 |
+
[step 2080] tok=136.3M R=8 lm=2.7901 12.5k tok/s
|
| 116 |
+
[step 2100] tok=137.6M R=8 lm=2.8317 12.5k tok/s
|
| 117 |
+
[step 2120] tok=138.9M R=8 lm=2.8179 12.5k tok/s
|
| 118 |
+
[step 2140] tok=140.2M R=8 lm=2.6815 12.5k tok/s
|
| 119 |
+
[step 2160] tok=141.6M R=8 lm=2.8411 12.5k tok/s
|
| 120 |
+
[step 2180] tok=142.9M R=8 lm=2.7767 12.5k tok/s
|
| 121 |
+
[step 2200] tok=144.2M R=8 lm=2.7837 12.5k tok/s
|
| 122 |
+
[step 2220] tok=145.5M R=8 lm=2.8408 12.5k tok/s
|
| 123 |
+
[step 2240] tok=146.8M R=8 lm=2.6935 12.5k tok/s
|
| 124 |
+
[step 2260] tok=148.1M R=8 lm=2.6410 12.5k tok/s
|
| 125 |
+
[step 2280] tok=149.4M R=8 lm=2.8572 12.5k tok/s
|
| 126 |
+
[eval] bake_E arm=E tok=150M trainR=8 BPB=0.8549 swap=0.9997 ent=0.9975
|
| 127 |
+
[step 2300] tok=150.7M R=8 lm=2.7941 12.4k tok/s
|
| 128 |
+
[step 2320] tok=152.0M R=8 lm=2.7511 12.4k tok/s
|
| 129 |
+
[step 2340] tok=153.4M R=8 lm=2.7677 12.4k tok/s
|
| 130 |
+
[step 2360] tok=154.7M R=8 lm=2.6721 12.4k tok/s
|
| 131 |
+
[step 2380] tok=156.0M R=8 lm=2.7973 12.4k tok/s
|
| 132 |
+
[step 2400] tok=157.3M R=8 lm=2.7878 12.4k tok/s
|
| 133 |
+
[step 2420] tok=158.6M R=8 lm=2.8131 12.4k tok/s
|
| 134 |
+
[step 2440] tok=159.9M R=8 lm=2.7991 12.4k tok/s
|
| 135 |
+
[step 2460] tok=161.2M R=8 lm=2.8426 12.4k tok/s
|
| 136 |
+
[step 2480] tok=162.5M R=8 lm=2.6125 12.4k tok/s
|
| 137 |
+
[step 2500] tok=163.8M R=8 lm=2.6993 12.4k tok/s
|
| 138 |
+
[step 2520] tok=165.2M R=8 lm=2.7192 12.5k tok/s
|
| 139 |
+
[step 2540] tok=166.5M R=8 lm=2.8282 12.5k tok/s
|
| 140 |
+
[step 2560] tok=167.8M R=8 lm=2.6752 12.5k tok/s
|
| 141 |
+
[step 2580] tok=169.1M R=8 lm=2.7770 12.5k tok/s
|
| 142 |
+
[step 2600] tok=170.4M R=8 lm=2.5896 12.5k tok/s
|
| 143 |
+
[step 2620] tok=171.7M R=8 lm=2.8081 12.5k tok/s
|
| 144 |
+
[step 2640] tok=173.0M R=8 lm=2.7121 12.5k tok/s
|
| 145 |
+
[step 2660] tok=174.3M R=8 lm=2.8913 12.5k tok/s
|
| 146 |
+
[step 2680] tok=175.6M R=8 lm=2.7989 12.5k tok/s
|
| 147 |
+
[step 2700] tok=176.9M R=8 lm=2.7113 12.5k tok/s
|
| 148 |
+
[step 2720] tok=178.3M R=8 lm=2.7910 12.5k tok/s
|
| 149 |
+
[step 2740] tok=179.6M R=8 lm=2.6708 12.5k tok/s
|
| 150 |
+
[step 2760] tok=180.9M R=8 lm=2.8105 12.5k tok/s
|
| 151 |
+
[step 2780] tok=182.2M R=8 lm=2.7017 12.5k tok/s
|
| 152 |
+
[step 2800] tok=183.5M R=8 lm=2.7738 12.5k tok/s
|
| 153 |
+
[step 2820] tok=184.8M R=8 lm=2.8282 12.5k tok/s
|
| 154 |
+
[step 2840] tok=186.1M R=8 lm=2.5647 12.5k tok/s
|
| 155 |
+
[step 2860] tok=187.4M R=8 lm=2.7219 12.5k tok/s
|
| 156 |
+
[step 2880] tok=188.7M R=8 lm=2.8591 12.5k tok/s
|
| 157 |
+
[step 2900] tok=190.1M R=8 lm=2.8264 12.5k tok/s
|
| 158 |
+
[step 2920] tok=191.4M R=8 lm=2.7223 12.5k tok/s
|
| 159 |
+
[step 2940] tok=192.7M R=8 lm=2.8673 12.5k tok/s
|
| 160 |
+
[step 2960] tok=194.0M R=8 lm=2.5764 12.5k tok/s
|
| 161 |
+
[step 2980] tok=195.3M R=8 lm=2.7147 12.5k tok/s
|
| 162 |
+
[step 3000] tok=196.6M R=8 lm=2.5708 12.5k tok/s
|
| 163 |
+
[step 3020] tok=197.9M R=8 lm=2.7084 12.5k tok/s
|
| 164 |
+
[step 3040] tok=199.2M R=8 lm=2.8331 12.5k tok/s
|
| 165 |
+
[eval] bake_E arm=E tok=200M trainR=8 BPB=0.8513 swap=0.9997 ent=0.9974
|
| 166 |
+
[step 3060] tok=200.5M R=8 lm=2.7425 12.4k tok/s
|
| 167 |
+
[step 3080] tok=201.9M R=8 lm=2.7916 12.4k tok/s
|
| 168 |
+
[step 3100] tok=203.2M R=8 lm=2.7857 12.4k tok/s
|
| 169 |
+
[step 3120] tok=204.5M R=8 lm=2.6652 12.4k tok/s
|
| 170 |
+
[step 3140] tok=205.8M R=8 lm=2.7446 12.5k tok/s
|
| 171 |
+
[step 3160] tok=207.1M R=8 lm=2.6853 12.5k tok/s
|
| 172 |
+
[step 3180] tok=208.4M R=8 lm=2.6811 12.5k tok/s
|
| 173 |
+
[step 3200] tok=209.7M R=8 lm=2.7073 12.5k tok/s
|
| 174 |
+
[step 3220] tok=211.0M R=8 lm=2.5805 12.5k tok/s
|
| 175 |
+
[step 3240] tok=212.3M R=8 lm=2.7334 12.5k tok/s
|
| 176 |
+
[step 3260] tok=213.6M R=8 lm=2.7725 12.5k tok/s
|
| 177 |
+
[step 3280] tok=215.0M R=8 lm=2.7283 12.5k tok/s
|
| 178 |
+
[step 3300] tok=216.3M R=8 lm=2.7305 12.5k tok/s
|
| 179 |
+
[step 3320] tok=217.6M R=8 lm=2.7572 12.5k tok/s
|
| 180 |
+
[step 3340] tok=218.9M R=8 lm=2.7391 12.5k tok/s
|
| 181 |
+
[step 3360] tok=220.2M R=8 lm=2.7243 12.5k tok/s
|
| 182 |
+
[step 3380] tok=221.5M R=8 lm=2.7581 12.5k tok/s
|
| 183 |
+
[step 3400] tok=222.8M R=8 lm=2.7590 12.5k tok/s
|
| 184 |
+
[step 3420] tok=224.1M R=8 lm=2.7166 12.5k tok/s
|
| 185 |
+
[step 3440] tok=225.4M R=8 lm=2.6815 12.5k tok/s
|
| 186 |
+
[step 3460] tok=226.8M R=8 lm=2.7891 12.5k tok/s
|
| 187 |
+
[step 3480] tok=228.1M R=8 lm=2.7806 12.5k tok/s
|
| 188 |
+
[step 3500] tok=229.4M R=8 lm=2.7291 12.5k tok/s
|
| 189 |
+
[step 3520] tok=230.7M R=8 lm=2.7274 12.5k tok/s
|
| 190 |
+
[step 3540] tok=232.0M R=8 lm=2.7539 12.5k tok/s
|
| 191 |
+
[step 3560] tok=233.3M R=8 lm=2.7594 12.5k tok/s
|
| 192 |
+
[step 3580] tok=234.6M R=8 lm=2.9015 12.5k tok/s
|
| 193 |
+
[step 3600] tok=235.9M R=8 lm=2.7853 12.5k tok/s
|
| 194 |
+
[step 3620] tok=237.2M R=8 lm=3.0034 12.5k tok/s
|
| 195 |
+
[step 3640] tok=238.6M R=8 lm=2.7360 12.5k tok/s
|
| 196 |
+
[step 3660] tok=239.9M R=8 lm=2.7924 12.5k tok/s
|
| 197 |
+
[step 3680] tok=241.2M R=8 lm=2.7879 12.5k tok/s
|
| 198 |
+
[step 3700] tok=242.5M R=8 lm=2.7076 12.5k tok/s
|
| 199 |
+
[step 3720] tok=243.8M R=8 lm=2.7205 12.5k tok/s
|
| 200 |
+
[step 3740] tok=245.1M R=8 lm=2.7443 12.5k tok/s
|
| 201 |
+
[step 3760] tok=246.4M R=8 lm=2.8030 12.5k tok/s
|
| 202 |
+
[step 3780] tok=247.7M R=8 lm=2.7057 12.5k tok/s
|
| 203 |
+
[step 3800] tok=249.0M R=8 lm=2.8551 12.5k tok/s
|
| 204 |
+
[eval] bake_E arm=E tok=250M trainR=8 BPB=0.8507 swap=0.9997 ent=0.9977
|
| 205 |
+
[DONE] arm=E final BPB=0.8507 swap=0.9997 ent=0.9977
|
olmoe_adapt/logs/bake_bake_Er64.log
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=Er64 tag=bake_Er64 lr=0.0003 trainable=471859200 (router=2097152 extra=469762048) tokens=50000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:53: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:55: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M R=8 lm=3.4014 12.0k tok/s
|
| 11 |
+
[step 40] tok=2.6M R=8 lm=3.2025 12.2k tok/s
|
| 12 |
+
[step 60] tok=3.9M R=8 lm=2.9764 12.2k tok/s
|
| 13 |
+
[step 80] tok=5.2M R=8 lm=2.9050 12.3k tok/s
|
| 14 |
+
[step 100] tok=6.6M R=8 lm=2.9510 12.3k tok/s
|
| 15 |
+
[step 120] tok=7.9M R=8 lm=2.9142 12.3k tok/s
|
| 16 |
+
[step 140] tok=9.2M R=8 lm=2.8502 12.4k tok/s
|
| 17 |
+
[eval] bake_Er64 arm=Er64 tok=10M trainR=8 BPB=0.8896 swap=0.9997 ent=0.9973
|
| 18 |
+
[step 160] tok=10.5M R=8 lm=2.9239 10.8k tok/s
|
| 19 |
+
[step 180] tok=11.8M R=8 lm=2.8601 10.9k tok/s
|
| 20 |
+
[step 200] tok=13.1M R=8 lm=2.8947 11.1k tok/s
|
| 21 |
+
[step 220] tok=14.4M R=8 lm=2.7773 11.2k tok/s
|
| 22 |
+
[step 240] tok=15.7M R=8 lm=2.6895 11.3k tok/s
|
| 23 |
+
[step 260] tok=17.0M R=8 lm=2.8055 11.4k tok/s
|
| 24 |
+
[step 280] tok=18.4M R=8 lm=2.8321 11.4k tok/s
|
| 25 |
+
[step 300] tok=19.7M R=8 lm=2.7299 11.5k tok/s
|
| 26 |
+
[eval] bake_Er64 arm=Er64 tok=20M trainR=8 BPB=0.8720 swap=0.9997 ent=0.9975
|
| 27 |
+
[step 320] tok=21.0M R=8 lm=2.7279 10.8k tok/s
|
| 28 |
+
[step 340] tok=22.3M R=8 lm=2.7761 10.9k tok/s
|
| 29 |
+
[step 360] tok=23.6M R=8 lm=2.7741 11.0k tok/s
|
| 30 |
+
[step 380] tok=24.9M R=8 lm=2.8305 11.1k tok/s
|
| 31 |
+
[step 400] tok=26.2M R=8 lm=2.7645 11.1k tok/s
|
| 32 |
+
[step 420] tok=27.5M R=8 lm=2.8134 11.2k tok/s
|
| 33 |
+
[step 440] tok=28.8M R=8 lm=2.7118 11.2k tok/s
|
| 34 |
+
[eval] bake_Er64 arm=Er64 tok=30M trainR=8 BPB=0.8653 swap=0.9997 ent=0.9974
|
| 35 |
+
[step 460] tok=30.1M R=8 lm=2.7065 10.8k tok/s
|
| 36 |
+
[step 480] tok=31.5M R=8 lm=2.7809 10.9k tok/s
|
| 37 |
+
[step 500] tok=32.8M R=8 lm=2.7415 10.9k tok/s
|
| 38 |
+
[step 520] tok=34.1M R=8 lm=2.8234 11.0k tok/s
|
| 39 |
+
[step 540] tok=35.4M R=8 lm=2.7567 11.0k tok/s
|
| 40 |
+
[step 560] tok=36.7M R=8 lm=2.8085 11.1k tok/s
|
| 41 |
+
[step 580] tok=38.0M R=8 lm=2.7745 11.1k tok/s
|
| 42 |
+
[step 600] tok=39.3M R=8 lm=2.9042 11.1k tok/s
|
| 43 |
+
[eval] bake_Er64 arm=Er64 tok=40M trainR=8 BPB=0.8619 swap=0.9997 ent=0.9972
|
| 44 |
+
[step 620] tok=40.6M R=8 lm=2.7714 10.8k tok/s
|
| 45 |
+
[step 640] tok=41.9M R=8 lm=2.5991 10.9k tok/s
|
| 46 |
+
[step 660] tok=43.3M R=8 lm=2.8151 10.9k tok/s
|
| 47 |
+
[step 680] tok=44.6M R=8 lm=2.6564 10.9k tok/s
|
| 48 |
+
[step 700] tok=45.9M R=8 lm=2.8144 11.0k tok/s
|
| 49 |
+
[step 720] tok=47.2M R=8 lm=2.7558 11.0k tok/s
|
| 50 |
+
[step 740] tok=48.5M R=8 lm=2.9449 11.0k tok/s
|
| 51 |
+
[step 760] tok=49.8M R=8 lm=2.8617 11.1k tok/s
|
| 52 |
+
[eval] bake_Er64 arm=Er64 tok=50M trainR=8 BPB=0.8592 swap=0.9997 ent=0.9974
|
| 53 |
+
[DONE] arm=Er64 final BPB=0.8592 swap=0.9997 ent=0.9974
|
olmoe_adapt/logs/bake_bake_Er8.log
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=Er8 tag=bake_Er8 lr=0.0003 trainable=60817408 (router=2097152 extra=58720256) tokens=50000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:53: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:55: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M R=8 lm=3.6125 12.5k tok/s
|
| 11 |
+
[step 40] tok=2.6M R=8 lm=3.3798 12.6k tok/s
|
| 12 |
+
[step 60] tok=3.9M R=8 lm=3.0978 12.6k tok/s
|
| 13 |
+
[step 80] tok=5.2M R=8 lm=3.0039 12.6k tok/s
|
| 14 |
+
[step 100] tok=6.6M R=8 lm=3.0453 12.6k tok/s
|
| 15 |
+
[step 120] tok=7.9M R=8 lm=3.0048 12.6k tok/s
|
| 16 |
+
[step 140] tok=9.2M R=8 lm=2.9397 12.6k tok/s
|
| 17 |
+
[eval] bake_Er8 arm=Er8 tok=10M trainR=8 BPB=0.9155 swap=0.9997 ent=0.9967
|
| 18 |
+
[step 160] tok=10.5M R=8 lm=3.0103 11.1k tok/s
|
| 19 |
+
[step 180] tok=11.8M R=8 lm=2.9573 11.3k tok/s
|
| 20 |
+
[step 200] tok=13.1M R=8 lm=2.9760 11.4k tok/s
|
| 21 |
+
[step 220] tok=14.4M R=8 lm=2.8795 11.5k tok/s
|
| 22 |
+
[step 240] tok=15.7M R=8 lm=2.7995 11.6k tok/s
|
| 23 |
+
[step 260] tok=17.0M R=8 lm=2.9043 11.7k tok/s
|
| 24 |
+
[step 280] tok=18.4M R=8 lm=2.9328 11.8k tok/s
|
| 25 |
+
[step 300] tok=19.7M R=8 lm=2.8297 11.8k tok/s
|
| 26 |
+
[eval] bake_Er8 arm=Er8 tok=20M trainR=8 BPB=0.9034 swap=0.9997 ent=0.9972
|
| 27 |
+
[step 320] tok=21.0M R=8 lm=2.8309 11.2k tok/s
|
| 28 |
+
[step 340] tok=22.3M R=8 lm=2.8656 11.3k tok/s
|
| 29 |
+
[step 360] tok=23.6M R=8 lm=2.8649 11.3k tok/s
|
| 30 |
+
[step 380] tok=24.9M R=8 lm=2.9298 11.4k tok/s
|
| 31 |
+
[step 400] tok=26.2M R=8 lm=2.8745 11.4k tok/s
|
| 32 |
+
[step 420] tok=27.5M R=8 lm=2.9303 11.5k tok/s
|
| 33 |
+
[step 440] tok=28.8M R=8 lm=2.8228 11.5k tok/s
|
| 34 |
+
[eval] bake_Er8 arm=Er8 tok=30M trainR=8 BPB=0.8998 swap=0.9997 ent=0.9974
|
| 35 |
+
[step 460] tok=30.1M R=8 lm=2.8116 11.1k tok/s
|
| 36 |
+
[step 480] tok=31.5M R=8 lm=2.8842 11.2k tok/s
|
| 37 |
+
[step 500] tok=32.8M R=8 lm=2.8629 11.2k tok/s
|
| 38 |
+
[step 520] tok=34.1M R=8 lm=2.9296 11.3k tok/s
|
| 39 |
+
[step 540] tok=35.4M R=8 lm=2.8690 11.3k tok/s
|
| 40 |
+
[step 560] tok=36.7M R=8 lm=2.9286 11.4k tok/s
|
| 41 |
+
[step 580] tok=38.0M R=8 lm=2.8981 11.4k tok/s
|
| 42 |
+
[step 600] tok=39.3M R=8 lm=3.0218 11.4k tok/s
|
| 43 |
+
[eval] bake_Er8 arm=Er8 tok=40M trainR=8 BPB=0.8980 swap=0.9997 ent=0.9973
|
| 44 |
+
[step 620] tok=40.6M R=8 lm=2.8898 11.1k tok/s
|
| 45 |
+
[step 640] tok=41.9M R=8 lm=2.7018 11.2k tok/s
|
| 46 |
+
[step 660] tok=43.3M R=8 lm=2.9338 11.2k tok/s
|
| 47 |
+
[step 680] tok=44.6M R=8 lm=2.7633 11.2k tok/s
|
| 48 |
+
[step 700] tok=45.9M R=8 lm=2.9274 11.3k tok/s
|
| 49 |
+
[step 720] tok=47.2M R=8 lm=2.8549 11.3k tok/s
|
| 50 |
+
[step 740] tok=48.5M R=8 lm=3.0609 11.3k tok/s
|
| 51 |
+
[step 760] tok=49.8M R=8 lm=2.9727 11.4k tok/s
|
| 52 |
+
[eval] bake_Er8 arm=Er8 tok=50M trainR=8 BPB=0.8956 swap=0.9997 ent=0.9972
|
| 53 |
+
[DONE] arm=Er8 final BPB=0.8956 swap=0.9997 ent=0.9972
|
olmoe_adapt/logs/bake_bake_Fprime.log
ADDED
|
@@ -0,0 +1,633 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
/workspace/olmoe-adapt/scripts/train_fprime.py:34: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 3 |
+
ck = torch.load(CE_CKPT, map_location="cuda")
|
| 4 |
+
[fprime] loaded CE deltas from data/ckpt_bake_CE.pt (seen=250M, 145 tensors)
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_fprime.py:40: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
[fprime] identity check: pre-merge(LoRA active) BPB=0.8149 post-merge(baked) BPB=0.8147 |delta|=0.00024
|
| 9 |
+
[fprime] merged parent BPB=0.8147 (rec 0.9317); bar = beat by >2sigma (0.012). Now FULL finetune, 8-bit Adam lr=1e-05.
|
| 10 |
+
[fprime] full-FT trainable params: 6.92B
|
| 11 |
+
/workspace/olmoe-adapt/scripts/train_fprime.py:87: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 12 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 13 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 14 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 15 |
+
[step 20] tok=0.3M lm=2.6134 13.3k tok/s
|
| 16 |
+
[step 40] tok=0.7M lm=2.5254 13.4k tok/s
|
| 17 |
+
[step 60] tok=1.0M lm=2.6837 13.4k tok/s
|
| 18 |
+
[step 80] tok=1.3M lm=2.6775 13.5k tok/s
|
| 19 |
+
[step 100] tok=1.6M lm=2.1230 13.5k tok/s
|
| 20 |
+
[step 120] tok=2.0M lm=2.7663 13.5k tok/s
|
| 21 |
+
[step 140] tok=2.3M lm=2.4637 13.5k tok/s
|
| 22 |
+
[step 160] tok=2.6M lm=2.4521 13.5k tok/s
|
| 23 |
+
[step 180] tok=2.9M lm=2.4507 13.5k tok/s
|
| 24 |
+
[step 200] tok=3.3M lm=2.5093 13.5k tok/s
|
| 25 |
+
[step 220] tok=3.6M lm=2.4410 13.5k tok/s
|
| 26 |
+
[step 240] tok=3.9M lm=2.7519 13.5k tok/s
|
| 27 |
+
[step 260] tok=4.3M lm=2.4605 13.5k tok/s
|
| 28 |
+
[step 280] tok=4.6M lm=2.6593 13.5k tok/s
|
| 29 |
+
[step 300] tok=4.9M lm=2.3406 13.5k tok/s
|
| 30 |
+
[step 320] tok=5.2M lm=2.7126 13.5k tok/s
|
| 31 |
+
[step 340] tok=5.6M lm=2.5524 13.5k tok/s
|
| 32 |
+
[step 360] tok=5.9M lm=2.5135 13.5k tok/s
|
| 33 |
+
[step 380] tok=6.2M lm=2.5878 13.5k tok/s
|
| 34 |
+
[step 400] tok=6.6M lm=2.5200 13.5k tok/s
|
| 35 |
+
[step 420] tok=6.9M lm=2.6316 13.5k tok/s
|
| 36 |
+
[step 440] tok=7.2M lm=2.5327 13.5k tok/s
|
| 37 |
+
[step 460] tok=7.5M lm=2.3817 13.5k tok/s
|
| 38 |
+
[step 480] tok=7.9M lm=2.6682 13.5k tok/s
|
| 39 |
+
[step 500] tok=8.2M lm=2.5608 13.5k tok/s
|
| 40 |
+
[step 520] tok=8.5M lm=2.6949 13.5k tok/s
|
| 41 |
+
[step 540] tok=8.8M lm=2.4622 13.5k tok/s
|
| 42 |
+
[step 560] tok=9.2M lm=2.6124 13.5k tok/s
|
| 43 |
+
[step 580] tok=9.5M lm=2.4582 13.5k tok/s
|
| 44 |
+
[step 600] tok=9.8M lm=2.2236 13.5k tok/s
|
| 45 |
+
[step 620] tok=10.2M lm=2.5181 13.5k tok/s
|
| 46 |
+
[step 640] tok=10.5M lm=2.7234 13.5k tok/s
|
| 47 |
+
[step 660] tok=10.8M lm=2.7449 13.5k tok/s
|
| 48 |
+
[step 680] tok=11.1M lm=2.4097 13.5k tok/s
|
| 49 |
+
[step 700] tok=11.5M lm=2.4254 13.5k tok/s
|
| 50 |
+
[step 720] tok=11.8M lm=2.5745 13.5k tok/s
|
| 51 |
+
[step 740] tok=12.1M lm=2.6179 13.5k tok/s
|
| 52 |
+
[step 760] tok=12.5M lm=2.6674 13.5k tok/s
|
| 53 |
+
[step 780] tok=12.8M lm=2.7567 13.5k tok/s
|
| 54 |
+
[step 800] tok=13.1M lm=2.7333 13.5k tok/s
|
| 55 |
+
[step 820] tok=13.4M lm=2.2649 13.5k tok/s
|
| 56 |
+
[step 840] tok=13.8M lm=2.7801 13.5k tok/s
|
| 57 |
+
[step 860] tok=14.1M lm=2.4491 13.5k tok/s
|
| 58 |
+
[step 880] tok=14.4M lm=2.6914 13.5k tok/s
|
| 59 |
+
[step 900] tok=14.7M lm=2.5782 13.5k tok/s
|
| 60 |
+
[step 920] tok=15.1M lm=2.5372 13.5k tok/s
|
| 61 |
+
[step 940] tok=15.4M lm=2.4731 13.5k tok/s
|
| 62 |
+
[step 960] tok=15.7M lm=2.4933 13.5k tok/s
|
| 63 |
+
[step 980] tok=16.1M lm=2.8227 13.5k tok/s
|
| 64 |
+
[step 1000] tok=16.4M lm=2.5169 13.5k tok/s
|
| 65 |
+
[step 1020] tok=16.7M lm=2.7046 13.5k tok/s
|
| 66 |
+
[step 1040] tok=17.0M lm=2.5632 13.5k tok/s
|
| 67 |
+
[step 1060] tok=17.4M lm=2.9032 13.5k tok/s
|
| 68 |
+
[step 1080] tok=17.7M lm=2.5478 13.5k tok/s
|
| 69 |
+
[step 1100] tok=18.0M lm=2.4593 13.5k tok/s
|
| 70 |
+
[step 1120] tok=18.4M lm=2.7273 13.5k tok/s
|
| 71 |
+
[step 1140] tok=18.7M lm=2.4948 13.5k tok/s
|
| 72 |
+
[step 1160] tok=19.0M lm=2.2541 13.5k tok/s
|
| 73 |
+
[step 1180] tok=19.3M lm=2.7083 13.5k tok/s
|
| 74 |
+
[step 1200] tok=19.7M lm=2.6458 13.5k tok/s
|
| 75 |
+
[step 1220] tok=20.0M lm=2.5267 13.5k tok/s
|
| 76 |
+
[step 1240] tok=20.3M lm=2.7572 13.5k tok/s
|
| 77 |
+
[step 1260] tok=20.6M lm=2.5481 13.5k tok/s
|
| 78 |
+
[step 1280] tok=21.0M lm=2.5618 13.5k tok/s
|
| 79 |
+
[step 1300] tok=21.3M lm=2.5684 13.5k tok/s
|
| 80 |
+
[step 1320] tok=21.6M lm=2.7430 13.5k tok/s
|
| 81 |
+
[step 1340] tok=22.0M lm=2.3221 13.5k tok/s
|
| 82 |
+
[step 1360] tok=22.3M lm=2.6060 13.5k tok/s
|
| 83 |
+
[step 1380] tok=22.6M lm=2.5181 13.5k tok/s
|
| 84 |
+
[step 1400] tok=22.9M lm=2.7453 13.5k tok/s
|
| 85 |
+
[step 1420] tok=23.3M lm=2.5047 13.5k tok/s
|
| 86 |
+
[step 1440] tok=23.6M lm=2.6929 13.5k tok/s
|
| 87 |
+
[step 1460] tok=23.9M lm=2.5333 13.5k tok/s
|
| 88 |
+
[step 1480] tok=24.2M lm=2.4800 13.5k tok/s
|
| 89 |
+
[step 1500] tok=24.6M lm=2.5179 13.5k tok/s
|
| 90 |
+
[step 1520] tok=24.9M lm=2.7473 13.5k tok/s
|
| 91 |
+
[eval] fprime arm=Fprime tok=25M BPB=0.8125 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 92 |
+
[step 1540] tok=25.2M lm=2.4211 13.3k tok/s
|
| 93 |
+
[step 1560] tok=25.6M lm=2.5736 13.3k tok/s
|
| 94 |
+
[step 1580] tok=25.9M lm=2.4299 13.3k tok/s
|
| 95 |
+
[step 1600] tok=26.2M lm=2.4751 13.3k tok/s
|
| 96 |
+
[step 1620] tok=26.5M lm=2.4777 13.3k tok/s
|
| 97 |
+
[step 1640] tok=26.9M lm=2.6171 13.3k tok/s
|
| 98 |
+
[step 1660] tok=27.2M lm=2.7397 13.3k tok/s
|
| 99 |
+
[step 1680] tok=27.5M lm=2.4449 13.3k tok/s
|
| 100 |
+
[step 1700] tok=27.9M lm=2.5398 13.3k tok/s
|
| 101 |
+
[step 1720] tok=28.2M lm=2.7683 13.3k tok/s
|
| 102 |
+
[step 1740] tok=28.5M lm=2.6051 13.3k tok/s
|
| 103 |
+
[step 1760] tok=28.8M lm=2.5081 13.3k tok/s
|
| 104 |
+
[step 1780] tok=29.2M lm=2.3773 13.4k tok/s
|
| 105 |
+
[step 1800] tok=29.5M lm=2.5413 13.4k tok/s
|
| 106 |
+
[step 1820] tok=29.8M lm=2.5288 13.4k tok/s
|
| 107 |
+
[step 1840] tok=30.1M lm=2.5430 13.4k tok/s
|
| 108 |
+
[step 1860] tok=30.5M lm=2.5205 13.4k tok/s
|
| 109 |
+
[step 1880] tok=30.8M lm=2.4150 13.4k tok/s
|
| 110 |
+
[step 1900] tok=31.1M lm=2.4115 13.4k tok/s
|
| 111 |
+
[step 1920] tok=31.5M lm=2.6640 13.4k tok/s
|
| 112 |
+
[step 1940] tok=31.8M lm=2.7854 13.4k tok/s
|
| 113 |
+
[step 1960] tok=32.1M lm=2.4874 13.4k tok/s
|
| 114 |
+
[step 1980] tok=32.4M lm=2.6333 13.4k tok/s
|
| 115 |
+
[step 2000] tok=32.8M lm=2.4721 13.4k tok/s
|
| 116 |
+
[step 2020] tok=33.1M lm=2.3789 13.4k tok/s
|
| 117 |
+
[step 2040] tok=33.4M lm=2.4808 13.4k tok/s
|
| 118 |
+
[step 2060] tok=33.8M lm=2.3437 13.4k tok/s
|
| 119 |
+
[step 2080] tok=34.1M lm=2.6316 13.4k tok/s
|
| 120 |
+
[step 2100] tok=34.4M lm=2.6771 13.4k tok/s
|
| 121 |
+
[step 2120] tok=34.7M lm=2.7450 13.4k tok/s
|
| 122 |
+
[step 2140] tok=35.1M lm=2.9379 13.4k tok/s
|
| 123 |
+
[step 2160] tok=35.4M lm=2.4731 13.4k tok/s
|
| 124 |
+
[step 2180] tok=35.7M lm=2.4004 13.4k tok/s
|
| 125 |
+
[step 2200] tok=36.0M lm=2.6569 13.4k tok/s
|
| 126 |
+
[step 2220] tok=36.4M lm=2.9185 13.4k tok/s
|
| 127 |
+
[step 2240] tok=36.7M lm=2.6892 13.4k tok/s
|
| 128 |
+
[step 2260] tok=37.0M lm=2.6143 13.4k tok/s
|
| 129 |
+
[step 2280] tok=37.4M lm=2.4471 13.4k tok/s
|
| 130 |
+
[step 2300] tok=37.7M lm=2.5439 13.4k tok/s
|
| 131 |
+
[step 2320] tok=38.0M lm=2.9399 13.4k tok/s
|
| 132 |
+
[step 2340] tok=38.3M lm=2.7544 13.4k tok/s
|
| 133 |
+
[step 2360] tok=38.7M lm=2.4024 13.4k tok/s
|
| 134 |
+
[step 2380] tok=39.0M lm=2.4509 13.4k tok/s
|
| 135 |
+
[step 2400] tok=39.3M lm=2.9240 13.4k tok/s
|
| 136 |
+
[step 2420] tok=39.6M lm=2.5835 13.4k tok/s
|
| 137 |
+
[step 2440] tok=40.0M lm=2.5647 13.4k tok/s
|
| 138 |
+
[step 2460] tok=40.3M lm=2.6651 13.4k tok/s
|
| 139 |
+
[step 2480] tok=40.6M lm=2.3905 13.4k tok/s
|
| 140 |
+
[step 2500] tok=41.0M lm=2.4820 13.4k tok/s
|
| 141 |
+
[step 2520] tok=41.3M lm=2.6637 13.4k tok/s
|
| 142 |
+
[step 2540] tok=41.6M lm=2.6891 13.4k tok/s
|
| 143 |
+
[step 2560] tok=41.9M lm=2.2957 13.4k tok/s
|
| 144 |
+
[step 2580] tok=42.3M lm=2.7233 13.4k tok/s
|
| 145 |
+
[step 2600] tok=42.6M lm=2.5233 13.4k tok/s
|
| 146 |
+
[step 2620] tok=42.9M lm=2.4608 13.4k tok/s
|
| 147 |
+
[step 2640] tok=43.3M lm=2.4601 13.4k tok/s
|
| 148 |
+
[step 2660] tok=43.6M lm=2.7211 13.4k tok/s
|
| 149 |
+
[step 2680] tok=43.9M lm=2.4797 13.4k tok/s
|
| 150 |
+
[step 2700] tok=44.2M lm=2.6443 13.4k tok/s
|
| 151 |
+
[step 2720] tok=44.6M lm=2.3143 13.4k tok/s
|
| 152 |
+
[step 2740] tok=44.9M lm=2.4238 13.4k tok/s
|
| 153 |
+
[step 2760] tok=45.2M lm=2.4579 13.4k tok/s
|
| 154 |
+
[step 2780] tok=45.5M lm=2.5905 13.4k tok/s
|
| 155 |
+
[step 2800] tok=45.9M lm=2.8926 13.4k tok/s
|
| 156 |
+
[step 2820] tok=46.2M lm=2.3259 13.4k tok/s
|
| 157 |
+
[step 2840] tok=46.5M lm=1.9622 13.4k tok/s
|
| 158 |
+
[step 2860] tok=46.9M lm=2.4161 13.4k tok/s
|
| 159 |
+
[step 2880] tok=47.2M lm=2.6749 13.4k tok/s
|
| 160 |
+
[step 2900] tok=47.5M lm=2.6591 13.4k tok/s
|
| 161 |
+
[step 2920] tok=47.8M lm=2.5580 13.4k tok/s
|
| 162 |
+
[step 2940] tok=48.2M lm=2.4672 13.4k tok/s
|
| 163 |
+
[step 2960] tok=48.5M lm=2.7107 13.4k tok/s
|
| 164 |
+
[step 2980] tok=48.8M lm=2.3923 13.4k tok/s
|
| 165 |
+
[step 3000] tok=49.2M lm=2.6935 13.4k tok/s
|
| 166 |
+
[step 3020] tok=49.5M lm=2.5851 13.4k tok/s
|
| 167 |
+
[step 3040] tok=49.8M lm=2.6690 13.4k tok/s
|
| 168 |
+
[eval] fprime arm=Fprime tok=50M BPB=0.8116 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 169 |
+
[step 3060] tok=50.1M lm=2.5447 13.3k tok/s
|
| 170 |
+
[step 3080] tok=50.5M lm=2.5419 13.3k tok/s
|
| 171 |
+
[step 3100] tok=50.8M lm=2.4576 13.3k tok/s
|
| 172 |
+
[step 3120] tok=51.1M lm=2.6669 13.3k tok/s
|
| 173 |
+
[step 3140] tok=51.4M lm=3.1364 13.3k tok/s
|
| 174 |
+
[step 3160] tok=51.8M lm=2.2603 13.3k tok/s
|
| 175 |
+
[step 3180] tok=52.1M lm=2.6018 13.3k tok/s
|
| 176 |
+
[step 3200] tok=52.4M lm=2.6199 13.3k tok/s
|
| 177 |
+
[step 3220] tok=52.8M lm=2.4953 13.3k tok/s
|
| 178 |
+
[step 3240] tok=53.1M lm=2.3243 13.3k tok/s
|
| 179 |
+
[step 3260] tok=53.4M lm=2.1277 13.3k tok/s
|
| 180 |
+
[step 3280] tok=53.7M lm=2.6497 13.3k tok/s
|
| 181 |
+
[step 3300] tok=54.1M lm=2.5927 13.4k tok/s
|
| 182 |
+
[step 3320] tok=54.4M lm=2.4515 13.4k tok/s
|
| 183 |
+
[step 3340] tok=54.7M lm=2.4784 13.4k tok/s
|
| 184 |
+
[step 3360] tok=55.1M lm=2.4802 13.4k tok/s
|
| 185 |
+
[step 3380] tok=55.4M lm=2.7821 13.4k tok/s
|
| 186 |
+
[step 3400] tok=55.7M lm=2.4017 13.4k tok/s
|
| 187 |
+
[step 3420] tok=56.0M lm=2.2797 13.4k tok/s
|
| 188 |
+
[step 3440] tok=56.4M lm=2.5447 13.4k tok/s
|
| 189 |
+
[step 3460] tok=56.7M lm=2.5486 13.4k tok/s
|
| 190 |
+
[step 3480] tok=57.0M lm=2.5163 13.4k tok/s
|
| 191 |
+
[step 3500] tok=57.3M lm=2.5908 13.4k tok/s
|
| 192 |
+
[step 3520] tok=57.7M lm=2.4649 13.4k tok/s
|
| 193 |
+
[step 3540] tok=58.0M lm=2.5919 13.4k tok/s
|
| 194 |
+
[step 3560] tok=58.3M lm=2.6889 13.4k tok/s
|
| 195 |
+
[step 3580] tok=58.7M lm=2.8319 13.4k tok/s
|
| 196 |
+
[step 3600] tok=59.0M lm=2.4297 13.4k tok/s
|
| 197 |
+
[step 3620] tok=59.3M lm=2.5256 13.4k tok/s
|
| 198 |
+
[step 3640] tok=59.6M lm=2.5188 13.4k tok/s
|
| 199 |
+
[step 3660] tok=60.0M lm=2.3199 13.4k tok/s
|
| 200 |
+
[step 3680] tok=60.3M lm=2.5415 13.4k tok/s
|
| 201 |
+
[step 3700] tok=60.6M lm=2.5508 13.4k tok/s
|
| 202 |
+
[step 3720] tok=60.9M lm=2.6631 13.4k tok/s
|
| 203 |
+
[step 3740] tok=61.3M lm=2.5828 13.4k tok/s
|
| 204 |
+
[step 3760] tok=61.6M lm=2.5140 13.4k tok/s
|
| 205 |
+
[step 3780] tok=61.9M lm=2.6322 13.4k tok/s
|
| 206 |
+
[step 3800] tok=62.3M lm=2.3547 13.4k tok/s
|
| 207 |
+
[step 3820] tok=62.6M lm=2.3146 13.4k tok/s
|
| 208 |
+
[step 3840] tok=62.9M lm=2.5714 13.4k tok/s
|
| 209 |
+
[step 3860] tok=63.2M lm=2.2302 13.4k tok/s
|
| 210 |
+
[step 3880] tok=63.6M lm=2.5309 13.4k tok/s
|
| 211 |
+
[step 3900] tok=63.9M lm=2.6259 13.4k tok/s
|
| 212 |
+
[step 3920] tok=64.2M lm=2.5107 13.4k tok/s
|
| 213 |
+
[step 3940] tok=64.6M lm=2.6135 13.4k tok/s
|
| 214 |
+
[step 3960] tok=64.9M lm=2.5628 13.4k tok/s
|
| 215 |
+
[step 3980] tok=65.2M lm=2.5224 13.4k tok/s
|
| 216 |
+
[step 4000] tok=65.5M lm=2.4229 13.4k tok/s
|
| 217 |
+
[step 4020] tok=65.9M lm=2.4825 13.4k tok/s
|
| 218 |
+
[step 4040] tok=66.2M lm=2.9558 13.4k tok/s
|
| 219 |
+
[step 4060] tok=66.5M lm=2.5046 13.4k tok/s
|
| 220 |
+
[step 4080] tok=66.8M lm=2.3368 13.4k tok/s
|
| 221 |
+
[step 4100] tok=67.2M lm=2.3663 13.4k tok/s
|
| 222 |
+
[step 4120] tok=67.5M lm=2.5299 13.4k tok/s
|
| 223 |
+
[step 4140] tok=67.8M lm=2.5373 13.4k tok/s
|
| 224 |
+
[step 4160] tok=68.2M lm=2.5632 13.4k tok/s
|
| 225 |
+
[step 4180] tok=68.5M lm=2.3705 13.4k tok/s
|
| 226 |
+
[step 4200] tok=68.8M lm=2.6630 13.4k tok/s
|
| 227 |
+
[step 4220] tok=69.1M lm=2.6132 13.4k tok/s
|
| 228 |
+
[step 4240] tok=69.5M lm=2.4277 13.4k tok/s
|
| 229 |
+
[step 4260] tok=69.8M lm=2.3883 13.4k tok/s
|
| 230 |
+
[step 4280] tok=70.1M lm=2.2609 13.4k tok/s
|
| 231 |
+
[step 4300] tok=70.5M lm=2.4571 13.4k tok/s
|
| 232 |
+
[step 4320] tok=70.8M lm=2.6039 13.4k tok/s
|
| 233 |
+
[step 4340] tok=71.1M lm=2.4779 13.4k tok/s
|
| 234 |
+
[step 4360] tok=71.4M lm=2.6246 13.4k tok/s
|
| 235 |
+
[step 4380] tok=71.8M lm=2.3683 13.4k tok/s
|
| 236 |
+
[step 4400] tok=72.1M lm=2.4617 13.4k tok/s
|
| 237 |
+
[step 4420] tok=72.4M lm=2.6253 13.4k tok/s
|
| 238 |
+
[step 4440] tok=72.7M lm=2.4285 13.4k tok/s
|
| 239 |
+
[step 4460] tok=73.1M lm=2.2835 13.4k tok/s
|
| 240 |
+
[step 4480] tok=73.4M lm=2.5708 13.4k tok/s
|
| 241 |
+
[step 4500] tok=73.7M lm=2.4828 13.4k tok/s
|
| 242 |
+
[step 4520] tok=74.1M lm=2.7196 13.4k tok/s
|
| 243 |
+
[step 4540] tok=74.4M lm=2.5722 13.4k tok/s
|
| 244 |
+
[step 4560] tok=74.7M lm=2.7011 13.4k tok/s
|
| 245 |
+
[eval] fprime arm=Fprime tok=75M BPB=0.8108 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 246 |
+
[step 4580] tok=75.0M lm=2.4526 13.3k tok/s
|
| 247 |
+
[step 4600] tok=75.4M lm=2.5173 13.3k tok/s
|
| 248 |
+
[step 4620] tok=75.7M lm=2.5810 13.3k tok/s
|
| 249 |
+
[step 4640] tok=76.0M lm=2.3337 13.3k tok/s
|
| 250 |
+
[step 4660] tok=76.3M lm=2.3820 13.3k tok/s
|
| 251 |
+
[step 4680] tok=76.7M lm=2.7456 13.3k tok/s
|
| 252 |
+
[step 4700] tok=77.0M lm=2.6327 13.3k tok/s
|
| 253 |
+
[step 4720] tok=77.3M lm=2.6005 13.3k tok/s
|
| 254 |
+
[step 4740] tok=77.7M lm=2.5538 13.3k tok/s
|
| 255 |
+
[step 4760] tok=78.0M lm=2.6342 13.3k tok/s
|
| 256 |
+
[step 4780] tok=78.3M lm=2.5003 13.3k tok/s
|
| 257 |
+
[step 4800] tok=78.6M lm=2.6289 13.3k tok/s
|
| 258 |
+
[step 4820] tok=79.0M lm=2.3995 13.3k tok/s
|
| 259 |
+
[step 4840] tok=79.3M lm=2.4856 13.4k tok/s
|
| 260 |
+
[step 4860] tok=79.6M lm=2.6219 13.4k tok/s
|
| 261 |
+
[step 4880] tok=80.0M lm=2.2619 13.4k tok/s
|
| 262 |
+
[step 4900] tok=80.3M lm=2.7152 13.4k tok/s
|
| 263 |
+
[step 4920] tok=80.6M lm=2.6155 13.4k tok/s
|
| 264 |
+
[step 4940] tok=80.9M lm=2.5760 13.4k tok/s
|
| 265 |
+
[step 4960] tok=81.3M lm=2.6755 13.4k tok/s
|
| 266 |
+
[step 4980] tok=81.6M lm=2.7069 13.4k tok/s
|
| 267 |
+
[step 5000] tok=81.9M lm=2.2134 13.4k tok/s
|
| 268 |
+
[step 5020] tok=82.2M lm=2.7218 13.4k tok/s
|
| 269 |
+
[step 5040] tok=82.6M lm=2.7638 13.4k tok/s
|
| 270 |
+
[step 5060] tok=82.9M lm=2.3864 13.4k tok/s
|
| 271 |
+
[step 5080] tok=83.2M lm=2.5083 13.4k tok/s
|
| 272 |
+
[step 5100] tok=83.6M lm=2.4163 13.4k tok/s
|
| 273 |
+
[step 5120] tok=83.9M lm=2.4608 13.4k tok/s
|
| 274 |
+
[step 5140] tok=84.2M lm=2.3577 13.4k tok/s
|
| 275 |
+
[step 5160] tok=84.5M lm=2.4164 13.4k tok/s
|
| 276 |
+
[step 5180] tok=84.9M lm=2.5495 13.4k tok/s
|
| 277 |
+
[step 5200] tok=85.2M lm=2.5370 13.4k tok/s
|
| 278 |
+
[step 5220] tok=85.5M lm=2.7037 13.4k tok/s
|
| 279 |
+
[step 5240] tok=85.9M lm=2.6938 13.4k tok/s
|
| 280 |
+
[step 5260] tok=86.2M lm=2.7492 13.4k tok/s
|
| 281 |
+
[step 5280] tok=86.5M lm=2.5090 13.4k tok/s
|
| 282 |
+
[step 5300] tok=86.8M lm=2.1507 13.4k tok/s
|
| 283 |
+
[step 5320] tok=87.2M lm=2.6025 13.4k tok/s
|
| 284 |
+
[step 5340] tok=87.5M lm=2.4317 13.4k tok/s
|
| 285 |
+
[step 5360] tok=87.8M lm=2.6780 13.4k tok/s
|
| 286 |
+
[step 5380] tok=88.1M lm=2.3375 13.4k tok/s
|
| 287 |
+
[step 5400] tok=88.5M lm=2.4797 13.4k tok/s
|
| 288 |
+
[step 5420] tok=88.8M lm=2.4009 13.4k tok/s
|
| 289 |
+
[step 5440] tok=89.1M lm=2.6293 13.4k tok/s
|
| 290 |
+
[step 5460] tok=89.5M lm=2.6727 13.4k tok/s
|
| 291 |
+
[step 5480] tok=89.8M lm=2.3927 13.4k tok/s
|
| 292 |
+
[step 5500] tok=90.1M lm=2.3927 13.4k tok/s
|
| 293 |
+
[step 5520] tok=90.4M lm=2.2821 13.4k tok/s
|
| 294 |
+
[step 5540] tok=90.8M lm=2.5467 13.4k tok/s
|
| 295 |
+
[step 5560] tok=91.1M lm=2.4386 13.4k tok/s
|
| 296 |
+
[step 5580] tok=91.4M lm=2.3882 13.4k tok/s
|
| 297 |
+
[step 5600] tok=91.8M lm=2.3638 13.4k tok/s
|
| 298 |
+
[step 5620] tok=92.1M lm=2.6353 13.4k tok/s
|
| 299 |
+
[step 5640] tok=92.4M lm=2.2628 13.4k tok/s
|
| 300 |
+
[step 5660] tok=92.7M lm=2.6364 13.4k tok/s
|
| 301 |
+
[step 5680] tok=93.1M lm=2.5255 13.4k tok/s
|
| 302 |
+
[step 5700] tok=93.4M lm=2.8294 13.4k tok/s
|
| 303 |
+
[step 5720] tok=93.7M lm=2.4010 13.4k tok/s
|
| 304 |
+
[step 5740] tok=94.0M lm=2.4100 13.4k tok/s
|
| 305 |
+
[step 5760] tok=94.4M lm=2.5831 13.4k tok/s
|
| 306 |
+
[step 5780] tok=94.7M lm=2.6139 13.4k tok/s
|
| 307 |
+
[step 5800] tok=95.0M lm=2.5018 13.4k tok/s
|
| 308 |
+
[step 5820] tok=95.4M lm=2.5250 13.4k tok/s
|
| 309 |
+
[step 5840] tok=95.7M lm=2.5471 13.4k tok/s
|
| 310 |
+
[step 5860] tok=96.0M lm=2.4800 13.4k tok/s
|
| 311 |
+
[step 5880] tok=96.3M lm=2.5008 13.4k tok/s
|
| 312 |
+
[step 5900] tok=96.7M lm=2.7407 13.4k tok/s
|
| 313 |
+
[step 5920] tok=97.0M lm=2.4610 13.4k tok/s
|
| 314 |
+
[step 5940] tok=97.3M lm=2.2647 13.4k tok/s
|
| 315 |
+
[step 5960] tok=97.6M lm=2.4538 13.4k tok/s
|
| 316 |
+
[step 5980] tok=98.0M lm=2.6735 13.4k tok/s
|
| 317 |
+
[step 6000] tok=98.3M lm=2.4299 13.4k tok/s
|
| 318 |
+
[step 6020] tok=98.6M lm=2.6015 13.4k tok/s
|
| 319 |
+
[step 6040] tok=99.0M lm=2.7054 13.4k tok/s
|
| 320 |
+
[step 6060] tok=99.3M lm=2.5014 13.4k tok/s
|
| 321 |
+
[step 6080] tok=99.6M lm=2.4014 13.4k tok/s
|
| 322 |
+
[step 6100] tok=99.9M lm=2.4718 13.4k tok/s
|
| 323 |
+
[eval] fprime arm=Fprime tok=100M BPB=0.8101 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 324 |
+
[step 6120] tok=100.3M lm=2.3468 13.3k tok/s
|
| 325 |
+
[step 6140] tok=100.6M lm=2.3530 13.3k tok/s
|
| 326 |
+
[step 6160] tok=100.9M lm=2.4408 13.3k tok/s
|
| 327 |
+
[step 6180] tok=101.3M lm=2.6210 13.3k tok/s
|
| 328 |
+
[step 6200] tok=101.6M lm=2.4005 13.3k tok/s
|
| 329 |
+
[step 6220] tok=101.9M lm=2.5549 13.3k tok/s
|
| 330 |
+
[step 6240] tok=102.2M lm=2.4658 13.3k tok/s
|
| 331 |
+
[step 6260] tok=102.6M lm=2.8827 13.3k tok/s
|
| 332 |
+
[step 6280] tok=102.9M lm=2.4726 13.3k tok/s
|
| 333 |
+
[step 6300] tok=103.2M lm=2.9897 13.3k tok/s
|
| 334 |
+
[step 6320] tok=103.5M lm=2.6882 13.3k tok/s
|
| 335 |
+
[step 6340] tok=103.9M lm=2.5352 13.3k tok/s
|
| 336 |
+
[step 6360] tok=104.2M lm=2.6320 13.3k tok/s
|
| 337 |
+
[step 6380] tok=104.5M lm=2.5958 13.3k tok/s
|
| 338 |
+
[step 6400] tok=104.9M lm=2.4939 13.3k tok/s
|
| 339 |
+
[step 6420] tok=105.2M lm=2.3654 13.3k tok/s
|
| 340 |
+
[step 6440] tok=105.5M lm=2.4524 13.3k tok/s
|
| 341 |
+
[step 6460] tok=105.8M lm=2.6176 13.3k tok/s
|
| 342 |
+
[step 6480] tok=106.2M lm=2.4999 13.3k tok/s
|
| 343 |
+
[step 6500] tok=106.5M lm=2.3875 13.3k tok/s
|
| 344 |
+
[step 6520] tok=106.8M lm=2.4161 13.3k tok/s
|
| 345 |
+
[step 6540] tok=107.2M lm=2.3806 13.3k tok/s
|
| 346 |
+
[step 6560] tok=107.5M lm=2.4804 13.3k tok/s
|
| 347 |
+
[step 6580] tok=107.8M lm=2.5090 13.3k tok/s
|
| 348 |
+
[step 6600] tok=108.1M lm=2.5073 13.3k tok/s
|
| 349 |
+
[step 6620] tok=108.5M lm=2.5146 13.4k tok/s
|
| 350 |
+
[step 6640] tok=108.8M lm=2.3964 13.4k tok/s
|
| 351 |
+
[step 6660] tok=109.1M lm=2.5842 13.4k tok/s
|
| 352 |
+
[step 6680] tok=109.4M lm=2.5442 13.4k tok/s
|
| 353 |
+
[step 6700] tok=109.8M lm=2.6478 13.4k tok/s
|
| 354 |
+
[step 6720] tok=110.1M lm=2.5073 13.4k tok/s
|
| 355 |
+
[step 6740] tok=110.4M lm=2.5078 13.4k tok/s
|
| 356 |
+
[step 6760] tok=110.8M lm=2.2728 13.4k tok/s
|
| 357 |
+
[step 6780] tok=111.1M lm=2.4202 13.4k tok/s
|
| 358 |
+
[step 6800] tok=111.4M lm=2.5759 13.4k tok/s
|
| 359 |
+
[step 6820] tok=111.7M lm=2.4435 13.4k tok/s
|
| 360 |
+
[step 6840] tok=112.1M lm=2.4539 13.4k tok/s
|
| 361 |
+
[step 6860] tok=112.4M lm=2.6953 13.4k tok/s
|
| 362 |
+
[step 6880] tok=112.7M lm=2.5318 13.4k tok/s
|
| 363 |
+
[step 6900] tok=113.0M lm=2.2857 13.4k tok/s
|
| 364 |
+
[step 6920] tok=113.4M lm=2.6975 13.4k tok/s
|
| 365 |
+
[step 6940] tok=113.7M lm=2.4892 13.4k tok/s
|
| 366 |
+
[step 6960] tok=114.0M lm=2.5265 13.4k tok/s
|
| 367 |
+
[step 6980] tok=114.4M lm=2.6560 13.4k tok/s
|
| 368 |
+
[step 7000] tok=114.7M lm=2.3385 13.4k tok/s
|
| 369 |
+
[step 7020] tok=115.0M lm=2.4512 13.4k tok/s
|
| 370 |
+
[step 7040] tok=115.3M lm=2.4118 13.4k tok/s
|
| 371 |
+
[step 7060] tok=115.7M lm=2.5234 13.4k tok/s
|
| 372 |
+
[step 7080] tok=116.0M lm=2.6145 13.4k tok/s
|
| 373 |
+
[step 7100] tok=116.3M lm=2.6380 13.4k tok/s
|
| 374 |
+
[step 7120] tok=116.7M lm=2.2944 13.4k tok/s
|
| 375 |
+
[step 7140] tok=117.0M lm=2.4558 13.4k tok/s
|
| 376 |
+
[step 7160] tok=117.3M lm=2.3923 13.4k tok/s
|
| 377 |
+
[step 7180] tok=117.6M lm=2.2982 13.4k tok/s
|
| 378 |
+
[step 7200] tok=118.0M lm=2.4927 13.4k tok/s
|
| 379 |
+
[step 7220] tok=118.3M lm=2.3564 13.4k tok/s
|
| 380 |
+
[step 7240] tok=118.6M lm=2.5827 13.4k tok/s
|
| 381 |
+
[step 7260] tok=118.9M lm=2.4834 13.4k tok/s
|
| 382 |
+
[step 7280] tok=119.3M lm=2.5949 13.4k tok/s
|
| 383 |
+
[step 7300] tok=119.6M lm=2.7407 13.4k tok/s
|
| 384 |
+
[step 7320] tok=119.9M lm=2.5267 13.4k tok/s
|
| 385 |
+
[step 7340] tok=120.3M lm=2.3066 13.4k tok/s
|
| 386 |
+
[step 7360] tok=120.6M lm=2.5088 13.4k tok/s
|
| 387 |
+
[step 7380] tok=120.9M lm=2.4880 13.4k tok/s
|
| 388 |
+
[step 7400] tok=121.2M lm=2.7667 13.4k tok/s
|
| 389 |
+
[step 7420] tok=121.6M lm=2.4958 13.4k tok/s
|
| 390 |
+
[step 7440] tok=121.9M lm=2.7444 13.4k tok/s
|
| 391 |
+
[step 7460] tok=122.2M lm=2.6257 13.4k tok/s
|
| 392 |
+
[step 7480] tok=122.6M lm=2.5004 13.4k tok/s
|
| 393 |
+
[step 7500] tok=122.9M lm=2.5335 13.4k tok/s
|
| 394 |
+
[step 7520] tok=123.2M lm=2.6300 13.4k tok/s
|
| 395 |
+
[step 7540] tok=123.5M lm=2.7652 13.4k tok/s
|
| 396 |
+
[step 7560] tok=123.9M lm=2.4357 13.4k tok/s
|
| 397 |
+
[step 7580] tok=124.2M lm=2.4507 13.4k tok/s
|
| 398 |
+
[step 7600] tok=124.5M lm=2.6888 13.4k tok/s
|
| 399 |
+
[step 7620] tok=124.8M lm=2.3355 13.4k tok/s
|
| 400 |
+
[eval] fprime arm=Fprime tok=125M BPB=0.8096 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 401 |
+
[step 7640] tok=125.2M lm=2.4953 13.3k tok/s
|
| 402 |
+
[step 7660] tok=125.5M lm=2.4981 13.3k tok/s
|
| 403 |
+
[step 7680] tok=125.8M lm=2.8222 13.3k tok/s
|
| 404 |
+
[step 7700] tok=126.2M lm=2.2608 13.3k tok/s
|
| 405 |
+
[step 7720] tok=126.5M lm=2.5182 13.3k tok/s
|
| 406 |
+
[step 7740] tok=126.8M lm=2.4253 13.3k tok/s
|
| 407 |
+
[step 7760] tok=127.1M lm=2.6021 13.3k tok/s
|
| 408 |
+
[step 7780] tok=127.5M lm=2.5411 13.3k tok/s
|
| 409 |
+
[step 7800] tok=127.8M lm=2.0349 13.3k tok/s
|
| 410 |
+
[step 7820] tok=128.1M lm=2.4921 13.3k tok/s
|
| 411 |
+
[step 7840] tok=128.5M lm=2.4162 13.3k tok/s
|
| 412 |
+
[step 7860] tok=128.8M lm=2.5121 13.3k tok/s
|
| 413 |
+
[step 7880] tok=129.1M lm=2.5885 13.3k tok/s
|
| 414 |
+
[step 7900] tok=129.4M lm=2.2912 13.3k tok/s
|
| 415 |
+
[step 7920] tok=129.8M lm=2.5048 13.3k tok/s
|
| 416 |
+
[step 7940] tok=130.1M lm=2.7940 13.3k tok/s
|
| 417 |
+
[step 7960] tok=130.4M lm=2.5545 13.3k tok/s
|
| 418 |
+
[step 7980] tok=130.7M lm=2.4651 13.3k tok/s
|
| 419 |
+
[step 8000] tok=131.1M lm=2.4456 13.3k tok/s
|
| 420 |
+
[step 8020] tok=131.4M lm=2.4400 13.3k tok/s
|
| 421 |
+
[step 8040] tok=131.7M lm=2.6032 13.3k tok/s
|
| 422 |
+
[step 8060] tok=132.1M lm=2.4012 13.3k tok/s
|
| 423 |
+
[step 8080] tok=132.4M lm=2.5427 13.3k tok/s
|
| 424 |
+
[step 8100] tok=132.7M lm=2.4175 13.3k tok/s
|
| 425 |
+
[step 8120] tok=133.0M lm=2.5686 13.3k tok/s
|
| 426 |
+
[step 8140] tok=133.4M lm=2.6883 13.3k tok/s
|
| 427 |
+
[step 8160] tok=133.7M lm=2.3408 13.3k tok/s
|
| 428 |
+
[step 8180] tok=134.0M lm=2.5588 13.3k tok/s
|
| 429 |
+
[step 8200] tok=134.3M lm=2.6404 13.3k tok/s
|
| 430 |
+
[step 8220] tok=134.7M lm=2.5033 13.3k tok/s
|
| 431 |
+
[step 8240] tok=135.0M lm=2.3719 13.3k tok/s
|
| 432 |
+
[step 8260] tok=135.3M lm=2.4220 13.3k tok/s
|
| 433 |
+
[step 8280] tok=135.7M lm=2.6103 13.4k tok/s
|
| 434 |
+
[step 8300] tok=136.0M lm=2.2200 13.4k tok/s
|
| 435 |
+
[step 8320] tok=136.3M lm=2.5588 13.4k tok/s
|
| 436 |
+
[step 8340] tok=136.6M lm=2.5820 13.4k tok/s
|
| 437 |
+
[step 8360] tok=137.0M lm=2.6018 13.4k tok/s
|
| 438 |
+
[step 8380] tok=137.3M lm=2.3102 13.4k tok/s
|
| 439 |
+
[step 8400] tok=137.6M lm=2.7655 13.4k tok/s
|
| 440 |
+
[step 8420] tok=138.0M lm=2.3033 13.4k tok/s
|
| 441 |
+
[step 8440] tok=138.3M lm=2.6555 13.4k tok/s
|
| 442 |
+
[step 8460] tok=138.6M lm=2.5261 13.4k tok/s
|
| 443 |
+
[step 8480] tok=138.9M lm=2.6223 13.4k tok/s
|
| 444 |
+
[step 8500] tok=139.3M lm=2.3682 13.4k tok/s
|
| 445 |
+
[step 8520] tok=139.6M lm=2.6099 13.4k tok/s
|
| 446 |
+
[step 8540] tok=139.9M lm=2.5694 13.4k tok/s
|
| 447 |
+
[step 8560] tok=140.2M lm=2.5344 13.4k tok/s
|
| 448 |
+
[step 8580] tok=140.6M lm=2.7825 13.4k tok/s
|
| 449 |
+
[step 8600] tok=140.9M lm=2.4607 13.4k tok/s
|
| 450 |
+
[step 8620] tok=141.2M lm=2.4182 13.4k tok/s
|
| 451 |
+
[step 8640] tok=141.6M lm=2.4242 13.4k tok/s
|
| 452 |
+
[step 8660] tok=141.9M lm=2.4662 13.4k tok/s
|
| 453 |
+
[step 8680] tok=142.2M lm=2.5675 13.4k tok/s
|
| 454 |
+
[step 8700] tok=142.5M lm=2.5115 13.4k tok/s
|
| 455 |
+
[step 8720] tok=142.9M lm=2.5554 13.4k tok/s
|
| 456 |
+
[step 8740] tok=143.2M lm=2.4682 13.4k tok/s
|
| 457 |
+
[step 8760] tok=143.5M lm=2.3725 13.4k tok/s
|
| 458 |
+
[step 8780] tok=143.9M lm=2.4686 13.4k tok/s
|
| 459 |
+
[step 8800] tok=144.2M lm=2.4986 13.4k tok/s
|
| 460 |
+
[step 8820] tok=144.5M lm=2.5493 13.4k tok/s
|
| 461 |
+
[step 8840] tok=144.8M lm=2.4300 13.4k tok/s
|
| 462 |
+
[step 8860] tok=145.2M lm=2.3500 13.4k tok/s
|
| 463 |
+
[step 8880] tok=145.5M lm=2.5456 13.4k tok/s
|
| 464 |
+
[step 8900] tok=145.8M lm=2.2538 13.4k tok/s
|
| 465 |
+
[step 8920] tok=146.1M lm=2.2531 13.4k tok/s
|
| 466 |
+
[step 8940] tok=146.5M lm=2.6319 13.4k tok/s
|
| 467 |
+
[step 8960] tok=146.8M lm=2.3060 13.4k tok/s
|
| 468 |
+
[step 8980] tok=147.1M lm=2.6314 13.4k tok/s
|
| 469 |
+
[step 9000] tok=147.5M lm=2.2753 13.4k tok/s
|
| 470 |
+
[step 9020] tok=147.8M lm=2.4199 13.4k tok/s
|
| 471 |
+
[step 9040] tok=148.1M lm=2.3571 13.4k tok/s
|
| 472 |
+
[step 9060] tok=148.4M lm=2.5279 13.4k tok/s
|
| 473 |
+
[step 9080] tok=148.8M lm=2.3306 13.4k tok/s
|
| 474 |
+
[step 9100] tok=149.1M lm=2.5291 13.4k tok/s
|
| 475 |
+
[step 9120] tok=149.4M lm=2.6429 13.4k tok/s
|
| 476 |
+
[step 9140] tok=149.7M lm=2.5343 13.4k tok/s
|
| 477 |
+
[eval] fprime arm=Fprime tok=150M BPB=0.8099 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 478 |
+
[step 9160] tok=150.1M lm=2.8660 13.3k tok/s
|
| 479 |
+
[step 9180] tok=150.4M lm=2.4013 13.3k tok/s
|
| 480 |
+
[step 9200] tok=150.7M lm=2.3495 13.3k tok/s
|
| 481 |
+
[step 9220] tok=151.1M lm=2.5169 13.3k tok/s
|
| 482 |
+
[step 9240] tok=151.4M lm=2.6926 13.3k tok/s
|
| 483 |
+
[step 9260] tok=151.7M lm=2.5001 13.3k tok/s
|
| 484 |
+
[step 9280] tok=152.0M lm=2.3136 13.3k tok/s
|
| 485 |
+
[step 9300] tok=152.4M lm=2.5111 13.3k tok/s
|
| 486 |
+
[step 9320] tok=152.7M lm=2.5047 13.3k tok/s
|
| 487 |
+
[step 9340] tok=153.0M lm=2.3352 13.3k tok/s
|
| 488 |
+
[step 9360] tok=153.4M lm=2.7247 13.3k tok/s
|
| 489 |
+
[step 9380] tok=153.7M lm=2.4280 13.3k tok/s
|
| 490 |
+
[step 9400] tok=154.0M lm=2.4506 13.3k tok/s
|
| 491 |
+
[step 9420] tok=154.3M lm=2.3871 13.3k tok/s
|
| 492 |
+
[step 9440] tok=154.7M lm=2.2589 13.3k tok/s
|
| 493 |
+
[step 9460] tok=155.0M lm=2.5757 13.3k tok/s
|
| 494 |
+
[step 9480] tok=155.3M lm=2.4984 13.3k tok/s
|
| 495 |
+
[step 9500] tok=155.6M lm=2.4816 13.3k tok/s
|
| 496 |
+
[step 9520] tok=156.0M lm=2.4237 13.3k tok/s
|
| 497 |
+
[step 9540] tok=156.3M lm=2.4891 13.3k tok/s
|
| 498 |
+
[step 9560] tok=156.6M lm=2.3370 13.3k tok/s
|
| 499 |
+
[step 9580] tok=157.0M lm=2.3651 13.3k tok/s
|
| 500 |
+
[step 9600] tok=157.3M lm=2.5134 13.3k tok/s
|
| 501 |
+
[step 9620] tok=157.6M lm=2.5695 13.3k tok/s
|
| 502 |
+
[step 9640] tok=157.9M lm=2.6374 13.3k tok/s
|
| 503 |
+
[step 9660] tok=158.3M lm=2.3151 13.3k tok/s
|
| 504 |
+
[step 9680] tok=158.6M lm=2.5344 13.3k tok/s
|
| 505 |
+
[step 9700] tok=158.9M lm=2.6940 13.3k tok/s
|
| 506 |
+
[step 9720] tok=159.3M lm=2.3280 13.3k tok/s
|
| 507 |
+
[step 9740] tok=159.6M lm=2.3067 13.3k tok/s
|
| 508 |
+
[step 9760] tok=159.9M lm=2.5536 13.3k tok/s
|
| 509 |
+
[step 9780] tok=160.2M lm=2.6377 13.3k tok/s
|
| 510 |
+
[step 9800] tok=160.6M lm=2.2190 13.3k tok/s
|
| 511 |
+
[step 9820] tok=160.9M lm=2.4605 13.3k tok/s
|
| 512 |
+
[step 9840] tok=161.2M lm=2.3928 13.3k tok/s
|
| 513 |
+
[step 9860] tok=161.5M lm=2.4181 13.3k tok/s
|
| 514 |
+
[step 9880] tok=161.9M lm=2.5043 13.3k tok/s
|
| 515 |
+
[step 9900] tok=162.2M lm=2.5612 13.3k tok/s
|
| 516 |
+
[step 9920] tok=162.5M lm=2.4591 13.3k tok/s
|
| 517 |
+
[step 9940] tok=162.9M lm=2.5874 13.3k tok/s
|
| 518 |
+
[step 9960] tok=163.2M lm=2.3502 13.3k tok/s
|
| 519 |
+
[step 9980] tok=163.5M lm=2.2758 13.3k tok/s
|
| 520 |
+
[step 10000] tok=163.8M lm=2.5662 13.3k tok/s
|
| 521 |
+
[step 10020] tok=164.2M lm=2.3104 13.3k tok/s
|
| 522 |
+
[step 10040] tok=164.5M lm=2.4325 13.4k tok/s
|
| 523 |
+
[step 10060] tok=164.8M lm=2.3505 13.4k tok/s
|
| 524 |
+
[step 10080] tok=165.2M lm=2.5564 13.4k tok/s
|
| 525 |
+
[step 10100] tok=165.5M lm=2.4739 13.4k tok/s
|
| 526 |
+
[step 10120] tok=165.8M lm=2.5550 13.4k tok/s
|
| 527 |
+
[step 10140] tok=166.1M lm=2.3205 13.4k tok/s
|
| 528 |
+
[step 10160] tok=166.5M lm=2.4941 13.4k tok/s
|
| 529 |
+
[step 10180] tok=166.8M lm=2.5342 13.4k tok/s
|
| 530 |
+
[step 10200] tok=167.1M lm=2.1702 13.4k tok/s
|
| 531 |
+
[step 10220] tok=167.4M lm=2.6143 13.4k tok/s
|
| 532 |
+
[step 10240] tok=167.8M lm=2.2998 13.4k tok/s
|
| 533 |
+
[step 10260] tok=168.1M lm=2.5405 13.4k tok/s
|
| 534 |
+
[step 10280] tok=168.4M lm=2.5911 13.4k tok/s
|
| 535 |
+
[step 10300] tok=168.8M lm=2.7042 13.4k tok/s
|
| 536 |
+
[step 10320] tok=169.1M lm=2.3643 13.4k tok/s
|
| 537 |
+
[step 10340] tok=169.4M lm=2.5094 13.4k tok/s
|
| 538 |
+
[step 10360] tok=169.7M lm=2.3738 13.4k tok/s
|
| 539 |
+
[step 10380] tok=170.1M lm=2.4041 13.4k tok/s
|
| 540 |
+
[step 10400] tok=170.4M lm=2.1738 13.4k tok/s
|
| 541 |
+
[step 10420] tok=170.7M lm=2.4832 13.4k tok/s
|
| 542 |
+
[step 10440] tok=171.0M lm=2.3381 13.4k tok/s
|
| 543 |
+
[step 10460] tok=171.4M lm=2.5244 13.4k tok/s
|
| 544 |
+
[step 10480] tok=171.7M lm=2.2544 13.4k tok/s
|
| 545 |
+
[step 10500] tok=172.0M lm=2.6307 13.4k tok/s
|
| 546 |
+
[step 10520] tok=172.4M lm=2.5491 13.4k tok/s
|
| 547 |
+
[step 10540] tok=172.7M lm=2.4960 13.4k tok/s
|
| 548 |
+
[step 10560] tok=173.0M lm=2.2016 13.4k tok/s
|
| 549 |
+
[step 10580] tok=173.3M lm=2.6630 13.4k tok/s
|
| 550 |
+
[step 10600] tok=173.7M lm=2.5375 13.4k tok/s
|
| 551 |
+
[step 10620] tok=174.0M lm=2.4039 13.4k tok/s
|
| 552 |
+
[step 10640] tok=174.3M lm=2.2915 13.4k tok/s
|
| 553 |
+
[step 10660] tok=174.7M lm=2.3568 13.4k tok/s
|
| 554 |
+
[step 10680] tok=175.0M lm=2.6058 13.4k tok/s
|
| 555 |
+
[eval] fprime arm=Fprime tok=175M BPB=0.8100 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 556 |
+
[step 10700] tok=175.3M lm=2.4212 13.3k tok/s
|
| 557 |
+
[step 10720] tok=175.6M lm=2.2855 13.3k tok/s
|
| 558 |
+
[step 10740] tok=176.0M lm=2.6300 13.3k tok/s
|
| 559 |
+
[step 10760] tok=176.3M lm=2.4741 13.3k tok/s
|
| 560 |
+
[step 10780] tok=176.6M lm=2.8113 13.3k tok/s
|
| 561 |
+
[step 10800] tok=176.9M lm=2.4372 13.3k tok/s
|
| 562 |
+
[step 10820] tok=177.3M lm=2.2422 13.3k tok/s
|
| 563 |
+
[step 10840] tok=177.6M lm=2.3691 13.3k tok/s
|
| 564 |
+
[step 10860] tok=177.9M lm=2.4079 13.3k tok/s
|
| 565 |
+
[step 10880] tok=178.3M lm=2.3223 13.3k tok/s
|
| 566 |
+
[step 10900] tok=178.6M lm=2.2587 13.3k tok/s
|
| 567 |
+
[step 10920] tok=178.9M lm=2.5151 13.3k tok/s
|
| 568 |
+
[step 10940] tok=179.2M lm=2.2665 13.3k tok/s
|
| 569 |
+
[step 10960] tok=179.6M lm=2.5030 13.3k tok/s
|
| 570 |
+
[step 10980] tok=179.9M lm=2.5778 13.3k tok/s
|
| 571 |
+
[step 11000] tok=180.2M lm=2.4250 13.3k tok/s
|
| 572 |
+
[step 11020] tok=180.6M lm=2.4066 13.3k tok/s
|
| 573 |
+
[step 11040] tok=180.9M lm=2.5185 13.3k tok/s
|
| 574 |
+
[step 11060] tok=181.2M lm=2.4517 13.3k tok/s
|
| 575 |
+
[step 11080] tok=181.5M lm=2.6144 13.3k tok/s
|
| 576 |
+
[step 11100] tok=181.9M lm=2.1355 13.3k tok/s
|
| 577 |
+
[step 11120] tok=182.2M lm=2.3572 13.3k tok/s
|
| 578 |
+
[step 11140] tok=182.5M lm=2.5518 13.3k tok/s
|
| 579 |
+
[step 11160] tok=182.8M lm=2.4026 13.3k tok/s
|
| 580 |
+
[step 11180] tok=183.2M lm=2.4167 13.3k tok/s
|
| 581 |
+
[step 11200] tok=183.5M lm=2.8972 13.3k tok/s
|
| 582 |
+
[step 11220] tok=183.8M lm=2.5184 13.3k tok/s
|
| 583 |
+
[step 11240] tok=184.2M lm=2.3077 13.3k tok/s
|
| 584 |
+
[step 11260] tok=184.5M lm=2.3921 13.3k tok/s
|
| 585 |
+
[step 11280] tok=184.8M lm=2.5144 13.3k tok/s
|
| 586 |
+
[step 11300] tok=185.1M lm=2.4928 13.3k tok/s
|
| 587 |
+
[step 11320] tok=185.5M lm=2.4512 13.3k tok/s
|
| 588 |
+
[step 11340] tok=185.8M lm=2.2730 13.3k tok/s
|
| 589 |
+
[step 11360] tok=186.1M lm=2.4599 13.3k tok/s
|
| 590 |
+
[step 11380] tok=186.4M lm=2.5106 13.3k tok/s
|
| 591 |
+
[step 11400] tok=186.8M lm=2.4368 13.3k tok/s
|
| 592 |
+
[step 11420] tok=187.1M lm=2.3510 13.3k tok/s
|
| 593 |
+
[step 11440] tok=187.4M lm=2.4279 13.3k tok/s
|
| 594 |
+
[step 11460] tok=187.8M lm=2.6128 13.3k tok/s
|
| 595 |
+
[step 11480] tok=188.1M lm=2.5663 13.3k tok/s
|
| 596 |
+
[step 11500] tok=188.4M lm=2.3172 13.3k tok/s
|
| 597 |
+
[step 11520] tok=188.7M lm=2.4390 13.3k tok/s
|
| 598 |
+
[step 11540] tok=189.1M lm=2.3106 13.3k tok/s
|
| 599 |
+
[step 11560] tok=189.4M lm=2.4641 13.3k tok/s
|
| 600 |
+
[step 11580] tok=189.7M lm=2.2332 13.3k tok/s
|
| 601 |
+
[step 11600] tok=190.1M lm=2.7902 13.3k tok/s
|
| 602 |
+
[step 11620] tok=190.4M lm=2.3680 13.3k tok/s
|
| 603 |
+
[step 11640] tok=190.7M lm=2.1870 13.3k tok/s
|
| 604 |
+
[step 11660] tok=191.0M lm=2.3297 13.3k tok/s
|
| 605 |
+
[step 11680] tok=191.4M lm=2.5743 13.3k tok/s
|
| 606 |
+
[step 11700] tok=191.7M lm=2.4116 13.4k tok/s
|
| 607 |
+
[step 11720] tok=192.0M lm=2.3385 13.4k tok/s
|
| 608 |
+
[step 11740] tok=192.3M lm=2.4277 13.4k tok/s
|
| 609 |
+
[step 11760] tok=192.7M lm=2.2349 13.4k tok/s
|
| 610 |
+
[step 11780] tok=193.0M lm=2.2148 13.4k tok/s
|
| 611 |
+
[step 11800] tok=193.3M lm=2.1694 13.4k tok/s
|
| 612 |
+
[step 11820] tok=193.7M lm=2.1965 13.4k tok/s
|
| 613 |
+
[step 11840] tok=194.0M lm=2.1788 13.4k tok/s
|
| 614 |
+
[step 11860] tok=194.3M lm=2.5099 13.4k tok/s
|
| 615 |
+
[step 11880] tok=194.6M lm=2.3539 13.4k tok/s
|
| 616 |
+
[step 11900] tok=195.0M lm=2.0159 13.4k tok/s
|
| 617 |
+
[step 11920] tok=195.3M lm=2.3335 13.4k tok/s
|
| 618 |
+
[step 11940] tok=195.6M lm=2.1208 13.4k tok/s
|
| 619 |
+
[step 11960] tok=196.0M lm=2.5584 13.4k tok/s
|
| 620 |
+
[step 11980] tok=196.3M lm=2.4616 13.4k tok/s
|
| 621 |
+
[step 12000] tok=196.6M lm=2.3926 13.4k tok/s
|
| 622 |
+
[step 12020] tok=196.9M lm=2.2521 13.4k tok/s
|
| 623 |
+
[step 12040] tok=197.3M lm=2.5742 13.4k tok/s
|
| 624 |
+
[step 12060] tok=197.6M lm=2.3208 13.4k tok/s
|
| 625 |
+
[step 12080] tok=197.9M lm=2.5256 13.4k tok/s
|
| 626 |
+
[step 12100] tok=198.2M lm=2.2798 13.4k tok/s
|
| 627 |
+
[step 12120] tok=198.6M lm=2.5291 13.4k tok/s
|
| 628 |
+
[step 12140] tok=198.9M lm=2.1410 13.4k tok/s
|
| 629 |
+
[step 12160] tok=199.2M lm=2.5130 13.4k tok/s
|
| 630 |
+
[step 12180] tok=199.6M lm=2.6438 13.4k tok/s
|
| 631 |
+
[step 12200] tok=199.9M lm=2.3034 13.4k tok/s
|
| 632 |
+
[eval] fprime arm=Fprime tok=200M BPB=0.8106 swap=0.9997 ent=0.9988 (parent 0.8147)
|
| 633 |
+
[DONE] arm=Fprime final BPB=0.8106 parent=0.8147 -> SAME plateau (constraint price)
|
olmoe_adapt/logs/bake_bake_G.log
ADDED
|
@@ -0,0 +1,470 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=G tag=bake_G lr=0.0003 trainable=237111296 (router=2097152 extra=235014144) tokens=150000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:49: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:51: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=0.3M R=8 lm=3.6019 5.3k tok/s
|
| 11 |
+
[step 40] tok=0.7M R=8 lm=3.0386 5.4k tok/s
|
| 12 |
+
[step 60] tok=1.0M R=8 lm=3.0540 5.4k tok/s
|
| 13 |
+
[step 80] tok=1.3M R=8 lm=3.0043 5.4k tok/s
|
| 14 |
+
[step 100] tok=1.6M R=8 lm=2.4422 5.4k tok/s
|
| 15 |
+
[step 120] tok=2.0M R=8 lm=3.0401 5.4k tok/s
|
| 16 |
+
[step 140] tok=2.3M R=8 lm=2.7283 5.4k tok/s
|
| 17 |
+
[step 160] tok=2.6M R=8 lm=2.6405 5.4k tok/s
|
| 18 |
+
[step 180] tok=2.9M R=8 lm=2.6733 5.4k tok/s
|
| 19 |
+
[step 200] tok=3.3M R=8 lm=2.7035 5.4k tok/s
|
| 20 |
+
[step 220] tok=3.6M R=8 lm=2.6223 5.4k tok/s
|
| 21 |
+
[step 240] tok=3.9M R=8 lm=2.9475 5.4k tok/s
|
| 22 |
+
[step 260] tok=4.3M R=8 lm=2.6715 5.4k tok/s
|
| 23 |
+
[step 280] tok=4.6M R=8 lm=2.8180 5.4k tok/s
|
| 24 |
+
[step 300] tok=4.9M R=8 lm=2.4985 5.4k tok/s
|
| 25 |
+
[step 320] tok=5.2M R=8 lm=2.8847 5.4k tok/s
|
| 26 |
+
[step 340] tok=5.6M R=8 lm=2.7191 5.4k tok/s
|
| 27 |
+
[step 360] tok=5.9M R=8 lm=2.6737 5.4k tok/s
|
| 28 |
+
[step 380] tok=6.2M R=8 lm=2.7207 5.4k tok/s
|
| 29 |
+
[step 400] tok=6.6M R=8 lm=2.6624 5.4k tok/s
|
| 30 |
+
[step 420] tok=6.9M R=8 lm=2.7939 5.4k tok/s
|
| 31 |
+
[step 440] tok=7.2M R=8 lm=2.7112 5.4k tok/s
|
| 32 |
+
[step 460] tok=7.5M R=8 lm=2.5109 5.4k tok/s
|
| 33 |
+
[step 480] tok=7.9M R=8 lm=2.8494 5.4k tok/s
|
| 34 |
+
[step 500] tok=8.2M R=8 lm=2.7067 5.4k tok/s
|
| 35 |
+
[step 520] tok=8.5M R=8 lm=2.8376 5.4k tok/s
|
| 36 |
+
[step 540] tok=8.8M R=8 lm=2.6008 5.4k tok/s
|
| 37 |
+
[step 560] tok=9.2M R=8 lm=2.7442 5.4k tok/s
|
| 38 |
+
[step 580] tok=9.5M R=8 lm=2.6003 5.4k tok/s
|
| 39 |
+
[step 600] tok=9.8M R=8 lm=2.3654 5.4k tok/s
|
| 40 |
+
[step 620] tok=10.2M R=8 lm=2.6361 5.4k tok/s
|
| 41 |
+
[step 640] tok=10.5M R=8 lm=2.8589 5.4k tok/s
|
| 42 |
+
[step 660] tok=10.8M R=8 lm=2.8702 5.4k tok/s
|
| 43 |
+
[step 680] tok=11.1M R=8 lm=2.5499 5.4k tok/s
|
| 44 |
+
[step 700] tok=11.5M R=8 lm=2.5698 5.4k tok/s
|
| 45 |
+
[step 720] tok=11.8M R=8 lm=2.7132 5.4k tok/s
|
| 46 |
+
[step 740] tok=12.1M R=8 lm=2.7273 5.4k tok/s
|
| 47 |
+
[step 760] tok=12.5M R=8 lm=2.7993 5.4k tok/s
|
| 48 |
+
[step 780] tok=12.8M R=8 lm=2.8882 5.4k tok/s
|
| 49 |
+
[step 800] tok=13.1M R=8 lm=2.8587 5.4k tok/s
|
| 50 |
+
[step 820] tok=13.4M R=8 lm=2.3921 5.4k tok/s
|
| 51 |
+
[step 840] tok=13.8M R=8 lm=2.8969 5.4k tok/s
|
| 52 |
+
[step 860] tok=14.1M R=8 lm=2.6257 5.4k tok/s
|
| 53 |
+
[step 880] tok=14.4M R=8 lm=2.8190 5.4k tok/s
|
| 54 |
+
[step 900] tok=14.7M R=8 lm=2.7096 5.4k tok/s
|
| 55 |
+
[step 920] tok=15.1M R=8 lm=2.6604 5.4k tok/s
|
| 56 |
+
[step 940] tok=15.4M R=8 lm=2.6338 5.4k tok/s
|
| 57 |
+
[step 960] tok=15.7M R=8 lm=2.6598 5.4k tok/s
|
| 58 |
+
[step 980] tok=16.1M R=8 lm=2.9367 5.4k tok/s
|
| 59 |
+
[step 1000] tok=16.4M R=8 lm=2.6245 5.4k tok/s
|
| 60 |
+
[step 1020] tok=16.7M R=8 lm=2.8269 5.4k tok/s
|
| 61 |
+
[step 1040] tok=17.0M R=8 lm=2.6874 5.4k tok/s
|
| 62 |
+
[step 1060] tok=17.4M R=8 lm=3.0796 5.4k tok/s
|
| 63 |
+
[step 1080] tok=17.7M R=8 lm=2.6822 5.4k tok/s
|
| 64 |
+
[step 1100] tok=18.0M R=8 lm=2.6129 5.4k tok/s
|
| 65 |
+
[step 1120] tok=18.4M R=8 lm=2.8648 5.4k tok/s
|
| 66 |
+
[step 1140] tok=18.7M R=8 lm=2.6017 5.4k tok/s
|
| 67 |
+
[step 1160] tok=19.0M R=8 lm=2.4064 5.4k tok/s
|
| 68 |
+
[step 1180] tok=19.3M R=8 lm=2.8491 5.4k tok/s
|
| 69 |
+
[step 1200] tok=19.7M R=8 lm=2.7915 5.4k tok/s
|
| 70 |
+
[step 1220] tok=20.0M R=8 lm=2.6770 5.4k tok/s
|
| 71 |
+
[step 1240] tok=20.3M R=8 lm=2.8754 5.4k tok/s
|
| 72 |
+
[step 1260] tok=20.6M R=8 lm=2.6603 5.4k tok/s
|
| 73 |
+
[step 1280] tok=21.0M R=8 lm=2.6750 5.4k tok/s
|
| 74 |
+
[step 1300] tok=21.3M R=8 lm=2.7534 5.4k tok/s
|
| 75 |
+
[step 1320] tok=21.6M R=8 lm=2.8766 5.4k tok/s
|
| 76 |
+
[step 1340] tok=22.0M R=8 lm=2.4411 5.4k tok/s
|
| 77 |
+
[step 1360] tok=22.3M R=8 lm=2.7386 5.4k tok/s
|
| 78 |
+
[step 1380] tok=22.6M R=8 lm=2.6159 5.4k tok/s
|
| 79 |
+
[step 1400] tok=22.9M R=8 lm=2.8959 5.4k tok/s
|
| 80 |
+
[step 1420] tok=23.3M R=8 lm=2.6571 5.4k tok/s
|
| 81 |
+
[step 1440] tok=23.6M R=8 lm=2.8019 5.4k tok/s
|
| 82 |
+
[step 1460] tok=23.9M R=8 lm=2.6383 5.4k tok/s
|
| 83 |
+
[step 1480] tok=24.2M R=8 lm=2.6253 5.4k tok/s
|
| 84 |
+
[step 1500] tok=24.6M R=8 lm=2.6680 5.4k tok/s
|
| 85 |
+
[step 1520] tok=24.9M R=8 lm=2.8650 5.4k tok/s
|
| 86 |
+
[step 1540] tok=25.2M R=8 lm=2.5618 5.4k tok/s
|
| 87 |
+
[step 1560] tok=25.6M R=8 lm=2.7045 5.4k tok/s
|
| 88 |
+
[step 1580] tok=25.9M R=8 lm=2.5662 5.4k tok/s
|
| 89 |
+
[step 1600] tok=26.2M R=8 lm=2.6075 5.4k tok/s
|
| 90 |
+
[step 1620] tok=26.5M R=8 lm=2.6034 5.4k tok/s
|
| 91 |
+
[step 1640] tok=26.9M R=8 lm=2.7287 5.4k tok/s
|
| 92 |
+
[step 1660] tok=27.2M R=8 lm=2.8897 5.4k tok/s
|
| 93 |
+
[step 1680] tok=27.5M R=8 lm=2.5767 5.4k tok/s
|
| 94 |
+
[step 1700] tok=27.9M R=8 lm=2.6749 5.4k tok/s
|
| 95 |
+
[step 1720] tok=28.2M R=8 lm=2.8753 5.4k tok/s
|
| 96 |
+
[step 1740] tok=28.5M R=8 lm=2.7079 5.4k tok/s
|
| 97 |
+
[step 1760] tok=28.8M R=8 lm=2.6338 5.4k tok/s
|
| 98 |
+
[step 1780] tok=29.2M R=8 lm=2.5011 5.4k tok/s
|
| 99 |
+
[step 1800] tok=29.5M R=8 lm=2.6799 5.4k tok/s
|
| 100 |
+
[step 1820] tok=29.8M R=8 lm=2.6853 5.4k tok/s
|
| 101 |
+
[step 1840] tok=30.1M R=8 lm=2.6720 5.4k tok/s
|
| 102 |
+
[step 1860] tok=30.5M R=8 lm=2.6596 5.4k tok/s
|
| 103 |
+
[step 1880] tok=30.8M R=8 lm=2.5761 5.4k tok/s
|
| 104 |
+
[step 1900] tok=31.1M R=8 lm=2.5446 5.4k tok/s
|
| 105 |
+
[step 1920] tok=31.5M R=8 lm=2.7738 5.4k tok/s
|
| 106 |
+
[step 1940] tok=31.8M R=8 lm=2.8934 5.4k tok/s
|
| 107 |
+
[step 1960] tok=32.1M R=8 lm=2.6247 5.4k tok/s
|
| 108 |
+
[step 1980] tok=32.4M R=8 lm=2.7698 5.4k tok/s
|
| 109 |
+
[step 2000] tok=32.8M R=8 lm=2.6231 5.4k tok/s
|
| 110 |
+
[step 2020] tok=33.1M R=8 lm=2.5023 5.4k tok/s
|
| 111 |
+
[step 2040] tok=33.4M R=8 lm=2.6073 5.4k tok/s
|
| 112 |
+
[step 2060] tok=33.8M R=8 lm=2.4878 5.4k tok/s
|
| 113 |
+
[step 2080] tok=34.1M R=8 lm=2.7694 5.4k tok/s
|
| 114 |
+
[step 2100] tok=34.4M R=8 lm=2.8014 5.4k tok/s
|
| 115 |
+
[step 2120] tok=34.7M R=8 lm=2.8858 5.4k tok/s
|
| 116 |
+
[step 2140] tok=35.1M R=8 lm=3.0807 5.4k tok/s
|
| 117 |
+
[step 2160] tok=35.4M R=8 lm=2.6410 5.4k tok/s
|
| 118 |
+
[step 2180] tok=35.7M R=8 lm=2.5189 5.4k tok/s
|
| 119 |
+
[step 2200] tok=36.0M R=8 lm=2.7630 5.4k tok/s
|
| 120 |
+
[step 2220] tok=36.4M R=8 lm=3.2119 5.4k tok/s
|
| 121 |
+
[step 2240] tok=36.7M R=8 lm=2.8439 5.4k tok/s
|
| 122 |
+
[step 2260] tok=37.0M R=8 lm=2.7645 5.4k tok/s
|
| 123 |
+
[step 2280] tok=37.4M R=8 lm=2.5836 5.4k tok/s
|
| 124 |
+
[step 2300] tok=37.7M R=8 lm=2.7030 5.4k tok/s
|
| 125 |
+
[step 2320] tok=38.0M R=8 lm=3.1094 5.4k tok/s
|
| 126 |
+
[step 2340] tok=38.3M R=8 lm=2.8917 5.4k tok/s
|
| 127 |
+
[step 2360] tok=38.7M R=8 lm=2.5557 5.4k tok/s
|
| 128 |
+
[step 2380] tok=39.0M R=8 lm=2.5868 5.4k tok/s
|
| 129 |
+
[step 2400] tok=39.3M R=8 lm=3.0752 5.4k tok/s
|
| 130 |
+
[step 2420] tok=39.6M R=8 lm=2.7294 5.4k tok/s
|
| 131 |
+
[step 2440] tok=40.0M R=8 lm=2.7034 5.4k tok/s
|
| 132 |
+
[step 2460] tok=40.3M R=8 lm=2.7633 5.4k tok/s
|
| 133 |
+
[step 2480] tok=40.6M R=8 lm=2.5684 5.4k tok/s
|
| 134 |
+
[step 2500] tok=41.0M R=8 lm=2.6083 5.4k tok/s
|
| 135 |
+
[step 2520] tok=41.3M R=8 lm=2.8046 5.4k tok/s
|
| 136 |
+
[step 2540] tok=41.6M R=8 lm=2.7993 5.4k tok/s
|
| 137 |
+
[step 2560] tok=41.9M R=8 lm=2.4257 5.4k tok/s
|
| 138 |
+
[step 2580] tok=42.3M R=8 lm=2.8834 5.4k tok/s
|
| 139 |
+
[step 2600] tok=42.6M R=8 lm=2.6741 5.4k tok/s
|
| 140 |
+
[step 2620] tok=42.9M R=8 lm=2.6044 5.4k tok/s
|
| 141 |
+
[step 2640] tok=43.3M R=8 lm=2.6287 5.4k tok/s
|
| 142 |
+
[step 2660] tok=43.6M R=8 lm=2.8463 5.4k tok/s
|
| 143 |
+
[step 2680] tok=43.9M R=8 lm=2.6587 5.4k tok/s
|
| 144 |
+
[step 2700] tok=44.2M R=8 lm=2.7925 5.4k tok/s
|
| 145 |
+
[step 2720] tok=44.6M R=8 lm=2.4418 5.4k tok/s
|
| 146 |
+
[step 2740] tok=44.9M R=8 lm=2.5998 5.4k tok/s
|
| 147 |
+
[step 2760] tok=45.2M R=8 lm=2.6169 5.4k tok/s
|
| 148 |
+
[step 2780] tok=45.5M R=8 lm=2.7065 5.4k tok/s
|
| 149 |
+
[step 2800] tok=45.9M R=8 lm=3.0430 5.4k tok/s
|
| 150 |
+
[step 2820] tok=46.2M R=8 lm=2.4820 5.4k tok/s
|
| 151 |
+
[step 2840] tok=46.5M R=8 lm=2.1529 5.4k tok/s
|
| 152 |
+
[step 2860] tok=46.9M R=8 lm=2.5733 5.4k tok/s
|
| 153 |
+
[step 2880] tok=47.2M R=8 lm=2.7997 5.4k tok/s
|
| 154 |
+
[step 2900] tok=47.5M R=8 lm=2.7809 5.4k tok/s
|
| 155 |
+
[step 2920] tok=47.8M R=8 lm=2.7312 5.4k tok/s
|
| 156 |
+
[step 2940] tok=48.2M R=8 lm=2.5959 5.4k tok/s
|
| 157 |
+
[step 2960] tok=48.5M R=8 lm=2.8557 5.4k tok/s
|
| 158 |
+
[step 2980] tok=48.8M R=8 lm=2.5332 5.4k tok/s
|
| 159 |
+
[step 3000] tok=49.2M R=8 lm=2.7908 5.4k tok/s
|
| 160 |
+
[step 3020] tok=49.5M R=8 lm=2.7107 5.4k tok/s
|
| 161 |
+
[step 3040] tok=49.8M R=8 lm=2.8028 5.4k tok/s
|
| 162 |
+
[eval] bake_G arm=G tok=50M trainR=8 BPB=0.8358 swap=0.9998 ent=0.9984
|
| 163 |
+
[step 3060] tok=50.1M R=8 lm=2.6639 5.4k tok/s
|
| 164 |
+
[step 3080] tok=50.5M R=8 lm=2.6720 5.4k tok/s
|
| 165 |
+
[step 3100] tok=50.8M R=8 lm=2.5831 5.4k tok/s
|
| 166 |
+
[step 3120] tok=51.1M R=8 lm=2.7825 5.4k tok/s
|
| 167 |
+
[step 3140] tok=51.4M R=8 lm=3.2525 5.4k tok/s
|
| 168 |
+
[step 3160] tok=51.8M R=8 lm=2.3931 5.4k tok/s
|
| 169 |
+
[step 3180] tok=52.1M R=8 lm=2.7371 5.4k tok/s
|
| 170 |
+
[step 3200] tok=52.4M R=8 lm=2.7521 5.4k tok/s
|
| 171 |
+
[step 3220] tok=52.8M R=8 lm=2.6311 5.4k tok/s
|
| 172 |
+
[step 3240] tok=53.1M R=8 lm=2.4337 5.4k tok/s
|
| 173 |
+
[step 3260] tok=53.4M R=8 lm=2.2602 5.4k tok/s
|
| 174 |
+
[step 3280] tok=53.7M R=8 lm=2.8003 5.4k tok/s
|
| 175 |
+
[step 3300] tok=54.1M R=8 lm=2.7289 5.4k tok/s
|
| 176 |
+
[step 3320] tok=54.4M R=8 lm=2.5883 5.4k tok/s
|
| 177 |
+
[step 3340] tok=54.7M R=8 lm=2.6468 5.4k tok/s
|
| 178 |
+
[step 3360] tok=55.1M R=8 lm=2.6367 5.4k tok/s
|
| 179 |
+
[step 3380] tok=55.4M R=8 lm=2.9377 5.4k tok/s
|
| 180 |
+
[step 3400] tok=55.7M R=8 lm=2.5282 5.4k tok/s
|
| 181 |
+
[step 3420] tok=56.0M R=8 lm=2.4420 5.4k tok/s
|
| 182 |
+
[step 3440] tok=56.4M R=8 lm=2.7026 5.4k tok/s
|
| 183 |
+
[step 3460] tok=56.7M R=8 lm=2.6819 5.4k tok/s
|
| 184 |
+
[step 3480] tok=57.0M R=8 lm=2.6538 5.4k tok/s
|
| 185 |
+
[step 3500] tok=57.3M R=8 lm=2.7534 5.4k tok/s
|
| 186 |
+
[step 3520] tok=57.7M R=8 lm=2.6040 5.4k tok/s
|
| 187 |
+
[step 3540] tok=58.0M R=8 lm=2.7339 5.4k tok/s
|
| 188 |
+
[step 3560] tok=58.3M R=8 lm=2.7544 5.4k tok/s
|
| 189 |
+
[step 3580] tok=58.7M R=8 lm=2.9717 5.4k tok/s
|
| 190 |
+
[step 3600] tok=59.0M R=8 lm=2.5694 5.4k tok/s
|
| 191 |
+
[step 3620] tok=59.3M R=8 lm=2.6605 5.4k tok/s
|
| 192 |
+
[step 3640] tok=59.6M R=8 lm=2.6541 5.4k tok/s
|
| 193 |
+
[step 3660] tok=60.0M R=8 lm=2.5150 5.4k tok/s
|
| 194 |
+
[step 3680] tok=60.3M R=8 lm=2.6725 5.4k tok/s
|
| 195 |
+
[step 3700] tok=60.6M R=8 lm=2.6822 5.4k tok/s
|
| 196 |
+
[step 3720] tok=60.9M R=8 lm=2.7957 5.4k tok/s
|
| 197 |
+
[step 3740] tok=61.3M R=8 lm=2.7291 5.4k tok/s
|
| 198 |
+
[step 3760] tok=61.6M R=8 lm=2.6457 5.4k tok/s
|
| 199 |
+
[step 3780] tok=61.9M R=8 lm=2.7859 5.4k tok/s
|
| 200 |
+
[step 3800] tok=62.3M R=8 lm=2.5116 5.4k tok/s
|
| 201 |
+
[step 3820] tok=62.6M R=8 lm=2.4721 5.4k tok/s
|
| 202 |
+
[step 3840] tok=62.9M R=8 lm=2.6904 5.4k tok/s
|
| 203 |
+
[step 3860] tok=63.2M R=8 lm=2.3672 5.4k tok/s
|
| 204 |
+
[step 3880] tok=63.6M R=8 lm=2.6644 5.4k tok/s
|
| 205 |
+
[step 3900] tok=63.9M R=8 lm=2.7572 5.4k tok/s
|
| 206 |
+
[step 3920] tok=64.2M R=8 lm=2.6709 5.4k tok/s
|
| 207 |
+
[step 3940] tok=64.6M R=8 lm=2.7653 5.4k tok/s
|
| 208 |
+
[step 3960] tok=64.9M R=8 lm=2.6970 5.4k tok/s
|
| 209 |
+
[step 3980] tok=65.2M R=8 lm=2.6614 5.4k tok/s
|
| 210 |
+
[step 4000] tok=65.5M R=8 lm=2.5812 5.4k tok/s
|
| 211 |
+
[step 4020] tok=65.9M R=8 lm=2.6803 5.4k tok/s
|
| 212 |
+
[step 4040] tok=66.2M R=8 lm=3.1689 5.4k tok/s
|
| 213 |
+
[step 4060] tok=66.5M R=8 lm=2.6264 5.4k tok/s
|
| 214 |
+
[step 4080] tok=66.8M R=8 lm=2.4943 5.4k tok/s
|
| 215 |
+
[step 4100] tok=67.2M R=8 lm=2.5632 5.4k tok/s
|
| 216 |
+
[step 4120] tok=67.5M R=8 lm=2.6693 5.4k tok/s
|
| 217 |
+
[step 4140] tok=67.8M R=8 lm=2.6568 5.4k tok/s
|
| 218 |
+
[step 4160] tok=68.2M R=8 lm=2.6952 5.4k tok/s
|
| 219 |
+
[step 4180] tok=68.5M R=8 lm=2.5554 5.4k tok/s
|
| 220 |
+
[step 4200] tok=68.8M R=8 lm=2.8027 5.4k tok/s
|
| 221 |
+
[step 4220] tok=69.1M R=8 lm=2.7503 5.4k tok/s
|
| 222 |
+
[step 4240] tok=69.5M R=8 lm=2.5508 5.4k tok/s
|
| 223 |
+
[step 4260] tok=69.8M R=8 lm=2.5404 5.4k tok/s
|
| 224 |
+
[step 4280] tok=70.1M R=8 lm=2.4188 5.4k tok/s
|
| 225 |
+
[step 4300] tok=70.5M R=8 lm=2.5959 5.4k tok/s
|
| 226 |
+
[step 4320] tok=70.8M R=8 lm=2.7567 5.4k tok/s
|
| 227 |
+
[step 4340] tok=71.1M R=8 lm=2.6335 5.4k tok/s
|
| 228 |
+
[step 4360] tok=71.4M R=8 lm=2.7411 5.4k tok/s
|
| 229 |
+
[step 4380] tok=71.8M R=8 lm=2.5468 5.4k tok/s
|
| 230 |
+
[step 4400] tok=72.1M R=8 lm=2.6159 5.4k tok/s
|
| 231 |
+
[step 4420] tok=72.4M R=8 lm=2.7609 5.4k tok/s
|
| 232 |
+
[step 4440] tok=72.7M R=8 lm=2.5745 5.4k tok/s
|
| 233 |
+
[step 4460] tok=73.1M R=8 lm=2.4301 5.4k tok/s
|
| 234 |
+
[step 4480] tok=73.4M R=8 lm=2.7125 5.4k tok/s
|
| 235 |
+
[step 4500] tok=73.7M R=8 lm=2.6482 5.4k tok/s
|
| 236 |
+
[step 4520] tok=74.1M R=8 lm=2.9316 5.4k tok/s
|
| 237 |
+
[step 4540] tok=74.4M R=8 lm=2.7095 5.4k tok/s
|
| 238 |
+
[step 4560] tok=74.7M R=8 lm=2.8557 5.4k tok/s
|
| 239 |
+
[step 4580] tok=75.0M R=8 lm=2.5757 5.4k tok/s
|
| 240 |
+
[step 4600] tok=75.4M R=8 lm=2.6710 5.4k tok/s
|
| 241 |
+
[step 4620] tok=75.7M R=8 lm=2.7533 5.4k tok/s
|
| 242 |
+
[step 4640] tok=76.0M R=8 lm=2.5945 5.4k tok/s
|
| 243 |
+
[step 4660] tok=76.3M R=8 lm=2.5103 5.4k tok/s
|
| 244 |
+
[step 4680] tok=76.7M R=8 lm=2.8816 5.4k tok/s
|
| 245 |
+
[step 4700] tok=77.0M R=8 lm=2.7835 5.4k tok/s
|
| 246 |
+
[step 4720] tok=77.3M R=8 lm=2.7414 5.4k tok/s
|
| 247 |
+
[step 4740] tok=77.7M R=8 lm=2.6950 5.4k tok/s
|
| 248 |
+
[step 4760] tok=78.0M R=8 lm=2.7837 5.4k tok/s
|
| 249 |
+
[step 4780] tok=78.3M R=8 lm=2.6757 5.4k tok/s
|
| 250 |
+
[step 4800] tok=78.6M R=8 lm=2.7706 5.4k tok/s
|
| 251 |
+
[step 4820] tok=79.0M R=8 lm=2.5729 5.4k tok/s
|
| 252 |
+
[step 4840] tok=79.3M R=8 lm=2.6976 5.4k tok/s
|
| 253 |
+
[step 4860] tok=79.6M R=8 lm=2.7557 5.4k tok/s
|
| 254 |
+
[step 4880] tok=80.0M R=8 lm=2.4456 5.4k tok/s
|
| 255 |
+
[step 4900] tok=80.3M R=8 lm=2.8465 5.4k tok/s
|
| 256 |
+
[step 4920] tok=80.6M R=8 lm=2.7918 5.4k tok/s
|
| 257 |
+
[step 4940] tok=80.9M R=8 lm=2.7083 5.4k tok/s
|
| 258 |
+
[step 4960] tok=81.3M R=8 lm=2.8456 5.4k tok/s
|
| 259 |
+
[step 4980] tok=81.6M R=8 lm=2.8636 5.4k tok/s
|
| 260 |
+
[step 5000] tok=81.9M R=8 lm=2.4538 5.4k tok/s
|
| 261 |
+
[step 5020] tok=82.2M R=8 lm=2.8942 5.4k tok/s
|
| 262 |
+
[step 5040] tok=82.6M R=8 lm=2.9292 5.4k tok/s
|
| 263 |
+
[step 5060] tok=82.9M R=8 lm=2.5295 5.4k tok/s
|
| 264 |
+
[step 5080] tok=83.2M R=8 lm=2.6237 5.4k tok/s
|
| 265 |
+
[step 5100] tok=83.6M R=8 lm=2.5823 5.4k tok/s
|
| 266 |
+
[step 5120] tok=83.9M R=8 lm=2.5995 5.4k tok/s
|
| 267 |
+
[step 5140] tok=84.2M R=8 lm=2.4956 5.4k tok/s
|
| 268 |
+
[step 5160] tok=84.5M R=8 lm=2.5795 5.4k tok/s
|
| 269 |
+
[step 5180] tok=84.9M R=8 lm=2.7110 5.4k tok/s
|
| 270 |
+
[step 5200] tok=85.2M R=8 lm=2.7122 5.4k tok/s
|
| 271 |
+
[step 5220] tok=85.5M R=8 lm=2.8741 5.4k tok/s
|
| 272 |
+
[step 5240] tok=85.9M R=8 lm=2.8497 5.4k tok/s
|
| 273 |
+
[step 5260] tok=86.2M R=8 lm=2.9170 5.4k tok/s
|
| 274 |
+
[step 5280] tok=86.5M R=8 lm=2.6476 5.4k tok/s
|
| 275 |
+
[step 5300] tok=86.8M R=8 lm=2.2833 5.4k tok/s
|
| 276 |
+
[step 5320] tok=87.2M R=8 lm=2.7197 5.4k tok/s
|
| 277 |
+
[step 5340] tok=87.5M R=8 lm=2.6149 5.4k tok/s
|
| 278 |
+
[step 5360] tok=87.8M R=8 lm=2.8218 5.4k tok/s
|
| 279 |
+
[step 5380] tok=88.1M R=8 lm=2.5544 5.4k tok/s
|
| 280 |
+
[step 5400] tok=88.5M R=8 lm=2.6228 5.4k tok/s
|
| 281 |
+
[step 5420] tok=88.8M R=8 lm=2.5896 5.4k tok/s
|
| 282 |
+
[step 5440] tok=89.1M R=8 lm=2.7996 5.4k tok/s
|
| 283 |
+
[step 5460] tok=89.5M R=8 lm=2.8751 5.4k tok/s
|
| 284 |
+
[step 5480] tok=89.8M R=8 lm=2.5507 5.4k tok/s
|
| 285 |
+
[step 5500] tok=90.1M R=8 lm=2.5540 5.4k tok/s
|
| 286 |
+
[step 5520] tok=90.4M R=8 lm=2.4347 5.4k tok/s
|
| 287 |
+
[step 5540] tok=90.8M R=8 lm=2.6771 5.4k tok/s
|
| 288 |
+
[step 5560] tok=91.1M R=8 lm=2.5859 5.4k tok/s
|
| 289 |
+
[step 5580] tok=91.4M R=8 lm=2.5547 5.4k tok/s
|
| 290 |
+
[step 5600] tok=91.8M R=8 lm=2.6045 5.4k tok/s
|
| 291 |
+
[step 5620] tok=92.1M R=8 lm=2.7788 5.4k tok/s
|
| 292 |
+
[step 5640] tok=92.4M R=8 lm=2.4206 5.4k tok/s
|
| 293 |
+
[step 5660] tok=92.7M R=8 lm=2.7922 5.4k tok/s
|
| 294 |
+
[step 5680] tok=93.1M R=8 lm=2.6478 5.4k tok/s
|
| 295 |
+
[step 5700] tok=93.4M R=8 lm=2.9799 5.4k tok/s
|
| 296 |
+
[step 5720] tok=93.7M R=8 lm=2.5620 5.4k tok/s
|
| 297 |
+
[step 5740] tok=94.0M R=8 lm=2.5737 5.4k tok/s
|
| 298 |
+
[step 5760] tok=94.4M R=8 lm=2.7355 5.4k tok/s
|
| 299 |
+
[step 5780] tok=94.7M R=8 lm=2.7583 5.4k tok/s
|
| 300 |
+
[step 5800] tok=95.0M R=8 lm=2.6505 5.4k tok/s
|
| 301 |
+
[step 5820] tok=95.4M R=8 lm=2.7143 5.4k tok/s
|
| 302 |
+
[step 5840] tok=95.7M R=8 lm=2.7085 5.4k tok/s
|
| 303 |
+
[step 5860] tok=96.0M R=8 lm=2.6012 5.4k tok/s
|
| 304 |
+
[step 5880] tok=96.3M R=8 lm=2.6526 5.4k tok/s
|
| 305 |
+
[step 5900] tok=96.7M R=8 lm=2.8989 5.4k tok/s
|
| 306 |
+
[step 5920] tok=97.0M R=8 lm=2.6238 5.4k tok/s
|
| 307 |
+
[step 5940] tok=97.3M R=8 lm=2.4651 5.4k tok/s
|
| 308 |
+
[step 5960] tok=97.6M R=8 lm=2.5566 5.4k tok/s
|
| 309 |
+
[step 5980] tok=98.0M R=8 lm=2.8226 5.4k tok/s
|
| 310 |
+
[step 6000] tok=98.3M R=8 lm=2.5837 5.4k tok/s
|
| 311 |
+
[step 6020] tok=98.6M R=8 lm=2.7223 5.4k tok/s
|
| 312 |
+
[step 6040] tok=99.0M R=8 lm=2.8389 5.4k tok/s
|
| 313 |
+
[step 6060] tok=99.3M R=8 lm=2.6357 5.4k tok/s
|
| 314 |
+
[step 6080] tok=99.6M R=8 lm=2.5519 5.4k tok/s
|
| 315 |
+
[step 6100] tok=99.9M R=8 lm=2.6150 5.4k tok/s
|
| 316 |
+
[eval] bake_G arm=G tok=100M trainR=8 BPB=0.8325 swap=0.9998 ent=0.9984
|
| 317 |
+
[step 6120] tok=100.3M R=8 lm=2.5573 5.4k tok/s
|
| 318 |
+
[step 6140] tok=100.6M R=8 lm=2.5208 5.4k tok/s
|
| 319 |
+
[step 6160] tok=100.9M R=8 lm=2.6214 5.4k tok/s
|
| 320 |
+
[step 6180] tok=101.3M R=8 lm=2.7941 5.4k tok/s
|
| 321 |
+
[step 6200] tok=101.6M R=8 lm=2.5430 5.4k tok/s
|
| 322 |
+
[step 6220] tok=101.9M R=8 lm=2.7059 5.4k tok/s
|
| 323 |
+
[step 6240] tok=102.2M R=8 lm=2.6482 5.4k tok/s
|
| 324 |
+
[step 6260] tok=102.6M R=8 lm=3.0462 5.4k tok/s
|
| 325 |
+
[step 6280] tok=102.9M R=8 lm=2.6500 5.4k tok/s
|
| 326 |
+
[step 6300] tok=103.2M R=8 lm=3.1380 5.4k tok/s
|
| 327 |
+
[step 6320] tok=103.5M R=8 lm=2.8398 5.4k tok/s
|
| 328 |
+
[step 6340] tok=103.9M R=8 lm=2.6881 5.4k tok/s
|
| 329 |
+
[step 6360] tok=104.2M R=8 lm=2.7680 5.4k tok/s
|
| 330 |
+
[step 6380] tok=104.5M R=8 lm=2.7555 5.4k tok/s
|
| 331 |
+
[step 6400] tok=104.9M R=8 lm=2.6671 5.4k tok/s
|
| 332 |
+
[step 6420] tok=105.2M R=8 lm=2.5576 5.4k tok/s
|
| 333 |
+
[step 6440] tok=105.5M R=8 lm=2.6536 5.4k tok/s
|
| 334 |
+
[step 6460] tok=105.8M R=8 lm=2.8060 5.4k tok/s
|
| 335 |
+
[step 6480] tok=106.2M R=8 lm=2.6633 5.4k tok/s
|
| 336 |
+
[step 6500] tok=106.5M R=8 lm=2.5764 5.4k tok/s
|
| 337 |
+
[step 6520] tok=106.8M R=8 lm=2.5817 5.4k tok/s
|
| 338 |
+
[step 6540] tok=107.2M R=8 lm=2.5452 5.4k tok/s
|
| 339 |
+
[step 6560] tok=107.5M R=8 lm=2.7138 5.4k tok/s
|
| 340 |
+
[step 6580] tok=107.8M R=8 lm=2.6645 5.4k tok/s
|
| 341 |
+
[step 6600] tok=108.1M R=8 lm=2.6755 5.4k tok/s
|
| 342 |
+
[step 6620] tok=108.5M R=8 lm=2.6629 5.4k tok/s
|
| 343 |
+
[step 6640] tok=108.8M R=8 lm=2.5482 5.4k tok/s
|
| 344 |
+
[step 6660] tok=109.1M R=8 lm=2.7305 5.4k tok/s
|
| 345 |
+
[step 6680] tok=109.4M R=8 lm=2.7118 5.4k tok/s
|
| 346 |
+
[step 6700] tok=109.8M R=8 lm=2.8276 5.4k tok/s
|
| 347 |
+
[step 6720] tok=110.1M R=8 lm=2.6955 5.4k tok/s
|
| 348 |
+
[step 6740] tok=110.4M R=8 lm=2.6676 5.4k tok/s
|
| 349 |
+
[step 6760] tok=110.8M R=8 lm=2.4215 5.4k tok/s
|
| 350 |
+
[step 6780] tok=111.1M R=8 lm=2.5993 5.4k tok/s
|
| 351 |
+
[step 6800] tok=111.4M R=8 lm=2.7345 5.4k tok/s
|
| 352 |
+
[step 6820] tok=111.7M R=8 lm=2.5904 5.4k tok/s
|
| 353 |
+
[step 6840] tok=112.1M R=8 lm=2.6083 5.4k tok/s
|
| 354 |
+
[step 6860] tok=112.4M R=8 lm=2.8690 5.4k tok/s
|
| 355 |
+
[step 6880] tok=112.7M R=8 lm=2.6952 5.4k tok/s
|
| 356 |
+
[step 6900] tok=113.0M R=8 lm=2.4511 5.4k tok/s
|
| 357 |
+
[step 6920] tok=113.4M R=8 lm=2.9834 5.4k tok/s
|
| 358 |
+
[step 6940] tok=113.7M R=8 lm=2.6664 5.4k tok/s
|
| 359 |
+
[step 6960] tok=114.0M R=8 lm=2.6884 5.4k tok/s
|
| 360 |
+
[step 6980] tok=114.4M R=8 lm=2.8766 5.4k tok/s
|
| 361 |
+
[step 7000] tok=114.7M R=8 lm=2.5464 5.4k tok/s
|
| 362 |
+
[step 7020] tok=115.0M R=8 lm=2.6088 5.4k tok/s
|
| 363 |
+
[step 7040] tok=115.3M R=8 lm=2.5819 5.4k tok/s
|
| 364 |
+
[step 7060] tok=115.7M R=8 lm=2.6846 5.4k tok/s
|
| 365 |
+
[step 7080] tok=116.0M R=8 lm=2.7658 5.4k tok/s
|
| 366 |
+
[step 7100] tok=116.3M R=8 lm=2.8379 5.4k tok/s
|
| 367 |
+
[step 7120] tok=116.7M R=8 lm=2.5408 5.4k tok/s
|
| 368 |
+
[step 7140] tok=117.0M R=8 lm=2.6608 5.4k tok/s
|
| 369 |
+
[step 7160] tok=117.3M R=8 lm=2.5441 5.4k tok/s
|
| 370 |
+
[step 7180] tok=117.6M R=8 lm=2.4927 5.4k tok/s
|
| 371 |
+
[step 7200] tok=118.0M R=8 lm=2.6706 5.4k tok/s
|
| 372 |
+
[step 7220] tok=118.3M R=8 lm=2.5447 5.4k tok/s
|
| 373 |
+
[step 7240] tok=118.6M R=8 lm=2.7641 5.4k tok/s
|
| 374 |
+
[step 7260] tok=118.9M R=8 lm=2.6738 5.4k tok/s
|
| 375 |
+
[step 7280] tok=119.3M R=8 lm=2.7657 5.4k tok/s
|
| 376 |
+
[step 7300] tok=119.6M R=8 lm=2.8983 5.4k tok/s
|
| 377 |
+
[step 7320] tok=119.9M R=8 lm=2.6733 5.4k tok/s
|
| 378 |
+
[step 7340] tok=120.3M R=8 lm=2.4818 5.4k tok/s
|
| 379 |
+
[step 7360] tok=120.6M R=8 lm=2.6765 5.4k tok/s
|
| 380 |
+
[step 7380] tok=120.9M R=8 lm=2.6735 5.4k tok/s
|
| 381 |
+
[step 7400] tok=121.2M R=8 lm=2.9138 5.4k tok/s
|
| 382 |
+
[step 7420] tok=121.6M R=8 lm=2.6891 5.4k tok/s
|
| 383 |
+
[step 7440] tok=121.9M R=8 lm=2.8974 5.4k tok/s
|
| 384 |
+
[step 7460] tok=122.2M R=8 lm=2.8166 5.4k tok/s
|
| 385 |
+
[step 7480] tok=122.6M R=8 lm=2.7147 5.4k tok/s
|
| 386 |
+
[step 7500] tok=122.9M R=8 lm=2.6964 5.4k tok/s
|
| 387 |
+
[step 7520] tok=123.2M R=8 lm=2.8025 5.4k tok/s
|
| 388 |
+
[step 7540] tok=123.5M R=8 lm=2.9287 5.4k tok/s
|
| 389 |
+
[step 7560] tok=123.9M R=8 lm=2.6386 5.4k tok/s
|
| 390 |
+
[step 7580] tok=124.2M R=8 lm=2.6092 5.4k tok/s
|
| 391 |
+
[step 7600] tok=124.5M R=8 lm=2.8603 5.4k tok/s
|
| 392 |
+
[step 7620] tok=124.8M R=8 lm=2.5386 5.4k tok/s
|
| 393 |
+
[step 7640] tok=125.2M R=8 lm=2.6356 5.4k tok/s
|
| 394 |
+
[step 7660] tok=125.5M R=8 lm=2.7183 5.4k tok/s
|
| 395 |
+
[step 7680] tok=125.8M R=8 lm=3.0788 5.4k tok/s
|
| 396 |
+
[step 7700] tok=126.2M R=8 lm=2.4403 5.4k tok/s
|
| 397 |
+
[step 7720] tok=126.5M R=8 lm=2.6682 5.4k tok/s
|
| 398 |
+
[step 7740] tok=126.8M R=8 lm=2.5845 5.4k tok/s
|
| 399 |
+
[step 7760] tok=127.1M R=8 lm=2.8256 5.4k tok/s
|
| 400 |
+
[step 7780] tok=127.5M R=8 lm=2.6969 5.4k tok/s
|
| 401 |
+
[step 7800] tok=127.8M R=8 lm=2.2414 5.4k tok/s
|
| 402 |
+
[step 7820] tok=128.1M R=8 lm=2.6268 5.4k tok/s
|
| 403 |
+
[step 7840] tok=128.5M R=8 lm=2.5981 5.4k tok/s
|
| 404 |
+
[step 7860] tok=128.8M R=8 lm=2.6741 5.4k tok/s
|
| 405 |
+
[step 7880] tok=129.1M R=8 lm=2.7685 5.4k tok/s
|
| 406 |
+
[step 7900] tok=129.4M R=8 lm=2.4935 5.4k tok/s
|
| 407 |
+
[step 7920] tok=129.8M R=8 lm=2.6693 5.4k tok/s
|
| 408 |
+
[step 7940] tok=130.1M R=8 lm=2.9663 5.4k tok/s
|
| 409 |
+
[step 7960] tok=130.4M R=8 lm=2.7004 5.4k tok/s
|
| 410 |
+
[step 7980] tok=130.7M R=8 lm=2.6457 5.4k tok/s
|
| 411 |
+
[step 8000] tok=131.1M R=8 lm=2.6542 5.4k tok/s
|
| 412 |
+
[step 8020] tok=131.4M R=8 lm=2.6315 5.4k tok/s
|
| 413 |
+
[step 8040] tok=131.7M R=8 lm=2.7842 5.4k tok/s
|
| 414 |
+
[step 8060] tok=132.1M R=8 lm=2.5842 5.4k tok/s
|
| 415 |
+
[step 8080] tok=132.4M R=8 lm=2.7254 5.4k tok/s
|
| 416 |
+
[step 8100] tok=132.7M R=8 lm=2.5738 5.4k tok/s
|
| 417 |
+
[step 8120] tok=133.0M R=8 lm=2.7756 5.4k tok/s
|
| 418 |
+
[step 8140] tok=133.4M R=8 lm=2.9245 5.4k tok/s
|
| 419 |
+
[step 8160] tok=133.7M R=8 lm=2.5088 5.4k tok/s
|
| 420 |
+
[step 8180] tok=134.0M R=8 lm=2.6304 5.4k tok/s
|
| 421 |
+
[step 8200] tok=134.3M R=8 lm=2.8031 5.4k tok/s
|
| 422 |
+
[step 8220] tok=134.7M R=8 lm=2.7121 5.4k tok/s
|
| 423 |
+
[step 8240] tok=135.0M R=8 lm=2.5556 5.4k tok/s
|
| 424 |
+
[step 8260] tok=135.3M R=8 lm=2.5858 5.4k tok/s
|
| 425 |
+
[step 8280] tok=135.7M R=8 lm=2.7826 5.4k tok/s
|
| 426 |
+
[step 8300] tok=136.0M R=8 lm=2.3885 5.4k tok/s
|
| 427 |
+
[step 8320] tok=136.3M R=8 lm=2.7513 5.4k tok/s
|
| 428 |
+
[step 8340] tok=136.6M R=8 lm=2.7539 5.4k tok/s
|
| 429 |
+
[step 8360] tok=137.0M R=8 lm=2.7967 5.4k tok/s
|
| 430 |
+
[step 8380] tok=137.3M R=8 lm=2.4991 5.4k tok/s
|
| 431 |
+
[step 8400] tok=137.6M R=8 lm=2.9108 5.4k tok/s
|
| 432 |
+
[step 8420] tok=138.0M R=8 lm=2.4620 5.4k tok/s
|
| 433 |
+
[step 8440] tok=138.3M R=8 lm=2.8634 5.4k tok/s
|
| 434 |
+
[step 8460] tok=138.6M R=8 lm=2.7355 5.4k tok/s
|
| 435 |
+
[step 8480] tok=138.9M R=8 lm=2.8701 5.4k tok/s
|
| 436 |
+
[step 8500] tok=139.3M R=8 lm=2.5176 5.4k tok/s
|
| 437 |
+
[step 8520] tok=139.6M R=8 lm=2.8011 5.4k tok/s
|
| 438 |
+
[step 8540] tok=139.9M R=8 lm=2.7492 5.4k tok/s
|
| 439 |
+
[step 8560] tok=140.2M R=8 lm=2.7317 5.4k tok/s
|
| 440 |
+
[step 8580] tok=140.6M R=8 lm=2.9385 5.4k tok/s
|
| 441 |
+
[step 8600] tok=140.9M R=8 lm=2.6303 5.4k tok/s
|
| 442 |
+
[step 8620] tok=141.2M R=8 lm=2.6095 5.4k tok/s
|
| 443 |
+
[step 8640] tok=141.6M R=8 lm=2.6191 5.4k tok/s
|
| 444 |
+
[step 8660] tok=141.9M R=8 lm=2.6420 5.4k tok/s
|
| 445 |
+
[step 8680] tok=142.2M R=8 lm=2.7196 5.4k tok/s
|
| 446 |
+
[step 8700] tok=142.5M R=8 lm=2.6768 5.4k tok/s
|
| 447 |
+
[step 8720] tok=142.9M R=8 lm=2.7533 5.4k tok/s
|
| 448 |
+
[step 8740] tok=143.2M R=8 lm=2.6829 5.4k tok/s
|
| 449 |
+
[step 8760] tok=143.5M R=8 lm=2.5596 5.4k tok/s
|
| 450 |
+
[step 8780] tok=143.9M R=8 lm=2.6669 5.4k tok/s
|
| 451 |
+
[step 8800] tok=144.2M R=8 lm=2.7141 5.4k tok/s
|
| 452 |
+
[step 8820] tok=144.5M R=8 lm=2.7363 5.4k tok/s
|
| 453 |
+
[step 8840] tok=144.8M R=8 lm=2.5783 5.4k tok/s
|
| 454 |
+
[step 8860] tok=145.2M R=8 lm=2.5362 5.4k tok/s
|
| 455 |
+
[step 8880] tok=145.5M R=8 lm=2.7207 5.4k tok/s
|
| 456 |
+
[step 8900] tok=145.8M R=8 lm=2.4546 5.4k tok/s
|
| 457 |
+
[step 8920] tok=146.1M R=8 lm=2.4234 5.4k tok/s
|
| 458 |
+
[step 8940] tok=146.5M R=8 lm=2.8665 5.4k tok/s
|
| 459 |
+
[step 8960] tok=146.8M R=8 lm=2.4970 5.4k tok/s
|
| 460 |
+
[step 8980] tok=147.1M R=8 lm=2.7820 5.4k tok/s
|
| 461 |
+
[step 9000] tok=147.5M R=8 lm=2.5198 5.4k tok/s
|
| 462 |
+
[step 9020] tok=147.8M R=8 lm=2.5996 5.4k tok/s
|
| 463 |
+
[step 9040] tok=148.1M R=8 lm=2.5454 5.4k tok/s
|
| 464 |
+
[step 9060] tok=148.4M R=8 lm=2.7242 5.4k tok/s
|
| 465 |
+
[step 9080] tok=148.8M R=8 lm=2.5667 5.4k tok/s
|
| 466 |
+
[step 9100] tok=149.1M R=8 lm=2.7551 5.4k tok/s
|
| 467 |
+
[step 9120] tok=149.4M R=8 lm=2.8176 5.4k tok/s
|
| 468 |
+
[step 9140] tok=149.7M R=8 lm=2.7713 5.4k tok/s
|
| 469 |
+
[eval] bake_G arm=G tok=150M trainR=8 BPB=0.8306 swap=0.9998 ent=0.9984
|
| 470 |
+
[DONE] arm=G final BPB=0.8306 swap=0.9998 ent=0.9984
|
olmoe_adapt/logs/bake_bake_H.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[bakeoff] arm=H tag=bake_H lr=0.0003 trainable=236978176 (router=2097152 extra=234881024) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:55: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_bakeoff.py:57: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M R=24 lm=2.4115 12.3k tok/s
|
| 11 |
+
[step 40] tok=2.6M R=24 lm=2.5372 12.5k tok/s
|
| 12 |
+
[step 60] tok=3.9M R=23 lm=2.4462 12.5k tok/s
|
| 13 |
+
[step 80] tok=5.2M R=23 lm=2.3587 12.5k tok/s
|
| 14 |
+
[step 100] tok=6.6M R=22 lm=2.4966 12.6k tok/s
|
| 15 |
+
[step 120] tok=7.9M R=22 lm=2.4682 12.6k tok/s
|
| 16 |
+
[step 140] tok=9.2M R=22 lm=2.4547 12.6k tok/s
|
| 17 |
+
[step 160] tok=10.5M R=21 lm=2.5284 12.6k tok/s
|
| 18 |
+
[step 180] tok=11.8M R=21 lm=2.4651 12.6k tok/s
|
| 19 |
+
[step 200] tok=13.1M R=20 lm=2.5416 12.6k tok/s
|
| 20 |
+
[step 220] tok=14.4M R=20 lm=2.4251 12.6k tok/s
|
| 21 |
+
[step 240] tok=15.7M R=19 lm=2.3439 12.6k tok/s
|
| 22 |
+
[step 260] tok=17.0M R=19 lm=2.4905 12.6k tok/s
|
| 23 |
+
[step 280] tok=18.4M R=19 lm=2.5155 12.6k tok/s
|
| 24 |
+
[step 300] tok=19.7M R=18 lm=2.4446 12.6k tok/s
|
| 25 |
+
[step 320] tok=21.0M R=18 lm=2.4407 12.6k tok/s
|
| 26 |
+
[step 340] tok=22.3M R=17 lm=2.5251 12.6k tok/s
|
| 27 |
+
[step 360] tok=23.6M R=17 lm=2.5381 12.6k tok/s
|
| 28 |
+
[step 380] tok=24.9M R=17 lm=2.5733 12.6k tok/s
|
| 29 |
+
[step 400] tok=26.2M R=16 lm=2.5341 12.6k tok/s
|
| 30 |
+
[step 420] tok=27.5M R=16 lm=2.5811 12.6k tok/s
|
| 31 |
+
[step 440] tok=28.8M R=15 lm=2.5066 12.6k tok/s
|
| 32 |
+
[step 460] tok=30.1M R=15 lm=2.5167 12.6k tok/s
|
| 33 |
+
[step 480] tok=31.5M R=14 lm=2.6134 12.6k tok/s
|
| 34 |
+
[step 500] tok=32.8M R=14 lm=2.5465 12.6k tok/s
|
| 35 |
+
[step 520] tok=34.1M R=14 lm=2.6591 12.6k tok/s
|
| 36 |
+
[step 540] tok=35.4M R=13 lm=2.6003 12.6k tok/s
|
| 37 |
+
[step 560] tok=36.7M R=13 lm=2.6549 12.6k tok/s
|
| 38 |
+
[step 580] tok=38.0M R=12 lm=2.6590 12.6k tok/s
|
| 39 |
+
[step 600] tok=39.3M R=12 lm=2.7870 12.6k tok/s
|
| 40 |
+
[step 620] tok=40.6M R=12 lm=2.6451 12.6k tok/s
|
| 41 |
+
[step 640] tok=41.9M R=11 lm=2.5215 12.6k tok/s
|
| 42 |
+
[step 660] tok=43.3M R=11 lm=2.7388 12.6k tok/s
|
| 43 |
+
[step 680] tok=44.6M R=10 lm=2.6295 12.6k tok/s
|
| 44 |
+
[step 700] tok=45.9M R=10 lm=2.7713 12.6k tok/s
|
| 45 |
+
[step 720] tok=47.2M R=9 lm=2.7668 12.6k tok/s
|
| 46 |
+
[step 740] tok=48.5M R=9 lm=2.9538 12.6k tok/s
|
| 47 |
+
[step 760] tok=49.8M R=9 lm=2.8675 12.6k tok/s
|
| 48 |
+
[eval] bake_H arm=H tok=50M trainR=8 BPB=0.8790 swap=0.9998 ent=0.9981
|
| 49 |
+
[step 780] tok=51.1M R=8 lm=2.8804 12.2k tok/s
|
| 50 |
+
[step 800] tok=52.4M R=8 lm=2.7676 12.2k tok/s
|
| 51 |
+
[step 820] tok=53.7M R=8 lm=2.8699 12.2k tok/s
|
| 52 |
+
[step 840] tok=55.1M R=8 lm=2.7491 12.3k tok/s
|
| 53 |
+
[step 860] tok=56.4M R=8 lm=2.8694 12.3k tok/s
|
| 54 |
+
[step 880] tok=57.7M R=8 lm=2.8369 12.3k tok/s
|
| 55 |
+
[step 900] tok=59.0M R=8 lm=2.8046 12.3k tok/s
|
| 56 |
+
[step 920] tok=60.3M R=8 lm=2.7897 12.3k tok/s
|
| 57 |
+
[step 940] tok=61.6M R=8 lm=2.8369 12.3k tok/s
|
| 58 |
+
[step 960] tok=62.9M R=8 lm=2.7213 12.3k tok/s
|
| 59 |
+
[step 980] tok=64.2M R=8 lm=2.9566 12.3k tok/s
|
| 60 |
+
[step 1000] tok=65.5M R=8 lm=2.7377 12.3k tok/s
|
| 61 |
+
[step 1020] tok=66.8M R=8 lm=2.6443 12.3k tok/s
|
| 62 |
+
[step 1040] tok=68.2M R=8 lm=2.7296 12.3k tok/s
|
| 63 |
+
[step 1060] tok=69.5M R=8 lm=2.8306 12.3k tok/s
|
| 64 |
+
[step 1080] tok=70.8M R=8 lm=2.7324 12.3k tok/s
|
| 65 |
+
[step 1100] tok=72.1M R=8 lm=2.7284 12.3k tok/s
|
| 66 |
+
[step 1120] tok=73.4M R=8 lm=2.9718 12.4k tok/s
|
| 67 |
+
[step 1140] tok=74.7M R=8 lm=2.9045 12.4k tok/s
|
| 68 |
+
[step 1160] tok=76.0M R=8 lm=2.7529 12.4k tok/s
|
| 69 |
+
[step 1180] tok=77.3M R=8 lm=2.7837 12.4k tok/s
|
| 70 |
+
[step 1200] tok=78.6M R=8 lm=2.7244 12.4k tok/s
|
| 71 |
+
[step 1220] tok=80.0M R=8 lm=2.7533 12.4k tok/s
|
| 72 |
+
[step 1240] tok=81.3M R=8 lm=2.8953 12.4k tok/s
|
| 73 |
+
[step 1260] tok=82.6M R=8 lm=2.8395 12.4k tok/s
|
| 74 |
+
[step 1280] tok=83.9M R=8 lm=2.8253 12.4k tok/s
|
| 75 |
+
[step 1300] tok=85.2M R=8 lm=2.7805 12.4k tok/s
|
| 76 |
+
[step 1320] tok=86.5M R=8 lm=2.7463 12.4k tok/s
|
| 77 |
+
[step 1340] tok=87.8M R=8 lm=2.7714 12.4k tok/s
|
| 78 |
+
[step 1360] tok=89.1M R=8 lm=2.8648 12.4k tok/s
|
| 79 |
+
[step 1380] tok=90.4M R=8 lm=2.7323 12.4k tok/s
|
| 80 |
+
[step 1400] tok=91.8M R=8 lm=2.7078 12.4k tok/s
|
| 81 |
+
[step 1420] tok=93.1M R=8 lm=2.8032 12.4k tok/s
|
| 82 |
+
[step 1440] tok=94.4M R=8 lm=2.7320 12.4k tok/s
|
| 83 |
+
[step 1460] tok=95.7M R=8 lm=2.8029 12.4k tok/s
|
| 84 |
+
[step 1480] tok=97.0M R=8 lm=2.8060 12.4k tok/s
|
| 85 |
+
[step 1500] tok=98.3M R=8 lm=2.8400 12.4k tok/s
|
| 86 |
+
[step 1520] tok=99.6M R=8 lm=2.6759 12.5k tok/s
|
| 87 |
+
[eval] bake_H arm=H tok=100M trainR=8 BPB=0.8613 swap=0.9998 ent=0.9979
|
| 88 |
+
[step 1540] tok=100.9M R=8 lm=2.6890 12.3k tok/s
|
| 89 |
+
[step 1560] tok=102.2M R=8 lm=2.8331 12.3k tok/s
|
| 90 |
+
[step 1580] tok=103.5M R=8 lm=2.7965 12.3k tok/s
|
| 91 |
+
[step 1600] tok=104.9M R=8 lm=2.6749 12.3k tok/s
|
| 92 |
+
[step 1620] tok=106.2M R=8 lm=2.6731 12.3k tok/s
|
| 93 |
+
[step 1640] tok=107.5M R=8 lm=2.7673 12.3k tok/s
|
| 94 |
+
[step 1660] tok=108.8M R=8 lm=2.8267 12.3k tok/s
|
| 95 |
+
[step 1680] tok=110.1M R=8 lm=2.7591 12.3k tok/s
|
| 96 |
+
[step 1700] tok=111.4M R=8 lm=2.7316 12.3k tok/s
|
| 97 |
+
[step 1720] tok=112.7M R=8 lm=2.8157 12.3k tok/s
|
| 98 |
+
[step 1740] tok=114.0M R=8 lm=2.7356 12.3k tok/s
|
| 99 |
+
[step 1760] tok=115.3M R=8 lm=2.7716 12.3k tok/s
|
| 100 |
+
[step 1780] tok=116.7M R=8 lm=2.7751 12.3k tok/s
|
| 101 |
+
[step 1800] tok=118.0M R=8 lm=2.8721 12.4k tok/s
|
| 102 |
+
[step 1820] tok=119.3M R=8 lm=2.7806 12.4k tok/s
|
| 103 |
+
[step 1840] tok=120.6M R=8 lm=2.7789 12.4k tok/s
|
| 104 |
+
[step 1860] tok=121.9M R=8 lm=2.8404 12.4k tok/s
|
| 105 |
+
[step 1880] tok=123.2M R=8 lm=2.7953 12.4k tok/s
|
| 106 |
+
[step 1900] tok=124.5M R=8 lm=2.7963 12.4k tok/s
|
| 107 |
+
[step 1920] tok=125.8M R=8 lm=2.7505 12.4k tok/s
|
| 108 |
+
[step 1940] tok=127.1M R=8 lm=2.8516 12.4k tok/s
|
| 109 |
+
[step 1960] tok=128.5M R=8 lm=2.7410 12.4k tok/s
|
| 110 |
+
[step 1980] tok=129.8M R=8 lm=2.7386 12.4k tok/s
|
| 111 |
+
[step 2000] tok=131.1M R=8 lm=2.7347 12.4k tok/s
|
| 112 |
+
[step 2020] tok=132.4M R=8 lm=2.7397 12.4k tok/s
|
| 113 |
+
[step 2040] tok=133.7M R=8 lm=2.6546 12.4k tok/s
|
| 114 |
+
[step 2060] tok=135.0M R=8 lm=2.6619 12.4k tok/s
|
| 115 |
+
[step 2080] tok=136.3M R=8 lm=2.8026 12.4k tok/s
|
| 116 |
+
[step 2100] tok=137.6M R=8 lm=2.8325 12.4k tok/s
|
| 117 |
+
[step 2120] tok=138.9M R=8 lm=2.8264 12.4k tok/s
|
| 118 |
+
[step 2140] tok=140.2M R=8 lm=2.6929 12.4k tok/s
|
| 119 |
+
[step 2160] tok=141.6M R=8 lm=2.8458 12.4k tok/s
|
| 120 |
+
[step 2180] tok=142.9M R=8 lm=2.7872 12.4k tok/s
|
| 121 |
+
[step 2200] tok=144.2M R=8 lm=2.7947 12.4k tok/s
|
| 122 |
+
[step 2220] tok=145.5M R=8 lm=2.8468 12.4k tok/s
|
| 123 |
+
[step 2240] tok=146.8M R=8 lm=2.7056 12.4k tok/s
|
| 124 |
+
[step 2260] tok=148.1M R=8 lm=2.6489 12.4k tok/s
|
| 125 |
+
[step 2280] tok=149.4M R=8 lm=2.8588 12.4k tok/s
|
| 126 |
+
[eval] bake_H arm=H tok=150M trainR=8 BPB=0.8572 swap=0.9998 ent=0.9976
|
| 127 |
+
[step 2300] tok=150.7M R=8 lm=2.8063 12.3k tok/s
|
| 128 |
+
[step 2320] tok=152.0M R=8 lm=2.7545 12.3k tok/s
|
| 129 |
+
[step 2340] tok=153.4M R=8 lm=2.7740 12.3k tok/s
|
| 130 |
+
[step 2360] tok=154.7M R=8 lm=2.6764 12.3k tok/s
|
| 131 |
+
[step 2380] tok=156.0M R=8 lm=2.8018 12.3k tok/s
|
| 132 |
+
[step 2400] tok=157.3M R=8 lm=2.7985 12.3k tok/s
|
| 133 |
+
[step 2420] tok=158.6M R=8 lm=2.8184 12.4k tok/s
|
| 134 |
+
[step 2440] tok=159.9M R=8 lm=2.7618 12.4k tok/s
|
| 135 |
+
[step 2460] tok=161.2M R=8 lm=2.8350 12.4k tok/s
|
| 136 |
+
[step 2480] tok=162.5M R=8 lm=2.6146 12.4k tok/s
|
| 137 |
+
[step 2500] tok=163.8M R=8 lm=2.7112 12.4k tok/s
|
| 138 |
+
[step 2520] tok=165.2M R=8 lm=2.7219 12.4k tok/s
|
| 139 |
+
[step 2540] tok=166.5M R=8 lm=2.8445 12.4k tok/s
|
| 140 |
+
[step 2560] tok=167.8M R=8 lm=2.6780 12.4k tok/s
|
| 141 |
+
[step 2580] tok=169.1M R=8 lm=2.7786 12.4k tok/s
|
| 142 |
+
[step 2600] tok=170.4M R=8 lm=2.6167 12.4k tok/s
|
| 143 |
+
[step 2620] tok=171.7M R=8 lm=2.8083 12.4k tok/s
|
| 144 |
+
[step 2640] tok=173.0M R=8 lm=2.7200 12.4k tok/s
|
| 145 |
+
[step 2660] tok=174.3M R=8 lm=2.8940 12.4k tok/s
|
| 146 |
+
[step 2680] tok=175.6M R=8 lm=2.7977 12.4k tok/s
|
| 147 |
+
[step 2700] tok=176.9M R=8 lm=2.7134 12.4k tok/s
|
| 148 |
+
[step 2720] tok=178.3M R=8 lm=2.7955 12.4k tok/s
|
| 149 |
+
[step 2740] tok=179.6M R=8 lm=2.6780 12.4k tok/s
|
| 150 |
+
[step 2760] tok=180.9M R=8 lm=2.8102 12.4k tok/s
|
| 151 |
+
[step 2780] tok=182.2M R=8 lm=2.7021 12.4k tok/s
|
| 152 |
+
[step 2800] tok=183.5M R=8 lm=2.7732 12.4k tok/s
|
| 153 |
+
[step 2820] tok=184.8M R=8 lm=2.8351 12.4k tok/s
|
| 154 |
+
[step 2840] tok=186.1M R=8 lm=2.5647 12.4k tok/s
|
| 155 |
+
[step 2860] tok=187.4M R=8 lm=2.7292 12.4k tok/s
|
| 156 |
+
[step 2880] tok=188.7M R=8 lm=2.8595 12.4k tok/s
|
| 157 |
+
[step 2900] tok=190.1M R=8 lm=2.8296 12.4k tok/s
|
| 158 |
+
[step 2920] tok=191.4M R=8 lm=2.7274 12.4k tok/s
|
| 159 |
+
[step 2940] tok=192.7M R=8 lm=2.8726 12.4k tok/s
|
| 160 |
+
[step 2960] tok=194.0M R=8 lm=2.5821 12.4k tok/s
|
| 161 |
+
[step 2980] tok=195.3M R=8 lm=2.7240 12.4k tok/s
|
| 162 |
+
[step 3000] tok=196.6M R=8 lm=2.5739 12.4k tok/s
|
| 163 |
+
[step 3020] tok=197.9M R=8 lm=2.7083 12.4k tok/s
|
| 164 |
+
[step 3040] tok=199.2M R=8 lm=2.8344 12.4k tok/s
|
| 165 |
+
[eval] bake_H arm=H tok=200M trainR=8 BPB=0.8527 swap=0.9998 ent=0.9980
|
| 166 |
+
[step 3060] tok=200.5M R=8 lm=2.7422 12.3k tok/s
|
| 167 |
+
[step 3080] tok=201.9M R=8 lm=2.7925 12.3k tok/s
|
| 168 |
+
[step 3100] tok=203.2M R=8 lm=2.7804 12.3k tok/s
|
| 169 |
+
[step 3120] tok=204.5M R=8 lm=2.6697 12.4k tok/s
|
| 170 |
+
[step 3140] tok=205.8M R=8 lm=2.7438 12.4k tok/s
|
| 171 |
+
[step 3160] tok=207.1M R=8 lm=2.6844 12.4k tok/s
|
| 172 |
+
[step 3180] tok=208.4M R=8 lm=2.6812 12.4k tok/s
|
| 173 |
+
[step 3200] tok=209.7M R=8 lm=2.7102 12.4k tok/s
|
| 174 |
+
[step 3220] tok=211.0M R=8 lm=2.5807 12.4k tok/s
|
| 175 |
+
[step 3240] tok=212.3M R=8 lm=2.7405 12.4k tok/s
|
| 176 |
+
[step 3260] tok=213.6M R=8 lm=2.7733 12.4k tok/s
|
| 177 |
+
[step 3280] tok=215.0M R=8 lm=2.7234 12.4k tok/s
|
| 178 |
+
[step 3300] tok=216.3M R=8 lm=2.7262 12.4k tok/s
|
| 179 |
+
[step 3320] tok=217.6M R=8 lm=2.7573 12.4k tok/s
|
| 180 |
+
[step 3340] tok=218.9M R=8 lm=2.7405 12.4k tok/s
|
| 181 |
+
[step 3360] tok=220.2M R=8 lm=2.7332 12.4k tok/s
|
| 182 |
+
[step 3380] tok=221.5M R=8 lm=2.7628 12.4k tok/s
|
| 183 |
+
[step 3400] tok=222.8M R=8 lm=2.7604 12.4k tok/s
|
| 184 |
+
[step 3420] tok=224.1M R=8 lm=2.7233 12.4k tok/s
|
| 185 |
+
[step 3440] tok=225.4M R=8 lm=2.6865 12.4k tok/s
|
| 186 |
+
[step 3460] tok=226.8M R=8 lm=2.7861 12.4k tok/s
|
| 187 |
+
[step 3480] tok=228.1M R=8 lm=2.7777 12.4k tok/s
|
| 188 |
+
[step 3500] tok=229.4M R=8 lm=2.7360 12.4k tok/s
|
| 189 |
+
[step 3520] tok=230.7M R=8 lm=2.7299 12.4k tok/s
|
| 190 |
+
[step 3540] tok=232.0M R=8 lm=2.7525 12.4k tok/s
|
| 191 |
+
[step 3560] tok=233.3M R=8 lm=2.7656 12.4k tok/s
|
| 192 |
+
[step 3580] tok=234.6M R=8 lm=2.9039 12.4k tok/s
|
| 193 |
+
[step 3600] tok=235.9M R=8 lm=2.7819 12.4k tok/s
|
| 194 |
+
[step 3620] tok=237.2M R=8 lm=2.9994 12.4k tok/s
|
| 195 |
+
[step 3640] tok=238.6M R=8 lm=2.7335 12.4k tok/s
|
| 196 |
+
[step 3660] tok=239.9M R=8 lm=2.7987 12.4k tok/s
|
| 197 |
+
[step 3680] tok=241.2M R=8 lm=2.7924 12.4k tok/s
|
| 198 |
+
[step 3700] tok=242.5M R=8 lm=2.7114 12.4k tok/s
|
| 199 |
+
[step 3720] tok=243.8M R=8 lm=2.7220 12.4k tok/s
|
| 200 |
+
[step 3740] tok=245.1M R=8 lm=2.7391 12.4k tok/s
|
| 201 |
+
[step 3760] tok=246.4M R=8 lm=2.8013 12.4k tok/s
|
| 202 |
+
[step 3780] tok=247.7M R=8 lm=2.7092 12.4k tok/s
|
| 203 |
+
[step 3800] tok=249.0M R=8 lm=2.8541 12.4k tok/s
|
| 204 |
+
[eval] bake_H arm=H tok=250M trainR=8 BPB=0.8510 swap=0.9997 ent=0.9981
|
| 205 |
+
[DONE] arm=H final BPB=0.8510 swap=0.9997 ent=0.9981
|
olmoe_adapt/logs/bakeoff_run.log
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
=== eval-noise sigma 10:04 ===
|
| 2 |
+
sigma: [sigma] R8: [2.7505, 2.7363, 2.7451] mean=2.7440 std=0.0059
|
| 3 |
+
=== BAKE-OFF arm B (0.25B, lr=3e-4, eval every 50M) 10:10 ===
|
| 4 |
+
B done: [DONE] arm=B final BPB=1.2788 swap=0.9999 ent=0.9918
|
| 5 |
+
=== BAKE-OFF arm D (0.25B, lr=3e-4, eval every 50M) 13:45 ===
|
| 6 |
+
D done:
|
| 7 |
+
=== BAKE-OFF arm E (0.25B, lr=3e-4, eval every 50M) 13:49 ===
|
| 8 |
+
E done: [DONE] arm=E final BPB=0.8507 swap=0.9997 ent=0.9977
|
| 9 |
+
=== BAKE-OFF arm C (0.25B, lr=3e-4, eval every 50M) 19:27 ===
|
| 10 |
+
C done: [DONE] arm=C final BPB=0.8505 swap=0.9998 ent=0.9988
|
| 11 |
+
ALL BAKE-OFF ARMS DONE
|
olmoe_adapt/logs/c4val.log
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
|
| 2 |
+
warnings.warn(
|
| 3 |
+
|
| 4 |
+
[c4val] model loaded fp32, free top-8 routing (no residency). budget=2000000 tok, seq=4096
|
| 5 |
+
[c4val] packed 489 seqs = 2002944 tokens from 9043681 utf-8 bytes
|
| 6 |
+
[c4val] 50/489 seqs, running CE=2.5016 nats/tok
|
| 7 |
+
[c4val] 100/489 seqs, running CE=2.4589 nats/tok
|
| 8 |
+
[c4val] 150/489 seqs, running CE=2.4513 nats/tok
|
| 9 |
+
[c4val] 200/489 seqs, running CE=2.4520 nats/tok
|
| 10 |
+
[c4val] 250/489 seqs, running CE=2.4634 nats/tok
|
| 11 |
+
[c4val] 300/489 seqs, running CE=2.4587 nats/tok
|
| 12 |
+
[c4val] 350/489 seqs, running CE=2.4620 nats/tok
|
| 13 |
+
[c4val] 400/489 seqs, running CE=2.4701 nats/tok
|
| 14 |
+
[c4val] 450/489 seqs, running CE=2.4692 nats/tok
|
| 15 |
+
[c4val] RESULT {
|
| 16 |
+
"source": "c4val_ours",
|
| 17 |
+
"model": "OLMoE-1B-7B-0125",
|
| 18 |
+
"routing": "free_top8_no_residency",
|
| 19 |
+
"precision": "fp32",
|
| 20 |
+
"corpus": "allenai/c4 en-validation (streaming)",
|
| 21 |
+
"seq": 4096,
|
| 22 |
+
"n_seqs": 489,
|
| 23 |
+
"n_tokens_total": 2002944,
|
| 24 |
+
"n_pred_tokens": 2002455,
|
| 25 |
+
"n_bytes_utf8": 9043681,
|
| 26 |
+
"bytes_per_token": 4.515194134234407,
|
| 27 |
+
"CE_nats_per_token": 2.473013001052926,
|
| 28 |
+
"perplexity": 11.858120918273926,
|
| 29 |
+
"BPB": 0.790177229727896,
|
| 30 |
+
"note": "direct next-token CE == perplexity path; ppl=exp(CE)"
|
| 31 |
+
}
|
| 32 |
+
[c4val] DONE CE=2.4730 nats/tok ppl=11.858 BPB=0.7902 tokens=2002455 bytes=9043681
|
olmoe_adapt/logs/cal0.log
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[cal0] 65 RMSNorm sites tagged; stats on 512 packs
|
| 3 |
+
/workspace/olmoe-adapt/scripts/cal0.py:35: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
ids = torch.load(f"{OUT}/bpb_slice_ids.pt").long()
|
| 5 |
+
[cal0] pass A (free routing)...
|
| 6 |
+
[cal0] pass B (greedy R=8)...
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
[cal0] ratio stats: median=0.944 p1=0.451 p99=1.721 (per-channel RMS_free/RMS_masked)
|
| 9 |
+
[cal0] matched-norms @R8 BPB=2.7278 rec=0.011 | clipped @R8=2.0965 rec=0.315 | matched+free(sanity)=2.2264
|
| 10 |
+
[cal0] wrote /workspace/FLAME-MoE/results/ablations/olmoe_cal0.csv
|
| 11 |
+
[cal0] VERDICT: closed-form moment-match @R8 = 2.7278 (1.1%) vs arm C trained 0.8505 (91.4%). mean gain-direction cos vs C = 0.010 (20% of sites aligned).
|
olmoe_adapt/logs/cal2.log
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
/workspace/olmoe-adapt/scripts/train_cal2.py:52: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 3 |
+
ids = torch.load(f"{OUT}/bpb_slice_ids.pt").long()
|
| 4 |
+
[cal2] computing g' from 512-pack RMS passes (free vs R=8)...
|
| 5 |
+
[temporal] scan path: triton (verified == reference)
|
| 6 |
+
[cal2] arm-C surface trainable=2230272 (router=2097152 norms=133120) lr=0.0003 tok=50000000 eval_every=10000000
|
| 7 |
+
/workspace/olmoe-adapt/scripts/train_cal2.py:91: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 8 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt")
|
| 9 |
+
/workspace/olmoe-adapt/scripts/train_cal2.py:93: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 10 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 11 |
+
[cal2 eval] tok=0M BPB=2.0965 (g'-init sanity vs Cal-0 clipped 2.0965) rec=0.315 cos_prime=1.0000 cos_C=-0.0342
|
| 12 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 13 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 14 |
+
[step 20] tok=1.3M lm=4.2190 18.3k tok/s
|
| 15 |
+
[step 40] tok=2.6M lm=3.8695 18.4k tok/s
|
| 16 |
+
[step 60] tok=3.9M lm=3.5709 18.4k tok/s
|
| 17 |
+
[step 80] tok=5.2M lm=3.4328 18.4k tok/s
|
| 18 |
+
[step 100] tok=6.6M lm=3.4433 18.4k tok/s
|
| 19 |
+
[step 120] tok=7.9M lm=3.3845 18.4k tok/s
|
| 20 |
+
[step 140] tok=9.2M lm=3.2970 18.4k tok/s
|
| 21 |
+
[cal2 eval] tok=10M BPB=1.0241 rec=0.831 vs C@50M=0.8791 swap=0.9997 ent=0.9963 cos_prime=0.2822 cos_C=0.3087
|
| 22 |
+
[step 160] tok=10.5M lm=3.3360 17.3k tok/s
|
| 23 |
+
[step 180] tok=11.8M lm=3.2734 17.5k tok/s
|
| 24 |
+
[step 200] tok=13.1M lm=3.2751 17.5k tok/s
|
| 25 |
+
[step 220] tok=14.4M lm=3.1474 17.6k tok/s
|
| 26 |
+
[step 240] tok=15.7M lm=3.0735 17.7k tok/s
|
| 27 |
+
[step 260] tok=17.0M lm=3.1472 17.8k tok/s
|
| 28 |
+
[step 280] tok=18.4M lm=3.1683 17.9k tok/s
|
| 29 |
+
[step 300] tok=19.7M lm=3.0713 18.0k tok/s
|
| 30 |
+
[cal2 eval] tok=20M BPB=0.9730 rec=0.855 vs C@50M=0.8791 swap=0.9997 ent=0.9964 cos_prime=0.2337 cos_C=0.3570
|
| 31 |
+
[step 320] tok=21.0M lm=3.0420 17.7k tok/s
|
| 32 |
+
[step 340] tok=22.3M lm=3.0772 17.8k tok/s
|
| 33 |
+
[step 360] tok=23.6M lm=3.0638 17.9k tok/s
|
| 34 |
+
[step 380] tok=24.9M lm=3.1181 17.9k tok/s
|
| 35 |
+
[step 400] tok=26.2M lm=3.0614 18.0k tok/s
|
| 36 |
+
[step 420] tok=27.5M lm=3.0968 18.1k tok/s
|
| 37 |
+
[step 440] tok=28.8M lm=2.9957 18.1k tok/s
|
| 38 |
+
[cal2 eval] tok=30M BPB=0.9502 rec=0.866 vs C@50M=0.8791 swap=0.9997 ent=0.9968 cos_prime=0.2040 cos_C=0.3839
|
| 39 |
+
[step 460] tok=30.1M lm=2.9783 17.9k tok/s
|
| 40 |
+
[step 480] tok=31.5M lm=3.0604 18.0k tok/s
|
| 41 |
+
[step 500] tok=32.8M lm=3.0088 18.0k tok/s
|
| 42 |
+
[step 520] tok=34.1M lm=3.0907 18.1k tok/s
|
| 43 |
+
[step 540] tok=35.4M lm=2.9967 18.1k tok/s
|
| 44 |
+
[step 560] tok=36.7M lm=3.0582 18.2k tok/s
|
| 45 |
+
[step 580] tok=38.0M lm=3.0279 18.2k tok/s
|
| 46 |
+
[step 600] tok=39.3M lm=3.1312 18.3k tok/s
|
| 47 |
+
[cal2 eval] tok=40M BPB=0.9362 rec=0.873 vs C@50M=0.8791 swap=0.9998 ent=0.9971 cos_prime=0.1842 cos_C=0.4152
|
| 48 |
+
[step 620] tok=40.6M lm=2.9991 18.1k tok/s
|
| 49 |
+
[step 640] tok=41.9M lm=2.7999 18.1k tok/s
|
| 50 |
+
[step 660] tok=43.3M lm=3.0487 18.2k tok/s
|
| 51 |
+
[step 680] tok=44.6M lm=2.8951 18.2k tok/s
|
| 52 |
+
[step 700] tok=45.9M lm=3.0312 18.3k tok/s
|
| 53 |
+
[step 720] tok=47.2M lm=2.9801 18.3k tok/s
|
| 54 |
+
[step 740] tok=48.5M lm=3.1495 18.3k tok/s
|
| 55 |
+
[step 760] tok=49.8M lm=3.0673 18.4k tok/s
|
| 56 |
+
[cal2 eval] tok=50M BPB=0.9262 rec=0.878 vs C@50M=0.8791 swap=0.9998 ent=0.9975 cos_prime=0.1702 cos_C=0.4397
|
| 57 |
+
[cal2] wrote /workspace/FLAME-MoE/results/ablations/olmoe_cal2.csv
|
| 58 |
+
[cal2] NULL: g'-init WORSE by 0.0471 -> init axis CLOSED (arm C's from-base start is not improved)
|
| 59 |
+
[cal2] mechanism: final cos_prime=0.1702 cos_C=0.4397 -> UNDOES g' toward arm C (g_C)
|
olmoe_adapt/logs/ce_gate.log
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[ce-mp] 384 MinFlow solves over 24 workers ...
|
| 2 |
+
[ce-mp] 32/384 solves (158s)
|
| 3 |
+
[ce-mp] 64/384 solves (303s)
|
| 4 |
+
[ce-mp] 96/384 solves (434s)
|
| 5 |
+
[ce-mp] 128/384 solves (551s)
|
| 6 |
+
[ce-mp] 160/384 solves (689s)
|
| 7 |
+
[ce-mp] 192/384 solves (816s)
|
| 8 |
+
[ce-mp] 224/384 solves (936s)
|
| 9 |
+
[ce-mp] 256/384 solves (1071s)
|
| 10 |
+
[ce-mp] 288/384 solves (1193s)
|
| 11 |
+
[ce-mp] 320/384 solves (1331s)
|
| 12 |
+
[ce-mp] 352/384 solves (1459s)
|
| 13 |
+
[ce-mp] 384/384 solves (1586s)
|
| 14 |
+
[ce-mp] all solves done in 26.4 min, cached -> /workspace/olmoe-adapt/data/oseries_ce/ce_paired_masks_N24.npz. GPU forwards next.
|
| 15 |
+
|
| 16 |
+
/workspace/olmoe-adapt/scripts/oseries_ce_paired_mp.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 17 |
+
ids = torch.load("/workspace/olmoe-adapt/data/bpb_slice_ids.pt")[:N].long()
|
| 18 |
+
[temporal] scan path: triton (verified == reference)
|
| 19 |
+
[ce-gate] pack 1/24: scan=0.8829 oracle=0.8915 greedy=0.8992 (scan-oracle=-0.0085)
|
| 20 |
+
[ce-gate] pack 2/24: scan=0.8288 oracle=0.8288 greedy=0.8381 (scan-oracle=-0.0000)
|
| 21 |
+
[ce-gate] pack 3/24: scan=0.8555 oracle=0.8663 greedy=0.8736 (scan-oracle=-0.0107)
|
| 22 |
+
[ce-gate] pack 4/24: scan=0.8703 oracle=0.8756 greedy=0.8886 (scan-oracle=-0.0053)
|
| 23 |
+
[ce-gate] pack 5/24: scan=0.8265 oracle=0.8104 greedy=0.8398 (scan-oracle=+0.0161)
|
| 24 |
+
[ce-gate] pack 6/24: scan=0.8604 oracle=0.8603 greedy=0.8804 (scan-oracle=+0.0001)
|
| 25 |
+
[ce-gate] pack 7/24: scan=0.8012 oracle=0.8080 greedy=0.8172 (scan-oracle=-0.0068)
|
| 26 |
+
[ce-gate] pack 8/24: scan=0.7938 oracle=0.8071 greedy=0.8104 (scan-oracle=-0.0133)
|
| 27 |
+
[ce-gate] pack 9/24: scan=0.6623 oracle=0.6618 greedy=0.6748 (scan-oracle=+0.0005)
|
| 28 |
+
[ce-gate] pack 10/24: scan=0.8503 oracle=0.8623 greedy=0.8580 (scan-oracle=-0.0120)
|
| 29 |
+
[ce-gate] pack 11/24: scan=0.8896 oracle=0.9085 greedy=0.9099 (scan-oracle=-0.0189)
|
| 30 |
+
[ce-gate] pack 12/24: scan=0.8475 oracle=0.8572 greedy=0.8567 (scan-oracle=-0.0097)
|
| 31 |
+
[ce-gate] pack 13/24: scan=0.8153 oracle=0.8458 greedy=0.8275 (scan-oracle=-0.0304)
|
| 32 |
+
[ce-gate] pack 14/24: scan=0.8199 oracle=0.8309 greedy=0.8319 (scan-oracle=-0.0110)
|
| 33 |
+
[ce-gate] pack 15/24: scan=0.7113 oracle=0.7393 greedy=0.7454 (scan-oracle=-0.0280)
|
| 34 |
+
[ce-gate] pack 16/24: scan=0.8340 oracle=0.8426 greedy=0.8435 (scan-oracle=-0.0086)
|
| 35 |
+
[ce-gate] pack 17/24: scan=0.8657 oracle=0.8758 greedy=0.8812 (scan-oracle=-0.0101)
|
| 36 |
+
[ce-gate] pack 18/24: scan=0.7574 oracle=0.7745 greedy=0.7716 (scan-oracle=-0.0172)
|
| 37 |
+
[ce-gate] pack 19/24: scan=0.8538 oracle=0.8802 greedy=0.8726 (scan-oracle=-0.0264)
|
| 38 |
+
[ce-gate] pack 20/24: scan=0.7971 oracle=0.8177 greedy=0.8112 (scan-oracle=-0.0205)
|
| 39 |
+
[ce-gate] pack 21/24: scan=0.8159 oracle=0.8300 greedy=0.8305 (scan-oracle=-0.0141)
|
| 40 |
+
[ce-gate] pack 22/24: scan=0.8381 oracle=0.8403 greedy=0.8522 (scan-oracle=-0.0023)
|
| 41 |
+
[ce-gate] pack 23/24: scan=0.7764 oracle=0.7929 greedy=0.7911 (scan-oracle=-0.0166)
|
| 42 |
+
[ce-gate] pack 24/24: scan=0.7869 oracle=0.8035 greedy=0.7976 (scan-oracle=-0.0167)
|
| 43 |
+
[ce-gate] N=24 CE-scan=0.8184 CE-oracle=0.8296 CE-greedy=0.8335
|
| 44 |
+
[ce-gate] PAIRED scan-oracle = -0.0113 +/- 0.0021 (t=-5.4); scan-greedy = -0.0151
|
| 45 |
+
[ce-gate] VERDICT: O-3 ALIVE on winner: MinFlow-m1 oracle beats CE live scan by -0.0113 (t=-5.4)
|
| 46 |
+
[ce-gate] appended.
|
olmoe_adapt/logs/corpus_build.log
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
|
| 2 |
+
warnings.warn(
|
| 3 |
+
/workspace/olmoe-adapt/scripts/build_finetune_corpus.py:32: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
bpb = torch.load(f"{OUT}/bpb_slice_ids.pt").reshape(-1).tolist()
|
| 5 |
+
[dedup] 6246791 windows from BPB slice
|
| 6 |
+
[dedup] 6247478 windows total (BPB + eval task text)
|
| 7 |
+
[dclm] 170899 packs, 700002304 tok, 7680 dedup-dropped
|
| 8 |
+
[dolmino] 73242 packs, 299999232 tok, 5296 dedup-dropped
|
| 9 |
+
[corpus] 244141 packs = 1000001536 tokens; dedup dropped 12976
|
olmoe_adapt/logs/corpus_smoke.log
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
|
| 2 |
+
warnings.warn(
|
| 3 |
+
/workspace/olmoe-adapt/scripts/build_finetune_corpus.py:32: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
bpb = torch.load(f"{OUT}/bpb_slice_ids.pt").reshape(-1).tolist()
|
| 5 |
+
[dedup] 6246791 windows from BPB slice
|
| 6 |
+
[dedup] 6247478 windows total (BPB + eval task text)
|
| 7 |
+
[dclm] 3418 packs, 14000128 tok, 146 dedup-dropped
|
| 8 |
+
[dolmino] 1465 packs, 6000640 tok, 43 dedup-dropped
|
| 9 |
+
[corpus] 4883 packs = 20000768 tokens; dedup dropped 189
|
olmoe_adapt/logs/dense_bracket.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
olmoe_adapt/logs/hf_delex.log
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/workspace/olmoe-adapt/scripts/hf_delex.py:26: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 2 |
+
ids = torch.load("/workspace/olmoe-adapt/data/bpb_slice_ids.pt")[:N, :SL].long()
|
| 3 |
+
|
| 4 |
+
[temporal] scan path: triton (verified == reference)
|
| 5 |
+
[base_impose_R8] PR_med=0.0652 generalist=0.0250 router_H=0.4708 | demand_AUC=0.9673 | token_AUC=0.6069 context_AUC=0.6028 ctx-tok=-0.0041
|
| 6 |
+
|
| 7 |
+
[CE_adapt_R8] PR_med=0.0876 generalist=0.0000 router_H=0.4598 | demand_AUC=0.9606 | token_AUC=0.6235 context_AUC=0.6728 ctx-tok=+0.0493
|
| 8 |
+
[delex] wrote /workspace/FLAME-MoE/results/ablations/olmoe_adapt_forensics.csv
|
| 9 |
+
[delex] DE-LEX SIGNATURE (CE-adapt vs base-impose): ctx-tok -0.0041->+0.0493, demand_AUC 0.967->0.961, PR_med 0.065->0.088
|
olmoe_adapt/logs/impose.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
IMPOSE (provisional, wikitext-103 val, 8x4096 tok, byte-divisor D=3.1550):
|
| 6 |
+
base CE=1.8914 BPB=0.5995
|
| 7 |
+
+mask CE=9.1604 BPB=2.9034 (R=k=8 of 64)
|
| 8 |
+
IMPOSE GAP: dCE=+7.2690 dBPB=+2.3039
|
olmoe_adapt/logs/live_greedy.log
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
<string>:7: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 3 |
+
[temporal] scan path: triton (verified == reference)
|
| 4 |
+
[live-greedy] base model LIVE impose (drifted-logit scan) on 24 packs: mean BPB=2.7047 (per-pack std 0.1390)
|
| 5 |
+
[live-greedy] saved per-pack
|
olmoe_adapt/logs/lmeval_downstream.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
olmoe_adapt/logs/lmeval_impose.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
olmoe_adapt/logs/merge_ce.log
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
/workspace/olmoe-adapt/scripts/merge_ce.py:19: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 3 |
+
ck = torch.load(f"{OUT}/ckpt_bake_CE.pt", map_location="cuda")
|
| 4 |
+
[merge] loaded CE deltas (seen=250M)
|
| 5 |
+
/workspace/olmoe-adapt/scripts/merge_ce.py:24: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
[merge] identity check: pre-merge(LoRA) BPB=0.8149 post-merge(baked) BPB=0.8147 |delta|=0.00024
|
| 9 |
+
[merge] VERIFIED merged-CE reproduces CE-final (0.8147 ~= 0.8149) at R=8.
|
| 10 |
+
|
| 11 |
+
[merge] saved merged-CE HF model -> /workspace/olmoe-adapt/merged_ce_model
|
olmoe_adapt/logs/model_download.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Hint: A new version of huggingface_hub (1.24.0) is available! You are using version 1.20.1.
|
| 2 |
+
To update, run: hf update
|
| 3 |
+
Still waiting to acquire lock on /workspace/olmoe-adapt/model/.cache/huggingface/.gitignore.lock (elapsed: 0.1 seconds)
|
| 4 |
+
Still waiting to acquire lock on /workspace/olmoe-adapt/model/.cache/huggingface/.gitignore.lock (elapsed: 0.1 seconds)
|
| 5 |
+
Still waiting to acquire lock on /workspace/olmoe-adapt/model/.cache/huggingface/.gitignore.lock (elapsed: 0.1 seconds)
|
| 6 |
+
Still waiting to acquire lock on /workspace/olmoe-adapt/model/.cache/huggingface/.gitignore.lock (elapsed: 0.1 seconds)
|
| 7 |
+
Still waiting to acquire lock on /workspace/olmoe-adapt/model/.cache/huggingface/.gitignore.lock (elapsed: 0.1 seconds)
|
| 8 |
+
path=/workspace/olmoe-adapt/model
|
olmoe_adapt/logs/oseries_full.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[full] 488 stored, using 12 seqs FULL T=4096; prune top-C=16; m=[1, 2, 4]
|
| 2 |
+
[full] seq 1/12 done (2086.3s/seq)
|
| 3 |
+
[full] seq 2/12 done (2085.1s/seq)
|
| 4 |
+
[full] seq 3/12 done (2174.5s/seq)
|
| 5 |
+
[full] seq 4/12 done (2223.9s/seq)
|
| 6 |
+
[full] seq 5/12 done (2279.6s/seq)
|
| 7 |
+
[full] seq 6/12 done (2555.3s/seq)
|
| 8 |
+
[full] seq 7/12 done (2493.6s/seq)
|
| 9 |
+
[full] seq 8/12 done (2443.8s/seq)
|
| 10 |
+
[full] seq 9/12 done (2521.8s/seq)
|
| 11 |
+
[full] seq 10/12 done (2471.6s/seq)
|
| 12 |
+
[full] seq 11/12 done (2462.9s/seq)
|
| 13 |
+
[full] seq 12/12 done (2440.3s/seq)
|
| 14 |
+
[full] wrote /workspace/FLAME-MoE/results/ablations/olmoe_minflow_full.csv
|
| 15 |
+
[full] AGG static=0.16664 greedy=0.26617 m1=0.23856 m2=0.32147 m4=0.38637 bound=0.40490
|
| 16 |
+
[full] KILL-CHECK m=1: minflow-greedy=+-0.02760 (~=greedy STOP)
|
olmoe_adapt/logs/oseries_ladder.log
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[ladder] 488 stored, using 6 seqs x 16 windows of 256; L=16 E=64 m=[1]
|
| 2 |
+
[ladder] seq 1/6 done
|
| 3 |
+
[ladder] seq 2/6 done
|
| 4 |
+
[ladder] seq 3/6 done
|
| 5 |
+
[ladder] seq 4/6 done
|
| 6 |
+
[ladder] seq 5/6 done
|
| 7 |
+
[ladder] seq 6/6 done
|
| 8 |
+
[ladder] wrote /workspace/FLAME-MoE/results/ablations/olmoe_minflow_calib.csv
|
| 9 |
+
[ladder] AGG static=0.21331 greedy=0.27658 minflow_m1=0.32708 top8bound=0.40537
|
| 10 |
+
[ladder] KILL-CHECK m=1: minflow-greedy gain=0.05050 (MinFlow > greedy -> proceed)
|
olmoe_adapt/logs/oseries_msweep.log
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[ladder] 488 stored, using 6 seqs x 16 windows of 256; L=16 E=64 m=[1, 2, 4]
|
| 2 |
+
[ladder] seq 1/6 done
|
| 3 |
+
[ladder] seq 2/6 done
|
| 4 |
+
[ladder] seq 3/6 done
|
| 5 |
+
[ladder] seq 4/6 done
|
| 6 |
+
[ladder] seq 5/6 done
|
| 7 |
+
[ladder] seq 6/6 done
|
| 8 |
+
[ladder] wrote /workspace/FLAME-MoE/results/ablations/olmoe_minflow_msweep.csv
|
| 9 |
+
[ladder] AGG static=0.21331 greedy=0.27658 minflow_m1=0.32708 top8bound=0.40537
|
| 10 |
+
[ladder] KILL-CHECK m=1: minflow-greedy gain=0.05050 (MinFlow > greedy -> proceed)
|
olmoe_adapt/logs/oseries_o2.log
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[o2] 16 packs, TW=256 (16 win), schedules: static, greedy, MinFlow m[1, 2, 4]
|
| 2 |
+
|
| 3 |
+
/workspace/olmoe-adapt/scripts/oseries_o2.py:91: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
ids = torch.load("/workspace/olmoe-adapt/data/bpb_slice_ids.pt")[:n].long()
|
| 5 |
+
[o2] pack 1/16 done
|
| 6 |
+
[o2] pack 2/16 done
|
| 7 |
+
[o2] pack 3/16 done
|
| 8 |
+
[o2] pack 4/16 done
|
| 9 |
+
[o2] pack 5/16 done
|
| 10 |
+
[o2] pack 6/16 done
|
| 11 |
+
[o2] pack 7/16 done
|
| 12 |
+
[o2] pack 8/16 done
|
| 13 |
+
[o2] pack 9/16 done
|
| 14 |
+
[o2] pack 10/16 done
|
| 15 |
+
[o2] pack 11/16 done
|
| 16 |
+
[o2] pack 12/16 done
|
| 17 |
+
[o2] pack 13/16 done
|
| 18 |
+
[o2] pack 14/16 done
|
| 19 |
+
[o2] pack 15/16 done
|
| 20 |
+
[o2] pack 16/16 done
|
| 21 |
+
[o2] wrote /workspace/FLAME-MoE/results/ablations/olmoe_minflow_bpb.csv
|
| 22 |
+
free_top8 BPB=0.6703
|
| 23 |
+
static BPB=2.7544
|
| 24 |
+
greedy BPB=2.2953
|
| 25 |
+
minflow_m1 BPB=2.2881
|
| 26 |
+
minflow_m2 BPB=2.0051
|
| 27 |
+
minflow_m4 BPB=1.8668
|
| 28 |
+
[o2] SCHEDULABILITY: MinFlow m1 BPB 2.2881 vs windowed-greedy 2.2953 = -0.0072 (better scheduling shrinks BPB)
|
olmoe_adapt/logs/oseries_o2full.log
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[o2full] 8 packs, FULL T=4096 single cold-fill; schedules: free, greedy, MinFlow m=1 (C=32 exact)
|
| 2 |
+
|
| 3 |
+
/workspace/olmoe-adapt/scripts/oseries_o2_full.py:58: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
ids = torch.load("/workspace/olmoe-adapt/data/bpb_slice_ids.pt")[:n].long()
|
| 5 |
+
[o2full] pack 1/8 done
|
| 6 |
+
[o2full] pack 2/8 done
|
| 7 |
+
[o2full] pack 3/8 done
|
| 8 |
+
[o2full] pack 4/8 done
|
| 9 |
+
[o2full] pack 5/8 done
|
| 10 |
+
[o2full] pack 6/8 done
|
| 11 |
+
[o2full] pack 7/8 done
|
| 12 |
+
[o2full] pack 8/8 done
|
| 13 |
+
[o2full] wrote /workspace/FLAME-MoE/results/ablations/olmoe_minflow_bpb_full.csv
|
| 14 |
+
free_top8 BPB=0.6850
|
| 15 |
+
greedy BPB=nan
|
| 16 |
+
minflow_m1 BPB=2.2522
|
| 17 |
+
[o2full] UNWINDOWED HEADLINE: greedy nan vs MinFlow m1 2.2522 = +nan BPB (greedy ~= hindsight-optimal <=1-swap)
|
olmoe_adapt/logs/oseries_paired.log
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[mp] 384 MinFlow solves over 24 workers ...
|
| 2 |
+
[mp] 32/384 solves (210s)
|
| 3 |
+
[mp] 64/384 solves (332s)
|
| 4 |
+
[mp] 96/384 solves (451s)
|
| 5 |
+
[mp] 128/384 solves (583s)
|
| 6 |
+
[mp] 160/384 solves (690s)
|
| 7 |
+
[mp] 192/384 solves (787s)
|
| 8 |
+
[mp] 224/384 solves (923s)
|
| 9 |
+
[mp] 256/384 solves (1043s)
|
| 10 |
+
[mp] 288/384 solves (1164s)
|
| 11 |
+
[mp] 320/384 solves (1283s)
|
| 12 |
+
[mp] 352/384 solves (1411s)
|
| 13 |
+
[mp] 384/384 solves (1527s)
|
| 14 |
+
[mp] all solves done in 25.5 min, cached -> /workspace/olmoe-adapt/data/oseries/paired_masks_N24.npz. GPU forwards next.
|
| 15 |
+
|
| 16 |
+
/workspace/olmoe-adapt/scripts/oseries_paired_mp.py:83: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 17 |
+
ids = torch.load("/workspace/olmoe-adapt/data/bpb_slice_ids.pt")[:N].long()
|
| 18 |
+
[paired] pack 1/24: greedy=2.3991 minflow=2.3774 diff=+0.0217
|
| 19 |
+
[paired] pack 2/24: greedy=2.2999 minflow=2.2996 diff=+0.0003
|
| 20 |
+
[paired] pack 3/24: greedy=2.3461 minflow=2.3272 diff=+0.0189
|
| 21 |
+
[paired] pack 4/24: greedy=2.3605 minflow=2.2850 diff=+0.0756
|
| 22 |
+
[paired] pack 5/24: greedy=2.0993 minflow=2.0665 diff=+0.0328
|
| 23 |
+
[paired] pack 6/24: greedy=2.1577 minflow=2.1283 diff=+0.0293
|
| 24 |
+
[paired] pack 7/24: greedy=2.3197 minflow=2.2607 diff=+0.0590
|
| 25 |
+
[paired] pack 8/24: greedy=2.3149 minflow=2.2731 diff=+0.0418
|
| 26 |
+
[paired] pack 9/24: greedy=2.2079 minflow=2.1455 diff=+0.0624
|
| 27 |
+
[paired] pack 10/24: greedy=2.2475 minflow=2.2319 diff=+0.0156
|
| 28 |
+
[paired] pack 11/24: greedy=2.3762 minflow=2.4608 diff=-0.0846
|
| 29 |
+
[paired] pack 12/24: greedy=2.4054 minflow=2.4220 diff=-0.0166
|
| 30 |
+
[paired] pack 13/24: greedy=2.2497 minflow=2.1887 diff=+0.0610
|
| 31 |
+
[paired] pack 14/24: greedy=2.3394 minflow=2.3063 diff=+0.0331
|
| 32 |
+
[paired] pack 15/24: greedy=2.2812 minflow=2.1890 diff=+0.0922
|
| 33 |
+
[paired] pack 16/24: greedy=2.3334 minflow=2.2919 diff=+0.0415
|
| 34 |
+
[paired] pack 17/24: greedy=2.2952 minflow=2.2539 diff=+0.0413
|
| 35 |
+
[paired] pack 18/24: greedy=2.2832 minflow=2.2625 diff=+0.0207
|
| 36 |
+
[paired] pack 19/24: greedy=2.2748 minflow=2.2768 diff=-0.0019
|
| 37 |
+
[paired] pack 20/24: greedy=2.3599 minflow=2.3693 diff=-0.0094
|
| 38 |
+
[paired] pack 21/24: greedy=2.3420 minflow=2.3336 diff=+0.0084
|
| 39 |
+
[paired] pack 22/24: greedy=2.3432 minflow=2.3078 diff=+0.0355
|
| 40 |
+
[paired] pack 23/24: greedy=2.2870 minflow=2.3158 diff=-0.0288
|
| 41 |
+
[paired] pack 24/24: greedy=2.3063 minflow=2.3012 diff=+0.0051
|
| 42 |
+
[paired] N=24 greedy=2.3012 MinFlow_m1=2.2781
|
| 43 |
+
[paired] PAIRED diff (greedy - MinFlow) = 0.0231 +/- 0.0076 (paired SE); t=3.0
|
| 44 |
+
[paired] appended.
|
olmoe_adapt/logs/probe1.log
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[train] tag=probe1 lr=3e-05 trainable=2097152 (router only) tokens=5000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_router.py:34: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt") # [Nseq,4096] int32
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_router.py:37: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M lm=7.7350 aux=23.9744 z=5.2966 19.4k tok/s
|
| 11 |
+
[eval] probe1 lr=3e-05 tok=3M adapted_BPB=2.3725
|
| 12 |
+
[step 40] tok=2.6M lm=7.4505 aux=22.5345 z=5.2552 17.7k tok/s
|
olmoe_adapt/logs/rsweep.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
R-SWEEP (2x4096, base CE=1.6145):
|
| 3 |
+
R=48: CE=1.7146 dCE=+0.1001
|
| 4 |
+
R=32: CE=2.0134 dCE=+0.3989
|
| 5 |
+
R=16: CE=4.7862 dCE=+3.1717
|
| 6 |
+
R= 8: CE=9.1532 dCE=+7.5386
|
olmoe_adapt/logs/run_h.out
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
=== BAKE-OFF arm H (E-recipe + zone anneal 24->8, 250M, eval 50M) 02:44 ===
|
| 2 |
+
H done: [DONE] arm=H final BPB=0.8510 swap=0.9997 ent=0.9981
|
| 3 |
+
ARM H DONE. Optional-idle set complete.
|
olmoe_adapt/logs/run_post_c.out
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
=== BAKE-OFF arm CE screen (50M, lr=3e-4, eval every 10M) 23:05 ===
|
| 2 |
+
=== BAKE-OFF arm CE (250M, lr=3e-4, eval every 10M, checkpointed) 00:06 ===
|
| 3 |
+
CE done: [DONE] arm=CE final BPB=0.8149 swap=0.9997 ent=0.9988
|
| 4 |
+
=== BAKE-OFF arm D re-run (250M, lr=3e-4, eval every 50M, MB=4 self-distill) 06:28 ===
|
| 5 |
+
D done: [DONE] arm=D final BPB=1.2919 swap=0.9999 ent=0.9903
|
| 6 |
+
POST-C QUEUE DONE (CE-250M, D). G + F-prime next.
|
olmoe_adapt/logs/sweep.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
=== SWEEP sweep_lr3e-5 (0.25B, eval every 50M) 23:17 ===
|
| 2 |
+
sweep_lr3e-5 done: [DONE] tag=sweep_lr3e-5 lr=3e-05 final adapted BPB=1.3618
|
| 3 |
+
=== SWEEP sweep_lr1e-4 (0.25B, eval every 50M) 02:50 ===
|
| 4 |
+
sweep_lr1e-4 done: [DONE] tag=sweep_lr1e-4 lr=0.0001 final adapted BPB=1.2897
|
| 5 |
+
=== SWEEP sweep_lr3e-4 (0.25B, eval every 50M) 06:25 ===
|
| 6 |
+
sweep_lr3e-4 done: [DONE] tag=sweep_lr3e-4 lr=0.0003 final adapted BPB=1.2825
|
| 7 |
+
ALL SWEEP DONE
|
olmoe_adapt/logs/sweep_sweep_lr1e-4.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[train] tag=sweep_lr1e-4 lr=0.0001 trainable=2097152 (router only) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_router.py:34: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt") # [Nseq,4096] int32
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_router.py:37: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M lm=7.0505 aux=23.2133 z=5.2848 19.4k tok/s
|
| 11 |
+
[step 40] tok=2.6M lm=6.7039 aux=23.4000 z=5.1689 19.6k tok/s
|
| 12 |
+
[step 60] tok=3.9M lm=6.3717 aux=25.1007 z=4.9833 19.7k tok/s
|
| 13 |
+
[step 80] tok=5.2M lm=6.1197 aux=24.0699 z=4.7704 19.7k tok/s
|
| 14 |
+
[step 100] tok=6.6M lm=6.0398 aux=24.6068 z=4.5669 19.7k tok/s
|
| 15 |
+
[step 120] tok=7.9M lm=5.7898 aux=24.2193 z=4.4354 19.7k tok/s
|
| 16 |
+
[step 140] tok=9.2M lm=5.7165 aux=23.8981 z=4.3917 19.7k tok/s
|
| 17 |
+
[step 160] tok=10.5M lm=5.5849 aux=24.2645 z=4.3414 19.8k tok/s
|
| 18 |
+
[step 180] tok=11.8M lm=5.4875 aux=24.6742 z=4.1955 19.8k tok/s
|
| 19 |
+
[step 200] tok=13.1M lm=5.3893 aux=24.4849 z=4.1876 19.8k tok/s
|
| 20 |
+
[step 220] tok=14.4M lm=5.2579 aux=24.1587 z=4.1796 19.8k tok/s
|
| 21 |
+
[step 240] tok=15.7M lm=5.2059 aux=24.0792 z=4.1820 19.8k tok/s
|
| 22 |
+
[step 260] tok=17.0M lm=5.2185 aux=24.7691 z=4.0615 19.8k tok/s
|
| 23 |
+
[step 280] tok=18.4M lm=5.2216 aux=24.3004 z=4.1100 19.8k tok/s
|
| 24 |
+
[step 300] tok=19.7M lm=5.0809 aux=24.5588 z=4.0354 19.8k tok/s
|
| 25 |
+
[step 320] tok=21.0M lm=5.0115 aux=24.6316 z=4.0035 19.8k tok/s
|
| 26 |
+
[step 340] tok=22.3M lm=5.0041 aux=24.4643 z=3.9768 19.8k tok/s
|
| 27 |
+
[step 360] tok=23.6M lm=4.9116 aux=24.6666 z=3.9236 19.8k tok/s
|
| 28 |
+
[step 380] tok=24.9M lm=5.0135 aux=24.6890 z=3.9242 19.8k tok/s
|
| 29 |
+
[step 400] tok=26.2M lm=4.9379 aux=24.6580 z=3.8649 19.8k tok/s
|
| 30 |
+
[step 420] tok=27.5M lm=4.9426 aux=23.9360 z=3.9681 19.8k tok/s
|
| 31 |
+
[step 440] tok=28.8M lm=4.8780 aux=24.6623 z=3.8143 19.8k tok/s
|
| 32 |
+
[step 460] tok=30.1M lm=4.7972 aux=24.6568 z=3.8014 19.8k tok/s
|
| 33 |
+
[step 480] tok=31.5M lm=4.8158 aux=24.7071 z=3.8084 19.8k tok/s
|
| 34 |
+
[step 500] tok=32.8M lm=4.8031 aux=24.5117 z=3.7817 19.8k tok/s
|
| 35 |
+
[step 520] tok=34.1M lm=4.8319 aux=24.5037 z=3.7976 19.8k tok/s
|
| 36 |
+
[step 540] tok=35.4M lm=4.8324 aux=24.1335 z=3.8287 19.8k tok/s
|
| 37 |
+
[step 560] tok=36.7M lm=4.8378 aux=23.9812 z=3.7580 19.8k tok/s
|
| 38 |
+
[step 580] tok=38.0M lm=4.6656 aux=23.8223 z=3.6890 19.8k tok/s
|
| 39 |
+
[step 600] tok=39.3M lm=4.8041 aux=23.7214 z=3.6939 19.8k tok/s
|
| 40 |
+
[step 620] tok=40.6M lm=4.6599 aux=23.7973 z=3.6300 19.8k tok/s
|
| 41 |
+
[step 640] tok=41.9M lm=4.5004 aux=23.6788 z=3.5962 19.8k tok/s
|
| 42 |
+
[step 660] tok=43.3M lm=4.6493 aux=23.3202 z=3.6526 19.8k tok/s
|
| 43 |
+
[step 680] tok=44.6M lm=4.4868 aux=24.0239 z=3.5657 19.8k tok/s
|
| 44 |
+
[step 700] tok=45.9M lm=4.6571 aux=23.6509 z=3.6077 19.8k tok/s
|
| 45 |
+
[step 720] tok=47.2M lm=4.5041 aux=23.5902 z=3.5373 19.8k tok/s
|
| 46 |
+
[step 740] tok=48.5M lm=4.7033 aux=23.4132 z=3.5741 19.8k tok/s
|
| 47 |
+
[step 760] tok=49.8M lm=4.5482 aux=23.5188 z=3.5504 19.8k tok/s
|
| 48 |
+
[eval] sweep_lr1e-4 lr=0.0001 tok=50M adapted_BPB=1.4358
|
| 49 |
+
[step 780] tok=51.1M lm=4.6246 aux=23.4041 z=3.5162 19.7k tok/s
|
| 50 |
+
[step 800] tok=52.4M lm=4.5351 aux=23.0512 z=3.5789 19.7k tok/s
|
| 51 |
+
[step 820] tok=53.7M lm=4.5479 aux=23.2363 z=3.4839 19.7k tok/s
|
| 52 |
+
[step 840] tok=55.1M lm=4.4625 aux=22.9874 z=3.4909 19.7k tok/s
|
| 53 |
+
[step 860] tok=56.4M lm=4.5599 aux=23.1319 z=3.5023 19.7k tok/s
|
| 54 |
+
[step 880] tok=57.7M lm=4.4659 aux=23.1599 z=3.4440 19.7k tok/s
|
| 55 |
+
[step 900] tok=59.0M lm=4.4974 aux=23.1240 z=3.4254 19.7k tok/s
|
| 56 |
+
[step 920] tok=60.3M lm=4.4101 aux=23.0547 z=3.4830 19.7k tok/s
|
| 57 |
+
[step 940] tok=61.6M lm=4.4275 aux=22.9332 z=3.4179 19.7k tok/s
|
| 58 |
+
[step 960] tok=62.9M lm=4.4068 aux=22.5944 z=3.4432 19.7k tok/s
|
| 59 |
+
[step 980] tok=64.2M lm=4.5698 aux=22.5177 z=3.3803 19.7k tok/s
|
| 60 |
+
[step 1000] tok=65.5M lm=4.3389 aux=22.6435 z=3.3924 19.7k tok/s
|
| 61 |
+
[step 1020] tok=66.8M lm=4.3390 aux=22.4713 z=3.3847 19.7k tok/s
|
| 62 |
+
[step 1040] tok=68.2M lm=4.3084 aux=22.4403 z=3.4096 19.7k tok/s
|
| 63 |
+
[step 1060] tok=69.5M lm=4.4096 aux=22.4000 z=3.3622 19.7k tok/s
|
| 64 |
+
[step 1080] tok=70.8M lm=4.3702 aux=22.3764 z=3.3353 19.7k tok/s
|
| 65 |
+
[step 1100] tok=72.1M lm=4.3339 aux=22.2730 z=3.3675 19.7k tok/s
|
| 66 |
+
[step 1120] tok=73.4M lm=4.5193 aux=22.1778 z=3.2355 19.7k tok/s
|
| 67 |
+
[step 1140] tok=74.7M lm=4.4768 aux=22.4620 z=3.2535 19.7k tok/s
|
| 68 |
+
[step 1160] tok=76.0M lm=4.3661 aux=22.0561 z=3.2366 19.8k tok/s
|
| 69 |
+
[step 1180] tok=77.3M lm=4.3484 aux=21.9462 z=3.2469 19.8k tok/s
|
| 70 |
+
[step 1200] tok=78.6M lm=4.2808 aux=21.7393 z=3.2503 19.8k tok/s
|
| 71 |
+
[step 1220] tok=80.0M lm=4.3065 aux=21.9387 z=3.1706 19.8k tok/s
|
| 72 |
+
[step 1240] tok=81.3M lm=4.4828 aux=21.9089 z=3.2372 19.8k tok/s
|
| 73 |
+
[step 1260] tok=82.6M lm=4.3698 aux=21.7501 z=3.1708 19.8k tok/s
|
| 74 |
+
[step 1280] tok=83.9M lm=4.3466 aux=21.3361 z=3.2172 19.8k tok/s
|
| 75 |
+
[step 1300] tok=85.2M lm=4.2772 aux=21.4667 z=3.1529 19.8k tok/s
|
| 76 |
+
[step 1320] tok=86.5M lm=4.2679 aux=21.4646 z=3.1761 19.8k tok/s
|
| 77 |
+
[step 1340] tok=87.8M lm=4.2645 aux=21.4769 z=3.1264 19.8k tok/s
|
| 78 |
+
[step 1360] tok=89.1M lm=4.4070 aux=21.3114 z=3.1158 19.8k tok/s
|
| 79 |
+
[step 1380] tok=90.4M lm=4.2168 aux=21.1551 z=3.1438 19.8k tok/s
|
| 80 |
+
[step 1400] tok=91.8M lm=4.2194 aux=21.2773 z=3.1623 19.8k tok/s
|
| 81 |
+
[step 1420] tok=93.1M lm=4.3341 aux=21.5232 z=3.0813 19.8k tok/s
|
| 82 |
+
[step 1440] tok=94.4M lm=4.2343 aux=20.9665 z=3.0813 19.8k tok/s
|
| 83 |
+
[step 1460] tok=95.7M lm=4.3212 aux=21.4973 z=3.0687 19.8k tok/s
|
| 84 |
+
[step 1480] tok=97.0M lm=4.3493 aux=21.0096 z=3.1029 19.8k tok/s
|
| 85 |
+
[step 1500] tok=98.3M lm=4.2712 aux=21.2618 z=3.0417 19.8k tok/s
|
| 86 |
+
[step 1520] tok=99.6M lm=4.1987 aux=20.8409 z=3.0499 19.8k tok/s
|
| 87 |
+
[eval] sweep_lr1e-4 lr=0.0001 tok=100M adapted_BPB=1.3475
|
| 88 |
+
[step 1540] tok=100.9M lm=4.2503 aux=20.9403 z=3.0470 19.7k tok/s
|
| 89 |
+
[step 1560] tok=102.2M lm=4.3377 aux=21.1156 z=3.0234 19.7k tok/s
|
| 90 |
+
[step 1580] tok=103.5M lm=4.3053 aux=21.0140 z=3.0466 19.7k tok/s
|
| 91 |
+
[step 1600] tok=104.9M lm=4.1923 aux=20.8105 z=3.0006 19.7k tok/s
|
| 92 |
+
[step 1620] tok=106.2M lm=4.2151 aux=20.7867 z=3.0273 19.7k tok/s
|
| 93 |
+
[step 1640] tok=107.5M lm=4.3349 aux=20.7737 z=3.0492 19.7k tok/s
|
| 94 |
+
[step 1660] tok=108.8M lm=4.3228 aux=20.5971 z=3.0193 19.7k tok/s
|
| 95 |
+
[step 1680] tok=110.1M lm=4.2328 aux=20.8205 z=2.9795 19.7k tok/s
|
| 96 |
+
[step 1700] tok=111.4M lm=4.2232 aux=20.8656 z=2.9783 19.7k tok/s
|
| 97 |
+
[step 1720] tok=112.7M lm=4.3012 aux=20.5731 z=2.9711 19.7k tok/s
|
| 98 |
+
[step 1740] tok=114.0M lm=4.1803 aux=20.5730 z=2.9758 19.7k tok/s
|
| 99 |
+
[step 1760] tok=115.3M lm=4.2289 aux=20.7685 z=2.9000 19.7k tok/s
|
| 100 |
+
[step 1780] tok=116.7M lm=4.1670 aux=20.5798 z=3.0170 19.7k tok/s
|
| 101 |
+
[step 1800] tok=118.0M lm=4.3606 aux=20.4922 z=2.8806 19.7k tok/s
|
| 102 |
+
[step 1820] tok=119.3M lm=4.1889 aux=20.6348 z=2.9014 19.7k tok/s
|
| 103 |
+
[step 1840] tok=120.6M lm=4.2016 aux=20.5798 z=2.8976 19.7k tok/s
|
| 104 |
+
[step 1860] tok=121.9M lm=4.2860 aux=20.4315 z=2.8850 19.7k tok/s
|
| 105 |
+
[step 1880] tok=123.2M lm=4.2557 aux=20.3729 z=2.8543 19.7k tok/s
|
| 106 |
+
[step 1900] tok=124.5M lm=4.2201 aux=20.6108 z=2.8076 19.7k tok/s
|
| 107 |
+
[step 1920] tok=125.8M lm=4.2374 aux=20.2602 z=2.9535 19.7k tok/s
|
| 108 |
+
[step 1940] tok=127.1M lm=4.2884 aux=20.5737 z=2.8613 19.7k tok/s
|
| 109 |
+
[step 1960] tok=128.5M lm=4.1558 aux=20.4266 z=2.8356 19.7k tok/s
|
| 110 |
+
[step 1980] tok=129.8M lm=4.1646 aux=20.2553 z=2.8352 19.7k tok/s
|
| 111 |
+
[step 2000] tok=131.1M lm=4.1775 aux=20.0495 z=2.8017 19.7k tok/s
|
| 112 |
+
[step 2020] tok=132.4M lm=4.1728 aux=20.2551 z=2.8100 19.7k tok/s
|
| 113 |
+
[step 2040] tok=133.7M lm=4.1213 aux=20.2691 z=2.8396 19.7k tok/s
|
| 114 |
+
[step 2060] tok=135.0M lm=4.1508 aux=20.1618 z=2.7812 19.7k tok/s
|
| 115 |
+
[step 2080] tok=136.3M lm=4.2306 aux=20.1989 z=2.8132 19.7k tok/s
|
| 116 |
+
[step 2100] tok=137.6M lm=4.2320 aux=20.0853 z=2.7512 19.7k tok/s
|
| 117 |
+
[step 2120] tok=138.9M lm=4.3642 aux=20.1884 z=2.7794 19.8k tok/s
|
| 118 |
+
[step 2140] tok=140.2M lm=4.1664 aux=20.1959 z=2.7862 19.8k tok/s
|
| 119 |
+
[step 2160] tok=141.6M lm=4.2690 aux=19.8836 z=2.7444 19.8k tok/s
|
| 120 |
+
[step 2180] tok=142.9M lm=4.2595 aux=19.5705 z=2.7480 19.8k tok/s
|
| 121 |
+
[step 2200] tok=144.2M lm=4.2574 aux=20.0747 z=2.7426 19.8k tok/s
|
| 122 |
+
[step 2220] tok=145.5M lm=4.2965 aux=20.0618 z=2.7195 19.8k tok/s
|
| 123 |
+
[step 2240] tok=146.8M lm=4.1962 aux=19.8072 z=2.7319 19.8k tok/s
|
| 124 |
+
[step 2260] tok=148.1M lm=4.1156 aux=19.9144 z=2.7472 19.8k tok/s
|
| 125 |
+
[step 2280] tok=149.4M lm=4.2646 aux=19.9578 z=2.7204 19.8k tok/s
|
| 126 |
+
[eval] sweep_lr1e-4 lr=0.0001 tok=150M adapted_BPB=1.3180
|
| 127 |
+
[step 2300] tok=150.7M lm=4.2300 aux=19.7042 z=2.7179 19.7k tok/s
|
| 128 |
+
[step 2320] tok=152.0M lm=4.1764 aux=19.9169 z=2.6955 19.7k tok/s
|
| 129 |
+
[step 2340] tok=153.4M lm=4.2454 aux=19.9376 z=2.6671 19.7k tok/s
|
| 130 |
+
[step 2360] tok=154.7M lm=4.1598 aux=19.5766 z=2.6791 19.7k tok/s
|
| 131 |
+
[step 2380] tok=156.0M lm=4.2964 aux=19.6305 z=2.7002 19.7k tok/s
|
| 132 |
+
[step 2400] tok=157.3M lm=4.2586 aux=19.8045 z=2.6922 19.7k tok/s
|
| 133 |
+
[step 2420] tok=158.6M lm=4.2497 aux=20.0250 z=2.6421 19.7k tok/s
|
| 134 |
+
[step 2440] tok=159.9M lm=4.1661 aux=19.5792 z=2.7034 19.7k tok/s
|
| 135 |
+
[step 2460] tok=161.2M lm=4.2395 aux=19.6022 z=2.6691 19.7k tok/s
|
| 136 |
+
[step 2480] tok=162.5M lm=4.0539 aux=19.6246 z=2.6563 19.7k tok/s
|
| 137 |
+
[step 2500] tok=163.8M lm=4.1547 aux=19.7737 z=2.6397 19.7k tok/s
|
| 138 |
+
[step 2520] tok=165.2M lm=4.1145 aux=19.5293 z=2.6162 19.7k tok/s
|
| 139 |
+
[step 2540] tok=166.5M lm=4.2877 aux=19.5281 z=2.6624 19.7k tok/s
|
| 140 |
+
[step 2560] tok=167.8M lm=4.1477 aux=19.8851 z=2.6490 19.7k tok/s
|
| 141 |
+
[step 2580] tok=169.1M lm=4.2014 aux=19.8712 z=2.5944 19.7k tok/s
|
| 142 |
+
[step 2600] tok=170.4M lm=4.0060 aux=19.8247 z=2.6765 19.7k tok/s
|
| 143 |
+
[step 2620] tok=171.7M lm=4.2297 aux=19.6749 z=2.6405 19.7k tok/s
|
| 144 |
+
[step 2640] tok=173.0M lm=4.1241 aux=19.6047 z=2.6430 19.7k tok/s
|
| 145 |
+
[step 2660] tok=174.3M lm=4.3418 aux=19.9085 z=2.6763 19.7k tok/s
|
| 146 |
+
[step 2680] tok=175.6M lm=4.2502 aux=19.4607 z=2.5841 19.7k tok/s
|
| 147 |
+
[step 2700] tok=176.9M lm=4.1237 aux=19.6981 z=2.5679 19.7k tok/s
|
| 148 |
+
[step 2720] tok=178.3M lm=4.2003 aux=19.4613 z=2.5922 19.7k tok/s
|
| 149 |
+
[step 2740] tok=179.6M lm=4.1392 aux=19.4307 z=2.5536 19.7k tok/s
|
| 150 |
+
[step 2760] tok=180.9M lm=4.2244 aux=19.5637 z=2.5520 19.7k tok/s
|
| 151 |
+
[step 2780] tok=182.2M lm=4.1371 aux=19.2386 z=2.5794 19.7k tok/s
|
| 152 |
+
[step 2800] tok=183.5M lm=4.1780 aux=19.5682 z=2.5370 19.7k tok/s
|
| 153 |
+
[step 2820] tok=184.8M lm=4.2056 aux=19.9188 z=2.4668 19.7k tok/s
|
| 154 |
+
[step 2840] tok=186.1M lm=4.0339 aux=19.3594 z=2.5541 19.7k tok/s
|
| 155 |
+
[step 2860] tok=187.4M lm=4.1697 aux=19.3876 z=2.5253 19.7k tok/s
|
| 156 |
+
[step 2880] tok=188.7M lm=4.2831 aux=19.4405 z=2.5166 19.7k tok/s
|
| 157 |
+
[step 2900] tok=190.1M lm=4.2280 aux=19.7177 z=2.5413 19.7k tok/s
|
| 158 |
+
[step 2920] tok=191.4M lm=4.1267 aux=19.2357 z=2.4845 19.7k tok/s
|
| 159 |
+
[step 2940] tok=192.7M lm=4.2764 aux=19.3415 z=2.4909 19.7k tok/s
|
| 160 |
+
[step 2960] tok=194.0M lm=4.0190 aux=19.3303 z=2.5073 19.7k tok/s
|
| 161 |
+
[step 2980] tok=195.3M lm=4.1295 aux=19.4968 z=2.4714 19.7k tok/s
|
| 162 |
+
[step 3000] tok=196.6M lm=4.0814 aux=19.4063 z=2.5580 19.7k tok/s
|
| 163 |
+
[step 3020] tok=197.9M lm=4.0854 aux=19.1890 z=2.4869 19.7k tok/s
|
| 164 |
+
[step 3040] tok=199.2M lm=4.2200 aux=19.0686 z=2.5025 19.7k tok/s
|
| 165 |
+
[eval] sweep_lr1e-4 lr=0.0001 tok=200M adapted_BPB=1.2994
|
| 166 |
+
[step 3060] tok=200.5M lm=4.1530 aux=19.2669 z=2.4482 19.7k tok/s
|
| 167 |
+
[step 3080] tok=201.9M lm=4.1783 aux=19.2128 z=2.4597 19.7k tok/s
|
| 168 |
+
[step 3100] tok=203.2M lm=4.1981 aux=19.2221 z=2.4493 19.7k tok/s
|
| 169 |
+
[step 3120] tok=204.5M lm=4.1135 aux=19.2164 z=2.4503 19.7k tok/s
|
| 170 |
+
[step 3140] tok=205.8M lm=4.1237 aux=19.1977 z=2.4004 19.7k tok/s
|
| 171 |
+
[step 3160] tok=207.1M lm=4.1242 aux=19.1981 z=2.4520 19.7k tok/s
|
| 172 |
+
[step 3180] tok=208.4M lm=4.0960 aux=19.3525 z=2.4416 19.7k tok/s
|
| 173 |
+
[step 3200] tok=209.7M lm=4.0821 aux=19.1154 z=2.4054 19.7k tok/s
|
| 174 |
+
[step 3220] tok=211.0M lm=4.0247 aux=19.1984 z=2.4189 19.7k tok/s
|
| 175 |
+
[step 3240] tok=212.3M lm=4.0962 aux=18.9526 z=2.4089 19.7k tok/s
|
| 176 |
+
[step 3260] tok=213.6M lm=4.1834 aux=19.2983 z=2.3959 19.7k tok/s
|
| 177 |
+
[step 3280] tok=215.0M lm=4.1293 aux=18.9004 z=2.4124 19.7k tok/s
|
| 178 |
+
[step 3300] tok=216.3M lm=4.1894 aux=19.2172 z=2.4227 19.7k tok/s
|
| 179 |
+
[step 3320] tok=217.6M lm=4.1698 aux=18.8687 z=2.4068 19.7k tok/s
|
| 180 |
+
[step 3340] tok=218.9M lm=4.0663 aux=19.4885 z=2.4049 19.7k tok/s
|
| 181 |
+
[step 3360] tok=220.2M lm=4.1159 aux=19.0358 z=2.3591 19.7k tok/s
|
| 182 |
+
[step 3380] tok=221.5M lm=4.2619 aux=19.2687 z=2.4234 19.7k tok/s
|
| 183 |
+
[step 3400] tok=222.8M lm=4.1746 aux=19.1928 z=2.3751 19.7k tok/s
|
| 184 |
+
[step 3420] tok=224.1M lm=4.1479 aux=18.9248 z=2.3651 19.7k tok/s
|
| 185 |
+
[step 3440] tok=225.4M lm=4.1132 aux=19.0453 z=2.3395 19.7k tok/s
|
| 186 |
+
[step 3460] tok=226.8M lm=4.1487 aux=19.1758 z=2.3582 19.7k tok/s
|
| 187 |
+
[step 3480] tok=228.1M lm=4.1299 aux=18.9651 z=2.3507 19.7k tok/s
|
| 188 |
+
[step 3500] tok=229.4M lm=4.0968 aux=18.9351 z=2.3418 19.7k tok/s
|
| 189 |
+
[step 3520] tok=230.7M lm=4.1239 aux=18.9472 z=2.3356 19.7k tok/s
|
| 190 |
+
[step 3540] tok=232.0M lm=4.0812 aux=19.0589 z=2.3375 19.7k tok/s
|
| 191 |
+
[step 3560] tok=233.3M lm=4.1156 aux=18.9621 z=2.3190 19.7k tok/s
|
| 192 |
+
[step 3580] tok=234.6M lm=4.2647 aux=18.9993 z=2.3049 19.7k tok/s
|
| 193 |
+
[step 3600] tok=235.9M lm=4.1537 aux=19.1472 z=2.3164 19.7k tok/s
|
| 194 |
+
[step 3620] tok=237.2M lm=4.3360 aux=19.0182 z=2.3010 19.7k tok/s
|
| 195 |
+
[step 3640] tok=238.6M lm=4.0941 aux=18.8745 z=2.2809 19.7k tok/s
|
| 196 |
+
[step 3660] tok=239.9M lm=4.2224 aux=19.0788 z=2.2844 19.7k tok/s
|
| 197 |
+
[step 3680] tok=241.2M lm=4.1888 aux=19.2541 z=2.3173 19.7k tok/s
|
| 198 |
+
[step 3700] tok=242.5M lm=4.0446 aux=18.9378 z=2.2688 19.7k tok/s
|
| 199 |
+
[step 3720] tok=243.8M lm=4.1129 aux=18.8927 z=2.2903 19.7k tok/s
|
| 200 |
+
[step 3740] tok=245.1M lm=4.1695 aux=18.7984 z=2.2737 19.7k tok/s
|
| 201 |
+
[step 3760] tok=246.4M lm=4.2014 aux=18.8010 z=2.2582 19.7k tok/s
|
| 202 |
+
[step 3780] tok=247.7M lm=4.0922 aux=19.1960 z=2.2646 19.7k tok/s
|
| 203 |
+
[step 3800] tok=249.0M lm=4.2009 aux=18.7170 z=2.2808 19.7k tok/s
|
| 204 |
+
[eval] sweep_lr1e-4 lr=0.0001 tok=250M adapted_BPB=1.2897
|
| 205 |
+
[DONE] tag=sweep_lr1e-4 lr=0.0001 final adapted BPB=1.2897
|
olmoe_adapt/logs/sweep_sweep_lr3e-4.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[train] tag=sweep_lr3e-4 lr=0.0003 trainable=2097152 (router only) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_router.py:34: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt") # [Nseq,4096] int32
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_router.py:37: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M lm=6.3803 aux=24.9245 z=5.1941 19.4k tok/s
|
| 11 |
+
[step 40] tok=2.6M lm=6.0390 aux=24.5390 z=4.8926 19.6k tok/s
|
| 12 |
+
[step 60] tok=3.9M lm=5.7039 aux=24.8121 z=4.6908 19.7k tok/s
|
| 13 |
+
[step 80] tok=5.2M lm=5.4975 aux=24.6249 z=4.6480 19.7k tok/s
|
| 14 |
+
[step 100] tok=6.6M lm=5.3927 aux=25.4975 z=4.4854 19.7k tok/s
|
| 15 |
+
[step 120] tok=7.9M lm=5.1935 aux=25.7361 z=4.4110 19.8k tok/s
|
| 16 |
+
[step 140] tok=9.2M lm=5.1469 aux=25.2406 z=4.3663 19.8k tok/s
|
| 17 |
+
[step 160] tok=10.5M lm=5.0982 aux=24.9565 z=4.3480 19.8k tok/s
|
| 18 |
+
[step 180] tok=11.8M lm=5.0259 aux=25.3920 z=4.2574 19.8k tok/s
|
| 19 |
+
[step 200] tok=13.1M lm=4.9351 aux=25.5971 z=4.2455 19.8k tok/s
|
| 20 |
+
[step 220] tok=14.4M lm=4.7896 aux=25.1147 z=4.2221 19.8k tok/s
|
| 21 |
+
[step 240] tok=15.7M lm=4.7395 aux=25.3412 z=4.1309 19.8k tok/s
|
| 22 |
+
[step 260] tok=17.0M lm=4.7317 aux=25.2586 z=4.0112 19.8k tok/s
|
| 23 |
+
[step 280] tok=18.4M lm=4.7375 aux=24.8281 z=4.0029 19.8k tok/s
|
| 24 |
+
[step 300] tok=19.7M lm=4.5808 aux=24.6241 z=3.9122 19.8k tok/s
|
| 25 |
+
[step 320] tok=21.0M lm=4.5259 aux=24.4288 z=3.8478 19.8k tok/s
|
| 26 |
+
[step 340] tok=22.3M lm=4.5579 aux=24.5277 z=3.7664 19.8k tok/s
|
| 27 |
+
[step 360] tok=23.6M lm=4.4584 aux=24.3552 z=3.7336 19.8k tok/s
|
| 28 |
+
[step 380] tok=24.9M lm=4.5723 aux=23.7857 z=3.7437 19.8k tok/s
|
| 29 |
+
[step 400] tok=26.2M lm=4.4798 aux=23.6969 z=3.6843 19.8k tok/s
|
| 30 |
+
[step 420] tok=27.5M lm=4.5136 aux=23.2035 z=3.7191 19.8k tok/s
|
| 31 |
+
[step 440] tok=28.8M lm=4.3978 aux=23.5050 z=3.5903 19.8k tok/s
|
| 32 |
+
[step 460] tok=30.1M lm=4.3478 aux=23.1641 z=3.5273 19.8k tok/s
|
| 33 |
+
[step 480] tok=31.5M lm=4.3895 aux=23.1672 z=3.5074 19.8k tok/s
|
| 34 |
+
[step 500] tok=32.8M lm=4.3687 aux=23.1219 z=3.4750 19.8k tok/s
|
| 35 |
+
[step 520] tok=34.1M lm=4.4028 aux=22.6110 z=3.4685 19.8k tok/s
|
| 36 |
+
[step 540] tok=35.4M lm=4.4171 aux=22.5504 z=3.4834 19.8k tok/s
|
| 37 |
+
[step 560] tok=36.7M lm=4.4548 aux=22.5527 z=3.4405 19.8k tok/s
|
| 38 |
+
[step 580] tok=38.0M lm=4.3320 aux=22.1000 z=3.3831 19.8k tok/s
|
| 39 |
+
[step 600] tok=39.3M lm=4.4593 aux=21.7477 z=3.3513 19.8k tok/s
|
| 40 |
+
[step 620] tok=40.6M lm=4.3041 aux=21.6124 z=3.3237 19.8k tok/s
|
| 41 |
+
[step 640] tok=41.9M lm=4.1365 aux=21.7900 z=3.2727 19.8k tok/s
|
| 42 |
+
[step 660] tok=43.3M lm=4.3216 aux=21.4911 z=3.3267 19.8k tok/s
|
| 43 |
+
[step 680] tok=44.6M lm=4.1715 aux=21.5802 z=3.2011 19.8k tok/s
|
| 44 |
+
[step 700] tok=45.9M lm=4.3354 aux=21.6361 z=3.2569 19.8k tok/s
|
| 45 |
+
[step 720] tok=47.2M lm=4.2096 aux=21.4642 z=3.1734 19.8k tok/s
|
| 46 |
+
[step 740] tok=48.5M lm=4.4438 aux=21.4142 z=3.1934 19.8k tok/s
|
| 47 |
+
[step 760] tok=49.8M lm=4.2749 aux=20.9627 z=3.1468 19.8k tok/s
|
| 48 |
+
[eval] sweep_lr3e-4 lr=0.0003 tok=50M adapted_BPB=1.3468
|
| 49 |
+
[step 780] tok=51.1M lm=4.3563 aux=21.2253 z=3.1138 19.7k tok/s
|
| 50 |
+
[step 800] tok=52.4M lm=4.2639 aux=20.9708 z=3.1804 19.7k tok/s
|
| 51 |
+
[step 820] tok=53.7M lm=4.3001 aux=20.8384 z=3.0899 19.7k tok/s
|
| 52 |
+
[step 840] tok=55.1M lm=4.2029 aux=20.7933 z=3.0558 19.7k tok/s
|
| 53 |
+
[step 860] tok=56.4M lm=4.3280 aux=20.4809 z=3.0557 19.7k tok/s
|
| 54 |
+
[step 880] tok=57.7M lm=4.2457 aux=20.6378 z=3.0048 19.7k tok/s
|
| 55 |
+
[step 900] tok=59.0M lm=4.2740 aux=20.8484 z=2.9894 19.7k tok/s
|
| 56 |
+
[step 920] tok=60.3M lm=4.2002 aux=20.5642 z=2.9863 19.7k tok/s
|
| 57 |
+
[step 940] tok=61.6M lm=4.2128 aux=20.5854 z=2.9599 19.7k tok/s
|
| 58 |
+
[step 960] tok=62.9M lm=4.2000 aux=20.2791 z=3.0020 19.7k tok/s
|
| 59 |
+
[step 980] tok=64.2M lm=4.3855 aux=20.6241 z=2.9703 19.7k tok/s
|
| 60 |
+
[step 1000] tok=65.5M lm=4.1419 aux=20.3419 z=2.9698 19.7k tok/s
|
| 61 |
+
[step 1020] tok=66.8M lm=4.1455 aux=20.2948 z=2.9327 19.7k tok/s
|
| 62 |
+
[step 1040] tok=68.2M lm=4.1307 aux=20.4855 z=2.9393 19.7k tok/s
|
| 63 |
+
[step 1060] tok=69.5M lm=4.2523 aux=20.2435 z=2.9010 19.7k tok/s
|
| 64 |
+
[step 1080] tok=70.8M lm=4.1865 aux=19.9444 z=2.8875 19.7k tok/s
|
| 65 |
+
[step 1100] tok=72.1M lm=4.1378 aux=20.0772 z=2.9142 19.7k tok/s
|
| 66 |
+
[step 1120] tok=73.4M lm=4.3602 aux=20.3769 z=2.8021 19.8k tok/s
|
| 67 |
+
[step 1140] tok=74.7M lm=4.3047 aux=20.1947 z=2.8253 19.8k tok/s
|
| 68 |
+
[step 1160] tok=76.0M lm=4.1833 aux=20.1863 z=2.8122 19.8k tok/s
|
| 69 |
+
[step 1180] tok=77.3M lm=4.2030 aux=20.0550 z=2.8108 19.8k tok/s
|
| 70 |
+
[step 1200] tok=78.6M lm=4.1243 aux=19.9076 z=2.7846 19.8k tok/s
|
| 71 |
+
[step 1220] tok=80.0M lm=4.1742 aux=20.3111 z=2.7244 19.8k tok/s
|
| 72 |
+
[step 1240] tok=81.3M lm=4.3491 aux=20.3325 z=2.7730 19.8k tok/s
|
| 73 |
+
[step 1260] tok=82.6M lm=4.2328 aux=19.9641 z=2.7415 19.8k tok/s
|
| 74 |
+
[step 1280] tok=83.9M lm=4.2065 aux=19.8315 z=2.7719 19.8k tok/s
|
| 75 |
+
[step 1300] tok=85.2M lm=4.1276 aux=20.0654 z=2.7092 19.8k tok/s
|
| 76 |
+
[step 1320] tok=86.5M lm=4.1149 aux=20.1432 z=2.7443 19.8k tok/s
|
| 77 |
+
[step 1340] tok=87.8M lm=4.1271 aux=19.9270 z=2.6924 19.8k tok/s
|
| 78 |
+
[step 1360] tok=89.1M lm=4.2793 aux=19.8811 z=2.6754 19.8k tok/s
|
| 79 |
+
[step 1380] tok=90.4M lm=4.0993 aux=20.1420 z=2.7141 19.8k tok/s
|
| 80 |
+
[step 1400] tok=91.8M lm=4.0815 aux=20.0381 z=2.7113 19.8k tok/s
|
| 81 |
+
[step 1420] tok=93.1M lm=4.1983 aux=20.3238 z=2.6326 19.8k tok/s
|
| 82 |
+
[step 1440] tok=94.4M lm=4.0888 aux=19.8110 z=2.6302 19.8k tok/s
|
| 83 |
+
[step 1460] tok=95.7M lm=4.1931 aux=19.9174 z=2.6092 19.8k tok/s
|
| 84 |
+
[step 1480] tok=97.0M lm=4.2106 aux=19.5394 z=2.6211 19.8k tok/s
|
| 85 |
+
[step 1500] tok=98.3M lm=4.1521 aux=20.0117 z=2.5738 19.8k tok/s
|
| 86 |
+
[step 1520] tok=99.6M lm=4.0723 aux=19.5716 z=2.5658 19.8k tok/s
|
| 87 |
+
[eval] sweep_lr3e-4 lr=0.0003 tok=100M adapted_BPB=1.3067
|
| 88 |
+
[step 1540] tok=100.9M lm=4.1370 aux=20.0426 z=2.5612 19.7k tok/s
|
| 89 |
+
[step 1560] tok=102.2M lm=4.2328 aux=19.8346 z=2.5508 19.7k tok/s
|
| 90 |
+
[step 1580] tok=103.5M lm=4.1846 aux=19.5341 z=2.5664 19.7k tok/s
|
| 91 |
+
[step 1600] tok=104.9M lm=4.0698 aux=19.7367 z=2.5400 19.7k tok/s
|
| 92 |
+
[step 1620] tok=106.2M lm=4.1000 aux=19.3297 z=2.5417 19.7k tok/s
|
| 93 |
+
[step 1640] tok=107.5M lm=4.2188 aux=19.3009 z=2.5583 19.7k tok/s
|
| 94 |
+
[step 1660] tok=108.8M lm=4.2375 aux=19.5241 z=2.5490 19.7k tok/s
|
| 95 |
+
[step 1680] tok=110.1M lm=4.1272 aux=19.6577 z=2.4956 19.7k tok/s
|
| 96 |
+
[step 1700] tok=111.4M lm=4.1229 aux=19.7091 z=2.4909 19.7k tok/s
|
| 97 |
+
[step 1720] tok=112.7M lm=4.1896 aux=19.6520 z=2.4693 19.7k tok/s
|
| 98 |
+
[step 1740] tok=114.0M lm=4.0765 aux=19.6193 z=2.4638 19.7k tok/s
|
| 99 |
+
[step 1760] tok=115.3M lm=4.1309 aux=19.8079 z=2.4212 19.7k tok/s
|
| 100 |
+
[step 1780] tok=116.7M lm=4.0888 aux=19.5344 z=2.5113 19.7k tok/s
|
| 101 |
+
[step 1800] tok=118.0M lm=4.2431 aux=19.3894 z=2.4192 19.7k tok/s
|
| 102 |
+
[step 1820] tok=119.3M lm=4.1018 aux=19.2149 z=2.3876 19.7k tok/s
|
| 103 |
+
[step 1840] tok=120.6M lm=4.1094 aux=19.6443 z=2.3933 19.7k tok/s
|
| 104 |
+
[step 1860] tok=121.9M lm=4.1790 aux=19.3661 z=2.3763 19.7k tok/s
|
| 105 |
+
[step 1880] tok=123.2M lm=4.1693 aux=19.4534 z=2.3581 19.7k tok/s
|
| 106 |
+
[step 1900] tok=124.5M lm=4.1429 aux=19.4671 z=2.3154 19.7k tok/s
|
| 107 |
+
[step 1920] tok=125.8M lm=4.1291 aux=19.5047 z=2.4220 19.7k tok/s
|
| 108 |
+
[step 1940] tok=127.1M lm=4.1839 aux=19.3793 z=2.3425 19.7k tok/s
|
| 109 |
+
[step 1960] tok=128.5M lm=4.0737 aux=19.3147 z=2.3237 19.7k tok/s
|
| 110 |
+
[step 1980] tok=129.8M lm=4.0674 aux=19.5524 z=2.3065 19.7k tok/s
|
| 111 |
+
[step 2000] tok=131.1M lm=4.0912 aux=19.4530 z=2.2874 19.7k tok/s
|
| 112 |
+
[step 2020] tok=132.4M lm=4.0930 aux=19.3719 z=2.2911 19.7k tok/s
|
| 113 |
+
[step 2040] tok=133.7M lm=4.0226 aux=19.4879 z=2.3300 19.7k tok/s
|
| 114 |
+
[step 2060] tok=135.0M lm=4.0391 aux=19.4044 z=2.2920 19.7k tok/s
|
| 115 |
+
[step 2080] tok=136.3M lm=4.1481 aux=19.3698 z=2.2998 19.7k tok/s
|
| 116 |
+
[step 2100] tok=137.6M lm=4.1176 aux=19.2338 z=2.2538 19.7k tok/s
|
| 117 |
+
[step 2120] tok=138.9M lm=4.2768 aux=19.7286 z=2.2640 19.7k tok/s
|
| 118 |
+
[step 2140] tok=140.2M lm=4.0800 aux=19.4296 z=2.2609 19.7k tok/s
|
| 119 |
+
[step 2160] tok=141.6M lm=4.1814 aux=18.8959 z=2.2377 19.7k tok/s
|
| 120 |
+
[step 2180] tok=142.9M lm=4.1627 aux=19.0693 z=2.2404 19.7k tok/s
|
| 121 |
+
[step 2200] tok=144.2M lm=4.1695 aux=19.0796 z=2.2239 19.7k tok/s
|
| 122 |
+
[step 2220] tok=145.5M lm=4.2142 aux=19.2656 z=2.1968 19.7k tok/s
|
| 123 |
+
[step 2240] tok=146.8M lm=4.1056 aux=19.1102 z=2.2065 19.8k tok/s
|
| 124 |
+
[step 2260] tok=148.1M lm=4.0431 aux=19.1844 z=2.2145 19.8k tok/s
|
| 125 |
+
[step 2280] tok=149.4M lm=4.1777 aux=19.2286 z=2.1882 19.8k tok/s
|
| 126 |
+
[eval] sweep_lr3e-4 lr=0.0003 tok=150M adapted_BPB=1.2901
|
| 127 |
+
[step 2300] tok=150.7M lm=4.1460 aux=19.0786 z=2.1819 19.7k tok/s
|
| 128 |
+
[step 2320] tok=152.0M lm=4.0973 aux=19.1128 z=2.1489 19.7k tok/s
|
| 129 |
+
[step 2340] tok=153.4M lm=4.1621 aux=19.1441 z=2.1352 19.7k tok/s
|
| 130 |
+
[step 2360] tok=154.7M lm=4.0712 aux=19.1432 z=2.1385 19.7k tok/s
|
| 131 |
+
[step 2380] tok=156.0M lm=4.2144 aux=19.0860 z=2.1478 19.7k tok/s
|
| 132 |
+
[step 2400] tok=157.3M lm=4.1789 aux=19.4888 z=2.1353 19.7k tok/s
|
| 133 |
+
[step 2420] tok=158.6M lm=4.1805 aux=19.3682 z=2.0810 19.7k tok/s
|
| 134 |
+
[step 2440] tok=159.9M lm=4.1160 aux=18.8145 z=2.1379 19.7k tok/s
|
| 135 |
+
[step 2460] tok=161.2M lm=4.1615 aux=19.0998 z=2.1266 19.7k tok/s
|
| 136 |
+
[step 2480] tok=162.5M lm=3.9650 aux=18.8911 z=2.1094 19.7k tok/s
|
| 137 |
+
[step 2500] tok=163.8M lm=4.0652 aux=19.3149 z=2.0897 19.7k tok/s
|
| 138 |
+
[step 2520] tok=165.2M lm=4.0465 aux=19.0829 z=2.0652 19.7k tok/s
|
| 139 |
+
[step 2540] tok=166.5M lm=4.2250 aux=18.8297 z=2.0936 19.7k tok/s
|
| 140 |
+
[step 2560] tok=167.8M lm=4.0694 aux=19.1746 z=2.0647 19.7k tok/s
|
| 141 |
+
[step 2580] tok=169.1M lm=4.1574 aux=19.4556 z=2.0359 19.7k tok/s
|
| 142 |
+
[step 2600] tok=170.4M lm=3.9366 aux=19.6625 z=2.0985 19.7k tok/s
|
| 143 |
+
[step 2620] tok=171.7M lm=4.1465 aux=19.2764 z=2.0731 19.7k tok/s
|
| 144 |
+
[step 2640] tok=173.0M lm=4.0411 aux=18.9603 z=2.0621 19.7k tok/s
|
| 145 |
+
[step 2660] tok=174.3M lm=4.2908 aux=19.4417 z=2.1016 19.7k tok/s
|
| 146 |
+
[step 2680] tok=175.6M lm=4.1832 aux=19.0982 z=2.0319 19.7k tok/s
|
| 147 |
+
[step 2700] tok=176.9M lm=4.0648 aux=18.8917 z=1.9704 19.7k tok/s
|
| 148 |
+
[step 2720] tok=178.3M lm=4.1338 aux=19.0308 z=2.0119 19.7k tok/s
|
| 149 |
+
[step 2740] tok=179.6M lm=4.0719 aux=18.9062 z=1.9761 19.7k tok/s
|
| 150 |
+
[step 2760] tok=180.9M lm=4.1648 aux=19.2842 z=1.9820 19.7k tok/s
|
| 151 |
+
[step 2780] tok=182.2M lm=4.0765 aux=19.2806 z=2.0016 19.7k tok/s
|
| 152 |
+
[step 2800] tok=183.5M lm=4.1104 aux=19.3995 z=1.9732 19.7k tok/s
|
| 153 |
+
[step 2820] tok=184.8M lm=4.1280 aux=19.4497 z=1.9109 19.7k tok/s
|
| 154 |
+
[step 2840] tok=186.1M lm=3.9834 aux=18.8844 z=1.9720 19.7k tok/s
|
| 155 |
+
[step 2860] tok=187.4M lm=4.1029 aux=18.8719 z=1.9448 19.7k tok/s
|
| 156 |
+
[step 2880] tok=188.7M lm=4.2276 aux=19.0810 z=1.9394 19.7k tok/s
|
| 157 |
+
[step 2900] tok=190.1M lm=4.1626 aux=19.3587 z=1.9594 19.7k tok/s
|
| 158 |
+
[step 2920] tok=191.4M lm=4.0713 aux=18.9823 z=1.9193 19.7k tok/s
|
| 159 |
+
[step 2940] tok=192.7M lm=4.2140 aux=19.1440 z=1.9095 19.7k tok/s
|
| 160 |
+
[step 2960] tok=194.0M lm=3.9756 aux=19.0877 z=1.9278 19.7k tok/s
|
| 161 |
+
[step 2980] tok=195.3M lm=4.0814 aux=19.2233 z=1.8837 19.7k tok/s
|
| 162 |
+
[step 3000] tok=196.6M lm=4.0104 aux=19.1991 z=1.9682 19.7k tok/s
|
| 163 |
+
[step 3020] tok=197.9M lm=4.0377 aux=18.7176 z=1.8865 19.7k tok/s
|
| 164 |
+
[step 3040] tok=199.2M lm=4.1714 aux=19.0001 z=1.9137 19.7k tok/s
|
| 165 |
+
[eval] sweep_lr3e-4 lr=0.0003 tok=200M adapted_BPB=1.2824
|
| 166 |
+
[step 3060] tok=200.5M lm=4.1026 aux=19.2270 z=1.8594 19.7k tok/s
|
| 167 |
+
[step 3080] tok=201.9M lm=4.1178 aux=18.9961 z=1.8702 19.7k tok/s
|
| 168 |
+
[step 3100] tok=203.2M lm=4.1547 aux=18.9106 z=1.8549 19.7k tok/s
|
| 169 |
+
[step 3120] tok=204.5M lm=4.0659 aux=18.6604 z=1.8577 19.7k tok/s
|
| 170 |
+
[step 3140] tok=205.8M lm=4.0718 aux=18.9275 z=1.8067 19.7k tok/s
|
| 171 |
+
[step 3160] tok=207.1M lm=4.0755 aux=19.0170 z=1.8615 19.7k tok/s
|
| 172 |
+
[step 3180] tok=208.4M lm=4.0380 aux=18.8694 z=1.8492 19.7k tok/s
|
| 173 |
+
[step 3200] tok=209.7M lm=4.0434 aux=18.5742 z=1.8224 19.7k tok/s
|
| 174 |
+
[step 3220] tok=211.0M lm=3.9809 aux=18.9036 z=1.8284 19.7k tok/s
|
| 175 |
+
[step 3240] tok=212.3M lm=4.0659 aux=18.8423 z=1.8017 19.7k tok/s
|
| 176 |
+
[step 3260] tok=213.6M lm=4.1536 aux=18.8924 z=1.8218 19.7k tok/s
|
| 177 |
+
[step 3280] tok=215.0M lm=4.0908 aux=18.3933 z=1.8036 19.7k tok/s
|
| 178 |
+
[step 3300] tok=216.3M lm=4.1516 aux=18.7095 z=1.8097 19.7k tok/s
|
| 179 |
+
[step 3320] tok=217.6M lm=4.1449 aux=18.8322 z=1.8105 19.7k tok/s
|
| 180 |
+
[step 3340] tok=218.9M lm=4.0337 aux=19.1977 z=1.8143 19.7k tok/s
|
| 181 |
+
[step 3360] tok=220.2M lm=4.0692 aux=18.9926 z=1.7718 19.7k tok/s
|
| 182 |
+
[step 3380] tok=221.5M lm=4.2125 aux=19.1008 z=1.8189 19.7k tok/s
|
| 183 |
+
[step 3400] tok=222.8M lm=4.1311 aux=18.7343 z=1.7891 19.7k tok/s
|
| 184 |
+
[step 3420] tok=224.1M lm=4.0991 aux=18.6782 z=1.7681 19.7k tok/s
|
| 185 |
+
[step 3440] tok=225.4M lm=4.0756 aux=18.7949 z=1.7546 19.7k tok/s
|
| 186 |
+
[step 3460] tok=226.8M lm=4.1367 aux=19.1672 z=1.7717 19.7k tok/s
|
| 187 |
+
[step 3480] tok=228.1M lm=4.1063 aux=18.7251 z=1.7538 19.7k tok/s
|
| 188 |
+
[step 3500] tok=229.4M lm=4.0598 aux=18.9345 z=1.7499 19.7k tok/s
|
| 189 |
+
[step 3520] tok=230.7M lm=4.0959 aux=18.8973 z=1.7479 19.7k tok/s
|
| 190 |
+
[step 3540] tok=232.0M lm=4.0535 aux=18.8046 z=1.7291 19.7k tok/s
|
| 191 |
+
[step 3560] tok=233.3M lm=4.0858 aux=18.9393 z=1.7070 19.7k tok/s
|
| 192 |
+
[step 3580] tok=234.6M lm=4.2064 aux=18.7263 z=1.7243 19.7k tok/s
|
| 193 |
+
[step 3600] tok=235.9M lm=4.1225 aux=18.8769 z=1.7063 19.7k tok/s
|
| 194 |
+
[step 3620] tok=237.2M lm=4.3033 aux=18.9200 z=1.7145 19.7k tok/s
|
| 195 |
+
[step 3640] tok=238.6M lm=4.0509 aux=18.5954 z=1.6872 19.7k tok/s
|
| 196 |
+
[step 3660] tok=239.9M lm=4.2001 aux=19.2464 z=1.7024 19.7k tok/s
|
| 197 |
+
[step 3680] tok=241.2M lm=4.1757 aux=19.3083 z=1.7309 19.7k tok/s
|
| 198 |
+
[step 3700] tok=242.5M lm=4.0182 aux=18.7596 z=1.6845 19.7k tok/s
|
| 199 |
+
[step 3720] tok=243.8M lm=4.0808 aux=18.7544 z=1.6867 19.7k tok/s
|
| 200 |
+
[step 3740] tok=245.1M lm=4.1368 aux=18.6616 z=1.6855 19.7k tok/s
|
| 201 |
+
[step 3760] tok=246.4M lm=4.1692 aux=18.9519 z=1.6707 19.7k tok/s
|
| 202 |
+
[step 3780] tok=247.7M lm=4.0489 aux=18.8494 z=1.6780 19.7k tok/s
|
| 203 |
+
[step 3800] tok=249.0M lm=4.1721 aux=18.9447 z=1.6806 19.7k tok/s
|
| 204 |
+
[eval] sweep_lr3e-4 lr=0.0003 tok=250M adapted_BPB=1.2825
|
| 205 |
+
[DONE] tag=sweep_lr3e-4 lr=0.0003 final adapted BPB=1.2825
|
olmoe_adapt/logs/sweep_sweep_lr3e-5.log
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
[train] tag=sweep_lr3e-5 lr=3e-05 trainable=2097152 (router only) tokens=250000000
|
| 3 |
+
/workspace/olmoe-adapt/scripts/train_router.py:34: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 4 |
+
corpus = torch.load(f"{OUT}/finetune_ids.pt") # [Nseq,4096] int32
|
| 5 |
+
/workspace/olmoe-adapt/scripts/train_router.py:37: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 6 |
+
bpb_ids = torch.load(f"{OUT}/bpb_slice_ids.pt")
|
| 7 |
+
[temporal] scan path: triton (verified == reference)
|
| 8 |
+
/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
|
| 9 |
+
with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined]
|
| 10 |
+
[step 20] tok=1.3M lm=7.7521 aux=24.0060 z=5.3060 19.4k tok/s
|
| 11 |
+
[step 40] tok=2.6M lm=7.4688 aux=22.5866 z=5.2494 19.6k tok/s
|
| 12 |
+
[step 60] tok=3.9M lm=7.1639 aux=23.0941 z=5.1882 19.7k tok/s
|
| 13 |
+
[step 80] tok=5.2M lm=6.9098 aux=22.2838 z=5.2068 19.7k tok/s
|
| 14 |
+
[step 100] tok=6.6M lm=6.8816 aux=23.6393 z=5.0969 19.7k tok/s
|
| 15 |
+
[step 120] tok=7.9M lm=6.6161 aux=23.8530 z=4.9800 19.7k tok/s
|
| 16 |
+
[step 140] tok=9.2M lm=6.5786 aux=23.5897 z=4.9440 19.8k tok/s
|
| 17 |
+
[step 160] tok=10.5M lm=6.4141 aux=23.9364 z=4.8870 19.8k tok/s
|
| 18 |
+
[step 180] tok=11.8M lm=6.3783 aux=24.5971 z=4.7448 19.8k tok/s
|
| 19 |
+
[step 200] tok=13.1M lm=6.2775 aux=24.2590 z=4.6999 19.8k tok/s
|
| 20 |
+
[step 220] tok=14.4M lm=6.1755 aux=23.5246 z=4.6037 19.8k tok/s
|
| 21 |
+
[step 240] tok=15.7M lm=6.0618 aux=23.1875 z=4.5119 19.8k tok/s
|
| 22 |
+
[step 260] tok=17.0M lm=6.0393 aux=23.8986 z=4.4176 19.8k tok/s
|
| 23 |
+
[step 280] tok=18.4M lm=6.0132 aux=23.5291 z=4.3912 19.8k tok/s
|
| 24 |
+
[step 300] tok=19.7M lm=5.8250 aux=23.8413 z=4.2921 19.8k tok/s
|
| 25 |
+
[step 320] tok=21.0M lm=5.7641 aux=23.5449 z=4.2553 19.8k tok/s
|
| 26 |
+
[step 340] tok=22.3M lm=5.7252 aux=23.5341 z=4.2018 19.8k tok/s
|
| 27 |
+
[step 360] tok=23.6M lm=5.6061 aux=23.9836 z=4.1271 19.8k tok/s
|
| 28 |
+
[step 380] tok=24.9M lm=5.7354 aux=24.1844 z=4.1153 19.8k tok/s
|
| 29 |
+
[step 400] tok=26.2M lm=5.6309 aux=24.1909 z=4.0696 19.8k tok/s
|
| 30 |
+
[step 420] tok=27.5M lm=5.6499 aux=23.0504 z=4.1738 19.8k tok/s
|
| 31 |
+
[step 440] tok=28.8M lm=5.5792 aux=24.1606 z=3.9948 19.8k tok/s
|
| 32 |
+
[step 460] tok=30.1M lm=5.4959 aux=23.9023 z=3.9781 19.8k tok/s
|
| 33 |
+
[step 480] tok=31.5M lm=5.4892 aux=23.9734 z=3.9781 19.8k tok/s
|
| 34 |
+
[step 500] tok=32.8M lm=5.4534 aux=23.9790 z=3.9648 19.8k tok/s
|
| 35 |
+
[step 520] tok=34.1M lm=5.4946 aux=24.0937 z=4.0108 19.8k tok/s
|
| 36 |
+
[step 540] tok=35.4M lm=5.4621 aux=23.5834 z=3.9996 19.8k tok/s
|
| 37 |
+
[step 560] tok=36.7M lm=5.4992 aux=23.7114 z=3.9843 19.8k tok/s
|
| 38 |
+
[step 580] tok=38.0M lm=5.3217 aux=23.5179 z=3.9488 19.8k tok/s
|
| 39 |
+
[step 600] tok=39.3M lm=5.4186 aux=23.6374 z=3.9503 19.8k tok/s
|
| 40 |
+
[step 620] tok=40.6M lm=5.3004 aux=23.9012 z=3.9016 19.8k tok/s
|
| 41 |
+
[step 640] tok=41.9M lm=5.1836 aux=23.5702 z=3.8459 19.8k tok/s
|
| 42 |
+
[step 660] tok=43.3M lm=5.2652 aux=23.6198 z=3.9385 19.8k tok/s
|
| 43 |
+
[step 680] tok=44.6M lm=5.1364 aux=24.2929 z=3.8586 19.8k tok/s
|
| 44 |
+
[step 700] tok=45.9M lm=5.2919 aux=23.7973 z=3.8406 19.8k tok/s
|
| 45 |
+
[step 720] tok=47.2M lm=5.1345 aux=24.3297 z=3.8254 19.8k tok/s
|
| 46 |
+
[step 740] tok=48.5M lm=5.3089 aux=24.0647 z=3.8895 19.8k tok/s
|
| 47 |
+
[step 760] tok=49.8M lm=5.1586 aux=24.0431 z=3.8089 19.8k tok/s
|
| 48 |
+
[eval] sweep_lr3e-5 lr=3e-05 tok=50M adapted_BPB=1.6406
|
| 49 |
+
[step 780] tok=51.1M lm=5.2672 aux=24.0876 z=3.8262 19.7k tok/s
|
| 50 |
+
[step 800] tok=52.4M lm=5.2116 aux=23.3228 z=3.8950 19.7k tok/s
|
| 51 |
+
[step 820] tok=53.7M lm=5.1895 aux=23.8672 z=3.8073 19.7k tok/s
|
| 52 |
+
[step 840] tok=55.1M lm=5.0964 aux=23.8042 z=3.8138 19.7k tok/s
|
| 53 |
+
[step 860] tok=56.4M lm=5.1935 aux=23.6581 z=3.8213 19.7k tok/s
|
| 54 |
+
[step 880] tok=57.7M lm=5.1102 aux=24.0364 z=3.7545 19.7k tok/s
|
| 55 |
+
[step 900] tok=59.0M lm=5.1331 aux=24.1238 z=3.7526 19.7k tok/s
|
| 56 |
+
[step 920] tok=60.3M lm=5.0076 aux=24.2830 z=3.7942 19.7k tok/s
|
| 57 |
+
[step 940] tok=61.6M lm=5.0394 aux=23.9340 z=3.7432 19.7k tok/s
|
| 58 |
+
[step 960] tok=62.9M lm=5.0328 aux=23.6537 z=3.8228 19.7k tok/s
|
| 59 |
+
[step 980] tok=64.2M lm=5.1620 aux=23.5845 z=3.7524 19.7k tok/s
|
| 60 |
+
[step 1000] tok=65.5M lm=4.9668 aux=23.7523 z=3.7606 19.7k tok/s
|
| 61 |
+
[step 1020] tok=66.8M lm=4.9886 aux=23.5917 z=3.8033 19.7k tok/s
|
| 62 |
+
[step 1040] tok=68.2M lm=4.9184 aux=23.6831 z=3.8067 19.7k tok/s
|
| 63 |
+
[step 1060] tok=69.5M lm=5.0309 aux=24.1709 z=3.7891 19.7k tok/s
|
| 64 |
+
[step 1080] tok=70.8M lm=5.0182 aux=23.7583 z=3.7390 19.7k tok/s
|
| 65 |
+
[step 1100] tok=72.1M lm=4.9554 aux=23.7651 z=3.8027 19.7k tok/s
|
| 66 |
+
[step 1120] tok=73.4M lm=5.1101 aux=23.9705 z=3.6773 19.7k tok/s
|
| 67 |
+
[step 1140] tok=74.7M lm=5.0445 aux=24.2486 z=3.7105 19.7k tok/s
|
| 68 |
+
[step 1160] tok=76.0M lm=4.9700 aux=24.0562 z=3.7174 19.7k tok/s
|
| 69 |
+
[step 1180] tok=77.3M lm=4.9558 aux=24.0377 z=3.6867 19.7k tok/s
|
| 70 |
+
[step 1200] tok=78.6M lm=4.8866 aux=23.8071 z=3.7360 19.7k tok/s
|
| 71 |
+
[step 1220] tok=80.0M lm=4.9140 aux=24.6021 z=3.6331 19.8k tok/s
|
| 72 |
+
[step 1240] tok=81.3M lm=5.0415 aux=24.0703 z=3.7370 19.8k tok/s
|
| 73 |
+
[step 1260] tok=82.6M lm=4.9414 aux=24.1093 z=3.6448 19.8k tok/s
|
| 74 |
+
[step 1280] tok=83.9M lm=4.9373 aux=23.5242 z=3.7013 19.8k tok/s
|
| 75 |
+
[step 1300] tok=85.2M lm=4.8409 aux=23.9295 z=3.6295 19.8k tok/s
|
| 76 |
+
[step 1320] tok=86.5M lm=4.8452 aux=24.2080 z=3.6916 19.8k tok/s
|
| 77 |
+
[step 1340] tok=87.8M lm=4.7938 aux=24.2042 z=3.6420 19.8k tok/s
|
| 78 |
+
[step 1360] tok=89.1M lm=4.9717 aux=24.0715 z=3.6224 19.8k tok/s
|
| 79 |
+
[step 1380] tok=90.4M lm=4.7561 aux=23.9782 z=3.6897 19.8k tok/s
|
| 80 |
+
[step 1400] tok=91.8M lm=4.7631 aux=23.7805 z=3.6797 19.8k tok/s
|
| 81 |
+
[step 1420] tok=93.1M lm=4.9115 aux=24.0962 z=3.6183 19.8k tok/s
|
| 82 |
+
[step 1440] tok=94.4M lm=4.7621 aux=23.5366 z=3.6059 19.8k tok/s
|
| 83 |
+
[step 1460] tok=95.7M lm=4.9019 aux=24.1316 z=3.6155 19.8k tok/s
|
| 84 |
+
[step 1480] tok=97.0M lm=4.8833 aux=23.7381 z=3.6576 19.8k tok/s
|
| 85 |
+
[step 1500] tok=98.3M lm=4.8186 aux=24.3369 z=3.5914 19.8k tok/s
|
| 86 |
+
[step 1520] tok=99.6M lm=4.7448 aux=23.6144 z=3.5994 19.8k tok/s
|
| 87 |
+
[eval] sweep_lr3e-5 lr=3e-05 tok=100M adapted_BPB=1.5211
|
| 88 |
+
[step 1540] tok=100.9M lm=4.8095 aux=24.0122 z=3.6136 19.7k tok/s
|
| 89 |
+
[step 1560] tok=102.2M lm=4.8705 aux=24.2564 z=3.5881 19.7k tok/s
|
| 90 |
+
[step 1580] tok=103.5M lm=4.8539 aux=23.7418 z=3.6204 19.7k tok/s
|
| 91 |
+
[step 1600] tok=104.9M lm=4.7389 aux=23.8723 z=3.5683 19.7k tok/s
|
| 92 |
+
[step 1620] tok=106.2M lm=4.7380 aux=23.5365 z=3.6213 19.7k tok/s
|
| 93 |
+
[step 1640] tok=107.5M lm=4.8432 aux=23.3159 z=3.6573 19.7k tok/s
|
| 94 |
+
[step 1660] tok=108.8M lm=4.8279 aux=23.5420 z=3.6130 19.7k tok/s
|
| 95 |
+
[step 1680] tok=110.1M lm=4.7192 aux=23.7558 z=3.5785 19.7k tok/s
|
| 96 |
+
[step 1700] tok=111.4M lm=4.7274 aux=23.7471 z=3.5722 19.7k tok/s
|
| 97 |
+
[step 1720] tok=112.7M lm=4.7686 aux=23.8433 z=3.5781 19.7k tok/s
|
| 98 |
+
[step 1740] tok=114.0M lm=4.6602 aux=23.2951 z=3.5300 19.7k tok/s
|
| 99 |
+
[step 1760] tok=115.3M lm=4.6977 aux=23.8038 z=3.4818 19.7k tok/s
|
| 100 |
+
[step 1780] tok=116.7M lm=4.6205 aux=23.1623 z=3.6358 19.7k tok/s
|
| 101 |
+
[step 1800] tok=118.0M lm=4.8158 aux=23.3768 z=3.4784 19.7k tok/s
|
| 102 |
+
[step 1820] tok=119.3M lm=4.6642 aux=23.7871 z=3.4623 19.7k tok/s
|
| 103 |
+
[step 1840] tok=120.6M lm=4.5870 aux=23.3848 z=3.4949 19.7k tok/s
|
| 104 |
+
[step 1860] tok=121.9M lm=4.7054 aux=23.5615 z=3.4558 19.7k tok/s
|
| 105 |
+
[step 1880] tok=123.2M lm=4.7116 aux=23.6199 z=3.4448 19.7k tok/s
|
| 106 |
+
[step 1900] tok=124.5M lm=4.6597 aux=24.1684 z=3.4317 19.7k tok/s
|
| 107 |
+
[step 1920] tok=125.8M lm=4.6905 aux=23.0209 z=3.5537 19.7k tok/s
|
| 108 |
+
[step 1940] tok=127.1M lm=4.6882 aux=23.7775 z=3.4771 19.7k tok/s
|
| 109 |
+
[step 1960] tok=128.5M lm=4.5773 aux=23.4738 z=3.4122 19.7k tok/s
|
| 110 |
+
[step 1980] tok=129.8M lm=4.5794 aux=23.3787 z=3.4224 19.7k tok/s
|
| 111 |
+
[step 2000] tok=131.1M lm=4.6031 aux=23.3394 z=3.3930 19.7k tok/s
|
| 112 |
+
[step 2020] tok=132.4M lm=4.5851 aux=23.3520 z=3.4082 19.7k tok/s
|
| 113 |
+
[step 2040] tok=133.7M lm=4.5180 aux=22.9633 z=3.4353 19.7k tok/s
|
| 114 |
+
[step 2060] tok=135.0M lm=4.5546 aux=23.1551 z=3.3887 19.7k tok/s
|
| 115 |
+
[step 2080] tok=136.3M lm=4.6284 aux=23.2592 z=3.4179 19.7k tok/s
|
| 116 |
+
[step 2100] tok=137.6M lm=4.5905 aux=23.1306 z=3.3588 19.7k tok/s
|
| 117 |
+
[step 2120] tok=138.9M lm=4.7294 aux=23.2788 z=3.3897 19.7k tok/s
|
| 118 |
+
[step 2140] tok=140.2M lm=4.5453 aux=23.1502 z=3.4036 19.7k tok/s
|
| 119 |
+
[step 2160] tok=141.6M lm=4.6536 aux=22.8934 z=3.3599 19.7k tok/s
|
| 120 |
+
[step 2180] tok=142.9M lm=4.6222 aux=22.8403 z=3.3969 19.7k tok/s
|
| 121 |
+
[step 2200] tok=144.2M lm=4.6180 aux=23.0656 z=3.3566 19.8k tok/s
|
| 122 |
+
[step 2220] tok=145.5M lm=4.6489 aux=23.1766 z=3.3479 19.8k tok/s
|
| 123 |
+
[step 2240] tok=146.8M lm=4.5684 aux=22.9291 z=3.3510 19.8k tok/s
|
| 124 |
+
[step 2260] tok=148.1M lm=4.5008 aux=22.7761 z=3.3688 19.8k tok/s
|
| 125 |
+
[step 2280] tok=149.4M lm=4.6034 aux=23.0790 z=3.3946 19.8k tok/s
|
| 126 |
+
[eval] sweep_lr3e-5 lr=3e-05 tok=150M adapted_BPB=1.4340
|
| 127 |
+
[step 2300] tok=150.7M lm=4.5967 aux=22.8089 z=3.3551 19.7k tok/s
|
| 128 |
+
[step 2320] tok=152.0M lm=4.5347 aux=23.1857 z=3.3204 19.7k tok/s
|
| 129 |
+
[step 2340] tok=153.4M lm=4.6008 aux=23.0620 z=3.3140 19.7k tok/s
|
| 130 |
+
[step 2360] tok=154.7M lm=4.5150 aux=22.6630 z=3.2984 19.7k tok/s
|
| 131 |
+
[step 2380] tok=156.0M lm=4.6591 aux=22.7077 z=3.3554 19.7k tok/s
|
| 132 |
+
[step 2400] tok=157.3M lm=4.5642 aux=22.9867 z=3.3437 19.7k tok/s
|
| 133 |
+
[step 2420] tok=158.6M lm=4.5827 aux=23.1791 z=3.2780 19.7k tok/s
|
| 134 |
+
[step 2440] tok=159.9M lm=4.4759 aux=22.5628 z=3.3475 19.7k tok/s
|
| 135 |
+
[step 2460] tok=161.2M lm=4.5639 aux=22.7338 z=3.3541 19.7k tok/s
|
| 136 |
+
[step 2480] tok=162.5M lm=4.3806 aux=22.5732 z=3.3291 19.7k tok/s
|
| 137 |
+
[step 2500] tok=163.8M lm=4.4748 aux=22.8350 z=3.3037 19.7k tok/s
|
| 138 |
+
[step 2520] tok=165.2M lm=4.4736 aux=22.6692 z=3.2559 19.7k tok/s
|
| 139 |
+
[step 2540] tok=166.5M lm=4.6126 aux=22.4374 z=3.3639 19.7k tok/s
|
| 140 |
+
[step 2560] tok=167.8M lm=4.4816 aux=22.8600 z=3.3362 19.7k tok/s
|
| 141 |
+
[step 2580] tok=169.1M lm=4.5337 aux=22.8693 z=3.2712 19.7k tok/s
|
| 142 |
+
[step 2600] tok=170.4M lm=4.3143 aux=22.4241 z=3.3436 19.7k tok/s
|
| 143 |
+
[step 2620] tok=171.7M lm=4.5262 aux=22.7638 z=3.3354 19.7k tok/s
|
| 144 |
+
[step 2640] tok=173.0M lm=4.4228 aux=22.5412 z=3.3418 19.7k tok/s
|
| 145 |
+
[step 2660] tok=174.3M lm=4.6698 aux=22.4901 z=3.3638 19.7k tok/s
|
| 146 |
+
[step 2680] tok=175.6M lm=4.5352 aux=22.4670 z=3.2637 19.7k tok/s
|
| 147 |
+
[step 2700] tok=176.9M lm=4.4436 aux=22.8082 z=3.2543 19.7k tok/s
|
| 148 |
+
[step 2720] tok=178.3M lm=4.5023 aux=22.4529 z=3.2867 19.7k tok/s
|
| 149 |
+
[step 2740] tok=179.6M lm=4.4542 aux=22.6227 z=3.2144 19.7k tok/s
|
| 150 |
+
[step 2760] tok=180.9M lm=4.5279 aux=22.6763 z=3.2495 19.7k tok/s
|
| 151 |
+
[step 2780] tok=182.2M lm=4.4469 aux=22.2748 z=3.2685 19.7k tok/s
|
| 152 |
+
[step 2800] tok=183.5M lm=4.4513 aux=22.3703 z=3.2213 19.7k tok/s
|
| 153 |
+
[step 2820] tok=184.8M lm=4.4742 aux=22.8690 z=3.1545 19.7k tok/s
|
| 154 |
+
[step 2840] tok=186.1M lm=4.3275 aux=22.2309 z=3.2396 19.7k tok/s
|
| 155 |
+
[step 2860] tok=187.4M lm=4.4496 aux=22.4419 z=3.2101 19.7k tok/s
|
| 156 |
+
[step 2880] tok=188.7M lm=4.5590 aux=22.3057 z=3.2146 19.7k tok/s
|
| 157 |
+
[step 2900] tok=190.1M lm=4.5004 aux=22.4933 z=3.2274 19.7k tok/s
|
| 158 |
+
[step 2920] tok=191.4M lm=4.4027 aux=22.2764 z=3.1858 19.7k tok/s
|
| 159 |
+
[step 2940] tok=192.7M lm=4.5542 aux=22.3621 z=3.1986 19.7k tok/s
|
| 160 |
+
[step 2960] tok=194.0M lm=4.2990 aux=22.2780 z=3.2168 19.7k tok/s
|
| 161 |
+
[step 2980] tok=195.3M lm=4.4180 aux=22.3155 z=3.1651 19.7k tok/s
|
| 162 |
+
[step 3000] tok=196.6M lm=4.3470 aux=22.1451 z=3.2768 19.7k tok/s
|
| 163 |
+
[step 3020] tok=197.9M lm=4.3533 aux=22.0714 z=3.1771 19.7k tok/s
|
| 164 |
+
[step 3040] tok=199.2M lm=4.4751 aux=22.0670 z=3.2227 19.7k tok/s
|
| 165 |
+
[eval] sweep_lr3e-5 lr=3e-05 tok=200M adapted_BPB=1.3904
|
| 166 |
+
[step 3060] tok=200.5M lm=4.4296 aux=22.4530 z=3.1557 19.7k tok/s
|
| 167 |
+
[step 3080] tok=201.9M lm=4.4167 aux=22.1853 z=3.1821 19.7k tok/s
|
| 168 |
+
[step 3100] tok=203.2M lm=4.4456 aux=22.2744 z=3.1613 19.7k tok/s
|
| 169 |
+
[step 3120] tok=204.5M lm=4.3498 aux=22.0402 z=3.1905 19.7k tok/s
|
| 170 |
+
[step 3140] tok=205.8M lm=4.3870 aux=22.5634 z=3.1201 19.7k tok/s
|
| 171 |
+
[step 3160] tok=207.1M lm=4.3571 aux=22.0412 z=3.1727 19.7k tok/s
|
| 172 |
+
[step 3180] tok=208.4M lm=4.3482 aux=22.3070 z=3.1866 19.7k tok/s
|
| 173 |
+
[step 3200] tok=209.7M lm=4.3503 aux=22.1161 z=3.1410 19.7k tok/s
|
| 174 |
+
[step 3220] tok=211.0M lm=4.2918 aux=21.9968 z=3.1685 19.7k tok/s
|
| 175 |
+
[step 3240] tok=212.3M lm=4.3585 aux=22.0265 z=3.1067 19.7k tok/s
|
| 176 |
+
[step 3260] tok=213.6M lm=4.4253 aux=22.2528 z=3.1349 19.7k tok/s
|
| 177 |
+
[step 3280] tok=215.0M lm=4.3851 aux=21.9208 z=3.1440 19.7k tok/s
|
| 178 |
+
[step 3300] tok=216.3M lm=4.4227 aux=22.0914 z=3.1652 19.7k tok/s
|
| 179 |
+
[step 3320] tok=217.6M lm=4.4191 aux=21.9152 z=3.1404 19.7k tok/s
|
| 180 |
+
[step 3340] tok=218.9M lm=4.3203 aux=22.2830 z=3.1348 19.7k tok/s
|
| 181 |
+
[step 3360] tok=220.2M lm=4.3532 aux=22.0526 z=3.1061 19.7k tok/s
|
| 182 |
+
[step 3380] tok=221.5M lm=4.4842 aux=21.9031 z=3.1523 19.7k tok/s
|
| 183 |
+
[step 3400] tok=222.8M lm=4.3977 aux=21.8634 z=3.0928 19.7k tok/s
|
| 184 |
+
[step 3420] tok=224.1M lm=4.4003 aux=21.8480 z=3.0958 19.7k tok/s
|
| 185 |
+
[step 3440] tok=225.4M lm=4.3662 aux=21.8988 z=3.0747 19.7k tok/s
|
| 186 |
+
[step 3460] tok=226.8M lm=4.3984 aux=22.0348 z=3.0983 19.7k tok/s
|
| 187 |
+
[step 3480] tok=228.1M lm=4.3448 aux=21.9246 z=3.0979 19.7k tok/s
|
| 188 |
+
[step 3500] tok=229.4M lm=4.3289 aux=21.9296 z=3.0817 19.7k tok/s
|
| 189 |
+
[step 3520] tok=230.7M lm=4.3681 aux=21.9144 z=3.0610 19.7k tok/s
|
| 190 |
+
[step 3540] tok=232.0M lm=4.3178 aux=21.8284 z=3.0716 19.7k tok/s
|
| 191 |
+
[step 3560] tok=233.3M lm=4.3226 aux=21.7442 z=3.0546 19.7k tok/s
|
| 192 |
+
[step 3580] tok=234.6M lm=4.4597 aux=21.5494 z=3.0447 19.7k tok/s
|
| 193 |
+
[step 3600] tok=235.9M lm=4.3938 aux=22.0100 z=3.0486 19.7k tok/s
|
| 194 |
+
[step 3620] tok=237.2M lm=4.5520 aux=21.6429 z=3.0173 19.7k tok/s
|
| 195 |
+
[step 3640] tok=238.6M lm=4.3063 aux=21.7061 z=3.0218 19.7k tok/s
|
| 196 |
+
[step 3660] tok=239.9M lm=4.4464 aux=21.9084 z=3.0477 19.7k tok/s
|
| 197 |
+
[step 3680] tok=241.2M lm=4.4161 aux=21.6854 z=3.0829 19.7k tok/s
|
| 198 |
+
[step 3700] tok=242.5M lm=4.2692 aux=21.6478 z=2.9947 19.7k tok/s
|
| 199 |
+
[step 3720] tok=243.8M lm=4.3305 aux=21.5907 z=3.0495 19.7k tok/s
|
| 200 |
+
[step 3740] tok=245.1M lm=4.3820 aux=21.6229 z=3.0303 19.7k tok/s
|
| 201 |
+
[step 3760] tok=246.4M lm=4.4255 aux=21.5070 z=3.0026 19.7k tok/s
|
| 202 |
+
[step 3780] tok=247.7M lm=4.3159 aux=21.4173 z=3.0155 19.7k tok/s
|
| 203 |
+
[step 3800] tok=249.0M lm=4.4128 aux=21.5892 z=3.0247 19.7k tok/s
|
| 204 |
+
[eval] sweep_lr3e-5 lr=3e-05 tok=250M adapted_BPB=1.3618
|
| 205 |
+
[DONE] tag=sweep_lr3e-5 lr=3e-05 final adapted BPB=1.3618
|
olmoe_adapt/oseries/meta.json
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_seq": 488,
|
| 3 |
+
"n_tokens": 1998848,
|
| 4 |
+
"L": 16,
|
| 5 |
+
"E": 64,
|
| 6 |
+
"topk": 24,
|
| 7 |
+
"seq": 4096,
|
| 8 |
+
"corpus": "stage1 audited slice (dolmino dclm/024*)",
|
| 9 |
+
"routing": "free_top8_no_residency",
|
| 10 |
+
"coverage_top8": 6.490237225308922,
|
| 11 |
+
"coverage_top24": 10.92119211237114,
|
| 12 |
+
"rank_mass_top32": [
|
| 13 |
+
1.475485,
|
| 14 |
+
1.109484,
|
| 15 |
+
0.924407,
|
| 16 |
+
0.787692,
|
| 17 |
+
0.675511,
|
| 18 |
+
0.579262,
|
| 19 |
+
0.499922,
|
| 20 |
+
0.438474,
|
| 21 |
+
0.394142,
|
| 22 |
+
0.361943,
|
| 23 |
+
0.337592,
|
| 24 |
+
0.318414,
|
| 25 |
+
0.302757,
|
| 26 |
+
0.289587,
|
| 27 |
+
0.278249,
|
| 28 |
+
0.268283,
|
| 29 |
+
0.259383,
|
| 30 |
+
0.251316,
|
| 31 |
+
0.243931,
|
| 32 |
+
0.237105,
|
| 33 |
+
0.230733,
|
| 34 |
+
0.224748,
|
| 35 |
+
0.219082,
|
| 36 |
+
0.213689,
|
| 37 |
+
0.208534,
|
| 38 |
+
0.203589,
|
| 39 |
+
0.198817,
|
| 40 |
+
0.1942,
|
| 41 |
+
0.189717,
|
| 42 |
+
0.185356,
|
| 43 |
+
0.1811,
|
| 44 |
+
0.176932
|
| 45 |
+
],
|
| 46 |
+
"note": "reward = plain base softmax mass over 64 experts, never renormalized after top-24 truncation"
|
| 47 |
+
}
|
olmoe_adapt/oseries_ce/meta.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_seq": 24,
|
| 3 |
+
"n_tokens": 98304,
|
| 4 |
+
"L": 16,
|
| 5 |
+
"E": 64,
|
| 6 |
+
"topk": 24,
|
| 7 |
+
"seq": 4096,
|
| 8 |
+
"model": "merged_ce_model (CE-adapted)",
|
| 9 |
+
"routing": "free_no_residency",
|
| 10 |
+
"coverage_top8": 5.095009203922624,
|
| 11 |
+
"coverage_top24": 9.745589475651895
|
| 12 |
+
}
|