diff --git a/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/finalize.json b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..12335b82ecb03a26b57ad2d8d2de6a36cda31a93 --- /dev/null +++ b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/finalize.json @@ -0,0 +1,29 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101", + "w_bits": null, + "a_bits": null, + "qkv_bits": null, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 4.0493755096973 + }, + "losses_eval_model": { + "val_dolma_cc": 4.0493755096973 + }, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.7319828271865845, + "loss_eval_model": 4.0493755096973, + "checks": { + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/perturb_all.json b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..ecd916ad274738e01fd120bbbb0d0a5f8d9ae4aa --- /dev/null +++ b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03457271680235863, + "blocks.0.attention.w_out.weight": 0.03741328418254852, + "blocks.0.attention.w_q.weight": 0.036707207560539246, + "blocks.0.attention.w_v.weight": 0.03474367409944534, + "blocks.0.attention_norm.weight": 0.7402628660202026, + "blocks.0.feed_forward.w1.weight": 0.03897462412714958, + "blocks.0.feed_forward.w2.weight": 0.03813355043530464, + "blocks.0.feed_forward.w3.weight": 0.03718235716223717, + "blocks.0.feed_forward_norm.weight": 0.7393606305122375, + "blocks.1.attention.w_k.weight": 0.03156030550599098, + "blocks.1.attention.w_out.weight": 0.0387837290763855, + "blocks.1.attention.w_q.weight": 0.03309893608093262, + "blocks.1.attention.w_v.weight": 0.04152149334549904, + "blocks.1.attention_norm.weight": 0.6954712271690369, + "blocks.1.feed_forward.w1.weight": 0.0488145537674427, + "blocks.1.feed_forward.w2.weight": 0.04625376686453819, + "blocks.1.feed_forward.w3.weight": 0.04682345688343048, + "blocks.1.feed_forward_norm.weight": 0.7512289881706238, + "blocks.2.attention.w_k.weight": 0.03609790280461311, + "blocks.2.attention.w_out.weight": 0.0422712042927742, + "blocks.2.attention.w_q.weight": 0.0382913276553154, + "blocks.2.attention.w_v.weight": 0.04444428160786629, + "blocks.2.attention_norm.weight": 0.6940568089485168, + "blocks.2.feed_forward.w1.weight": 0.051162414252758026, + "blocks.2.feed_forward.w2.weight": 0.04817779362201691, + "blocks.2.feed_forward.w3.weight": 0.04966016858816147, + "blocks.2.feed_forward_norm.weight": 0.7518685460090637, + "blocks.3.attention.w_k.weight": 0.03685362637042999, + "blocks.3.attention.w_out.weight": 0.042797308415174484, + "blocks.3.attention.w_q.weight": 0.040169961750507355, + "blocks.3.attention.w_v.weight": 0.041635192930698395, + "blocks.3.attention_norm.weight": 0.7284940481185913, + "blocks.3.feed_forward.w1.weight": 0.050651222467422485, + "blocks.3.feed_forward.w2.weight": 0.04761878401041031, + "blocks.3.feed_forward.w3.weight": 0.04890414699912071, + "blocks.3.feed_forward_norm.weight": 0.7569454908370972, + "blocks.4.attention.w_k.weight": 0.040658239275217056, + "blocks.4.attention.w_out.weight": 0.04410743713378906, + "blocks.4.attention.w_q.weight": 0.04254842922091484, + "blocks.4.attention.w_v.weight": 0.04303383082151413, + "blocks.4.attention_norm.weight": 0.703934371471405, + "blocks.4.feed_forward.w1.weight": 0.050434891134500504, + "blocks.4.feed_forward.w2.weight": 0.04856543242931366, + "blocks.4.feed_forward.w3.weight": 0.048799481242895126, + "blocks.4.feed_forward_norm.weight": 0.7514355182647705, + "blocks.5.attention.w_k.weight": 0.03925139456987381, + "blocks.5.attention.w_out.weight": 0.04608171060681343, + "blocks.5.attention.w_q.weight": 0.04282260686159134, + "blocks.5.attention.w_v.weight": 0.04206324368715286, + "blocks.5.attention_norm.weight": 0.7227079272270203, + "blocks.5.feed_forward.w1.weight": 0.05082806572318077, + "blocks.5.feed_forward.w2.weight": 0.05065442994236946, + "blocks.5.feed_forward.w3.weight": 0.05076573044061661, + "blocks.5.feed_forward_norm.weight": 0.7479311227798462, + "blocks.6.attention.w_k.weight": 0.039561763405799866, + "blocks.6.attention.w_out.weight": 0.05980084836483002, + "blocks.6.attention.w_q.weight": 0.04352796822786331, + "blocks.6.attention.w_v.weight": 0.04301363602280617, + "blocks.6.attention_norm.weight": 0.7296965718269348, + "blocks.6.feed_forward.w1.weight": 0.051500093191862106, + "blocks.6.feed_forward.w2.weight": 0.054001640528440475, + "blocks.6.feed_forward.w3.weight": 0.051999837160110474, + "blocks.6.feed_forward_norm.weight": 0.7462171912193298, + "blocks.7.attention.w_k.weight": 0.042883116751909256, + "blocks.7.attention.w_out.weight": 0.06064591929316521, + "blocks.7.attention.w_q.weight": 0.04550102725625038, + "blocks.7.attention.w_v.weight": 0.043729517608881, + "blocks.7.attention_norm.weight": 0.7210500836372375, + "blocks.7.feed_forward.w1.weight": 0.05071728676557541, + "blocks.7.feed_forward.w2.weight": 0.05820971727371216, + "blocks.7.feed_forward.w3.weight": 0.052353113889694214, + "blocks.7.feed_forward_norm.weight": 0.741530179977417, + "blocks.8.attention.w_k.weight": 0.040963493287563324, + "blocks.8.attention.w_out.weight": 0.08356135338544846, + "blocks.8.attention.w_q.weight": 0.04410872235894203, + "blocks.8.attention.w_v.weight": 0.04232929274439812, + "blocks.8.attention_norm.weight": 0.7226449251174927, + "blocks.8.feed_forward.w1.weight": 0.049077108502388, + "blocks.8.feed_forward.w2.weight": 0.06283104419708252, + "blocks.8.feed_forward.w3.weight": 0.05128471553325653, + "blocks.8.feed_forward_norm.weight": 0.7345075011253357, + "embeddings.weight": 0.05591508001089096, + "lm_head.norm.weight": 1.3878247737884521, + "lm_head.w_out.weight": 0.06863079965114594 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 4.0493755096973 + }, + "base_loss": 4.0493755096973, + "loss": { + "0.005": [ + 4.0495698425518905, + 4.049669285950546, + 4.049879588903228 + ], + "0.01": [ + 4.050270418126432, + 4.050327205479154, + 4.0508540152754575 + ], + "0.02": [ + 4.0527773479440095, + 4.053184565883751, + 4.05434469844073 + ], + "0.05": [ + 4.070846207219855, + 4.073331396264503, + 4.075627577124697 + ], + "0.1": [ + 4.139764054437487, + 4.152049371249179, + 4.152491287572062 + ], + "0.15": [ + 4.269614547246076, + 4.3033789691601365, + 4.291750445128339 + ], + "0.2": [ + 4.481975479068514, + 4.55160124920126, + 4.511998031100518 + ], + "0.3": [ + 5.234732680119546, + 5.418616669633772, + 5.279652465939638 + ], + "0.4": [ + 6.4647297840623645, + 6.775691639048803, + 6.493282746697875 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 4.0495698425518905, + 4.049669285950546, + 4.049879588903228 + ], + "0.01": [ + 4.050270418126432, + 4.050327205479154, + 4.0508540152754575 + ], + "0.02": [ + 4.0527773479440095, + 4.053184565883751, + 4.05434469844073 + ], + "0.05": [ + 4.070846207219855, + 4.073331396264503, + 4.075627577124697 + ], + "0.1": [ + 4.139764054437487, + 4.152049371249179, + 4.152491287572062 + ], + "0.15": [ + 4.269614547246076, + 4.3033789691601365, + 4.291750445128339 + ], + "0.2": [ + 4.481975479068514, + 4.55160124920126, + 4.511998031100518 + ], + "0.3": [ + 5.234732680119546, + 5.418616669633772, + 5.279652465939638 + ], + "0.4": [ + 6.4647297840623645, + 6.775691639048803, + 6.493282746697875 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00033072943792156195, + "0.01": 0.0011083699297156357, + "0.02": 0.004060027725530269, + "0.05": 0.02389288383905175, + "0.1": 0.09872606138894291, + "0.15": 0.23887247748088392, + "0.2": 0.4658160767594648, + "0.3": 1.2616250955336845, + "0.4": 2.528525880239048 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00033072943792156195, + "0.01": 0.0011083699297156357, + "0.02": 0.004060027725530269, + "0.05": 0.02389288383905175, + "0.1": 0.09872606138894291, + "0.15": 0.23887247748088392, + "0.2": 0.4658160767594648, + "0.3": 1.2616250955336845, + "0.4": 2.528525880239048 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 56.3 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/perturb_blocks.json b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..dd04f3bd32835bb8a566d2a0953f756494e615b4 --- /dev/null +++ b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03457271680235863, + "blocks.0.attention.w_out.weight": 0.03741328418254852, + "blocks.0.attention.w_q.weight": 0.036707207560539246, + "blocks.0.attention.w_v.weight": 0.03474367409944534, + "blocks.0.feed_forward.w1.weight": 0.03897462412714958, + "blocks.0.feed_forward.w2.weight": 0.03813355043530464, + "blocks.0.feed_forward.w3.weight": 0.03718235716223717, + "blocks.1.attention.w_k.weight": 0.03156030550599098, + "blocks.1.attention.w_out.weight": 0.0387837290763855, + "blocks.1.attention.w_q.weight": 0.03309893608093262, + "blocks.1.attention.w_v.weight": 0.04152149334549904, + "blocks.1.feed_forward.w1.weight": 0.0488145537674427, + "blocks.1.feed_forward.w2.weight": 0.04625376686453819, + "blocks.1.feed_forward.w3.weight": 0.04682345688343048, + "blocks.2.attention.w_k.weight": 0.03609790280461311, + "blocks.2.attention.w_out.weight": 0.0422712042927742, + "blocks.2.attention.w_q.weight": 0.0382913276553154, + "blocks.2.attention.w_v.weight": 0.04444428160786629, + "blocks.2.feed_forward.w1.weight": 0.051162414252758026, + "blocks.2.feed_forward.w2.weight": 0.04817779362201691, + "blocks.2.feed_forward.w3.weight": 0.04966016858816147, + "blocks.3.attention.w_k.weight": 0.03685362637042999, + "blocks.3.attention.w_out.weight": 0.042797308415174484, + "blocks.3.attention.w_q.weight": 0.040169961750507355, + "blocks.3.attention.w_v.weight": 0.041635192930698395, + "blocks.3.feed_forward.w1.weight": 0.050651222467422485, + "blocks.3.feed_forward.w2.weight": 0.04761878401041031, + "blocks.3.feed_forward.w3.weight": 0.04890414699912071, + "blocks.4.attention.w_k.weight": 0.040658239275217056, + "blocks.4.attention.w_out.weight": 0.04410743713378906, + "blocks.4.attention.w_q.weight": 0.04254842922091484, + "blocks.4.attention.w_v.weight": 0.04303383082151413, + "blocks.4.feed_forward.w1.weight": 0.050434891134500504, + "blocks.4.feed_forward.w2.weight": 0.04856543242931366, + "blocks.4.feed_forward.w3.weight": 0.048799481242895126, + "blocks.5.attention.w_k.weight": 0.03925139456987381, + "blocks.5.attention.w_out.weight": 0.04608171060681343, + "blocks.5.attention.w_q.weight": 0.04282260686159134, + "blocks.5.attention.w_v.weight": 0.04206324368715286, + "blocks.5.feed_forward.w1.weight": 0.05082806572318077, + "blocks.5.feed_forward.w2.weight": 0.05065442994236946, + "blocks.5.feed_forward.w3.weight": 0.05076573044061661, + "blocks.6.attention.w_k.weight": 0.039561763405799866, + "blocks.6.attention.w_out.weight": 0.05980084836483002, + "blocks.6.attention.w_q.weight": 0.04352796822786331, + "blocks.6.attention.w_v.weight": 0.04301363602280617, + "blocks.6.feed_forward.w1.weight": 0.051500093191862106, + "blocks.6.feed_forward.w2.weight": 0.054001640528440475, + "blocks.6.feed_forward.w3.weight": 0.051999837160110474, + "blocks.7.attention.w_k.weight": 0.042883116751909256, + "blocks.7.attention.w_out.weight": 0.06064591929316521, + "blocks.7.attention.w_q.weight": 0.04550102725625038, + "blocks.7.attention.w_v.weight": 0.043729517608881, + "blocks.7.feed_forward.w1.weight": 0.05071728676557541, + "blocks.7.feed_forward.w2.weight": 0.05820971727371216, + "blocks.7.feed_forward.w3.weight": 0.052353113889694214, + "blocks.8.attention.w_k.weight": 0.040963493287563324, + "blocks.8.attention.w_out.weight": 0.08356135338544846, + "blocks.8.attention.w_q.weight": 0.04410872235894203, + "blocks.8.attention.w_v.weight": 0.04232929274439812, + "blocks.8.feed_forward.w1.weight": 0.049077108502388, + "blocks.8.feed_forward.w2.weight": 0.06283104419708252, + "blocks.8.feed_forward.w3.weight": 0.05128471553325653 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 4.0493755096973 + }, + "base_loss": 4.0493755096973, + "loss": { + "0.005": [ + 4.049424695408524, + 4.04958523753866, + 4.049485700244279 + ], + "0.01": [ + 4.049717063225876, + 4.049925789512824, + 4.049838928141476 + ], + "0.02": [ + 4.050981975588614, + 4.051243275657571, + 4.0511587695040205 + ], + "0.05": [ + 4.060081461271643, + 4.0611475455219, + 4.060970325163404 + ], + "0.1": [ + 4.095681630703352, + 4.097457234096067, + 4.097584016800324 + ], + "0.15": [ + 4.163234809173837, + 4.163216593402962, + 4.165194847246677 + ], + "0.2": [ + 4.273525844141357, + 4.267101013429092, + 4.272839509418618 + ], + "0.3": [ + 4.675111350941662, + 4.636568509013489, + 4.653812177436882 + ], + "0.4": [ + 5.35069790032881, + 5.268898275435976, + 5.295578284372346 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 4.049424695408524, + 4.04958523753866, + 4.049485700244279 + ], + "0.01": [ + 4.049717063225876, + 4.049925789512824, + 4.049838928141476 + ], + "0.02": [ + 4.050981975588614, + 4.051243275657571, + 4.0511587695040205 + ], + "0.05": [ + 4.060081461271643, + 4.0611475455219, + 4.060970325163404 + ], + "0.1": [ + 4.095681630703352, + 4.097457234096067, + 4.097584016800324 + ], + "0.15": [ + 4.163234809173837, + 4.163216593402962, + 4.165194847246677 + ], + "0.2": [ + 4.273525844141357, + 4.267101013429092, + 4.272839509418618 + ], + "0.3": [ + 4.675111350941662, + 4.636568509013489, + 4.653812177436882 + ], + "0.4": [ + 5.35069790032881, + 5.268898275435976, + 5.295578284372346 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00012303469985397442, + "0.01": 0.0004517505960919266, + "0.02": 0.0017524972194351562, + "0.05": 0.011357600955015812, + "0.1": 0.04753211750261421, + "0.15": 0.11450657357719152, + "0.2": 0.22177994596572148, + "0.3": 0.6057885027667105, + "0.4": 1.2556826436817445 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00012303469985397442, + "0.01": 0.0004517505960919266, + "0.02": 0.0017524972194351562, + "0.05": 0.011357600955015812, + "0.1": 0.04753211750261421, + "0.15": 0.11450657357719152, + "0.2": 0.22177994596572148, + "0.3": 0.6057885027667105, + "0.4": 1.2556826436817445 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 50.2 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/ptq.json b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..041fd860c975a2ad85d96c08e4fde2f4ad43ae47 --- /dev/null +++ b/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101/ptq.json @@ -0,0 +1,48 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-adamw-lr2.4e-03-s101", + "w_bits": null, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 4.0493755096973 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.82802165992871 + }, + "4": { + "val_dolma_cc": 4.154064397006654 + }, + "5": { + "val_dolma_cc": 4.072646571698026 + }, + "6": { + "val_dolma_cc": 4.054745805333628 + }, + "8": { + "val_dolma_cc": 4.049746444763523 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 0.7786461502314097 + }, + "4": { + "val_dolma_cc": 0.10468888730935433 + }, + "5": { + "val_dolma_cc": 0.023271062000725706 + }, + "6": { + "val_dolma_cc": 0.00537029563632796 + }, + "8": { + "val_dolma_cc": 0.0003709350662228772 + } + }, + "checks": { + "weights_restored": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/bf16_start.json b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..23faa771e486e7c0128a1b79f5e690b24be8be8b --- /dev/null +++ b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": null, + "a": null, + "qkv": null + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.916410182665998, + "heldout_cc_en_tail_4M": 4.494598077772183 + }, + "loss_bf16_master": 3.916410182665998, + "losses_eval_model": { + "val_dolma_cc": 3.916410182665998 + }, + "bf16_minus_quant": 0.0, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-n5-28", + "gpu": "NVIDIA L40S", + "t_s": 18.2 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/finalize.json b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..d6513e960a1e8e284ca7fface03eee15e41276d7 --- /dev/null +++ b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/finalize.json @@ -0,0 +1,29 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "w_bits": null, + "a_bits": null, + "qkv_bits": null, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.916410182665998 + }, + "losses_eval_model": { + "val_dolma_cc": 3.916410182665998 + }, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.5616782903671265, + "loss_eval_model": 3.916410182665998, + "checks": { + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/launch.yaml b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91b30b198ec27738c4b878ba694c04bb45874379 --- /dev/null +++ b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,126 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: null + fq_act_bits: null + fq_qkv_bits: null + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.01 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 232 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1002327 + save_async: false +wandb: + enabled: false +init_seed: 101 +n_tokens: 610009088 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_all.json b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..d2acb81dc9d8a493dc864968b195910672f1f04a --- /dev/null +++ b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.033792030066251755, + "blocks.0.attention.w_out.weight": 0.026513146236538887, + "blocks.0.attention.w_q.weight": 0.033974722027778625, + "blocks.0.attention.w_v.weight": 0.030098984017968178, + "blocks.0.attention_norm.weight": 0.7212387323379517, + "blocks.0.feed_forward.w1.weight": 0.03262019529938698, + "blocks.0.feed_forward.w2.weight": 0.018550856038928032, + "blocks.0.feed_forward.w3.weight": 0.03300027176737785, + "blocks.0.feed_forward_norm.weight": 0.7314680218696594, + "blocks.1.attention.w_k.weight": 0.028769828379154205, + "blocks.1.attention.w_out.weight": 0.024300970137119293, + "blocks.1.attention.w_q.weight": 0.0290773157030344, + "blocks.1.attention.w_v.weight": 0.02681984379887581, + "blocks.1.attention_norm.weight": 0.7247515916824341, + "blocks.1.feed_forward.w1.weight": 0.035662610083818436, + "blocks.1.feed_forward.w2.weight": 0.019625242799520493, + "blocks.1.feed_forward.w3.weight": 0.036090947687625885, + "blocks.1.feed_forward_norm.weight": 0.7420072555541992, + "blocks.2.attention.w_k.weight": 0.031684815883636475, + "blocks.2.attention.w_out.weight": 0.0261231679469347, + "blocks.2.attention.w_q.weight": 0.03192731738090515, + "blocks.2.attention.w_v.weight": 0.029027849435806274, + "blocks.2.attention_norm.weight": 0.7313526272773743, + "blocks.2.feed_forward.w1.weight": 0.035158202052116394, + "blocks.2.feed_forward.w2.weight": 0.017811624333262444, + "blocks.2.feed_forward.w3.weight": 0.035606566816568375, + "blocks.2.feed_forward_norm.weight": 0.7424768209457397, + "blocks.3.attention.w_k.weight": 0.03212268650531769, + "blocks.3.attention.w_out.weight": 0.02810906246304512, + "blocks.3.attention.w_q.weight": 0.03197857737541199, + "blocks.3.attention.w_v.weight": 0.03150033950805664, + "blocks.3.attention_norm.weight": 0.7306333184242249, + "blocks.3.feed_forward.w1.weight": 0.03466830775141716, + "blocks.3.feed_forward.w2.weight": 0.018097171559929848, + "blocks.3.feed_forward.w3.weight": 0.03538702800869942, + "blocks.3.feed_forward_norm.weight": 0.7410686016082764, + "blocks.4.attention.w_k.weight": 0.03302117437124252, + "blocks.4.attention.w_out.weight": 0.02875415049493313, + "blocks.4.attention.w_q.weight": 0.032805606722831726, + "blocks.4.attention.w_v.weight": 0.031778451055288315, + "blocks.4.attention_norm.weight": 0.7310084104537964, + "blocks.4.feed_forward.w1.weight": 0.0345308855175972, + "blocks.4.feed_forward.w2.weight": 0.019194750115275383, + "blocks.4.feed_forward.w3.weight": 0.03569754585623741, + "blocks.4.feed_forward_norm.weight": 0.7415932416915894, + "blocks.5.attention.w_k.weight": 0.03010912798345089, + "blocks.5.attention.w_out.weight": 0.03453635796904564, + "blocks.5.attention.w_q.weight": 0.030799981206655502, + "blocks.5.attention.w_v.weight": 0.03295043483376503, + "blocks.5.attention_norm.weight": 0.7344293594360352, + "blocks.5.feed_forward.w1.weight": 0.03447722643613815, + "blocks.5.feed_forward.w2.weight": 0.020819514989852905, + "blocks.5.feed_forward.w3.weight": 0.03604985773563385, + "blocks.5.feed_forward_norm.weight": 0.742188036441803, + "blocks.6.attention.w_k.weight": 0.03257782757282257, + "blocks.6.attention.w_out.weight": 0.036244895309209824, + "blocks.6.attention.w_q.weight": 0.0317949503660202, + "blocks.6.attention.w_v.weight": 0.034245118498802185, + "blocks.6.attention_norm.weight": 0.7350813746452332, + "blocks.6.feed_forward.w1.weight": 0.03427771106362343, + "blocks.6.feed_forward.w2.weight": 0.02320433221757412, + "blocks.6.feed_forward.w3.weight": 0.036269668489694595, + "blocks.6.feed_forward_norm.weight": 0.7430891990661621, + "blocks.7.attention.w_k.weight": 0.033718906342983246, + "blocks.7.attention.w_out.weight": 0.040113773196935654, + "blocks.7.attention.w_q.weight": 0.0327695831656456, + "blocks.7.attention.w_v.weight": 0.034865327179431915, + "blocks.7.attention_norm.weight": 0.7363488674163818, + "blocks.7.feed_forward.w1.weight": 0.033236388117074966, + "blocks.7.feed_forward.w2.weight": 0.02615497075021267, + "blocks.7.feed_forward.w3.weight": 0.03580132871866226, + "blocks.7.feed_forward_norm.weight": 0.7429885268211365, + "blocks.8.attention.w_k.weight": 0.03233204782009125, + "blocks.8.attention.w_out.weight": 0.04406481236219406, + "blocks.8.attention.w_q.weight": 0.031116899102926254, + "blocks.8.attention.w_v.weight": 0.03517533093690872, + "blocks.8.attention_norm.weight": 0.7402186989784241, + "blocks.8.feed_forward.w1.weight": 0.03163241967558861, + "blocks.8.feed_forward.w2.weight": 0.030139151960611343, + "blocks.8.feed_forward.w3.weight": 0.03480131924152374, + "blocks.8.feed_forward_norm.weight": 0.741178035736084, + "embeddings.weight": 0.055930908769369125, + "lm_head.norm.weight": 1.363660454750061, + "lm_head.w_out.weight": 0.06779012084007263 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.916410182665998 + }, + "base_loss": 3.916410182665998, + "loss": { + "0.005": [ + 3.916753880146988, + 3.916703325123091, + 3.916709217079908 + ], + "0.01": [ + 3.9176285853457067, + 3.9174146400213132, + 3.917594133438847 + ], + "0.02": [ + 3.9209808701229307, + 3.92055286992913, + 3.920792448786767 + ], + "0.05": [ + 3.94461695175913, + 3.9424623265307757, + 3.944583939586739 + ], + "0.1": [ + 4.036349081743616, + 4.027967018093331, + 4.03692925952328 + ], + "0.15": [ + 4.225827218217674, + 4.199057738391233, + 4.224654679687876 + ], + "0.2": [ + 4.583495436479917, + 4.510621069837889, + 4.570480080397276 + ], + "0.3": [ + 6.054807350069808, + 5.9142904866477775, + 6.009911476361313 + ], + "0.4": [ + 7.959366780994102, + 7.961219625358557, + 7.720636466859065 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.916753880146988, + 3.916703325123091, + 3.916709217079908 + ], + "0.01": [ + 3.9176285853457067, + 3.9174146400213132, + 3.917594133438847 + ], + "0.02": [ + 3.9209808701229307, + 3.92055286992913, + 3.920792448786767 + ], + "0.05": [ + 3.94461695175913, + 3.9424623265307757, + 3.944583939586739 + ], + "0.1": [ + 4.036349081743616, + 4.027967018093331, + 4.03692925952328 + ], + "0.15": [ + 4.225827218217674, + 4.199057738391233, + 4.224654679687876 + ], + "0.2": [ + 4.583495436479917, + 4.510621069837889, + 4.570480080397276 + ], + "0.3": [ + 6.054807350069808, + 5.9142904866477775, + 6.009911476361313 + ], + "0.4": [ + 7.959366780994102, + 7.961219625358557, + 7.720636466859065 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00031195811733164547, + "0.01": 0.0011356036026244354, + "0.02": 0.004365213613611196, + "0.05": 0.027477556626217314, + "0.1": 0.11733827045407796, + "0.15": 0.3001030294329299, + "0.2": 0.6384553462390299, + "0.3": 2.076592921693635, + "0.4": 3.9639974417379102 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00031195811733164547, + "0.01": 0.0011356036026244354, + "0.02": 0.004365213613611196, + "0.05": 0.027477556626217314, + "0.1": 0.11733827045407796, + "0.15": 0.3001030294329299, + "0.2": 0.6384553462390299, + "0.3": 2.076592921693635, + "0.4": 3.9639974417379102 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 61.2 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..8b835c69fd1dde77a5a0b619b447dc509835f293 --- /dev/null +++ b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.033792030066251755, + "blocks.0.attention.w_out.weight": 0.026513146236538887, + "blocks.0.attention.w_q.weight": 0.033974722027778625, + "blocks.0.attention.w_v.weight": 0.030098984017968178, + "blocks.0.feed_forward.w1.weight": 0.03262019529938698, + "blocks.0.feed_forward.w2.weight": 0.018550856038928032, + "blocks.0.feed_forward.w3.weight": 0.03300027176737785, + "blocks.1.attention.w_k.weight": 0.028769828379154205, + "blocks.1.attention.w_out.weight": 0.024300970137119293, + "blocks.1.attention.w_q.weight": 0.0290773157030344, + "blocks.1.attention.w_v.weight": 0.02681984379887581, + "blocks.1.feed_forward.w1.weight": 0.035662610083818436, + "blocks.1.feed_forward.w2.weight": 0.019625242799520493, + "blocks.1.feed_forward.w3.weight": 0.036090947687625885, + "blocks.2.attention.w_k.weight": 0.031684815883636475, + "blocks.2.attention.w_out.weight": 0.0261231679469347, + "blocks.2.attention.w_q.weight": 0.03192731738090515, + "blocks.2.attention.w_v.weight": 0.029027849435806274, + "blocks.2.feed_forward.w1.weight": 0.035158202052116394, + "blocks.2.feed_forward.w2.weight": 0.017811624333262444, + "blocks.2.feed_forward.w3.weight": 0.035606566816568375, + "blocks.3.attention.w_k.weight": 0.03212268650531769, + "blocks.3.attention.w_out.weight": 0.02810906246304512, + "blocks.3.attention.w_q.weight": 0.03197857737541199, + "blocks.3.attention.w_v.weight": 0.03150033950805664, + "blocks.3.feed_forward.w1.weight": 0.03466830775141716, + "blocks.3.feed_forward.w2.weight": 0.018097171559929848, + "blocks.3.feed_forward.w3.weight": 0.03538702800869942, + "blocks.4.attention.w_k.weight": 0.03302117437124252, + "blocks.4.attention.w_out.weight": 0.02875415049493313, + "blocks.4.attention.w_q.weight": 0.032805606722831726, + "blocks.4.attention.w_v.weight": 0.031778451055288315, + "blocks.4.feed_forward.w1.weight": 0.0345308855175972, + "blocks.4.feed_forward.w2.weight": 0.019194750115275383, + "blocks.4.feed_forward.w3.weight": 0.03569754585623741, + "blocks.5.attention.w_k.weight": 0.03010912798345089, + "blocks.5.attention.w_out.weight": 0.03453635796904564, + "blocks.5.attention.w_q.weight": 0.030799981206655502, + "blocks.5.attention.w_v.weight": 0.03295043483376503, + "blocks.5.feed_forward.w1.weight": 0.03447722643613815, + "blocks.5.feed_forward.w2.weight": 0.020819514989852905, + "blocks.5.feed_forward.w3.weight": 0.03604985773563385, + "blocks.6.attention.w_k.weight": 0.03257782757282257, + "blocks.6.attention.w_out.weight": 0.036244895309209824, + "blocks.6.attention.w_q.weight": 0.0317949503660202, + "blocks.6.attention.w_v.weight": 0.034245118498802185, + "blocks.6.feed_forward.w1.weight": 0.03427771106362343, + "blocks.6.feed_forward.w2.weight": 0.02320433221757412, + "blocks.6.feed_forward.w3.weight": 0.036269668489694595, + "blocks.7.attention.w_k.weight": 0.033718906342983246, + "blocks.7.attention.w_out.weight": 0.040113773196935654, + "blocks.7.attention.w_q.weight": 0.0327695831656456, + "blocks.7.attention.w_v.weight": 0.034865327179431915, + "blocks.7.feed_forward.w1.weight": 0.033236388117074966, + "blocks.7.feed_forward.w2.weight": 0.02615497075021267, + "blocks.7.feed_forward.w3.weight": 0.03580132871866226, + "blocks.8.attention.w_k.weight": 0.03233204782009125, + "blocks.8.attention.w_out.weight": 0.04406481236219406, + "blocks.8.attention.w_q.weight": 0.031116899102926254, + "blocks.8.attention.w_v.weight": 0.03517533093690872, + "blocks.8.feed_forward.w1.weight": 0.03163241967558861, + "blocks.8.feed_forward.w2.weight": 0.030139151960611343, + "blocks.8.feed_forward.w3.weight": 0.03480131924152374 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.916410182665998 + }, + "base_loss": 3.916410182665998, + "loss": { + "0.005": [ + 3.9166031188216612, + 3.9166111077830292, + 3.9166282124544796 + ], + "0.01": [ + 3.9170695145428205, + 3.917076439352625, + 3.9171663953706153 + ], + "0.02": [ + 3.9188807005034314, + 3.918924248558763, + 3.919003539587189 + ], + "0.05": [ + 3.9309205856238885, + 3.9306903689526145, + 3.9308996703508847 + ], + "0.1": [ + 3.9758266704926046, + 3.973638188030323, + 3.974508139757904 + ], + "0.15": [ + 4.059841444233951, + 4.0517076565329395, + 4.055604896455549 + ], + "0.2": [ + 4.1971037213818345, + 4.177021170450288, + 4.188559100755623 + ], + "0.3": [ + 4.710997663686541, + 4.645984176817421, + 4.6946044558029065 + ], + "0.4": [ + 5.60428553083135, + 5.503434611767463, + 5.5852172258845725 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.9166031188216612, + 3.9166111077830292, + 3.9166282124544796 + ], + "0.01": [ + 3.9170695145428205, + 3.917076439352625, + 3.9171663953706153 + ], + "0.02": [ + 3.9188807005034314, + 3.918924248558763, + 3.919003539587189 + ], + "0.05": [ + 3.9309205856238885, + 3.9306903689526145, + 3.9308996703508847 + ], + "0.1": [ + 3.9758266704926046, + 3.973638188030323, + 3.974508139757904 + ], + "0.15": [ + 4.059841444233951, + 4.0517076565329395, + 4.055604896455549 + ], + "0.2": [ + 4.1971037213818345, + 4.177021170450288, + 4.188559100755623 + ], + "0.3": [ + 4.710997663686541, + 4.645984176817421, + 4.6946044558029065 + ], + "0.4": [ + 5.60428553083135, + 5.503434611767463, + 5.5852172258845725 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00020396368705899448, + "0.01": 0.0006939337560227798, + "0.02": 0.0025259802171300194, + "0.05": 0.014426692309798206, + "0.1": 0.05824748342761277, + "0.15": 0.1393078164081487, + "0.2": 0.2711511481965845, + "0.3": 0.7674519161029583, + "0.4": 1.6479022734951307 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00020396368705899448, + "0.01": 0.0006939337560227798, + "0.02": 0.0025259802171300194, + "0.05": 0.014426692309798206, + "0.1": 0.05824748342761277, + "0.15": 0.1393078164081487, + "0.2": 0.2711511481965845, + "0.3": 0.7674519161029583, + "0.4": 1.6479022734951307 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 52.6 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ptq.json b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..02992835b016667046b422ccf3055229d76442cd --- /dev/null +++ b/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ptq.json @@ -0,0 +1,48 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "w_bits": null, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.916410182665998 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.996126214447322 + }, + "4": { + "val_dolma_cc": 4.042027073674593 + }, + "5": { + "val_dolma_cc": 3.942661643683497 + }, + "6": { + "val_dolma_cc": 3.9220034376164308 + }, + "8": { + "val_dolma_cc": 3.9167275423957983 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 1.0797160317813246 + }, + "4": { + "val_dolma_cc": 0.12561689100859486 + }, + "5": { + "val_dolma_cc": 0.02625146101749909 + }, + "6": { + "val_dolma_cc": 0.0055932549504329465 + }, + "8": { + "val_dolma_cc": 0.00031735972980051486 + } + }, + "checks": { + "weights_restored": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/bf16_start.json b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..fb82d97ccc13e4fd70253b61cc884b79a7848a18 --- /dev/null +++ b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 4.116169639029774, + "heldout_cc_en_tail_4M": 4.678772911880955 + }, + "loss_bf16_master": 4.116169639029774, + "losses_eval_model": { + "val_dolma_cc": 4.133484409991807 + }, + "bf16_minus_quant": -0.01731477096203271, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-p9-24", + "gpu": "NVIDIA L40S", + "t_s": 17.5 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/finalize.json b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..94f61d2f91845d2ea701eed66f0c57a676661119 --- /dev/null +++ b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/finalize.json @@ -0,0 +1,102 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 4.1323714209140086 + }, + "losses_fakequant_master": { + "val_dolma_cc": 4.133484409991807 + }, + "losses_eval_model": { + "val_dolma_cc": 4.133484409991807 + }, + "max_distinct_values_per_row": 63, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.031120842322707176, + "blocks.0.attention.w_k": 0.03135550022125244, + "blocks.0.attention.w_v": 0.03434007987380028, + "blocks.0.attention.w_out": 0.03542300686240196, + "blocks.0.feed_forward.w1": 0.031323470175266266, + "blocks.0.feed_forward.w2": 0.035301655530929565, + "blocks.0.feed_forward.w3": 0.03124716505408287, + "blocks.1.attention.w_q": 0.030890315771102905, + "blocks.1.attention.w_k": 0.03229247033596039, + "blocks.1.attention.w_v": 0.033395953476428986, + "blocks.1.attention.w_out": 0.032106492668390274, + "blocks.1.feed_forward.w1": 0.031076418235898018, + "blocks.1.feed_forward.w2": 0.034724216908216476, + "blocks.1.feed_forward.w3": 0.03132103011012077, + "blocks.2.attention.w_q": 0.031059447675943375, + "blocks.2.attention.w_k": 0.032405510544776917, + "blocks.2.attention.w_v": 0.033812735229730606, + "blocks.2.attention.w_out": 0.03228583559393883, + "blocks.2.feed_forward.w1": 0.03101040981709957, + "blocks.2.feed_forward.w2": 0.03550992161035538, + "blocks.2.feed_forward.w3": 0.03121618926525116, + "blocks.3.attention.w_q": 0.03107432834804058, + "blocks.3.attention.w_k": 0.03350537270307541, + "blocks.3.attention.w_v": 0.03315924108028412, + "blocks.3.attention.w_out": 0.03204391151666641, + "blocks.3.feed_forward.w1": 0.03128368780016899, + "blocks.3.feed_forward.w2": 0.03541150316596031, + "blocks.3.feed_forward.w3": 0.03160535916686058, + "blocks.4.attention.w_q": 0.031151646748185158, + "blocks.4.attention.w_k": 0.032543595880270004, + "blocks.4.attention.w_v": 0.033398281782865524, + "blocks.4.attention.w_out": 0.03176643326878548, + "blocks.4.feed_forward.w1": 0.031281664967536926, + "blocks.4.feed_forward.w2": 0.03540990874171257, + "blocks.4.feed_forward.w3": 0.031895920634269714, + "blocks.5.attention.w_q": 0.03117757849395275, + "blocks.5.attention.w_k": 0.033542435616254807, + "blocks.5.attention.w_v": 0.033953651785850525, + "blocks.5.attention.w_out": 0.032777994871139526, + "blocks.5.feed_forward.w1": 0.03133861720561981, + "blocks.5.feed_forward.w2": 0.03533758595585823, + "blocks.5.feed_forward.w3": 0.0314180962741375, + "blocks.6.attention.w_q": 0.031007658690214157, + "blocks.6.attention.w_k": 0.03384607285261154, + "blocks.6.attention.w_v": 0.03334997221827507, + "blocks.6.attention.w_out": 0.033462539315223694, + "blocks.6.feed_forward.w1": 0.03140699863433838, + "blocks.6.feed_forward.w2": 0.03598141297698021, + "blocks.6.feed_forward.w3": 0.03141026571393013, + "blocks.7.attention.w_q": 0.03154123201966286, + "blocks.7.attention.w_k": 0.031651951372623444, + "blocks.7.attention.w_v": 0.035259682685136795, + "blocks.7.attention.w_out": 0.03749556094408035, + "blocks.7.feed_forward.w1": 0.031442634761333466, + "blocks.7.feed_forward.w2": 0.03741942718625069, + "blocks.7.feed_forward.w3": 0.03139324113726616, + "blocks.8.attention.w_q": 0.030701495707035065, + "blocks.8.attention.w_k": 0.031224187463521957, + "blocks.8.attention.w_v": 0.03455720096826553, + "blocks.8.attention.w_out": 0.03540285304188728, + "blocks.8.feed_forward.w1": 0.031160781159996986, + "blocks.8.feed_forward.w2": 0.0417548343539238, + "blocks.8.feed_forward.w3": 0.03098127990961075 + }, + "rel_quant_noise_mean": 0.03286899629211615, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.7896115779876709, + "loss_eval_model": 4.133484409991807, + "checks": { + "snap_reproduces_fakequant": true, + "grid_levels_ok": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/launch.yaml b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb49bce30fa0e48b978362e71149a6a15846ec0f --- /dev/null +++ b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/launch.yaml @@ -0,0 +1,118 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 102 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: olmo_core.optim.AdamWConfig + lr: 0.0024 + betas: + - 0.9 + - 0.95 + eps: 1.0e-15 + weight_decay: 0.1 + fused: true + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 232 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1002327 + save_async: false +wandb: + enabled: false +init_seed: 102 +n_tokens: 610009088 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/perturb_all.json b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..b47bc5289e4d6e6b1c6abb9248952b4716eb571b --- /dev/null +++ b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03452238067984581, + "blocks.0.attention.w_out.weight": 0.039284028112888336, + "blocks.0.attention.w_q.weight": 0.037009578198194504, + "blocks.0.attention.w_v.weight": 0.045134395360946655, + "blocks.0.attention_norm.weight": 0.739876389503479, + "blocks.0.feed_forward.w1.weight": 0.03798346593976021, + "blocks.0.feed_forward.w2.weight": 0.0359223298728466, + "blocks.0.feed_forward.w3.weight": 0.040152616798877716, + "blocks.0.feed_forward_norm.weight": 0.7368800044059753, + "blocks.1.attention.w_k.weight": 0.02928074263036251, + "blocks.1.attention.w_out.weight": 0.03693598881363869, + "blocks.1.attention.w_q.weight": 0.029001660645008087, + "blocks.1.attention.w_v.weight": 0.05250340327620506, + "blocks.1.attention_norm.weight": 0.7101855874061584, + "blocks.1.feed_forward.w1.weight": 0.047833625227212906, + "blocks.1.feed_forward.w2.weight": 0.04353787750005722, + "blocks.1.feed_forward.w3.weight": 0.048920538276433945, + "blocks.1.feed_forward_norm.weight": 0.7491304874420166, + "blocks.2.attention.w_k.weight": 0.03276506066322327, + "blocks.2.attention.w_out.weight": 0.03439931944012642, + "blocks.2.attention.w_q.weight": 0.03276437148451805, + "blocks.2.attention.w_v.weight": 0.0495532788336277, + "blocks.2.attention_norm.weight": 0.7206140756607056, + "blocks.2.feed_forward.w1.weight": 0.04991117864847183, + "blocks.2.feed_forward.w2.weight": 0.04640195146203041, + "blocks.2.feed_forward.w3.weight": 0.050042614340782166, + "blocks.2.feed_forward_norm.weight": 0.7519209980964661, + "blocks.3.attention.w_k.weight": 0.03559667617082596, + "blocks.3.attention.w_out.weight": 0.03937024250626564, + "blocks.3.attention.w_q.weight": 0.03479573875665665, + "blocks.3.attention.w_v.weight": 0.054586127400398254, + "blocks.3.attention_norm.weight": 0.7257862687110901, + "blocks.3.feed_forward.w1.weight": 0.048882536590099335, + "blocks.3.feed_forward.w2.weight": 0.04395680129528046, + "blocks.3.feed_forward.w3.weight": 0.04914790019392967, + "blocks.3.feed_forward_norm.weight": 0.754949152469635, + "blocks.4.attention.w_k.weight": 0.03553074598312378, + "blocks.4.attention.w_out.weight": 0.041324224323034286, + "blocks.4.attention.w_q.weight": 0.03667189180850983, + "blocks.4.attention.w_v.weight": 0.05485207587480545, + "blocks.4.attention_norm.weight": 0.7308045625686646, + "blocks.4.feed_forward.w1.weight": 0.04888234660029411, + "blocks.4.feed_forward.w2.weight": 0.04575841501355171, + "blocks.4.feed_forward.w3.weight": 0.049392424523830414, + "blocks.4.feed_forward_norm.weight": 0.752315878868103, + "blocks.5.attention.w_k.weight": 0.036597561091184616, + "blocks.5.attention.w_out.weight": 0.04749889299273491, + "blocks.5.attention.w_q.weight": 0.0354626402258873, + "blocks.5.attention.w_v.weight": 0.04642144963145256, + "blocks.5.attention_norm.weight": 0.7264080047607422, + "blocks.5.feed_forward.w1.weight": 0.04904254153370857, + "blocks.5.feed_forward.w2.weight": 0.04696989804506302, + "blocks.5.feed_forward.w3.weight": 0.05032774806022644, + "blocks.5.feed_forward_norm.weight": 0.7507573366165161, + "blocks.6.attention.w_k.weight": 0.03965156897902489, + "blocks.6.attention.w_out.weight": 0.05916905403137207, + "blocks.6.attention.w_q.weight": 0.03646329045295715, + "blocks.6.attention.w_v.weight": 0.04659885913133621, + "blocks.6.attention_norm.weight": 0.7252739667892456, + "blocks.6.feed_forward.w1.weight": 0.048663850873708725, + "blocks.6.feed_forward.w2.weight": 0.0510810911655426, + "blocks.6.feed_forward.w3.weight": 0.050724003463983536, + "blocks.6.feed_forward_norm.weight": 0.7489078640937805, + "blocks.7.attention.w_k.weight": 0.03603087738156319, + "blocks.7.attention.w_out.weight": 0.08853258937597275, + "blocks.7.attention.w_q.weight": 0.03710123151540756, + "blocks.7.attention.w_v.weight": 0.05319391191005707, + "blocks.7.attention_norm.weight": 0.7369636297225952, + "blocks.7.feed_forward.w1.weight": 0.049033112823963165, + "blocks.7.feed_forward.w2.weight": 0.06086074933409691, + "blocks.7.feed_forward.w3.weight": 0.05256909877061844, + "blocks.7.feed_forward_norm.weight": 0.7491270899772644, + "blocks.8.attention.w_k.weight": 0.04031990468502045, + "blocks.8.attention.w_out.weight": 0.09576456993818283, + "blocks.8.attention.w_q.weight": 0.03576497733592987, + "blocks.8.attention.w_v.weight": 0.042381834238767624, + "blocks.8.attention_norm.weight": 0.7316818237304688, + "blocks.8.feed_forward.w1.weight": 0.047424208372831345, + "blocks.8.feed_forward.w2.weight": 0.06759675592184067, + "blocks.8.feed_forward.w3.weight": 0.05208192765712738, + "blocks.8.feed_forward_norm.weight": 0.7457408905029297, + "embeddings.weight": 0.05595573037862778, + "lm_head.norm.weight": 1.3673397302627563, + "lm_head.w_out.weight": 0.07094171643257141 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 4.133484409991807 + }, + "base_loss": 4.133484409991807, + "loss": { + "0.005": [ + 4.134091336316463, + 4.13444897731279, + 4.133487833273491 + ], + "0.01": [ + 4.13518646564468, + 4.134274726406633, + 4.133956944146065 + ], + "0.02": [ + 4.135846947744149, + 4.136405912914343, + 4.1364804895948515 + ], + "0.05": [ + 4.147796513000677, + 4.148100367395197, + 4.1488774366879895 + ], + "0.1": [ + 4.191778945611088, + 4.190513446400063, + 4.193277110606912 + ], + "0.15": [ + 4.268052713123267, + 4.266817600916623, + 4.270579134128668 + ], + "0.2": [ + 4.384039275048625, + 4.381098965594603, + 4.388253307414357 + ], + "0.3": [ + 4.788946718744993, + 4.775071755113937, + 4.790860837895414 + ], + "0.4": [ + 5.53872043004493, + 5.510531493315125, + 5.542292638174446 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 4.134091336316463, + 4.13444897731279, + 4.133487833273491 + ], + "0.01": [ + 4.13518646564468, + 4.134274726406633, + 4.133956944146065 + ], + "0.02": [ + 4.135846947744149, + 4.136405912914343, + 4.1364804895948515 + ], + "0.05": [ + 4.147796513000677, + 4.148100367395197, + 4.1488774366879895 + ], + "0.1": [ + 4.191778945611088, + 4.190513446400063, + 4.193277110606912 + ], + "0.15": [ + 4.268052713123267, + 4.266817600916623, + 4.270579134128668 + ], + "0.2": [ + 4.384039275048625, + 4.381098965594603, + 4.388253307414357 + ], + "0.3": [ + 4.788946718744993, + 4.775071755113937, + 4.790860837895414 + ], + "0.4": [ + 5.53872043004493, + 5.510531493315125, + 5.542292638174446 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0005249723091074188, + "0.01": 0.000988302073985814, + "0.02": 0.0027600400926406365, + "0.05": 0.014773695702814393, + "0.1": 0.05837209088088091, + "0.15": 0.13499873939771145, + "0.2": 0.2509794393607212, + "0.3": 0.6514753605929746, + "0.4": 1.3970304438530272 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0005249723091074188, + "0.01": 0.000988302073985814, + "0.02": 0.0027600400926406365, + "0.05": 0.014773695702814393, + "0.1": 0.05837209088088091, + "0.15": 0.13499873939771145, + "0.2": 0.2509794393607212, + "0.3": 0.6514753605929746, + "0.4": 1.3970304438530272 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 77.9 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/perturb_blocks.json b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..da172b2c808a9856bfc6acaa869539a9288e0434 --- /dev/null +++ b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03452238067984581, + "blocks.0.attention.w_out.weight": 0.039284028112888336, + "blocks.0.attention.w_q.weight": 0.037009578198194504, + "blocks.0.attention.w_v.weight": 0.045134395360946655, + "blocks.0.feed_forward.w1.weight": 0.03798346593976021, + "blocks.0.feed_forward.w2.weight": 0.0359223298728466, + "blocks.0.feed_forward.w3.weight": 0.040152616798877716, + "blocks.1.attention.w_k.weight": 0.02928074263036251, + "blocks.1.attention.w_out.weight": 0.03693598881363869, + "blocks.1.attention.w_q.weight": 0.029001660645008087, + "blocks.1.attention.w_v.weight": 0.05250340327620506, + "blocks.1.feed_forward.w1.weight": 0.047833625227212906, + "blocks.1.feed_forward.w2.weight": 0.04353787750005722, + "blocks.1.feed_forward.w3.weight": 0.048920538276433945, + "blocks.2.attention.w_k.weight": 0.03276506066322327, + "blocks.2.attention.w_out.weight": 0.03439931944012642, + "blocks.2.attention.w_q.weight": 0.03276437148451805, + "blocks.2.attention.w_v.weight": 0.0495532788336277, + "blocks.2.feed_forward.w1.weight": 0.04991117864847183, + "blocks.2.feed_forward.w2.weight": 0.04640195146203041, + "blocks.2.feed_forward.w3.weight": 0.050042614340782166, + "blocks.3.attention.w_k.weight": 0.03559667617082596, + "blocks.3.attention.w_out.weight": 0.03937024250626564, + "blocks.3.attention.w_q.weight": 0.03479573875665665, + "blocks.3.attention.w_v.weight": 0.054586127400398254, + "blocks.3.feed_forward.w1.weight": 0.048882536590099335, + "blocks.3.feed_forward.w2.weight": 0.04395680129528046, + "blocks.3.feed_forward.w3.weight": 0.04914790019392967, + "blocks.4.attention.w_k.weight": 0.03553074598312378, + "blocks.4.attention.w_out.weight": 0.041324224323034286, + "blocks.4.attention.w_q.weight": 0.03667189180850983, + "blocks.4.attention.w_v.weight": 0.05485207587480545, + "blocks.4.feed_forward.w1.weight": 0.04888234660029411, + "blocks.4.feed_forward.w2.weight": 0.04575841501355171, + "blocks.4.feed_forward.w3.weight": 0.049392424523830414, + "blocks.5.attention.w_k.weight": 0.036597561091184616, + "blocks.5.attention.w_out.weight": 0.04749889299273491, + "blocks.5.attention.w_q.weight": 0.0354626402258873, + "blocks.5.attention.w_v.weight": 0.04642144963145256, + "blocks.5.feed_forward.w1.weight": 0.04904254153370857, + "blocks.5.feed_forward.w2.weight": 0.04696989804506302, + "blocks.5.feed_forward.w3.weight": 0.05032774806022644, + "blocks.6.attention.w_k.weight": 0.03965156897902489, + "blocks.6.attention.w_out.weight": 0.05916905403137207, + "blocks.6.attention.w_q.weight": 0.03646329045295715, + "blocks.6.attention.w_v.weight": 0.04659885913133621, + "blocks.6.feed_forward.w1.weight": 0.048663850873708725, + "blocks.6.feed_forward.w2.weight": 0.0510810911655426, + "blocks.6.feed_forward.w3.weight": 0.050724003463983536, + "blocks.7.attention.w_k.weight": 0.03603087738156319, + "blocks.7.attention.w_out.weight": 0.08853258937597275, + "blocks.7.attention.w_q.weight": 0.03710123151540756, + "blocks.7.attention.w_v.weight": 0.05319391191005707, + "blocks.7.feed_forward.w1.weight": 0.049033112823963165, + "blocks.7.feed_forward.w2.weight": 0.06086074933409691, + "blocks.7.feed_forward.w3.weight": 0.05256909877061844, + "blocks.8.attention.w_k.weight": 0.04031990468502045, + "blocks.8.attention.w_out.weight": 0.09576456993818283, + "blocks.8.attention.w_q.weight": 0.03576497733592987, + "blocks.8.attention.w_v.weight": 0.042381834238767624, + "blocks.8.feed_forward.w1.weight": 0.047424208372831345, + "blocks.8.feed_forward.w2.weight": 0.06759675592184067, + "blocks.8.feed_forward.w3.weight": 0.05208192765712738 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 4.133484409991807 + }, + "base_loss": 4.133484409991807, + "loss": { + "0.005": [ + 4.133970637272798, + 4.133880180477601, + 4.133772411657893 + ], + "0.01": [ + 4.133476006324315, + 4.134386121948764, + 4.133500517021167 + ], + "0.02": [ + 4.134247136712427, + 4.134187356433435, + 4.134905647001647 + ], + "0.05": [ + 4.1378708036122065, + 4.138091039950253, + 4.138317696408576 + ], + "0.1": [ + 4.151370927679759, + 4.150401763380517, + 4.15023772363458 + ], + "0.15": [ + 4.172548873977413, + 4.172452908632948, + 4.172826934433491 + ], + "0.2": [ + 4.205900721983018, + 4.205130561850742, + 4.20530034097294 + ], + "0.3": [ + 4.312236227343587, + 4.312655600867546, + 4.315095485224318 + ], + "0.4": [ + 4.502774966904886, + 4.495064891492847, + 4.509433621449239 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 4.133970637272798, + 4.133880180477601, + 4.133772411657893 + ], + "0.01": [ + 4.133476006324315, + 4.134386121948764, + 4.133500517021167 + ], + "0.02": [ + 4.134247136712427, + 4.134187356433435, + 4.134905647001647 + ], + "0.05": [ + 4.1378708036122065, + 4.138091039950253, + 4.138317696408576 + ], + "0.1": [ + 4.151370927679759, + 4.150401763380517, + 4.15023772363458 + ], + "0.15": [ + 4.172548873977413, + 4.172452908632948, + 4.172826934433491 + ], + "0.2": [ + 4.205900721983018, + 4.205130561850742, + 4.20530034097294 + ], + "0.3": [ + 4.312236227343587, + 4.312655600867546, + 4.315095485224318 + ], + "0.4": [ + 4.502774966904886, + 4.495064891492847, + 4.509433621449239 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00038999981095777514, + "0.01": 0.0003031384396088299, + "0.02": 0.0009623033906960288, + "0.05": 0.004608769998537987, + "0.1": 0.017185728239812015, + "0.15": 0.03912516235614394, + "0.2": 0.07195946494376049, + "0.3": 0.17984469448667717, + "0.4": 0.368940083290517 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00038999981095777514, + "0.01": 0.0003031384396088299, + "0.02": 0.0009623033906960288, + "0.05": 0.004608769998537987, + "0.1": 0.017185728239812015, + "0.15": 0.03912516235614394, + "0.2": 0.07195946494376049, + "0.3": 0.17984469448667717, + "0.4": 0.368940083290517 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 72.2 +} \ No newline at end of file diff --git a/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/ptq.json b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..1fdee9a08b930ca3c79422c9fbfb4aa17393670b --- /dev/null +++ b/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102/ptq.json @@ -0,0 +1,52 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D0.61B-f6-adamw-lr2.4e-03-s102", + "w_bits": 6, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 4.133484409991807 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.402245652440087 + }, + "4": { + "val_dolma_cc": 4.17207222014817 + }, + "5": { + "val_dolma_cc": 4.144966006773763 + }, + "6": { + "val_dolma_cc": 4.133246677750118 + }, + "8": { + "val_dolma_cc": 4.134107232080439 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 0.2687612424482797 + }, + "4": { + "val_dolma_cc": 0.038587810156363034 + }, + "5": { + "val_dolma_cc": 0.01148159678195615 + }, + "6": { + "val_dolma_cc": -0.00023773224168888163 + }, + "8": { + "val_dolma_cc": 0.0006228220886317359 + } + }, + "own_grid_max_rel_weight_change": 2.2988398029610835e-07, + "own_grid_delta_loss": -0.00023773224168888163, + "checks": { + "weights_restored": true, + "own_grid_weights_unchanged": true, + "own_grid_is_fixed_point": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/bf16_start.json b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..f90f82a63604f7a8f4249aaafadd68ee69d3ac83 --- /dev/null +++ b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": null, + "a": null, + "qkv": null + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.831054171073506, + "heldout_cc_en_tail_4M": 4.413598668423799 + }, + "loss_bf16_master": 3.831054171073506, + "losses_eval_model": { + "val_dolma_cc": 3.831054171073506 + }, + "bf16_minus_quant": 0.0, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-m5-16", + "gpu": "NVIDIA L40S", + "t_s": 17.8 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/finalize.json b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..aeaf8fe7dbb3a823537f342c613fe156692cb277 --- /dev/null +++ b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/finalize.json @@ -0,0 +1,29 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101", + "w_bits": null, + "a_bits": null, + "qkv_bits": null, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.831054171073506 + }, + "losses_eval_model": { + "val_dolma_cc": 3.831054171073506 + }, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 1.0510008335113525, + "loss_eval_model": 3.831054171073506, + "checks": { + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/launch.yaml b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fb1b52b9e21e02c868ceb987abcf3460e684276 --- /dev/null +++ b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/launch.yaml @@ -0,0 +1,118 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: null + fq_act_bits: null + fq_qkv_bits: null + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: olmo_core.optim.AdamWConfig + lr: 0.0012 + betas: + - 0.9 + - 0.95 + eps: 1.0e-15 + weight_decay: 0.1 + fused: true + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/perturb_all.json b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..4442a1bf4d990d9644358ce9719dfdbbd77a4d28 --- /dev/null +++ b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03268071636557579, + "blocks.0.attention.w_out.weight": 0.03668426722288132, + "blocks.0.attention.w_q.weight": 0.03406849503517151, + "blocks.0.attention.w_v.weight": 0.033527445048093796, + "blocks.0.attention_norm.weight": 0.6293824911117554, + "blocks.0.feed_forward.w1.weight": 0.038742586970329285, + "blocks.0.feed_forward.w2.weight": 0.039205584675073624, + "blocks.0.feed_forward.w3.weight": 0.04028899595141411, + "blocks.0.feed_forward_norm.weight": 0.6330354809761047, + "blocks.1.attention.w_k.weight": 0.03544138744473457, + "blocks.1.attention.w_out.weight": 0.03620398789644241, + "blocks.1.attention.w_q.weight": 0.0365966372191906, + "blocks.1.attention.w_v.weight": 0.03734150528907776, + "blocks.1.attention_norm.weight": 0.6099925637245178, + "blocks.1.feed_forward.w1.weight": 0.04364994540810585, + "blocks.1.feed_forward.w2.weight": 0.04219818487763405, + "blocks.1.feed_forward.w3.weight": 0.04411840811371803, + "blocks.1.feed_forward_norm.weight": 0.643448531627655, + "blocks.2.attention.w_k.weight": 0.03736107051372528, + "blocks.2.attention.w_out.weight": 0.03443260118365288, + "blocks.2.attention.w_q.weight": 0.03904962167143822, + "blocks.2.attention.w_v.weight": 0.03684396296739578, + "blocks.2.attention_norm.weight": 0.6134346723556519, + "blocks.2.feed_forward.w1.weight": 0.04324464499950409, + "blocks.2.feed_forward.w2.weight": 0.040622010827064514, + "blocks.2.feed_forward.w3.weight": 0.04401577636599541, + "blocks.2.feed_forward_norm.weight": 0.6434590220451355, + "blocks.3.attention.w_k.weight": 0.03984769433736801, + "blocks.3.attention.w_out.weight": 0.03913891315460205, + "blocks.3.attention.w_q.weight": 0.042075786739587784, + "blocks.3.attention.w_v.weight": 0.039377178996801376, + "blocks.3.attention_norm.weight": 0.6263216137886047, + "blocks.3.feed_forward.w1.weight": 0.04268060624599457, + "blocks.3.feed_forward.w2.weight": 0.0402805358171463, + "blocks.3.feed_forward.w3.weight": 0.04456881061196327, + "blocks.3.feed_forward_norm.weight": 0.6389124989509583, + "blocks.4.attention.w_k.weight": 0.04043733328580856, + "blocks.4.attention.w_out.weight": 0.04046373814344406, + "blocks.4.attention.w_q.weight": 0.04414784163236618, + "blocks.4.attention.w_v.weight": 0.0379972942173481, + "blocks.4.attention_norm.weight": 0.6256927251815796, + "blocks.4.feed_forward.w1.weight": 0.04161463677883148, + "blocks.4.feed_forward.w2.weight": 0.040296878665685654, + "blocks.4.feed_forward.w3.weight": 0.04442773759365082, + "blocks.4.feed_forward_norm.weight": 0.6358168721199036, + "blocks.5.attention.w_k.weight": 0.03878093510866165, + "blocks.5.attention.w_out.weight": 0.04189108684659004, + "blocks.5.attention.w_q.weight": 0.041319891810417175, + "blocks.5.attention.w_v.weight": 0.0382239855825901, + "blocks.5.attention_norm.weight": 0.6185463070869446, + "blocks.5.feed_forward.w1.weight": 0.041318804025650024, + "blocks.5.feed_forward.w2.weight": 0.04324207454919815, + "blocks.5.feed_forward.w3.weight": 0.04525141790509224, + "blocks.5.feed_forward_norm.weight": 0.6307767033576965, + "blocks.6.attention.w_k.weight": 0.04055728018283844, + "blocks.6.attention.w_out.weight": 0.055124517530202866, + "blocks.6.attention.w_q.weight": 0.04400692880153656, + "blocks.6.attention.w_v.weight": 0.04032757133245468, + "blocks.6.attention_norm.weight": 0.6209803819656372, + "blocks.6.feed_forward.w1.weight": 0.04073719307780266, + "blocks.6.feed_forward.w2.weight": 0.04550210386514664, + "blocks.6.feed_forward.w3.weight": 0.04522354528307915, + "blocks.6.feed_forward_norm.weight": 0.6327573657035828, + "blocks.7.attention.w_k.weight": 0.039778005331754684, + "blocks.7.attention.w_out.weight": 0.057591959834098816, + "blocks.7.attention.w_q.weight": 0.04154594987630844, + "blocks.7.attention.w_v.weight": 0.03982347622513771, + "blocks.7.attention_norm.weight": 0.6144234538078308, + "blocks.7.feed_forward.w1.weight": 0.040671318769454956, + "blocks.7.feed_forward.w2.weight": 0.0517294704914093, + "blocks.7.feed_forward.w3.weight": 0.04585808888077736, + "blocks.7.feed_forward_norm.weight": 0.6315277814865112, + "blocks.8.attention.w_k.weight": 0.03729722276329994, + "blocks.8.attention.w_out.weight": 0.0786689817905426, + "blocks.8.attention.w_q.weight": 0.04076381400227547, + "blocks.8.attention.w_v.weight": 0.03802540898323059, + "blocks.8.attention_norm.weight": 0.6195935606956482, + "blocks.8.feed_forward.w1.weight": 0.038343265652656555, + "blocks.8.feed_forward.w2.weight": 0.05855748802423477, + "blocks.8.feed_forward.w3.weight": 0.044858478009700775, + "blocks.8.feed_forward_norm.weight": 0.6258807182312012, + "embeddings.weight": 0.045032817870378494, + "lm_head.norm.weight": 1.8185430765151978, + "lm_head.w_out.weight": 0.05665469169616699 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.831054171073506 + }, + "base_loss": 3.831054171073506, + "loss": { + "0.005": [ + 3.8317018403186744, + 3.8312902014551584, + 3.8314713380479004 + ], + "0.01": [ + 3.8330465366624002, + 3.8324081983050196, + 3.8327434412583328 + ], + "0.02": [ + 3.838123752438785, + 3.836907145514839, + 3.8374857075055484 + ], + "0.05": [ + 3.872347300979871, + 3.8698361849778355, + 3.871757166340911 + ], + "0.1": [ + 4.003700027599084, + 4.0011724876696135, + 4.013152979830948 + ], + "0.15": [ + 4.276705020410427, + 4.271452419154528, + 4.3165530950910815 + ], + "0.2": [ + 4.786053736649279, + 4.758407978870707, + 4.870210199041412 + ], + "0.3": [ + 6.652791131836611, + 6.553454359603605, + 6.76614946240315 + ], + "0.4": [ + 8.568021647722265, + 8.522721748528717, + 8.586911589912544 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.8317018403186744, + 3.8312902014551584, + 3.8314713380479004 + ], + "0.01": [ + 3.8330465366624002, + 3.8324081983050196, + 3.8327434412583328 + ], + "0.02": [ + 3.838123752438785, + 3.836907145514839, + 3.8374857075055484 + ], + "0.05": [ + 3.872347300979871, + 3.8698361849778355, + 3.871757166340911 + ], + "0.1": [ + 4.003700027599084, + 4.0011724876696135, + 4.013152979830948 + ], + "0.15": [ + 4.276705020410427, + 4.271452419154528, + 4.3165530950910815 + ], + "0.2": [ + 4.786053736649279, + 4.758407978870707, + 4.870210199041412 + ], + "0.3": [ + 6.652791131836611, + 6.553454359603605, + 6.76614946240315 + ], + "0.4": [ + 8.568021647722265, + 8.522721748528717, + 8.586911589912544 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0004336222004055479, + "0.01": 0.0016785543350783527, + "0.02": 0.0064513640795516025, + "0.05": 0.040259379692700215, + "0.1": 0.1749543272930425, + "0.15": 0.4571826738118401, + "0.2": 0.9738364671136268, + "0.3": 2.82641081354095, + "0.4": 4.728164157647669 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0004336222004055479, + "0.01": 0.0016785543350783527, + "0.02": 0.0064513640795516025, + "0.05": 0.040259379692700215, + "0.1": 0.1749543272930425, + "0.15": 0.4571826738118401, + "0.2": 0.9738364671136268, + "0.3": 2.82641081354095, + "0.4": 4.728164157647669 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 56.0 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..367d228b7e49b2c42f7e1593b43e15c5cf3e7dbc --- /dev/null +++ b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03268071636557579, + "blocks.0.attention.w_out.weight": 0.03668426722288132, + "blocks.0.attention.w_q.weight": 0.03406849503517151, + "blocks.0.attention.w_v.weight": 0.033527445048093796, + "blocks.0.feed_forward.w1.weight": 0.038742586970329285, + "blocks.0.feed_forward.w2.weight": 0.039205584675073624, + "blocks.0.feed_forward.w3.weight": 0.04028899595141411, + "blocks.1.attention.w_k.weight": 0.03544138744473457, + "blocks.1.attention.w_out.weight": 0.03620398789644241, + "blocks.1.attention.w_q.weight": 0.0365966372191906, + "blocks.1.attention.w_v.weight": 0.03734150528907776, + "blocks.1.feed_forward.w1.weight": 0.04364994540810585, + "blocks.1.feed_forward.w2.weight": 0.04219818487763405, + "blocks.1.feed_forward.w3.weight": 0.04411840811371803, + "blocks.2.attention.w_k.weight": 0.03736107051372528, + "blocks.2.attention.w_out.weight": 0.03443260118365288, + "blocks.2.attention.w_q.weight": 0.03904962167143822, + "blocks.2.attention.w_v.weight": 0.03684396296739578, + "blocks.2.feed_forward.w1.weight": 0.04324464499950409, + "blocks.2.feed_forward.w2.weight": 0.040622010827064514, + "blocks.2.feed_forward.w3.weight": 0.04401577636599541, + "blocks.3.attention.w_k.weight": 0.03984769433736801, + "blocks.3.attention.w_out.weight": 0.03913891315460205, + "blocks.3.attention.w_q.weight": 0.042075786739587784, + "blocks.3.attention.w_v.weight": 0.039377178996801376, + "blocks.3.feed_forward.w1.weight": 0.04268060624599457, + "blocks.3.feed_forward.w2.weight": 0.0402805358171463, + "blocks.3.feed_forward.w3.weight": 0.04456881061196327, + "blocks.4.attention.w_k.weight": 0.04043733328580856, + "blocks.4.attention.w_out.weight": 0.04046373814344406, + "blocks.4.attention.w_q.weight": 0.04414784163236618, + "blocks.4.attention.w_v.weight": 0.0379972942173481, + "blocks.4.feed_forward.w1.weight": 0.04161463677883148, + "blocks.4.feed_forward.w2.weight": 0.040296878665685654, + "blocks.4.feed_forward.w3.weight": 0.04442773759365082, + "blocks.5.attention.w_k.weight": 0.03878093510866165, + "blocks.5.attention.w_out.weight": 0.04189108684659004, + "blocks.5.attention.w_q.weight": 0.041319891810417175, + "blocks.5.attention.w_v.weight": 0.0382239855825901, + "blocks.5.feed_forward.w1.weight": 0.041318804025650024, + "blocks.5.feed_forward.w2.weight": 0.04324207454919815, + "blocks.5.feed_forward.w3.weight": 0.04525141790509224, + "blocks.6.attention.w_k.weight": 0.04055728018283844, + "blocks.6.attention.w_out.weight": 0.055124517530202866, + "blocks.6.attention.w_q.weight": 0.04400692880153656, + "blocks.6.attention.w_v.weight": 0.04032757133245468, + "blocks.6.feed_forward.w1.weight": 0.04073719307780266, + "blocks.6.feed_forward.w2.weight": 0.04550210386514664, + "blocks.6.feed_forward.w3.weight": 0.04522354528307915, + "blocks.7.attention.w_k.weight": 0.039778005331754684, + "blocks.7.attention.w_out.weight": 0.057591959834098816, + "blocks.7.attention.w_q.weight": 0.04154594987630844, + "blocks.7.attention.w_v.weight": 0.03982347622513771, + "blocks.7.feed_forward.w1.weight": 0.040671318769454956, + "blocks.7.feed_forward.w2.weight": 0.0517294704914093, + "blocks.7.feed_forward.w3.weight": 0.04585808888077736, + "blocks.8.attention.w_k.weight": 0.03729722276329994, + "blocks.8.attention.w_out.weight": 0.0786689817905426, + "blocks.8.attention.w_q.weight": 0.04076381400227547, + "blocks.8.attention.w_v.weight": 0.03802540898323059, + "blocks.8.feed_forward.w1.weight": 0.038343265652656555, + "blocks.8.feed_forward.w2.weight": 0.05855748802423477, + "blocks.8.feed_forward.w3.weight": 0.044858478009700775 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.831054171073506 + }, + "base_loss": 3.831054171073506, + "loss": { + "0.005": [ + 3.83126655146913, + 3.831416064763763, + 3.8311541387232984 + ], + "0.01": [ + 3.8320505338347615, + 3.8322378597207867, + 3.8318906489747087 + ], + "0.02": [ + 3.835079523889827, + 3.8354311594720008, + 3.8344159588806828 + ], + "0.05": [ + 3.8555864952406136, + 3.857775685833163, + 3.8541448554719424 + ], + "0.1": [ + 3.9345608681548763, + 3.946372234551774, + 3.9354457101796987 + ], + "0.15": [ + 4.090792315924315, + 4.127439078418456, + 4.105478190432802 + ], + "0.2": [ + 4.360869604782873, + 4.4485717888851, + 4.4065429116301 + ], + "0.3": [ + 5.350963750428164, + 5.582059019661014, + 5.417273765133487 + ], + "0.4": [ + 6.654905765310262, + 6.920882303084387, + 6.549217831386812 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.83126655146913, + 3.831416064763763, + 3.8311541387232984 + ], + "0.01": [ + 3.8320505338347615, + 3.8322378597207867, + 3.8318906489747087 + ], + "0.02": [ + 3.835079523889827, + 3.8354311594720008, + 3.8344159588806828 + ], + "0.05": [ + 3.8555864952406136, + 3.857775685833163, + 3.8541448554719424 + ], + "0.1": [ + 3.9345608681548763, + 3.946372234551774, + 3.9354457101796987 + ], + "0.15": [ + 4.090792315924315, + 4.127439078418456, + 4.105478190432802 + ], + "0.2": [ + 4.360869604782873, + 4.4485717888851, + 4.4065429116301 + ], + "0.3": [ + 5.350963750428164, + 5.582059019661014, + 5.417273765133487 + ], + "0.4": [ + 6.654905765310262, + 6.920882303084387, + 6.549217831386812 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.000224747245224588, + "0.01": 0.0010055097699126492, + "0.02": 0.003921376340664473, + "0.05": 0.024781507775066824, + "0.1": 0.10773876655527692, + "0.15": 0.2768490238516841, + "0.2": 0.574273930692518, + "0.3": 1.6190446740007158, + "0.4": 2.8772811288536473 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.000224747245224588, + "0.01": 0.0010055097699126492, + "0.02": 0.003921376340664473, + "0.05": 0.024781507775066824, + "0.1": 0.10773876655527692, + "0.15": 0.2768490238516841, + "0.2": 0.574273930692518, + "0.3": 1.6190446740007158, + "0.4": 2.8772811288536473 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 49.8 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/ptq.json b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..9f27a4ddeaa6f8ef17ad715aaf81c7143484be1a --- /dev/null +++ b/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101/ptq.json @@ -0,0 +1,48 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-adamw-lr1.2e-03-s101", + "w_bits": null, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.831054171073506 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 6.164653690965879 + }, + "4": { + "val_dolma_cc": 4.078234822788718 + }, + "5": { + "val_dolma_cc": 3.8779693310580523 + }, + "6": { + "val_dolma_cc": 3.839041501003307 + }, + "8": { + "val_dolma_cc": 3.8320119113927316 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 2.3335995198923727 + }, + "4": { + "val_dolma_cc": 0.2471806517152122 + }, + "5": { + "val_dolma_cc": 0.04691515998454632 + }, + "6": { + "val_dolma_cc": 0.007987329929801223 + }, + "8": { + "val_dolma_cc": 0.0009577403192255751 + } + }, + "checks": { + "weights_restored": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..5589ce52c2457e6b9b1e49f6894ab9c309f50dd1 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": null, + "a": null, + "qkv": null + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.741282098723469, + "heldout_cc_en_tail_4M": 4.324322303835364 + }, + "loss_bf16_master": 3.741282098723469, + "losses_eval_model": { + "val_dolma_cc": 3.741282098723469 + }, + "bf16_minus_quant": 0.0, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-t5-28", + "gpu": "NVIDIA L40S", + "t_s": 17.7 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..6677fa9f39cf8b0db9a47ebb75d69b3ae973e103 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/finalize.json @@ -0,0 +1,29 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "w_bits": null, + "a_bits": null, + "qkv_bits": null, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.741282098723469 + }, + "losses_eval_model": { + "val_dolma_cc": 3.741282098723469 + }, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.7421518564224243, + "loss_eval_model": 3.741282098723469, + "checks": { + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..753aa0adc911994fddacb087fa0cadc0ed2ca329 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,126 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: null + fq_act_bits: null + fq_qkv_bits: null + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.01 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_all.json b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..4da46f94a6cf2b899a1fa4d2ec32f26aac21f7df --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.035397958010435104, + "blocks.0.attention.w_out.weight": 0.02164395898580551, + "blocks.0.attention.w_q.weight": 0.035328783094882965, + "blocks.0.attention.w_v.weight": 0.025519702583551407, + "blocks.0.attention_norm.weight": 0.38304951786994934, + "blocks.0.feed_forward.w1.weight": 0.030326496809720993, + "blocks.0.feed_forward.w2.weight": 0.01632818393409252, + "blocks.0.feed_forward.w3.weight": 0.030313409864902496, + "blocks.0.feed_forward_norm.weight": 0.39804232120513916, + "blocks.1.attention.w_k.weight": 0.027827803045511246, + "blocks.1.attention.w_out.weight": 0.01929481327533722, + "blocks.1.attention.w_q.weight": 0.028433021157979965, + "blocks.1.attention.w_v.weight": 0.021994709968566895, + "blocks.1.attention_norm.weight": 0.3855082094669342, + "blocks.1.feed_forward.w1.weight": 0.030362948775291443, + "blocks.1.feed_forward.w2.weight": 0.01613524742424488, + "blocks.1.feed_forward.w3.weight": 0.029716597869992256, + "blocks.1.feed_forward_norm.weight": 0.40586504340171814, + "blocks.2.attention.w_k.weight": 0.03229168802499771, + "blocks.2.attention.w_out.weight": 0.0226726196706295, + "blocks.2.attention.w_q.weight": 0.03159981593489647, + "blocks.2.attention.w_v.weight": 0.0259097870439291, + "blocks.2.attention_norm.weight": 0.3970188796520233, + "blocks.2.feed_forward.w1.weight": 0.029754923656582832, + "blocks.2.feed_forward.w2.weight": 0.01480945385992527, + "blocks.2.feed_forward.w3.weight": 0.02957199513912201, + "blocks.2.feed_forward_norm.weight": 0.40933650732040405, + "blocks.3.attention.w_k.weight": 0.032474976032972336, + "blocks.3.attention.w_out.weight": 0.02344447560608387, + "blocks.3.attention.w_q.weight": 0.03168802335858345, + "blocks.3.attention.w_v.weight": 0.026256276294589043, + "blocks.3.attention_norm.weight": 0.3984137773513794, + "blocks.3.feed_forward.w1.weight": 0.02918386459350586, + "blocks.3.feed_forward.w2.weight": 0.014518745243549347, + "blocks.3.feed_forward.w3.weight": 0.029343916103243828, + "blocks.3.feed_forward_norm.weight": 0.40580126643180847, + "blocks.4.attention.w_k.weight": 0.03286627680063248, + "blocks.4.attention.w_out.weight": 0.024900637567043304, + "blocks.4.attention.w_q.weight": 0.03240432217717171, + "blocks.4.attention.w_v.weight": 0.027380263432860374, + "blocks.4.attention_norm.weight": 0.3966514468193054, + "blocks.4.feed_forward.w1.weight": 0.02874669060111046, + "blocks.4.feed_forward.w2.weight": 0.01469415333122015, + "blocks.4.feed_forward.w3.weight": 0.029311303049325943, + "blocks.4.feed_forward_norm.weight": 0.40535956621170044, + "blocks.5.attention.w_k.weight": 0.03163629397749901, + "blocks.5.attention.w_out.weight": 0.026556408032774925, + "blocks.5.attention.w_q.weight": 0.0323525033891201, + "blocks.5.attention.w_v.weight": 0.027816232293844223, + "blocks.5.attention_norm.weight": 0.4023590683937073, + "blocks.5.feed_forward.w1.weight": 0.028784623369574547, + "blocks.5.feed_forward.w2.weight": 0.015200715512037277, + "blocks.5.feed_forward.w3.weight": 0.029435208067297935, + "blocks.5.feed_forward_norm.weight": 0.40602362155914307, + "blocks.6.attention.w_k.weight": 0.03339715674519539, + "blocks.6.attention.w_out.weight": 0.029964886605739594, + "blocks.6.attention.w_q.weight": 0.0327397957444191, + "blocks.6.attention.w_v.weight": 0.029403777793049812, + "blocks.6.attention_norm.weight": 0.40344297885894775, + "blocks.6.feed_forward.w1.weight": 0.02904530242085457, + "blocks.6.feed_forward.w2.weight": 0.017573032528162003, + "blocks.6.feed_forward.w3.weight": 0.02994123101234436, + "blocks.6.feed_forward_norm.weight": 0.408926784992218, + "blocks.7.attention.w_k.weight": 0.032194577157497406, + "blocks.7.attention.w_out.weight": 0.03255749121308327, + "blocks.7.attention.w_q.weight": 0.03139836713671684, + "blocks.7.attention.w_v.weight": 0.029072798788547516, + "blocks.7.attention_norm.weight": 0.40457212924957275, + "blocks.7.feed_forward.w1.weight": 0.028424231335520744, + "blocks.7.feed_forward.w2.weight": 0.02051093615591526, + "blocks.7.feed_forward.w3.weight": 0.029400411993265152, + "blocks.7.feed_forward_norm.weight": 0.4079188406467438, + "blocks.8.attention.w_k.weight": 0.030158329755067825, + "blocks.8.attention.w_out.weight": 0.038990020751953125, + "blocks.8.attention.w_q.weight": 0.029624801129102707, + "blocks.8.attention.w_v.weight": 0.02889910154044628, + "blocks.8.attention_norm.weight": 0.4107125699520111, + "blocks.8.feed_forward.w1.weight": 0.026936113834381104, + "blocks.8.feed_forward.w2.weight": 0.025364842265844345, + "blocks.8.feed_forward.w3.weight": 0.028375402092933655, + "blocks.8.feed_forward_norm.weight": 0.4011843800544739, + "embeddings.weight": 0.06977110356092453, + "lm_head.norm.weight": 1.3798266649246216, + "lm_head.w_out.weight": 0.07359108328819275 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.741282098723469 + }, + "base_loss": 3.741282098723469, + "loss": { + "0.005": [ + 3.7416292781709712, + 3.741794487701162, + 3.742089562845286 + ], + "0.01": [ + 3.74275158248727, + 3.7429588612151057, + 3.7436641863484628 + ], + "0.02": [ + 3.7473024344720507, + 3.747717246228647, + 3.749264851236619 + ], + "0.05": [ + 3.780543043428401, + 3.781887751509092, + 3.785293220390546 + ], + "0.1": [ + 3.916245888619137, + 3.919793292627658, + 3.924784930274286 + ], + "0.15": [ + 4.211604527414047, + 4.2234978373936345, + 4.22084451631432 + ], + "0.2": [ + 4.7966840943394145, + 4.855740092999335, + 4.809550477027847 + ], + "0.3": [ + 7.265636298785715, + 7.430777746750457, + 7.120484006803559 + ], + "0.4": [ + 9.54127430772559, + 9.48668455317653, + 9.123880935086254 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.7416292781709712, + 3.741794487701162, + 3.742089562845286 + ], + "0.01": [ + 3.74275158248727, + 3.7429588612151057, + 3.7436641863484628 + ], + "0.02": [ + 3.7473024344720507, + 3.747717246228647, + 3.749264851236619 + ], + "0.05": [ + 3.780543043428401, + 3.781887751509092, + 3.785293220390546 + ], + "0.1": [ + 3.916245888619137, + 3.919793292627658, + 3.924784930274286 + ], + "0.15": [ + 4.211604527414047, + 4.2234978373936345, + 4.22084451631432 + ], + "0.2": [ + 4.7966840943394145, + 4.855740092999335, + 4.809550477027847 + ], + "0.3": [ + 7.265636298785715, + 7.430777746750457, + 7.120484006803559 + ], + "0.4": [ + 9.54127430772559, + 9.48668455317653, + 9.123880935086254 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0005556775156705385, + "0.01": 0.00184277796014376, + "0.02": 0.0068127452556367984, + "0.05": 0.04129257305254397, + "0.1": 0.17899260511689175, + "0.15": 0.47736686165053177, + "0.2": 1.079376122732063, + "0.3": 3.5310172520564413, + "0.4": 5.642664499939322 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0005556775156705385, + "0.01": 0.00184277796014376, + "0.02": 0.0068127452556367984, + "0.05": 0.04129257305254397, + "0.1": 0.17899260511689175, + "0.15": 0.47736686165053177, + "0.2": 1.079376122732063, + "0.3": 3.5310172520564413, + "0.4": 5.642664499939322 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 56.5 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..f613dd919191204b3d266d8fc8eff81168a792f0 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.035397958010435104, + "blocks.0.attention.w_out.weight": 0.02164395898580551, + "blocks.0.attention.w_q.weight": 0.035328783094882965, + "blocks.0.attention.w_v.weight": 0.025519702583551407, + "blocks.0.feed_forward.w1.weight": 0.030326496809720993, + "blocks.0.feed_forward.w2.weight": 0.01632818393409252, + "blocks.0.feed_forward.w3.weight": 0.030313409864902496, + "blocks.1.attention.w_k.weight": 0.027827803045511246, + "blocks.1.attention.w_out.weight": 0.01929481327533722, + "blocks.1.attention.w_q.weight": 0.028433021157979965, + "blocks.1.attention.w_v.weight": 0.021994709968566895, + "blocks.1.feed_forward.w1.weight": 0.030362948775291443, + "blocks.1.feed_forward.w2.weight": 0.01613524742424488, + "blocks.1.feed_forward.w3.weight": 0.029716597869992256, + "blocks.2.attention.w_k.weight": 0.03229168802499771, + "blocks.2.attention.w_out.weight": 0.0226726196706295, + "blocks.2.attention.w_q.weight": 0.03159981593489647, + "blocks.2.attention.w_v.weight": 0.0259097870439291, + "blocks.2.feed_forward.w1.weight": 0.029754923656582832, + "blocks.2.feed_forward.w2.weight": 0.01480945385992527, + "blocks.2.feed_forward.w3.weight": 0.02957199513912201, + "blocks.3.attention.w_k.weight": 0.032474976032972336, + "blocks.3.attention.w_out.weight": 0.02344447560608387, + "blocks.3.attention.w_q.weight": 0.03168802335858345, + "blocks.3.attention.w_v.weight": 0.026256276294589043, + "blocks.3.feed_forward.w1.weight": 0.02918386459350586, + "blocks.3.feed_forward.w2.weight": 0.014518745243549347, + "blocks.3.feed_forward.w3.weight": 0.029343916103243828, + "blocks.4.attention.w_k.weight": 0.03286627680063248, + "blocks.4.attention.w_out.weight": 0.024900637567043304, + "blocks.4.attention.w_q.weight": 0.03240432217717171, + "blocks.4.attention.w_v.weight": 0.027380263432860374, + "blocks.4.feed_forward.w1.weight": 0.02874669060111046, + "blocks.4.feed_forward.w2.weight": 0.01469415333122015, + "blocks.4.feed_forward.w3.weight": 0.029311303049325943, + "blocks.5.attention.w_k.weight": 0.03163629397749901, + "blocks.5.attention.w_out.weight": 0.026556408032774925, + "blocks.5.attention.w_q.weight": 0.0323525033891201, + "blocks.5.attention.w_v.weight": 0.027816232293844223, + "blocks.5.feed_forward.w1.weight": 0.028784623369574547, + "blocks.5.feed_forward.w2.weight": 0.015200715512037277, + "blocks.5.feed_forward.w3.weight": 0.029435208067297935, + "blocks.6.attention.w_k.weight": 0.03339715674519539, + "blocks.6.attention.w_out.weight": 0.029964886605739594, + "blocks.6.attention.w_q.weight": 0.0327397957444191, + "blocks.6.attention.w_v.weight": 0.029403777793049812, + "blocks.6.feed_forward.w1.weight": 0.02904530242085457, + "blocks.6.feed_forward.w2.weight": 0.017573032528162003, + "blocks.6.feed_forward.w3.weight": 0.02994123101234436, + "blocks.7.attention.w_k.weight": 0.032194577157497406, + "blocks.7.attention.w_out.weight": 0.03255749121308327, + "blocks.7.attention.w_q.weight": 0.03139836713671684, + "blocks.7.attention.w_v.weight": 0.029072798788547516, + "blocks.7.feed_forward.w1.weight": 0.028424231335520744, + "blocks.7.feed_forward.w2.weight": 0.02051093615591526, + "blocks.7.feed_forward.w3.weight": 0.029400411993265152, + "blocks.8.attention.w_k.weight": 0.030158329755067825, + "blocks.8.attention.w_out.weight": 0.038990020751953125, + "blocks.8.attention.w_q.weight": 0.029624801129102707, + "blocks.8.attention.w_v.weight": 0.02889910154044628, + "blocks.8.feed_forward.w1.weight": 0.026936113834381104, + "blocks.8.feed_forward.w2.weight": 0.025364842265844345, + "blocks.8.feed_forward.w3.weight": 0.028375402092933655 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.741282098723469 + }, + "base_loss": 3.741282098723469, + "loss": { + "0.005": [ + 3.741544666384729, + 3.741646793636223, + 3.7415041934144293 + ], + "0.01": [ + 3.742238622310576, + 3.7425577856207837, + 3.7421255522946613 + ], + "0.02": [ + 3.7449042593793065, + 3.74560978012876, + 3.7447914123657418 + ], + "0.05": [ + 3.764693616993574, + 3.7656580669448987, + 3.7637789552510497 + ], + "0.1": [ + 3.8448218197141966, + 3.842812259470448, + 3.8400778085714027 + ], + "0.15": [ + 4.014115305391468, + 3.9956352023966195, + 3.998548066912348 + ], + "0.2": [ + 4.336877310147849, + 4.268974573538131, + 4.296562694990125 + ], + "0.3": [ + 5.680166579468835, + 5.429263905080685, + 5.506133190586791 + ], + "0.4": [ + 7.38970396117274, + 7.121290946674239, + 7.185266885769214 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.741544666384729, + 3.741646793636223, + 3.7415041934144293 + ], + "0.01": [ + 3.742238622310576, + 3.7425577856207837, + 3.7421255522946613 + ], + "0.02": [ + 3.7449042593793065, + 3.74560978012876, + 3.7447914123657418 + ], + "0.05": [ + 3.764693616993574, + 3.7656580669448987, + 3.7637789552510497 + ], + "0.1": [ + 3.8448218197141966, + 3.842812259470448, + 3.8400778085714027 + ], + "0.15": [ + 4.014115305391468, + 3.9956352023966195, + 3.998548066912348 + ], + "0.2": [ + 4.336877310147849, + 4.268974573538131, + 4.296562694990125 + ], + "0.3": [ + 5.680166579468835, + 5.429263905080685, + 5.506133190586791 + ], + "0.4": [ + 7.38970396117274, + 7.121290946674239, + 7.185266885769214 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0002831190883254564, + "0.01": 0.0010252213518713305, + "0.02": 0.0038197185678003898, + "0.05": 0.02342811433970482, + "0.1": 0.10128853052854714, + "0.15": 0.2614840928433435, + "0.2": 0.5595227608352329, + "0.3": 1.7972391263219678, + "0.4": 3.4908051658152615 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0002831190883254564, + "0.01": 0.0010252213518713305, + "0.02": 0.0038197185678003898, + "0.05": 0.02342811433970482, + "0.1": 0.10128853052854714, + "0.15": 0.2614840928433435, + "0.2": 0.5595227608352329, + "0.3": 1.7972391263219678, + "0.4": 3.4908051658152615 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 49.9 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ptq.json b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..68deac9fd272fa6f28589a379e9fb1347aa900ed --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101/ptq.json @@ -0,0 +1,48 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr1.0e-02-alr2.4e-03-s101", + "w_bits": null, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.741282098723469 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 5.78544710766109 + }, + "4": { + "val_dolma_cc": 3.9773186148965696 + }, + "5": { + "val_dolma_cc": 3.7802833630607204 + }, + "6": { + "val_dolma_cc": 3.751075317329996 + }, + "8": { + "val_dolma_cc": 3.7417057523276123 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 2.0441650089376213 + }, + "4": { + "val_dolma_cc": 0.23603651617310062 + }, + "5": { + "val_dolma_cc": 0.039001264337251484 + }, + "6": { + "val_dolma_cc": 0.009793218606527265 + }, + "8": { + "val_dolma_cc": 0.00042365360414331477 + } + }, + "checks": { + "weights_restored": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..f5af3a5f103f565e337cfaef9e4cc36a2fdf6dba --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": null, + "a": null, + "qkv": null + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.7980115686420324, + "heldout_cc_en_tail_4M": 4.380696555852657 + }, + "loss_bf16_master": 3.7980115686420324, + "losses_eval_model": { + "val_dolma_cc": 3.7980115686420324 + }, + "bf16_minus_quant": 0.0, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-q9-28", + "gpu": "NVIDIA L40S", + "t_s": 17.9 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..c64f914befeca34faaf6df7b845e7cbd719a5d94 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/finalize.json @@ -0,0 +1,29 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101", + "w_bits": null, + "a_bits": null, + "qkv_bits": null, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.7980115686420324 + }, + "losses_eval_model": { + "val_dolma_cc": 3.7980115686420324 + }, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.8346741199493408, + "loss_eval_model": 3.7980115686420324, + "checks": { + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71c5fe210878b5914c0d8012cb17d99439032f4a --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,126 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: null + fq_act_bits: null + fq_qkv_bits: null + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.0025 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/perturb_all.json b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..c395f275b7122e2de8b4e1a0dc78416ef3c596db --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.02823960967361927, + "blocks.0.attention.w_out.weight": 0.012285185977816582, + "blocks.0.attention.w_q.weight": 0.027905184775590897, + "blocks.0.attention.w_v.weight": 0.015026269480586052, + "blocks.0.attention_norm.weight": 0.36402034759521484, + "blocks.0.feed_forward.w1.weight": 0.018776489421725273, + "blocks.0.feed_forward.w2.weight": 0.011734351515769958, + "blocks.0.feed_forward.w3.weight": 0.01938738487660885, + "blocks.0.feed_forward_norm.weight": 0.40471863746643066, + "blocks.1.attention.w_k.weight": 0.021475251764059067, + "blocks.1.attention.w_out.weight": 0.013774644583463669, + "blocks.1.attention.w_q.weight": 0.02127489447593689, + "blocks.1.attention.w_v.weight": 0.01492380816489458, + "blocks.1.attention_norm.weight": 0.3935880959033966, + "blocks.1.feed_forward.w1.weight": 0.019992079585790634, + "blocks.1.feed_forward.w2.weight": 0.012382809072732925, + "blocks.1.feed_forward.w3.weight": 0.020290696993470192, + "blocks.1.feed_forward_norm.weight": 0.4071131944656372, + "blocks.2.attention.w_k.weight": 0.025532985106110573, + "blocks.2.attention.w_out.weight": 0.015362584963440895, + "blocks.2.attention.w_q.weight": 0.024703780189156532, + "blocks.2.attention.w_v.weight": 0.017002055421471596, + "blocks.2.attention_norm.weight": 0.404275506734848, + "blocks.2.feed_forward.w1.weight": 0.01942611113190651, + "blocks.2.feed_forward.w2.weight": 0.012278757989406586, + "blocks.2.feed_forward.w3.weight": 0.019887568429112434, + "blocks.2.feed_forward_norm.weight": 0.40656808018684387, + "blocks.3.attention.w_k.weight": 0.02614908292889595, + "blocks.3.attention.w_out.weight": 0.015408954583108425, + "blocks.3.attention.w_q.weight": 0.025211982429027557, + "blocks.3.attention.w_v.weight": 0.01707778498530388, + "blocks.3.attention_norm.weight": 0.40307778120040894, + "blocks.3.feed_forward.w1.weight": 0.018848182633519173, + "blocks.3.feed_forward.w2.weight": 0.011943705379962921, + "blocks.3.feed_forward.w3.weight": 0.01950244978070259, + "blocks.3.feed_forward_norm.weight": 0.4055529236793518, + "blocks.4.attention.w_k.weight": 0.026735495775938034, + "blocks.4.attention.w_out.weight": 0.016208117827773094, + "blocks.4.attention.w_q.weight": 0.025898175314068794, + "blocks.4.attention.w_v.weight": 0.017795512452721596, + "blocks.4.attention_norm.weight": 0.39835861325263977, + "blocks.4.feed_forward.w1.weight": 0.018612002953886986, + "blocks.4.feed_forward.w2.weight": 0.011872970499098301, + "blocks.4.feed_forward.w3.weight": 0.019427184015512466, + "blocks.4.feed_forward_norm.weight": 0.40461936593055725, + "blocks.5.attention.w_k.weight": 0.026231853291392326, + "blocks.5.attention.w_out.weight": 0.01922769658267498, + "blocks.5.attention.w_q.weight": 0.025986338034272194, + "blocks.5.attention.w_v.weight": 0.019273919984698296, + "blocks.5.attention_norm.weight": 0.4032162129878998, + "blocks.5.feed_forward.w1.weight": 0.018274476751685143, + "blocks.5.feed_forward.w2.weight": 0.011750449426472187, + "blocks.5.feed_forward.w3.weight": 0.019116736948490143, + "blocks.5.feed_forward_norm.weight": 0.40511777997016907, + "blocks.6.attention.w_k.weight": 0.027327269315719604, + "blocks.6.attention.w_out.weight": 0.017924083396792412, + "blocks.6.attention.w_q.weight": 0.026071004569530487, + "blocks.6.attention.w_v.weight": 0.019368091598153114, + "blocks.6.attention_norm.weight": 0.4013967216014862, + "blocks.6.feed_forward.w1.weight": 0.018336961045861244, + "blocks.6.feed_forward.w2.weight": 0.012955289334058762, + "blocks.6.feed_forward.w3.weight": 0.01925334520637989, + "blocks.6.feed_forward_norm.weight": 0.4079210162162781, + "blocks.7.attention.w_k.weight": 0.02898509055376053, + "blocks.7.attention.w_out.weight": 0.020589452236890793, + "blocks.7.attention.w_q.weight": 0.027352629229426384, + "blocks.7.attention.w_v.weight": 0.020123351365327835, + "blocks.7.attention_norm.weight": 0.4034270644187927, + "blocks.7.feed_forward.w1.weight": 0.01795170083642006, + "blocks.7.feed_forward.w2.weight": 0.01468669530004263, + "blocks.7.feed_forward.w3.weight": 0.019164539873600006, + "blocks.7.feed_forward_norm.weight": 0.40853628516197205, + "blocks.8.attention.w_k.weight": 0.02681003138422966, + "blocks.8.attention.w_out.weight": 0.025461845099925995, + "blocks.8.attention.w_q.weight": 0.025192417204380035, + "blocks.8.attention.w_v.weight": 0.0201339740306139, + "blocks.8.attention_norm.weight": 0.40670523047447205, + "blocks.8.feed_forward.w1.weight": 0.016974635422229767, + "blocks.8.feed_forward.w2.weight": 0.017902221530675888, + "blocks.8.feed_forward.w3.weight": 0.01865983195602894, + "blocks.8.feed_forward_norm.weight": 0.4039677381515503, + "embeddings.weight": 0.07120760530233383, + "lm_head.norm.weight": 1.4134982824325562, + "lm_head.w_out.weight": 0.0739491730928421 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.7980115686420324 + }, + "base_loss": 3.7980115686420324, + "loss": { + "0.005": [ + 3.798565494741589, + 3.798943397653158, + 3.7988173856763194 + ], + "0.01": [ + 3.8004938782177438, + 3.801210975527769, + 3.800666517247268 + ], + "0.02": [ + 3.808603229556021, + 3.8101174585456627, + 3.808464905502089 + ], + "0.05": [ + 3.869149998923329, + 3.873003010046592, + 3.8638678557245845 + ], + "0.1": [ + 4.129703283871569, + 4.14031559402067, + 4.10362852287375 + ], + "0.15": [ + 4.732539682837764, + 4.720558259543812, + 4.678776374996494 + ], + "0.2": [ + 5.877093123527715, + 5.69483182430382, + 5.735007019957474 + ], + "0.3": [ + 8.204661368819409, + 8.114015678896147, + 8.297994659586678 + ], + "0.4": [ + 9.525585677721825, + 9.636142527577801, + 10.106714495897632 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.798565494741589, + 3.798943397653158, + 3.7988173856763194 + ], + "0.01": [ + 3.8004938782177438, + 3.801210975527769, + 3.800666517247268 + ], + "0.02": [ + 3.808603229556021, + 3.8101174585456627, + 3.808464905502089 + ], + "0.05": [ + 3.869149998923329, + 3.873003010046592, + 3.8638678557245845 + ], + "0.1": [ + 4.129703283871569, + 4.14031559402067, + 4.10362852287375 + ], + "0.15": [ + 4.732539682837764, + 4.720558259543812, + 4.678776374996494 + ], + "0.2": [ + 5.877093123527715, + 5.69483182430382, + 5.735007019957474 + ], + "0.3": [ + 8.204661368819409, + 8.114015678896147, + 8.297994659586678 + ], + "0.4": [ + 9.525585677721825, + 9.636142527577801, + 10.106714495897632 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0007638573816564076, + "0.01": 0.002778888355561726, + "0.02": 0.011050295892558903, + "0.05": 0.07066205292280348, + "0.1": 0.32653756494663044, + "0.15": 0.9126132038173247, + "0.2": 1.9709657539543035, + "0.3": 4.407545667125378, + "0.4": 5.958135998423721 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0007638573816564076, + "0.01": 0.002778888355561726, + "0.02": 0.011050295892558903, + "0.05": 0.07066205292280348, + "0.1": 0.32653756494663044, + "0.15": 0.9126132038173247, + "0.2": 1.9709657539543035, + "0.3": 4.407545667125378, + "0.4": 5.958135998423721 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 57.6 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..ec75c895f0d901f39c97869e28814852e31f8b04 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.02823960967361927, + "blocks.0.attention.w_out.weight": 0.012285185977816582, + "blocks.0.attention.w_q.weight": 0.027905184775590897, + "blocks.0.attention.w_v.weight": 0.015026269480586052, + "blocks.0.feed_forward.w1.weight": 0.018776489421725273, + "blocks.0.feed_forward.w2.weight": 0.011734351515769958, + "blocks.0.feed_forward.w3.weight": 0.01938738487660885, + "blocks.1.attention.w_k.weight": 0.021475251764059067, + "blocks.1.attention.w_out.weight": 0.013774644583463669, + "blocks.1.attention.w_q.weight": 0.02127489447593689, + "blocks.1.attention.w_v.weight": 0.01492380816489458, + "blocks.1.feed_forward.w1.weight": 0.019992079585790634, + "blocks.1.feed_forward.w2.weight": 0.012382809072732925, + "blocks.1.feed_forward.w3.weight": 0.020290696993470192, + "blocks.2.attention.w_k.weight": 0.025532985106110573, + "blocks.2.attention.w_out.weight": 0.015362584963440895, + "blocks.2.attention.w_q.weight": 0.024703780189156532, + "blocks.2.attention.w_v.weight": 0.017002055421471596, + "blocks.2.feed_forward.w1.weight": 0.01942611113190651, + "blocks.2.feed_forward.w2.weight": 0.012278757989406586, + "blocks.2.feed_forward.w3.weight": 0.019887568429112434, + "blocks.3.attention.w_k.weight": 0.02614908292889595, + "blocks.3.attention.w_out.weight": 0.015408954583108425, + "blocks.3.attention.w_q.weight": 0.025211982429027557, + "blocks.3.attention.w_v.weight": 0.01707778498530388, + "blocks.3.feed_forward.w1.weight": 0.018848182633519173, + "blocks.3.feed_forward.w2.weight": 0.011943705379962921, + "blocks.3.feed_forward.w3.weight": 0.01950244978070259, + "blocks.4.attention.w_k.weight": 0.026735495775938034, + "blocks.4.attention.w_out.weight": 0.016208117827773094, + "blocks.4.attention.w_q.weight": 0.025898175314068794, + "blocks.4.attention.w_v.weight": 0.017795512452721596, + "blocks.4.feed_forward.w1.weight": 0.018612002953886986, + "blocks.4.feed_forward.w2.weight": 0.011872970499098301, + "blocks.4.feed_forward.w3.weight": 0.019427184015512466, + "blocks.5.attention.w_k.weight": 0.026231853291392326, + "blocks.5.attention.w_out.weight": 0.01922769658267498, + "blocks.5.attention.w_q.weight": 0.025986338034272194, + "blocks.5.attention.w_v.weight": 0.019273919984698296, + "blocks.5.feed_forward.w1.weight": 0.018274476751685143, + "blocks.5.feed_forward.w2.weight": 0.011750449426472187, + "blocks.5.feed_forward.w3.weight": 0.019116736948490143, + "blocks.6.attention.w_k.weight": 0.027327269315719604, + "blocks.6.attention.w_out.weight": 0.017924083396792412, + "blocks.6.attention.w_q.weight": 0.026071004569530487, + "blocks.6.attention.w_v.weight": 0.019368091598153114, + "blocks.6.feed_forward.w1.weight": 0.018336961045861244, + "blocks.6.feed_forward.w2.weight": 0.012955289334058762, + "blocks.6.feed_forward.w3.weight": 0.01925334520637989, + "blocks.7.attention.w_k.weight": 0.02898509055376053, + "blocks.7.attention.w_out.weight": 0.020589452236890793, + "blocks.7.attention.w_q.weight": 0.027352629229426384, + "blocks.7.attention.w_v.weight": 0.020123351365327835, + "blocks.7.feed_forward.w1.weight": 0.01795170083642006, + "blocks.7.feed_forward.w2.weight": 0.01468669530004263, + "blocks.7.feed_forward.w3.weight": 0.019164539873600006, + "blocks.8.attention.w_k.weight": 0.02681003138422966, + "blocks.8.attention.w_out.weight": 0.025461845099925995, + "blocks.8.attention.w_q.weight": 0.025192417204380035, + "blocks.8.attention.w_v.weight": 0.0201339740306139, + "blocks.8.feed_forward.w1.weight": 0.016974635422229767, + "blocks.8.feed_forward.w2.weight": 0.017902221530675888, + "blocks.8.feed_forward.w3.weight": 0.01865983195602894 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": null, + "base_losses": { + "val_dolma_cc": 3.7980115686420324 + }, + "base_loss": 3.7980115686420324, + "loss": { + "0.005": [ + 3.798558264770674, + 3.7982190821091835, + 3.798555240545827 + ], + "0.01": [ + 3.7999863209576263, + 3.7992850724637135, + 3.7998101021535424 + ], + "0.02": [ + 3.8054275391657852, + 3.8040605073761977, + 3.8053264134687033 + ], + "0.05": [ + 3.8446402723276525, + 3.8436552592122974, + 3.846435186941383 + ], + "0.1": [ + 4.01551527501933, + 4.025724475166709, + 4.028125819240211 + ], + "0.15": [ + 4.412840005031559, + 4.434557264906385, + 4.445230928087892 + ], + "0.2": [ + 5.227415800885474, + 5.085496949077468, + 5.149504562330737 + ], + "0.3": [ + 7.651915984545514, + 6.825729054930752, + 6.897205173366565 + ], + "0.4": [ + 8.87147847208868, + 8.323283079088746, + 8.34723364098581 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.798558264770674, + 3.7982190821091835, + 3.798555240545827 + ], + "0.01": [ + 3.7999863209576263, + 3.7992850724637135, + 3.7998101021535424 + ], + "0.02": [ + 3.8054275391657852, + 3.8040605073761977, + 3.8053264134687033 + ], + "0.05": [ + 3.8446402723276525, + 3.8436552592122974, + 3.846435186941383 + ], + "0.1": [ + 4.01551527501933, + 4.025724475166709, + 4.028125819240211 + ], + "0.15": [ + 4.412840005031559, + 4.434557264906385, + 4.445230928087892 + ], + "0.2": [ + 5.227415800885474, + 5.085496949077468, + 5.149504562330737 + ], + "0.3": [ + 7.651915984545514, + 6.825729054930752, + 6.897205173366565 + ], + "0.4": [ + 8.87147847208868, + 8.323283079088746, + 8.34723364098581 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0004326271665293646, + "0.01": 0.001682263216261859, + "0.02": 0.00692658469486318, + "0.05": 0.04689867085174537, + "0.1": 0.22511028783338416, + "0.15": 0.6328644973665791, + "0.2": 1.3561275354558604, + "0.3": 3.3269385023055773, + "0.4": 4.715986828745715 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0004326271665293646, + "0.01": 0.001682263216261859, + "0.02": 0.00692658469486318, + "0.05": 0.04689867085174537, + "0.1": 0.22511028783338416, + "0.15": 0.6328644973665791, + "0.2": 1.3561275354558604, + "0.3": 3.3269385023055773, + "0.4": 4.715986828745715 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 51.6 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/ptq.json b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..cbf559b92e8890b7d3eb07c8597c17ca365fc0a1 --- /dev/null +++ b/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101/ptq.json @@ -0,0 +1,48 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-bf16-muon-lr2.5e-03-alr2.4e-03-s101", + "w_bits": null, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.7980115686420324 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 7.490522650528621 + }, + "4": { + "val_dolma_cc": 4.375064463328336 + }, + "5": { + "val_dolma_cc": 3.8832731915557948 + }, + "6": { + "val_dolma_cc": 3.8167199790950557 + }, + "8": { + "val_dolma_cc": 3.7995107586662 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 3.6925110818865883 + }, + "4": { + "val_dolma_cc": 0.5770528946863034 + }, + "5": { + "val_dolma_cc": 0.08526162291376238 + }, + "6": { + "val_dolma_cc": 0.018708410453023294 + }, + "8": { + "val_dolma_cc": 0.0014991900241674294 + } + }, + "checks": { + "weights_restored": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..370c320b5ad59ab773a342bb1e9bcc217078c8db --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.872419299440231, + "heldout_cc_en_tail_4M": 4.45661268858849 + }, + "loss_bf16_master": 3.872419299440231, + "losses_eval_model": { + "val_dolma_cc": 3.904266601452955 + }, + "bf16_minus_quant": -0.03184730201272368, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-q9-24", + "gpu": "NVIDIA L40S", + "t_s": 18.6 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..91ae4bb7e86bbc426c495c46c38e2fda475cf07b --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/finalize.json @@ -0,0 +1,102 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.902572777325409 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.904266601452955 + }, + "losses_eval_model": { + "val_dolma_cc": 3.904266601452955 + }, + "max_distinct_values_per_row": 62, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.03132954612374306, + "blocks.0.attention.w_k": 0.032565560191869736, + "blocks.0.attention.w_v": 0.03376292809844017, + "blocks.0.attention.w_out": 0.035142719745635986, + "blocks.0.feed_forward.w1": 0.030101409181952477, + "blocks.0.feed_forward.w2": 0.03631531447172165, + "blocks.0.feed_forward.w3": 0.030498016625642776, + "blocks.1.attention.w_q": 0.03087511472404003, + "blocks.1.attention.w_k": 0.03758403658866882, + "blocks.1.attention.w_v": 0.031966060400009155, + "blocks.1.attention.w_out": 0.03447512537240982, + "blocks.1.feed_forward.w1": 0.03064761683344841, + "blocks.1.feed_forward.w2": 0.036250922828912735, + "blocks.1.feed_forward.w3": 0.03089948371052742, + "blocks.2.attention.w_q": 0.030825156718492508, + "blocks.2.attention.w_k": 0.03836624696850777, + "blocks.2.attention.w_v": 0.03155285865068436, + "blocks.2.attention.w_out": 0.03362602740526199, + "blocks.2.feed_forward.w1": 0.030612805858254433, + "blocks.2.feed_forward.w2": 0.036123599857091904, + "blocks.2.feed_forward.w3": 0.030936317518353462, + "blocks.3.attention.w_q": 0.03071513958275318, + "blocks.3.attention.w_k": 0.03686222434043884, + "blocks.3.attention.w_v": 0.03170269727706909, + "blocks.3.attention.w_out": 0.03152976930141449, + "blocks.3.feed_forward.w1": 0.030767356976866722, + "blocks.3.feed_forward.w2": 0.036432571709156036, + "blocks.3.feed_forward.w3": 0.031014615669846535, + "blocks.4.attention.w_q": 0.031317088752985, + "blocks.4.attention.w_k": 0.04460396245121956, + "blocks.4.attention.w_v": 0.032061345875263214, + "blocks.4.attention.w_out": 0.03140087053179741, + "blocks.4.feed_forward.w1": 0.03076915629208088, + "blocks.4.feed_forward.w2": 0.0370493121445179, + "blocks.4.feed_forward.w3": 0.031176608055830002, + "blocks.5.attention.w_q": 0.03135261312127113, + "blocks.5.attention.w_k": 0.04125480353832245, + "blocks.5.attention.w_v": 0.032418251037597656, + "blocks.5.attention.w_out": 0.03230922296643257, + "blocks.5.feed_forward.w1": 0.031060615554451942, + "blocks.5.feed_forward.w2": 0.037558216601610184, + "blocks.5.feed_forward.w3": 0.031182829290628433, + "blocks.6.attention.w_q": 0.031420376151800156, + "blocks.6.attention.w_k": 0.040330782532691956, + "blocks.6.attention.w_v": 0.0323045440018177, + "blocks.6.attention.w_out": 0.037702176719903946, + "blocks.6.feed_forward.w1": 0.030968982726335526, + "blocks.6.feed_forward.w2": 0.03673739731311798, + "blocks.6.feed_forward.w3": 0.031195146963000298, + "blocks.7.attention.w_q": 0.03101927787065506, + "blocks.7.attention.w_k": 0.04204888641834259, + "blocks.7.attention.w_v": 0.032408490777015686, + "blocks.7.attention.w_out": 0.03582685440778732, + "blocks.7.feed_forward.w1": 0.03105190210044384, + "blocks.7.feed_forward.w2": 0.03869321197271347, + "blocks.7.feed_forward.w3": 0.03149225935339928, + "blocks.8.attention.w_q": 0.03125115856528282, + "blocks.8.attention.w_k": 0.033950477838516235, + "blocks.8.attention.w_v": 0.03320341184735298, + "blocks.8.attention.w_out": 0.03774721175432205, + "blocks.8.feed_forward.w1": 0.031206922605633736, + "blocks.8.feed_forward.w2": 0.04580705240368843, + "blocks.8.feed_forward.w3": 0.03194732218980789 + }, + "rel_quant_noise_mean": 0.03376682516601351, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.8170008659362793, + "loss_eval_model": 3.904266601452955, + "checks": { + "snap_reproduces_fakequant": true, + "grid_levels_ok": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1bbe4d68f7105f6749f302d95aaf6456c0eb6df --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/launch.yaml @@ -0,0 +1,118 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: olmo_core.optim.AdamWConfig + lr: 0.0024 + betas: + - 0.9 + - 0.95 + eps: 1.0e-15 + weight_decay: 0.1 + fused: true + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/perturb_all.json b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..0e89bfe1900fdce295a8c1f6e4af779ff70cc9bc --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.04447223246097565, + "blocks.0.attention.w_out.weight": 0.06222694367170334, + "blocks.0.attention.w_q.weight": 0.04480505734682083, + "blocks.0.attention.w_v.weight": 0.11886989325284958, + "blocks.0.attention_norm.weight": 0.404301255941391, + "blocks.0.feed_forward.w1.weight": 0.057579103857278824, + "blocks.0.feed_forward.w2.weight": 0.04649105295538902, + "blocks.0.feed_forward.w3.weight": 0.06854056566953659, + "blocks.0.feed_forward_norm.weight": 0.398822158575058, + "blocks.1.attention.w_k.weight": 0.04462267458438873, + "blocks.1.attention.w_out.weight": 0.0488797165453434, + "blocks.1.attention.w_q.weight": 0.03434634581208229, + "blocks.1.attention.w_v.weight": 0.08584452420473099, + "blocks.1.attention_norm.weight": 0.36681878566741943, + "blocks.1.feed_forward.w1.weight": 0.062314797192811966, + "blocks.1.feed_forward.w2.weight": 0.05205081030726433, + "blocks.1.feed_forward.w3.weight": 0.07317005097866058, + "blocks.1.feed_forward_norm.weight": 0.4142328202724457, + "blocks.2.attention.w_k.weight": 0.044960957020521164, + "blocks.2.attention.w_out.weight": 0.05148233845829964, + "blocks.2.attention.w_q.weight": 0.03788594901561737, + "blocks.2.attention.w_v.weight": 0.10192523151636124, + "blocks.2.attention_norm.weight": 0.3903007209300995, + "blocks.2.feed_forward.w1.weight": 0.06271107494831085, + "blocks.2.feed_forward.w2.weight": 0.054983798414468765, + "blocks.2.feed_forward.w3.weight": 0.073421411216259, + "blocks.2.feed_forward_norm.weight": 0.4138107895851135, + "blocks.3.attention.w_k.weight": 0.0436176098883152, + "blocks.3.attention.w_out.weight": 0.05381327494978905, + "blocks.3.attention.w_q.weight": 0.03841737285256386, + "blocks.3.attention.w_v.weight": 0.09413135051727295, + "blocks.3.attention_norm.weight": 0.38852155208587646, + "blocks.3.feed_forward.w1.weight": 0.06022169068455696, + "blocks.3.feed_forward.w2.weight": 0.057319436222314835, + "blocks.3.feed_forward.w3.weight": 0.07215607166290283, + "blocks.3.feed_forward_norm.weight": 0.41180163621902466, + "blocks.4.attention.w_k.weight": 0.04765772819519043, + "blocks.4.attention.w_out.weight": 0.06548422574996948, + "blocks.4.attention.w_q.weight": 0.033922698348760605, + "blocks.4.attention.w_v.weight": 0.12088527530431747, + "blocks.4.attention_norm.weight": 0.40293774008750916, + "blocks.4.feed_forward.w1.weight": 0.05975082889199257, + "blocks.4.feed_forward.w2.weight": 0.06048689782619476, + "blocks.4.feed_forward.w3.weight": 0.07176288217306137, + "blocks.4.feed_forward_norm.weight": 0.41228407621383667, + "blocks.5.attention.w_k.weight": 0.045243971049785614, + "blocks.5.attention.w_out.weight": 0.08075490593910217, + "blocks.5.attention.w_q.weight": 0.034430962055921555, + "blocks.5.attention.w_v.weight": 0.07127781957387924, + "blocks.5.attention_norm.weight": 0.39218249917030334, + "blocks.5.feed_forward.w1.weight": 0.059570491313934326, + "blocks.5.feed_forward.w2.weight": 0.06447386741638184, + "blocks.5.feed_forward.w3.weight": 0.0718831717967987, + "blocks.5.feed_forward_norm.weight": 0.41259849071502686, + "blocks.6.attention.w_k.weight": 0.050092969089746475, + "blocks.6.attention.w_out.weight": 0.09478561580181122, + "blocks.6.attention.w_q.weight": 0.03283941373229027, + "blocks.6.attention.w_v.weight": 0.0808403417468071, + "blocks.6.attention_norm.weight": 0.40019938349723816, + "blocks.6.feed_forward.w1.weight": 0.0594840906560421, + "blocks.6.feed_forward.w2.weight": 0.075862817466259, + "blocks.6.feed_forward.w3.weight": 0.07260943949222565, + "blocks.6.feed_forward_norm.weight": 0.41187795996665955, + "blocks.7.attention.w_k.weight": 0.050687216222286224, + "blocks.7.attention.w_out.weight": 0.10306242853403091, + "blocks.7.attention.w_q.weight": 0.03203458711504936, + "blocks.7.attention.w_v.weight": 0.09186305850744247, + "blocks.7.attention_norm.weight": 0.40369895100593567, + "blocks.7.feed_forward.w1.weight": 0.057336632162332535, + "blocks.7.feed_forward.w2.weight": 0.08863422274589539, + "blocks.7.feed_forward.w3.weight": 0.07244992256164551, + "blocks.7.feed_forward_norm.weight": 0.4105563461780548, + "blocks.8.attention.w_k.weight": 0.0413111075758934, + "blocks.8.attention.w_out.weight": 0.17310740053653717, + "blocks.8.attention.w_q.weight": 0.03296327218413353, + "blocks.8.attention.w_v.weight": 0.09154683351516724, + "blocks.8.attention_norm.weight": 0.4069346785545349, + "blocks.8.feed_forward.w1.weight": 0.05462755262851715, + "blocks.8.feed_forward.w2.weight": 0.10933306068181992, + "blocks.8.feed_forward.w3.weight": 0.07172390073537827, + "blocks.8.feed_forward_norm.weight": 0.40633541345596313, + "embeddings.weight": 0.06951554119586945, + "lm_head.norm.weight": 1.401055932044983, + "lm_head.w_out.weight": 0.07445036619901657 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.904266601452955 + }, + "base_loss": 3.904266601452955, + "loss": { + "0.005": [ + 3.904142573023332, + 3.905856580819682, + 3.904488195366709 + ], + "0.01": [ + 3.906111492066938, + 3.905960675969104, + 3.9051861183869865 + ], + "0.02": [ + 3.909523917056772, + 3.909952218499361, + 3.9093763325368474 + ], + "0.05": [ + 3.925472618660411, + 3.9244579266709434, + 3.9253922126202365 + ], + "0.1": [ + 3.984940330517964, + 3.9796524988031425, + 3.978508801910565 + ], + "0.15": [ + 4.09187802438134, + 4.074926657265826, + 4.0745152218430505 + ], + "0.2": [ + 4.262114352255951, + 4.221008223512892, + 4.219886568642033 + ], + "0.3": [ + 4.900139414809284, + 4.741884334174138, + 4.7508327298968 + ], + "0.4": [ + 6.209186188214108, + 5.761083701905357, + 5.801262238681934 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.904142573023332, + 3.905856580819682, + 3.904488195366709 + ], + "0.01": [ + 3.906111492066938, + 3.905960675969104, + 3.9051861183869865 + ], + "0.02": [ + 3.909523917056772, + 3.909952218499361, + 3.9093763325368474 + ], + "0.05": [ + 3.925472618660411, + 3.9244579266709434, + 3.9253922126202365 + ], + "0.1": [ + 3.984940330517964, + 3.9796524988031425, + 3.978508801910565 + ], + "0.15": [ + 4.09187802438134, + 4.074926657265826, + 4.0745152218430505 + ], + "0.2": [ + 4.262114352255951, + 4.221008223512892, + 4.219886568642033 + ], + "0.3": [ + 4.900139414809284, + 4.741884334174138, + 4.7508327298968 + ], + "0.4": [ + 6.209186188214108, + 5.761083701905357, + 5.801262238681934 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0005625149502859195, + "0.01": 0.0014861606880547384, + "0.02": 0.005350887911372393, + "0.05": 0.020840984530908546, + "0.1": 0.076767275624269, + "0.15": 0.17617336637711745, + "0.2": 0.3300697800173369, + "0.3": 0.8933522248404531, + "0.4": 2.019577441480844 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0005625149502859195, + "0.01": 0.0014861606880547384, + "0.02": 0.005350887911372393, + "0.05": 0.020840984530908546, + "0.1": 0.076767275624269, + "0.15": 0.17617336637711745, + "0.2": 0.3300697800173369, + "0.3": 0.8933522248404531, + "0.4": 2.019577441480844 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 78.5 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..b6580ddf57fe2f1738f9150be5292025cf8b3a60 --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.04447223246097565, + "blocks.0.attention.w_out.weight": 0.06222694367170334, + "blocks.0.attention.w_q.weight": 0.04480505734682083, + "blocks.0.attention.w_v.weight": 0.11886989325284958, + "blocks.0.feed_forward.w1.weight": 0.057579103857278824, + "blocks.0.feed_forward.w2.weight": 0.04649105295538902, + "blocks.0.feed_forward.w3.weight": 0.06854056566953659, + "blocks.1.attention.w_k.weight": 0.04462267458438873, + "blocks.1.attention.w_out.weight": 0.0488797165453434, + "blocks.1.attention.w_q.weight": 0.03434634581208229, + "blocks.1.attention.w_v.weight": 0.08584452420473099, + "blocks.1.feed_forward.w1.weight": 0.062314797192811966, + "blocks.1.feed_forward.w2.weight": 0.05205081030726433, + "blocks.1.feed_forward.w3.weight": 0.07317005097866058, + "blocks.2.attention.w_k.weight": 0.044960957020521164, + "blocks.2.attention.w_out.weight": 0.05148233845829964, + "blocks.2.attention.w_q.weight": 0.03788594901561737, + "blocks.2.attention.w_v.weight": 0.10192523151636124, + "blocks.2.feed_forward.w1.weight": 0.06271107494831085, + "blocks.2.feed_forward.w2.weight": 0.054983798414468765, + "blocks.2.feed_forward.w3.weight": 0.073421411216259, + "blocks.3.attention.w_k.weight": 0.0436176098883152, + "blocks.3.attention.w_out.weight": 0.05381327494978905, + "blocks.3.attention.w_q.weight": 0.03841737285256386, + "blocks.3.attention.w_v.weight": 0.09413135051727295, + "blocks.3.feed_forward.w1.weight": 0.06022169068455696, + "blocks.3.feed_forward.w2.weight": 0.057319436222314835, + "blocks.3.feed_forward.w3.weight": 0.07215607166290283, + "blocks.4.attention.w_k.weight": 0.04765772819519043, + "blocks.4.attention.w_out.weight": 0.06548422574996948, + "blocks.4.attention.w_q.weight": 0.033922698348760605, + "blocks.4.attention.w_v.weight": 0.12088527530431747, + "blocks.4.feed_forward.w1.weight": 0.05975082889199257, + "blocks.4.feed_forward.w2.weight": 0.06048689782619476, + "blocks.4.feed_forward.w3.weight": 0.07176288217306137, + "blocks.5.attention.w_k.weight": 0.045243971049785614, + "blocks.5.attention.w_out.weight": 0.08075490593910217, + "blocks.5.attention.w_q.weight": 0.034430962055921555, + "blocks.5.attention.w_v.weight": 0.07127781957387924, + "blocks.5.feed_forward.w1.weight": 0.059570491313934326, + "blocks.5.feed_forward.w2.weight": 0.06447386741638184, + "blocks.5.feed_forward.w3.weight": 0.0718831717967987, + "blocks.6.attention.w_k.weight": 0.050092969089746475, + "blocks.6.attention.w_out.weight": 0.09478561580181122, + "blocks.6.attention.w_q.weight": 0.03283941373229027, + "blocks.6.attention.w_v.weight": 0.0808403417468071, + "blocks.6.feed_forward.w1.weight": 0.0594840906560421, + "blocks.6.feed_forward.w2.weight": 0.075862817466259, + "blocks.6.feed_forward.w3.weight": 0.07260943949222565, + "blocks.7.attention.w_k.weight": 0.050687216222286224, + "blocks.7.attention.w_out.weight": 0.10306242853403091, + "blocks.7.attention.w_q.weight": 0.03203458711504936, + "blocks.7.attention.w_v.weight": 0.09186305850744247, + "blocks.7.feed_forward.w1.weight": 0.057336632162332535, + "blocks.7.feed_forward.w2.weight": 0.08863422274589539, + "blocks.7.feed_forward.w3.weight": 0.07244992256164551, + "blocks.8.attention.w_k.weight": 0.0413111075758934, + "blocks.8.attention.w_out.weight": 0.17310740053653717, + "blocks.8.attention.w_q.weight": 0.03296327218413353, + "blocks.8.attention.w_v.weight": 0.09154683351516724, + "blocks.8.feed_forward.w1.weight": 0.05462755262851715, + "blocks.8.feed_forward.w2.weight": 0.10933306068181992, + "blocks.8.feed_forward.w3.weight": 0.07172390073537827 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.904266601452955 + }, + "base_loss": 3.904266601452955, + "loss": { + "0.005": [ + 3.9049811009812228, + 3.9050922656963607, + 3.9045786482495832 + ], + "0.01": [ + 3.9052768881679367, + 3.9059409226557116, + 3.9047522105870813 + ], + "0.02": [ + 3.9056773221414636, + 3.906056570891774, + 3.906350495813252 + ], + "0.05": [ + 3.9104463643199243, + 3.909633035627544, + 3.9087067855874817 + ], + "0.1": [ + 3.927618077318498, + 3.925788155248195, + 3.9249304138773375 + ], + "0.15": [ + 3.955862181699748, + 3.952781071692016, + 3.952633987949297 + ], + "0.2": [ + 4.0011487398755365, + 3.9967307581679266, + 3.995774160246703 + ], + "0.3": [ + 4.152392294090428, + 4.146487391400203, + 4.144532861876632 + ], + "0.4": [ + 4.442498969337913, + 4.438886389958466, + 4.428868185455077 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.9049811009812228, + 3.9050922656963607, + 3.9045786482495832 + ], + "0.01": [ + 3.9052768881679367, + 3.9059409226557116, + 3.9047522105870813 + ], + "0.02": [ + 3.9056773221414636, + 3.906056570891774, + 3.906350495813252 + ], + "0.05": [ + 3.9104463643199243, + 3.909633035627544, + 3.9087067855874817 + ], + "0.1": [ + 3.927618077318498, + 3.925788155248195, + 3.9249304138773375 + ], + "0.15": [ + 3.955862181699748, + 3.952781071692016, + 3.952633987949297 + ], + "0.2": [ + 4.0011487398755365, + 3.9967307581679266, + 3.995774160246703 + ], + "0.3": [ + 4.152392294090428, + 4.146487391400203, + 4.144532861876632 + ], + "0.4": [ + 4.442498969337913, + 4.438886389958466, + 4.428868185455077 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0006174035227672903, + "0.01": 0.0010567390172884217, + "0.02": 0.0017615281625413814, + "0.05": 0.00532879372536188, + "0.1": 0.02184561402838847, + "0.15": 0.04949247899406517, + "0.2": 0.09361795131043449, + "0.3": 0.24353758100280043, + "0.4": 0.5324845801308635 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0006174035227672903, + "0.01": 0.0010567390172884217, + "0.02": 0.0017615281625413814, + "0.05": 0.00532879372536188, + "0.1": 0.02184561402838847, + "0.15": 0.04949247899406517, + "0.2": 0.09361795131043449, + "0.3": 0.24353758100280043, + "0.4": 0.5324845801308635 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 72.9 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/ptq.json b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..13d717b21002726276d6bc9daf5df4aeae4bf70a --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101/ptq.json @@ -0,0 +1,52 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr2.4e-03-s101", + "w_bits": 6, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.904266601452955 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.34406060706216 + }, + "4": { + "val_dolma_cc": 3.9607311680540698 + }, + "5": { + "val_dolma_cc": 3.923247943308014 + }, + "6": { + "val_dolma_cc": 3.904266601452955 + }, + "8": { + "val_dolma_cc": 3.9050877665261488 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 0.43979400560920556 + }, + "4": { + "val_dolma_cc": 0.056464566601114985 + }, + "5": { + "val_dolma_cc": 0.018981341855059064 + }, + "6": { + "val_dolma_cc": 0.0 + }, + "8": { + "val_dolma_cc": 0.0008211650731939635 + } + }, + "own_grid_max_rel_weight_change": 2.2933392074264702e-07, + "own_grid_delta_loss": 0.0, + "checks": { + "weights_restored": true, + "own_grid_weights_unchanged": true, + "own_grid_is_fixed_point": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/finalize.json b/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..fd0390ba6f37d8e786efb7ab25f3a1470265b4d1 --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/finalize.json @@ -0,0 +1,102 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.9273053028299922 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.9291688200086536 + }, + "losses_eval_model": { + "val_dolma_cc": 3.9291688200086536 + }, + "max_distinct_values_per_row": 62, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.029958514496684074, + "blocks.0.attention.w_k": 0.030053334310650826, + "blocks.0.attention.w_v": 0.03284899517893791, + "blocks.0.attention.w_out": 0.03345663100481033, + "blocks.0.feed_forward.w1": 0.029894156381487846, + "blocks.0.feed_forward.w2": 0.03646356984972954, + "blocks.0.feed_forward.w3": 0.029902366921305656, + "blocks.1.attention.w_q": 0.03054300881922245, + "blocks.1.attention.w_k": 0.03423832729458809, + "blocks.1.attention.w_v": 0.03337271139025688, + "blocks.1.attention.w_out": 0.03670801594853401, + "blocks.1.feed_forward.w1": 0.03054652363061905, + "blocks.1.feed_forward.w2": 0.03641407936811447, + "blocks.1.feed_forward.w3": 0.0308720413595438, + "blocks.2.attention.w_q": 0.03073035180568695, + "blocks.2.attention.w_k": 0.0352826826274395, + "blocks.2.attention.w_v": 0.03186545521020889, + "blocks.2.attention.w_out": 0.03409954160451889, + "blocks.2.feed_forward.w1": 0.030620494857430458, + "blocks.2.feed_forward.w2": 0.039687350392341614, + "blocks.2.feed_forward.w3": 0.030994972214102745, + "blocks.3.attention.w_q": 0.03141950070858002, + "blocks.3.attention.w_k": 0.04081984609365463, + "blocks.3.attention.w_v": 0.032464489340782166, + "blocks.3.attention.w_out": 0.03311280161142349, + "blocks.3.feed_forward.w1": 0.030588535591959953, + "blocks.3.feed_forward.w2": 0.0373603031039238, + "blocks.3.feed_forward.w3": 0.031179826706647873, + "blocks.4.attention.w_q": 0.032007645815610886, + "blocks.4.attention.w_k": 0.04560428112745285, + "blocks.4.attention.w_v": 0.0322083942592144, + "blocks.4.attention.w_out": 0.03341170400381088, + "blocks.4.feed_forward.w1": 0.030829299241304398, + "blocks.4.feed_forward.w2": 0.03711560368537903, + "blocks.4.feed_forward.w3": 0.03147368133068085, + "blocks.5.attention.w_q": 0.03180655464529991, + "blocks.5.attention.w_k": 0.043339189141988754, + "blocks.5.attention.w_v": 0.032215967774391174, + "blocks.5.attention.w_out": 0.03398735448718071, + "blocks.5.feed_forward.w1": 0.03098096139729023, + "blocks.5.feed_forward.w2": 0.03750373050570488, + "blocks.5.feed_forward.w3": 0.031507816165685654, + "blocks.6.attention.w_q": 0.03224027529358864, + "blocks.6.attention.w_k": 0.038728248327970505, + "blocks.6.attention.w_v": 0.03304196894168854, + "blocks.6.attention.w_out": 0.036059584468603134, + "blocks.6.feed_forward.w1": 0.030856654047966003, + "blocks.6.feed_forward.w2": 0.038444433361291885, + "blocks.6.feed_forward.w3": 0.031284697353839874, + "blocks.7.attention.w_q": 0.031709592789411545, + "blocks.7.attention.w_k": 0.04747031256556511, + "blocks.7.attention.w_v": 0.032274387776851654, + "blocks.7.attention.w_out": 0.0332445427775383, + "blocks.7.feed_forward.w1": 0.030889054760336876, + "blocks.7.feed_forward.w2": 0.040012676268815994, + "blocks.7.feed_forward.w3": 0.03180297836661339, + "blocks.8.attention.w_q": 0.031886614859104156, + "blocks.8.attention.w_k": 0.03702918440103531, + "blocks.8.attention.w_v": 0.03320465236902237, + "blocks.8.attention.w_out": 0.03827323764562607, + "blocks.8.feed_forward.w1": 0.03127485141158104, + "blocks.8.feed_forward.w2": 0.04559102654457092, + "blocks.8.feed_forward.w3": 0.03272668644785881 + }, + "rel_quant_noise_mean": 0.03408787733623906, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.9966011643409729, + "loss_eval_model": 3.9291688200086536, + "checks": { + "snap_reproduces_fakequant": true, + "grid_levels_ok": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..9f69c6df506edda57787ea70a26673e2564ca498 --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.10295666009187698, + "blocks.0.attention.w_out.weight": 0.07433991879224777, + "blocks.0.attention.w_q.weight": 0.10253647714853287, + "blocks.0.attention.w_v.weight": 0.2449776977300644, + "blocks.0.feed_forward.w1.weight": 0.09247293323278427, + "blocks.0.feed_forward.w2.weight": 0.06845320016145706, + "blocks.0.feed_forward.w3.weight": 0.09594220668077469, + "blocks.1.attention.w_k.weight": 0.0940186083316803, + "blocks.1.attention.w_out.weight": 0.059166502207517624, + "blocks.1.attention.w_q.weight": 0.09678329527378082, + "blocks.1.attention.w_v.weight": 0.12398216873407364, + "blocks.1.feed_forward.w1.weight": 0.09601962566375732, + "blocks.1.feed_forward.w2.weight": 0.07874780148267746, + "blocks.1.feed_forward.w3.weight": 0.1040811538696289, + "blocks.2.attention.w_k.weight": 0.09405351430177689, + "blocks.2.attention.w_out.weight": 0.06419295817613602, + "blocks.2.attention.w_q.weight": 0.09307216107845306, + "blocks.2.attention.w_v.weight": 0.1580667495727539, + "blocks.2.feed_forward.w1.weight": 0.09151057153940201, + "blocks.2.feed_forward.w2.weight": 0.08589061349630356, + "blocks.2.feed_forward.w3.weight": 0.10345649719238281, + "blocks.3.attention.w_k.weight": 0.05170898512005806, + "blocks.3.attention.w_out.weight": 0.07905052602291107, + "blocks.3.attention.w_q.weight": 0.039911024272441864, + "blocks.3.attention.w_v.weight": 0.13418684899806976, + "blocks.3.feed_forward.w1.weight": 0.08949227631092072, + "blocks.3.feed_forward.w2.weight": 0.08585387468338013, + "blocks.3.feed_forward.w3.weight": 0.10214531421661377, + "blocks.4.attention.w_k.weight": 0.09089221805334091, + "blocks.4.attention.w_out.weight": 0.08829694986343384, + "blocks.4.attention.w_q.weight": 0.08905211836099625, + "blocks.4.attention.w_v.weight": 0.10798521339893341, + "blocks.4.feed_forward.w1.weight": 0.08801241219043732, + "blocks.4.feed_forward.w2.weight": 0.086526058614254, + "blocks.4.feed_forward.w3.weight": 0.09784262627363205, + "blocks.5.attention.w_k.weight": 0.05476023629307747, + "blocks.5.attention.w_out.weight": 0.09816227108240128, + "blocks.5.attention.w_q.weight": 0.03678838536143303, + "blocks.5.attention.w_v.weight": 0.14953851699829102, + "blocks.5.feed_forward.w1.weight": 0.08689950406551361, + "blocks.5.feed_forward.w2.weight": 0.09530370682477951, + "blocks.5.feed_forward.w3.weight": 0.09842880815267563, + "blocks.6.attention.w_k.weight": 0.046556148678064346, + "blocks.6.attention.w_out.weight": 0.14122274518013, + "blocks.6.attention.w_q.weight": 0.03567918762564659, + "blocks.6.attention.w_v.weight": 0.17810435593128204, + "blocks.6.feed_forward.w1.weight": 0.08768820762634277, + "blocks.6.feed_forward.w2.weight": 0.11142566800117493, + "blocks.6.feed_forward.w3.weight": 0.10013778507709503, + "blocks.7.attention.w_k.weight": 0.06066993623971939, + "blocks.7.attention.w_out.weight": 0.14124080538749695, + "blocks.7.attention.w_q.weight": 0.032068297266960144, + "blocks.7.attention.w_v.weight": 0.13702520728111267, + "blocks.7.feed_forward.w1.weight": 0.08460602909326553, + "blocks.7.feed_forward.w2.weight": 0.13135100901126862, + "blocks.7.feed_forward.w3.weight": 0.09995918720960617, + "blocks.8.attention.w_k.weight": 0.04496841877698898, + "blocks.8.attention.w_out.weight": 0.20446938276290894, + "blocks.8.attention.w_q.weight": 0.03546059504151344, + "blocks.8.attention.w_v.weight": 0.1397651731967926, + "blocks.8.feed_forward.w1.weight": 0.08004491031169891, + "blocks.8.feed_forward.w2.weight": 0.14966775476932526, + "blocks.8.feed_forward.w3.weight": 0.09858717769384384 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.9291688200086536 + }, + "base_loss": 3.9291688200086536, + "loss": { + "0.005": [ + 3.929990462385122, + 3.9300811226210577, + 3.9286797484696248 + ], + "0.01": [ + 3.9299742810216456, + 3.929023837182808, + 3.9296316007443552 + ], + "0.02": [ + 3.9296030525312786, + 3.929805515190826, + 3.930205507071976 + ], + "0.05": [ + 3.9350386917683746, + 3.934420873539483, + 3.935254033791695 + ], + "0.1": [ + 3.951707530556173, + 3.952103301041967, + 3.953444804931003 + ], + "0.15": [ + 3.984590967994828, + 3.9838400564864167, + 3.9871740785449017 + ], + "0.2": [ + 4.031742510469999, + 4.0354458752796845, + 4.036791694459802 + ], + "0.3": [ + 4.207225285518124, + 4.217448648270866, + 4.214980704741259 + ], + "0.4": [ + 4.549954341638222, + 4.537920336737617, + 4.557709361804521 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.929990462385122, + 3.9300811226210577, + 3.9286797484696248 + ], + "0.01": [ + 3.9299742810216456, + 3.929023837182808, + 3.9296316007443552 + ], + "0.02": [ + 3.9296030525312786, + 3.929805515190826, + 3.930205507071976 + ], + "0.05": [ + 3.9350386917683746, + 3.934420873539483, + 3.935254033791695 + ], + "0.1": [ + 3.951707530556173, + 3.952103301041967, + 3.953444804931003 + ], + "0.15": [ + 3.984590967994828, + 3.9838400564864167, + 3.9871740785449017 + ], + "0.2": [ + 4.031742510469999, + 4.0354458752796845, + 4.036791694459802 + ], + "0.3": [ + 4.207225285518124, + 4.217448648270866, + 4.214980704741259 + ], + "0.4": [ + 4.549954341638222, + 4.537920336737617, + 4.557709361804521 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00041495781661460285, + "0.01": 0.0003744196409489753, + "0.02": 0.0007025382560397198, + "0.05": 0.0057357130245310195, + "0.1": 0.023249725501060414, + "0.15": 0.056032881000061874, + "0.2": 0.10549120672784174, + "0.3": 0.2840493928347625, + "0.4": 0.6193591933847999 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00041495781661460285, + "0.01": 0.0003744196409489753, + "0.02": 0.0007025382560397198, + "0.05": 0.0057357130245310195, + "0.1": 0.023249725501060414, + "0.15": 0.056032881000061874, + "0.2": 0.10549120672784174, + "0.3": 0.2840493928347625, + "0.4": 0.6193591933847999 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 71.7 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/ptq.json b/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..28ee3f813344d35348ee7964b0fce051cb6375e5 --- /dev/null +++ b/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101/ptq.json @@ -0,0 +1,52 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-adamw-lr4.8e-03-s101", + "w_bits": 6, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.9291688200086536 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.449217951886328 + }, + "4": { + "val_dolma_cc": 3.9960233321176966 + }, + "5": { + "val_dolma_cc": 3.953879643863526 + }, + "6": { + "val_dolma_cc": 3.929098875517303 + }, + "8": { + "val_dolma_cc": 3.9300679185345655 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 0.5200491318776743 + }, + "4": { + "val_dolma_cc": 0.06685451210904292 + }, + "5": { + "val_dolma_cc": 0.024710823854872555 + }, + "6": { + "val_dolma_cc": -6.994449135078185e-05 + }, + "8": { + "val_dolma_cc": 0.0008990985259118567 + } + }, + "own_grid_max_rel_weight_change": 2.2763482832033333e-07, + "own_grid_delta_loss": -6.994449135078185e-05, + "checks": { + "weights_restored": true, + "own_grid_weights_unchanged": true, + "own_grid_is_fixed_point": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..45f5b76e6e7cda446b079127c71026893de74155 --- /dev/null +++ b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.7739331285905724, + "heldout_cc_en_tail_4M": 4.356792957085902 + }, + "loss_bf16_master": 3.7739331285905724, + "losses_eval_model": { + "val_dolma_cc": 3.8165860759648975 + }, + "bf16_minus_quant": -0.04265294737432512, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-m5-20", + "gpu": "NVIDIA L40S", + "t_s": 18.2 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..3948b1dc5be82a93e89058be4e2df654b8c21c72 --- /dev/null +++ b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json @@ -0,0 +1,102 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.816500349166907 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.8165860759648975 + }, + "losses_eval_model": { + "val_dolma_cc": 3.8165860759648975 + }, + "max_distinct_values_per_row": 62, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.030932195484638214, + "blocks.0.attention.w_k": 0.031067954376339912, + "blocks.0.attention.w_v": 0.031908489763736725, + "blocks.0.attention.w_out": 0.03268344700336456, + "blocks.0.feed_forward.w1": 0.030635656788945198, + "blocks.0.feed_forward.w2": 0.03498230129480362, + "blocks.0.feed_forward.w3": 0.030625954270362854, + "blocks.1.attention.w_q": 0.030392387881875038, + "blocks.1.attention.w_k": 0.030469106510281563, + "blocks.1.attention.w_v": 0.03046298213303089, + "blocks.1.attention.w_out": 0.03174380585551262, + "blocks.1.feed_forward.w1": 0.030810555443167686, + "blocks.1.feed_forward.w2": 0.03493629768490791, + "blocks.1.feed_forward.w3": 0.030692772939801216, + "blocks.2.attention.w_q": 0.03051871620118618, + "blocks.2.attention.w_k": 0.030241983011364937, + "blocks.2.attention.w_v": 0.030938543379306793, + "blocks.2.attention.w_out": 0.033308736979961395, + "blocks.2.feed_forward.w1": 0.030609318986535072, + "blocks.2.feed_forward.w2": 0.03414329141378403, + "blocks.2.feed_forward.w3": 0.030738672241568565, + "blocks.3.attention.w_q": 0.030858557671308517, + "blocks.3.attention.w_k": 0.030566640198230743, + "blocks.3.attention.w_v": 0.030706897377967834, + "blocks.3.attention.w_out": 0.032133810222148895, + "blocks.3.feed_forward.w1": 0.030591895803809166, + "blocks.3.feed_forward.w2": 0.03722628951072693, + "blocks.3.feed_forward.w3": 0.030861755833029747, + "blocks.4.attention.w_q": 0.030701402574777603, + "blocks.4.attention.w_k": 0.03091822750866413, + "blocks.4.attention.w_v": 0.031039241701364517, + "blocks.4.attention.w_out": 0.032929953187704086, + "blocks.4.feed_forward.w1": 0.030669309198856354, + "blocks.4.feed_forward.w2": 0.034518878906965256, + "blocks.4.feed_forward.w3": 0.030812233686447144, + "blocks.5.attention.w_q": 0.030762795358896255, + "blocks.5.attention.w_k": 0.0308926273137331, + "blocks.5.attention.w_v": 0.030903514474630356, + "blocks.5.attention.w_out": 0.033717043697834015, + "blocks.5.feed_forward.w1": 0.03070686198771, + "blocks.5.feed_forward.w2": 0.03584417328238487, + "blocks.5.feed_forward.w3": 0.030798662453889847, + "blocks.6.attention.w_q": 0.03058581054210663, + "blocks.6.attention.w_k": 0.030940260738134384, + "blocks.6.attention.w_v": 0.031143827363848686, + "blocks.6.attention.w_out": 0.03557548299431801, + "blocks.6.feed_forward.w1": 0.030763618648052216, + "blocks.6.feed_forward.w2": 0.0363113172352314, + "blocks.6.feed_forward.w3": 0.030922498553991318, + "blocks.7.attention.w_q": 0.031031783670186996, + "blocks.7.attention.w_k": 0.031099854037165642, + "blocks.7.attention.w_v": 0.03084469400346279, + "blocks.7.attention.w_out": 0.033200811594724655, + "blocks.7.feed_forward.w1": 0.03116525150835514, + "blocks.7.feed_forward.w2": 0.041475873440504074, + "blocks.7.feed_forward.w3": 0.031072761863470078, + "blocks.8.attention.w_q": 0.03149639070034027, + "blocks.8.attention.w_k": 0.03132655471563339, + "blocks.8.attention.w_v": 0.03121422417461872, + "blocks.8.attention.w_out": 0.037734467536211014, + "blocks.8.feed_forward.w1": 0.031303565949201584, + "blocks.8.feed_forward.w2": 0.04681382700800896, + "blocks.8.feed_forward.w3": 0.03129962459206581 + }, + "rel_quant_noise_mean": 0.03219565781690772, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.8291723132133484, + "loss_eval_model": 3.8165860759648975, + "checks": { + "snap_reproduces_fakequant": true, + "grid_levels_ok": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b2adbe774290b0d73fb08ecaba9d2ceec545976 --- /dev/null +++ b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,126 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.005 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/perturb_all.json b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..80bba98d88970c2fd9e2cf6fe77874b05e7540e9 --- /dev/null +++ b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.033884573727846146, + "blocks.0.attention.w_out.weight": 0.021545495837926865, + "blocks.0.attention.w_q.weight": 0.032501790672540665, + "blocks.0.attention.w_v.weight": 0.026910170912742615, + "blocks.0.attention_norm.weight": 0.3879252076148987, + "blocks.0.feed_forward.w1.weight": 0.02422112412750721, + "blocks.0.feed_forward.w2.weight": 0.008776023052632809, + "blocks.0.feed_forward.w3.weight": 0.024775676429271698, + "blocks.0.feed_forward_norm.weight": 0.4009745121002197, + "blocks.1.attention.w_k.weight": 0.027325544506311417, + "blocks.1.attention.w_out.weight": 0.017743699252605438, + "blocks.1.attention.w_q.weight": 0.026878027245402336, + "blocks.1.attention.w_v.weight": 0.02216923050582409, + "blocks.1.attention_norm.weight": 0.3890649080276489, + "blocks.1.feed_forward.w1.weight": 0.0247330479323864, + "blocks.1.feed_forward.w2.weight": 0.009560699574649334, + "blocks.1.feed_forward.w3.weight": 0.024706684052944183, + "blocks.1.feed_forward_norm.weight": 0.4063830077648163, + "blocks.2.attention.w_k.weight": 0.02852986566722393, + "blocks.2.attention.w_out.weight": 0.018322842195630074, + "blocks.2.attention.w_q.weight": 0.02715517021715641, + "blocks.2.attention.w_v.weight": 0.023582633584737778, + "blocks.2.attention_norm.weight": 0.3945653438568115, + "blocks.2.feed_forward.w1.weight": 0.02440229430794716, + "blocks.2.feed_forward.w2.weight": 0.010609530843794346, + "blocks.2.feed_forward.w3.weight": 0.024790987372398376, + "blocks.2.feed_forward_norm.weight": 0.40593409538269043, + "blocks.3.attention.w_k.weight": 0.029783764854073524, + "blocks.3.attention.w_out.weight": 0.02107260376214981, + "blocks.3.attention.w_q.weight": 0.027262361720204353, + "blocks.3.attention.w_v.weight": 0.024727392941713333, + "blocks.3.attention_norm.weight": 0.39631086587905884, + "blocks.3.feed_forward.w1.weight": 0.023838182911276817, + "blocks.3.feed_forward.w2.weight": 0.011221504770219326, + "blocks.3.feed_forward.w3.weight": 0.024804741144180298, + "blocks.3.feed_forward_norm.weight": 0.4052497446537018, + "blocks.4.attention.w_k.weight": 0.029391638934612274, + "blocks.4.attention.w_out.weight": 0.02368864417076111, + "blocks.4.attention.w_q.weight": 0.030474061146378517, + "blocks.4.attention.w_v.weight": 0.0250234492123127, + "blocks.4.attention_norm.weight": 0.40151700377464294, + "blocks.4.feed_forward.w1.weight": 0.023672061040997505, + "blocks.4.feed_forward.w2.weight": 0.012774117290973663, + "blocks.4.feed_forward.w3.weight": 0.024888446554541588, + "blocks.4.feed_forward_norm.weight": 0.40640202164649963, + "blocks.5.attention.w_k.weight": 0.0299989040941, + "blocks.5.attention.w_out.weight": 0.02527996338903904, + "blocks.5.attention.w_q.weight": 0.03006657026708126, + "blocks.5.attention.w_v.weight": 0.02713628113269806, + "blocks.5.attention_norm.weight": 0.40378764271736145, + "blocks.5.feed_forward.w1.weight": 0.023739339783787727, + "blocks.5.feed_forward.w2.weight": 0.01463079173117876, + "blocks.5.feed_forward.w3.weight": 0.025103239342570305, + "blocks.5.feed_forward_norm.weight": 0.4088801145553589, + "blocks.6.attention.w_k.weight": 0.029504111036658287, + "blocks.6.attention.w_out.weight": 0.02975505404174328, + "blocks.6.attention.w_q.weight": 0.02926630713045597, + "blocks.6.attention.w_v.weight": 0.027235006913542747, + "blocks.6.attention_norm.weight": 0.40453341603279114, + "blocks.6.feed_forward.w1.weight": 0.02357933484017849, + "blocks.6.feed_forward.w2.weight": 0.016710400581359863, + "blocks.6.feed_forward.w3.weight": 0.024904726073145866, + "blocks.6.feed_forward_norm.weight": 0.4096214175224304, + "blocks.7.attention.w_k.weight": 0.030117973685264587, + "blocks.7.attention.w_out.weight": 0.03328891843557358, + "blocks.7.attention.w_q.weight": 0.028999419882893562, + "blocks.7.attention.w_v.weight": 0.025352291762828827, + "blocks.7.attention_norm.weight": 0.40566807985305786, + "blocks.7.feed_forward.w1.weight": 0.022943098098039627, + "blocks.7.feed_forward.w2.weight": 0.019236372783780098, + "blocks.7.feed_forward.w3.weight": 0.024431584402918816, + "blocks.7.feed_forward_norm.weight": 0.41024506092071533, + "blocks.8.attention.w_k.weight": 0.028775880113244057, + "blocks.8.attention.w_out.weight": 0.039014291018247604, + "blocks.8.attention.w_q.weight": 0.026884691789746284, + "blocks.8.attention.w_v.weight": 0.02573145553469658, + "blocks.8.attention_norm.weight": 0.409767210483551, + "blocks.8.feed_forward.w1.weight": 0.02186930924654007, + "blocks.8.feed_forward.w2.weight": 0.023152999579906464, + "blocks.8.feed_forward.w3.weight": 0.02385779283940792, + "blocks.8.feed_forward_norm.weight": 0.4094424843788147, + "embeddings.weight": 0.07002507150173187, + "lm_head.norm.weight": 1.3853260278701782, + "lm_head.w_out.weight": 0.07471665740013123 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.8165860759648975 + }, + "base_loss": 3.8165860759648975, + "loss": { + "0.005": [ + 3.8172678371833086, + 3.8171116142565764, + 3.8159586725472936 + ], + "0.01": [ + 3.8176468903175227, + 3.818043408056804, + 3.8168794922845257 + ], + "0.02": [ + 3.8206880907293117, + 3.8205451970833733, + 3.820196484005677 + ], + "0.05": [ + 3.843052730337484, + 3.843897525835096, + 3.8416292241809287 + ], + "0.1": [ + 3.9249599206092856, + 3.929138217826567, + 3.921051436673467 + ], + "0.15": [ + 4.079257128156649, + 4.096721040742732, + 4.074365094313875 + ], + "0.2": [ + 4.34255490302199, + 4.388319989029223, + 4.333827130957245 + ], + "0.3": [ + 5.484848179687375, + 5.656619338838878, + 5.408889064578226 + ], + "0.4": [ + 7.629972741131737, + 7.77432493980658, + 7.303515376176326 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.8172678371833086, + 3.8171116142565764, + 3.8159586725472936 + ], + "0.01": [ + 3.8176468903175227, + 3.818043408056804, + 3.8168794922845257 + ], + "0.02": [ + 3.8206880907293117, + 3.8205451970833733, + 3.820196484005677 + ], + "0.05": [ + 3.843052730337484, + 3.843897525835096, + 3.8416292241809287 + ], + "0.1": [ + 3.9249599206092856, + 3.929138217826567, + 3.921051436673467 + ], + "0.15": [ + 4.079257128156649, + 4.096721040742732, + 4.074365094313875 + ], + "0.2": [ + 4.34255490302199, + 4.388319989029223, + 4.333827130957245 + ], + "0.3": [ + 5.484848179687375, + 5.656619338838878, + 5.408889064578226 + ], + "0.4": [ + 7.629972741131737, + 7.77432493980658, + 7.303515376176326 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0001932986974959583, + "0.01": 0.0009371875880535008, + "0.02": 0.003890514641223586, + "0.05": 0.026273750819605368, + "0.1": 0.10846378240487553, + "0.15": 0.266861678439521, + "0.2": 0.5383145983712554, + "0.3": 1.7001994517365953, + "0.4": 3.7526849430733167 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0001932986974959583, + "0.01": 0.0009371875880535008, + "0.02": 0.003890514641223586, + "0.05": 0.026273750819605368, + "0.1": 0.10846378240487553, + "0.15": 0.266861678439521, + "0.2": 0.5383145983712554, + "0.3": 1.7001994517365953, + "0.4": 3.7526849430733167 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 79.2 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..843030f1723b5ccad0a22c9382944dc036b8a68c --- /dev/null +++ b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.033884573727846146, + "blocks.0.attention.w_out.weight": 0.021545495837926865, + "blocks.0.attention.w_q.weight": 0.032501790672540665, + "blocks.0.attention.w_v.weight": 0.026910170912742615, + "blocks.0.feed_forward.w1.weight": 0.02422112412750721, + "blocks.0.feed_forward.w2.weight": 0.008776023052632809, + "blocks.0.feed_forward.w3.weight": 0.024775676429271698, + "blocks.1.attention.w_k.weight": 0.027325544506311417, + "blocks.1.attention.w_out.weight": 0.017743699252605438, + "blocks.1.attention.w_q.weight": 0.026878027245402336, + "blocks.1.attention.w_v.weight": 0.02216923050582409, + "blocks.1.feed_forward.w1.weight": 0.0247330479323864, + "blocks.1.feed_forward.w2.weight": 0.009560699574649334, + "blocks.1.feed_forward.w3.weight": 0.024706684052944183, + "blocks.2.attention.w_k.weight": 0.02852986566722393, + "blocks.2.attention.w_out.weight": 0.018322842195630074, + "blocks.2.attention.w_q.weight": 0.02715517021715641, + "blocks.2.attention.w_v.weight": 0.023582633584737778, + "blocks.2.feed_forward.w1.weight": 0.02440229430794716, + "blocks.2.feed_forward.w2.weight": 0.010609530843794346, + "blocks.2.feed_forward.w3.weight": 0.024790987372398376, + "blocks.3.attention.w_k.weight": 0.029783764854073524, + "blocks.3.attention.w_out.weight": 0.02107260376214981, + "blocks.3.attention.w_q.weight": 0.027262361720204353, + "blocks.3.attention.w_v.weight": 0.024727392941713333, + "blocks.3.feed_forward.w1.weight": 0.023838182911276817, + "blocks.3.feed_forward.w2.weight": 0.011221504770219326, + "blocks.3.feed_forward.w3.weight": 0.024804741144180298, + "blocks.4.attention.w_k.weight": 0.029391638934612274, + "blocks.4.attention.w_out.weight": 0.02368864417076111, + "blocks.4.attention.w_q.weight": 0.030474061146378517, + "blocks.4.attention.w_v.weight": 0.0250234492123127, + "blocks.4.feed_forward.w1.weight": 0.023672061040997505, + "blocks.4.feed_forward.w2.weight": 0.012774117290973663, + "blocks.4.feed_forward.w3.weight": 0.024888446554541588, + "blocks.5.attention.w_k.weight": 0.0299989040941, + "blocks.5.attention.w_out.weight": 0.02527996338903904, + "blocks.5.attention.w_q.weight": 0.03006657026708126, + "blocks.5.attention.w_v.weight": 0.02713628113269806, + "blocks.5.feed_forward.w1.weight": 0.023739339783787727, + "blocks.5.feed_forward.w2.weight": 0.01463079173117876, + "blocks.5.feed_forward.w3.weight": 0.025103239342570305, + "blocks.6.attention.w_k.weight": 0.029504111036658287, + "blocks.6.attention.w_out.weight": 0.02975505404174328, + "blocks.6.attention.w_q.weight": 0.02926630713045597, + "blocks.6.attention.w_v.weight": 0.027235006913542747, + "blocks.6.feed_forward.w1.weight": 0.02357933484017849, + "blocks.6.feed_forward.w2.weight": 0.016710400581359863, + "blocks.6.feed_forward.w3.weight": 0.024904726073145866, + "blocks.7.attention.w_k.weight": 0.030117973685264587, + "blocks.7.attention.w_out.weight": 0.03328891843557358, + "blocks.7.attention.w_q.weight": 0.028999419882893562, + "blocks.7.attention.w_v.weight": 0.025352291762828827, + "blocks.7.feed_forward.w1.weight": 0.022943098098039627, + "blocks.7.feed_forward.w2.weight": 0.019236372783780098, + "blocks.7.feed_forward.w3.weight": 0.024431584402918816, + "blocks.8.attention.w_k.weight": 0.028775880113244057, + "blocks.8.attention.w_out.weight": 0.039014291018247604, + "blocks.8.attention.w_q.weight": 0.026884691789746284, + "blocks.8.attention.w_v.weight": 0.02573145553469658, + "blocks.8.feed_forward.w1.weight": 0.02186930924654007, + "blocks.8.feed_forward.w2.weight": 0.023152999579906464, + "blocks.8.feed_forward.w3.weight": 0.02385779283940792 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.8165860759648975 + }, + "base_loss": 3.8165860759648975, + "loss": { + "0.005": [ + 3.817336541468609, + 3.817538655931506, + 3.8168800165356633 + ], + "0.01": [ + 3.8171605613161628, + 3.816650633188959, + 3.8168201697471975 + ], + "0.02": [ + 3.8179068758463135, + 3.8178052141611283, + 3.818238981117444 + ], + "0.05": [ + 3.8249593447154986, + 3.8250348603532682, + 3.8262709545527156 + ], + "0.1": [ + 3.8528304287629402, + 3.852761274560618, + 3.854550848854923 + ], + "0.15": [ + 3.9037916220974953, + 3.902684196341472, + 3.9077033297865786 + ], + "0.2": [ + 3.9867743644386087, + 3.98497371827326, + 3.987170623964643 + ], + "0.3": [ + 4.314607317431654, + 4.309899569600771, + 4.3006122689914745 + ], + "0.4": [ + 5.02026613272549, + 5.055846697512039, + 4.985979411922304 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.817336541468609, + 3.817538655931506, + 3.8168800165356633 + ], + "0.01": [ + 3.8171605613161628, + 3.816650633188959, + 3.8168201697471975 + ], + "0.02": [ + 3.8179068758463135, + 3.8178052141611283, + 3.818238981117444 + ], + "0.05": [ + 3.8249593447154986, + 3.8250348603532682, + 3.8262709545527156 + ], + "0.1": [ + 3.8528304287629402, + 3.852761274560618, + 3.854550848854923 + ], + "0.15": [ + 3.9037916220974953, + 3.902684196341472, + 3.9077033297865786 + ], + "0.2": [ + 3.9867743644386087, + 3.98497371827326, + 3.987170623964643 + ], + "0.3": [ + 4.314607317431654, + 4.309899569600771, + 4.3006122689914745 + ], + "0.4": [ + 5.02026613272549, + 5.055846697512039, + 4.985979411922304 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0006656620136951119, + "0.01": 0.00029104545254243774, + "0.02": 0.0013976144100644738, + "0.05": 0.008835643908929658, + "0.1": 0.03679477476126225, + "0.15": 0.08814030677695106, + "0.2": 0.1697201595939397, + "0.3": 0.49178697604306887, + "0.4": 1.2041113380883797 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0006656620136951119, + "0.01": 0.00029104545254243774, + "0.02": 0.0013976144100644738, + "0.05": 0.008835643908929658, + "0.1": 0.03679477476126225, + "0.15": 0.08814030677695106, + "0.2": 0.1697201595939397, + "0.3": 0.49178697604306887, + "0.4": 1.2041113380883797 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 71.7 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/ptq.json b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..064cbe8e88bfd587b807fc7d7025a0a64aac673e --- /dev/null +++ b/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101/ptq.json @@ -0,0 +1,52 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6-muon-lr5.0e-03-alr2.4e-03-s101", + "w_bits": 6, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.8165860759648975 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.549142319661365 + }, + "4": { + "val_dolma_cc": 3.9001482527445095 + }, + "5": { + "val_dolma_cc": 3.8357712028443145 + }, + "6": { + "val_dolma_cc": 3.8165860759648975 + }, + "8": { + "val_dolma_cc": 3.8169317139575623 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 0.7325562436964672 + }, + "4": { + "val_dolma_cc": 0.08356217677961197 + }, + "5": { + "val_dolma_cc": 0.01918512687941698 + }, + "6": { + "val_dolma_cc": 0.0 + }, + "8": { + "val_dolma_cc": 0.00034563799266473794 + } + }, + "own_grid_max_rel_weight_change": 2.2763893525734602e-07, + "own_grid_delta_loss": 0.0, + "checks": { + "weights_restored": true, + "own_grid_weights_unchanged": true, + "own_grid_is_fixed_point": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..f45ff5a0d3b190090f2fd7ea2bd000846f68c2e2 --- /dev/null +++ b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 4.706428408139432, + "heldout_cc_en_tail_4M": 5.2704657109951345 + }, + "loss_bf16_master": 4.706428408139432, + "losses_eval_model": { + "val_dolma_cc": 4.668224568905942 + }, + "bf16_minus_quant": 0.03820383923349002, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-o9-28", + "gpu": "NVIDIA L40S", + "t_s": 18.3 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c1b44b5a79994a7b3bcaff55805aeb44922dce --- /dev/null +++ b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json @@ -0,0 +1,100 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101", + "arm": "B", + "trust_mode": "halfstep:0.8", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_fakequant_master": { + "val_dolma_cc": 4.668224568905942 + }, + "losses_eval_model": { + "val_dolma_cc": 4.668224568905942 + }, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.03332328423857689, + "blocks.0.attention.w_k": 0.03641916811466217, + "blocks.0.attention.w_v": 0.031027039512991905, + "blocks.0.attention.w_out": 0.030954813584685326, + "blocks.0.feed_forward.w1": 0.03877241536974907, + "blocks.0.feed_forward.w2": 0.0469457283616066, + "blocks.0.feed_forward.w3": 0.03839801251888275, + "blocks.1.attention.w_q": 0.03360611945390701, + "blocks.1.attention.w_k": 0.035510193556547165, + "blocks.1.attention.w_v": 0.02801809273660183, + "blocks.1.attention.w_out": 0.0328420028090477, + "blocks.1.feed_forward.w1": 0.037539612501859665, + "blocks.1.feed_forward.w2": 0.045879002660512924, + "blocks.1.feed_forward.w3": 0.037243716418743134, + "blocks.2.attention.w_q": 0.03463498130440712, + "blocks.2.attention.w_k": 0.036013852804899216, + "blocks.2.attention.w_v": 0.028043702244758606, + "blocks.2.attention.w_out": 0.03151233121752739, + "blocks.2.feed_forward.w1": 0.03810415044426918, + "blocks.2.feed_forward.w2": 0.04473399743437767, + "blocks.2.feed_forward.w3": 0.037958044558763504, + "blocks.3.attention.w_q": 0.03387307748198509, + "blocks.3.attention.w_k": 0.03613683208823204, + "blocks.3.attention.w_v": 0.026545124128460884, + "blocks.3.attention.w_out": 0.029225513339042664, + "blocks.3.feed_forward.w1": 0.038329266011714935, + "blocks.3.feed_forward.w2": 0.044376637786626816, + "blocks.3.feed_forward.w3": 0.03779160976409912, + "blocks.4.attention.w_q": 0.03460658714175224, + "blocks.4.attention.w_k": 0.036429718136787415, + "blocks.4.attention.w_v": 0.026127543300390244, + "blocks.4.attention.w_out": 0.029204169288277626, + "blocks.4.feed_forward.w1": 0.03868156671524048, + "blocks.4.feed_forward.w2": 0.041954245418310165, + "blocks.4.feed_forward.w3": 0.038220085203647614, + "blocks.5.attention.w_q": 0.033795177936553955, + "blocks.5.attention.w_k": 0.03475954383611679, + "blocks.5.attention.w_v": 0.026510033756494522, + "blocks.5.attention.w_out": 0.028667427599430084, + "blocks.5.feed_forward.w1": 0.038451336324214935, + "blocks.5.feed_forward.w2": 0.04262901097536087, + "blocks.5.feed_forward.w3": 0.037854891270399094, + "blocks.6.attention.w_q": 0.03513786941766739, + "blocks.6.attention.w_k": 0.03596772626042366, + "blocks.6.attention.w_v": 0.027041783556342125, + "blocks.6.attention.w_out": 0.028368759900331497, + "blocks.6.feed_forward.w1": 0.03803660348057747, + "blocks.6.feed_forward.w2": 0.040330544114112854, + "blocks.6.feed_forward.w3": 0.03748779743909836, + "blocks.7.attention.w_q": 0.036245547235012054, + "blocks.7.attention.w_k": 0.038349855691194534, + "blocks.7.attention.w_v": 0.02823600172996521, + "blocks.7.attention.w_out": 0.033784590661525726, + "blocks.7.feed_forward.w1": 0.03777198866009712, + "blocks.7.feed_forward.w2": 0.03741661086678505, + "blocks.7.feed_forward.w3": 0.037181828171014786, + "blocks.8.attention.w_q": 0.035573069006204605, + "blocks.8.attention.w_k": 0.03772805258631706, + "blocks.8.attention.w_v": 0.027792055159807205, + "blocks.8.attention.w_out": 0.036074232310056686, + "blocks.8.feed_forward.w1": 0.037256933748722076, + "blocks.8.feed_forward.w2": 0.03773248940706253, + "blocks.8.feed_forward.w3": 0.036940693855285645 + }, + "rel_quant_noise_mean": 0.035430233216001875, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.9561190605163574, + "loss_eval_model": 4.668224568905942, + "checks": { + "snap_reproduces_fakequant": true, + "weights_on_arm_grid": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99a87055c0eeff60bee36501b9cf7ecbd717ecf0 --- /dev/null +++ b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,134 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.quest_train_module.QuestTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + fq_arm: B + fq_trust_mode: halfstep:0.8 + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.005 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false + osc_probe: + _CLASS_: precq.quest_callbacks.OscillationProbeCallback + interval: 100 + bits: 6 + arm: B + trust_mode: halfstep:0.8 +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/osc_probe.jsonl b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/osc_probe.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..541482d3ce22a39f7e21bbbb69c810034d8a42c4 --- /dev/null +++ b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/osc_probe.jsonl @@ -0,0 +1,72 @@ +{"start": true, "step": 0, "arm": "B", "bits": 6, "interval": 100} +{"step": 0, "w_mask_frac": 0.199476782964845} +{"step": 100, "changed_frac": 0.2764321025620159, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2870903015136719, "blocks.0.attention.w_k": 0.31298828125, "blocks.0.attention.w_v": 0.2553443908691406, "blocks.0.attention.w_out": 0.25687408447265625, "blocks.0.feed_forward.w1": 0.27880096435546875, "blocks.0.feed_forward.w2": 0.18207931518554688, "blocks.0.feed_forward.w3": 0.27885565161705017, "blocks.1.attention.w_q": 0.270538330078125, "blocks.1.attention.w_k": 0.2844696044921875, "blocks.1.attention.w_v": 0.274017333984375, "blocks.1.attention.w_out": 0.2741050720214844, "blocks.1.feed_forward.w1": 0.3186607360839844, "blocks.1.feed_forward.w2": 0.18595123291015625, "blocks.1.feed_forward.w3": 0.31733959913253784, "blocks.2.attention.w_q": 0.2806358337402344, "blocks.2.attention.w_k": 0.29892730712890625, "blocks.2.attention.w_v": 0.2764091491699219, "blocks.2.attention.w_out": 0.2776756286621094, "blocks.2.feed_forward.w1": 0.32444000244140625, "blocks.2.feed_forward.w2": 0.18469874560832977, "blocks.2.feed_forward.w3": 0.3253771662712097, "blocks.3.attention.w_q": 0.2869834899902344, "blocks.3.attention.w_k": 0.3018798828125, "blocks.3.attention.w_v": 0.276397705078125, "blocks.3.attention.w_out": 0.278350830078125, "blocks.3.feed_forward.w1": 0.3299598693847656, "blocks.3.feed_forward.w2": 0.18507640063762665, "blocks.3.feed_forward.w3": 0.33136749267578125, "blocks.4.attention.w_q": 0.28211212158203125, "blocks.4.attention.w_k": 0.29969024658203125, "blocks.4.attention.w_v": 0.2769660949707031, "blocks.4.attention.w_out": 0.277069091796875, "blocks.4.feed_forward.w1": 0.3306363523006439, "blocks.4.feed_forward.w2": 0.18302664160728455, "blocks.4.feed_forward.w3": 0.3329213559627533, "blocks.5.attention.w_q": 0.2841796875, "blocks.5.attention.w_k": 0.3020362854003906, "blocks.5.attention.w_v": 0.272796630859375, "blocks.5.attention.w_out": 0.2748146057128906, "blocks.5.feed_forward.w1": 0.3315238952636719, "blocks.5.feed_forward.w2": 0.18051275610923767, "blocks.5.feed_forward.w3": 0.33233898878097534, "blocks.6.attention.w_q": 0.28350830078125, "blocks.6.attention.w_k": 0.3023796081542969, "blocks.6.attention.w_v": 0.26938629150390625, "blocks.6.attention.w_out": 0.2672920227050781, "blocks.6.feed_forward.w1": 0.3307482600212097, "blocks.6.feed_forward.w2": 0.17713801562786102, "blocks.6.feed_forward.w3": 0.3309745788574219, "blocks.7.attention.w_q": 0.2773017883300781, "blocks.7.attention.w_k": 0.2952308654785156, "blocks.7.attention.w_v": 0.26317596435546875, "blocks.7.attention.w_out": 0.2611732482910156, "blocks.7.feed_forward.w1": 0.32949066162109375, "blocks.7.feed_forward.w2": 0.17301814258098602, "blocks.7.feed_forward.w3": 0.33029302954673767, "blocks.8.attention.w_q": 0.2777671813964844, "blocks.8.attention.w_k": 0.2972412109375, "blocks.8.attention.w_v": 0.24840927124023438, "blocks.8.attention.w_out": 0.251190185546875, "blocks.8.feed_forward.w1": 0.32846197485923767, "blocks.8.feed_forward.w2": 0.16553497314453125, "blocks.8.feed_forward.w3": 0.32948940992355347}, "w_mask_frac": 0.11091806319279549} +{"step": 200, "changed_frac": 0.5518783830169939, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5615234375, "blocks.0.attention.w_k": 0.6012954711914062, "blocks.0.attention.w_v": 0.5194206237792969, "blocks.0.attention.w_out": 0.5294647216796875, "blocks.0.feed_forward.w1": 0.5343246459960938, "blocks.0.feed_forward.w2": 0.40502041578292847, "blocks.0.feed_forward.w3": 0.5312665700912476, "blocks.1.attention.w_q": 0.5432510375976562, "blocks.1.attention.w_k": 0.5667037963867188, "blocks.1.attention.w_v": 0.5896682739257812, "blocks.1.attention.w_out": 0.5881004333496094, "blocks.1.feed_forward.w1": 0.6159960627555847, "blocks.1.feed_forward.w2": 0.4350852966308594, "blocks.1.feed_forward.w3": 0.6160532832145691, "blocks.2.attention.w_q": 0.5570831298828125, "blocks.2.attention.w_k": 0.5832328796386719, "blocks.2.attention.w_v": 0.5971717834472656, "blocks.2.attention.w_out": 0.5981216430664062, "blocks.2.feed_forward.w1": 0.635986328125, "blocks.2.feed_forward.w2": 0.4386075437068939, "blocks.2.feed_forward.w3": 0.6328239440917969, "blocks.3.attention.w_q": 0.5559120178222656, "blocks.3.attention.w_k": 0.5734481811523438, "blocks.3.attention.w_v": 0.5977630615234375, "blocks.3.attention.w_out": 0.599761962890625, "blocks.3.feed_forward.w1": 0.6400604248046875, "blocks.3.feed_forward.w2": 0.4378458857536316, "blocks.3.feed_forward.w3": 0.6380144953727722, "blocks.4.attention.w_q": 0.5585517883300781, "blocks.4.attention.w_k": 0.5731773376464844, "blocks.4.attention.w_v": 0.5924758911132812, "blocks.4.attention.w_out": 0.5955162048339844, "blocks.4.feed_forward.w1": 0.6359062194824219, "blocks.4.feed_forward.w2": 0.42449697852134705, "blocks.4.feed_forward.w3": 0.6368522644042969, "blocks.5.attention.w_q": 0.5618095397949219, "blocks.5.attention.w_k": 0.5754280090332031, "blocks.5.attention.w_v": 0.5855827331542969, "blocks.5.attention.w_out": 0.5775108337402344, "blocks.5.feed_forward.w1": 0.636908233165741, "blocks.5.feed_forward.w2": 0.4104042053222656, "blocks.5.feed_forward.w3": 0.6334457397460938, "blocks.6.attention.w_q": 0.5480690002441406, "blocks.6.attention.w_k": 0.56732177734375, "blocks.6.attention.w_v": 0.5700187683105469, "blocks.6.attention.w_out": 0.5661163330078125, "blocks.6.feed_forward.w1": 0.6314544677734375, "blocks.6.feed_forward.w2": 0.38972729444503784, "blocks.6.feed_forward.w3": 0.6283226013183594, "blocks.7.attention.w_q": 0.5296783447265625, "blocks.7.attention.w_k": 0.5486183166503906, "blocks.7.attention.w_v": 0.55633544921875, "blocks.7.attention.w_out": 0.5491867065429688, "blocks.7.feed_forward.w1": 0.6257833242416382, "blocks.7.feed_forward.w2": 0.3612009882926941, "blocks.7.feed_forward.w3": 0.6202799677848816, "blocks.8.attention.w_q": 0.5315895080566406, "blocks.8.attention.w_k": 0.5540428161621094, "blocks.8.attention.w_v": 0.5200881958007812, "blocks.8.attention.w_out": 0.5170059204101562, "blocks.8.feed_forward.w1": 0.6168467402458191, "blocks.8.feed_forward.w2": 0.31391143798828125, "blocks.8.feed_forward.w3": 0.6166166067123413}, "w_mask_frac": 0.13874791065851846} +{"step": 300, "changed_frac": 0.6551862537351429, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.64239501953125, "blocks.0.attention.w_k": 0.6777915954589844, "blocks.0.attention.w_v": 0.6415367126464844, "blocks.0.attention.w_out": 0.6438179016113281, "blocks.0.feed_forward.w1": 0.6549581289291382, "blocks.0.feed_forward.w2": 0.5793304443359375, "blocks.0.feed_forward.w3": 0.655157744884491, "blocks.1.attention.w_q": 0.6707916259765625, "blocks.1.attention.w_k": 0.681976318359375, "blocks.1.attention.w_v": 0.6849517822265625, "blocks.1.attention.w_out": 0.6840171813964844, "blocks.1.feed_forward.w1": 0.7149251699447632, "blocks.1.feed_forward.w2": 0.5932388305664062, "blocks.1.feed_forward.w3": 0.7139320373535156, "blocks.2.attention.w_q": 0.6744918823242188, "blocks.2.attention.w_k": 0.6911430358886719, "blocks.2.attention.w_v": 0.6802291870117188, "blocks.2.attention.w_out": 0.6791114807128906, "blocks.2.feed_forward.w1": 0.7191886901855469, "blocks.2.feed_forward.w2": 0.5903486013412476, "blocks.2.feed_forward.w3": 0.7179438471794128, "blocks.3.attention.w_q": 0.6626129150390625, "blocks.3.attention.w_k": 0.6774444580078125, "blocks.3.attention.w_v": 0.6790390014648438, "blocks.3.attention.w_out": 0.6768531799316406, "blocks.3.feed_forward.w1": 0.7220891714096069, "blocks.3.feed_forward.w2": 0.5826416015625, "blocks.3.feed_forward.w3": 0.720306396484375, "blocks.4.attention.w_q": 0.6467208862304688, "blocks.4.attention.w_k": 0.6633415222167969, "blocks.4.attention.w_v": 0.6781120300292969, "blocks.4.attention.w_out": 0.6746139526367188, "blocks.4.feed_forward.w1": 0.7167854309082031, "blocks.4.feed_forward.w2": 0.5714988708496094, "blocks.4.feed_forward.w3": 0.7167854309082031, "blocks.5.attention.w_q": 0.6342849731445312, "blocks.5.attention.w_k": 0.6535186767578125, "blocks.5.attention.w_v": 0.6721000671386719, "blocks.5.attention.w_out": 0.6733436584472656, "blocks.5.feed_forward.w1": 0.7142486572265625, "blocks.5.feed_forward.w2": 0.5570411682128906, "blocks.5.feed_forward.w3": 0.7116534113883972, "blocks.6.attention.w_q": 0.6229820251464844, "blocks.6.attention.w_k": 0.6402511596679688, "blocks.6.attention.w_v": 0.6608352661132812, "blocks.6.attention.w_out": 0.6590042114257812, "blocks.6.feed_forward.w1": 0.7098719477653503, "blocks.6.feed_forward.w2": 0.5322481989860535, "blocks.6.feed_forward.w3": 0.7099241018295288, "blocks.7.attention.w_q": 0.6067848205566406, "blocks.7.attention.w_k": 0.6244583129882812, "blocks.7.attention.w_v": 0.6503868103027344, "blocks.7.attention.w_out": 0.6467208862304688, "blocks.7.feed_forward.w1": 0.7046610713005066, "blocks.7.feed_forward.w2": 0.5113258361816406, "blocks.7.feed_forward.w3": 0.7024459838867188, "blocks.8.attention.w_q": 0.5982627868652344, "blocks.8.attention.w_k": 0.6143531799316406, "blocks.8.attention.w_v": 0.6295547485351562, "blocks.8.attention.w_out": 0.6315689086914062, "blocks.8.feed_forward.w1": 0.6984037160873413, "blocks.8.feed_forward.w2": 0.45581817626953125, "blocks.8.feed_forward.w3": 0.6990242004394531}, "w_mask_frac": 0.16305640091498694} +{"step": 400, "changed_frac": 0.6921879368969518, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6672019958496094, "blocks.0.attention.w_k": 0.6949577331542969, "blocks.0.attention.w_v": 0.71990966796875, "blocks.0.attention.w_out": 0.7281951904296875, "blocks.0.feed_forward.w1": 0.7172698974609375, "blocks.0.feed_forward.w2": 0.648352324962616, "blocks.0.feed_forward.w3": 0.7224388122558594, "blocks.1.attention.w_q": 0.6637420654296875, "blocks.1.attention.w_k": 0.6747093200683594, "blocks.1.attention.w_v": 0.735931396484375, "blocks.1.attention.w_out": 0.7365798950195312, "blocks.1.feed_forward.w1": 0.7526067495346069, "blocks.1.feed_forward.w2": 0.6342875361442566, "blocks.1.feed_forward.w3": 0.7590141296386719, "blocks.2.attention.w_q": 0.6674079895019531, "blocks.2.attention.w_k": 0.688568115234375, "blocks.2.attention.w_v": 0.7330551147460938, "blocks.2.attention.w_out": 0.7305793762207031, "blocks.2.feed_forward.w1": 0.7524185180664062, "blocks.2.feed_forward.w2": 0.6252988576889038, "blocks.2.feed_forward.w3": 0.7560234069824219, "blocks.3.attention.w_q": 0.6591606140136719, "blocks.3.attention.w_k": 0.6784744262695312, "blocks.3.attention.w_v": 0.7344703674316406, "blocks.3.attention.w_out": 0.7329444885253906, "blocks.3.feed_forward.w1": 0.748870849609375, "blocks.3.feed_forward.w2": 0.6161880493164062, "blocks.3.feed_forward.w3": 0.7490119934082031, "blocks.4.attention.w_q": 0.6583747863769531, "blocks.4.attention.w_k": 0.6850090026855469, "blocks.4.attention.w_v": 0.7303123474121094, "blocks.4.attention.w_out": 0.727508544921875, "blocks.4.feed_forward.w1": 0.7461764216423035, "blocks.4.feed_forward.w2": 0.5969403982162476, "blocks.4.feed_forward.w3": 0.7475090026855469, "blocks.5.attention.w_q": 0.6477088928222656, "blocks.5.attention.w_k": 0.6811485290527344, "blocks.5.attention.w_v": 0.7258796691894531, "blocks.5.attention.w_out": 0.7247848510742188, "blocks.5.feed_forward.w1": 0.7401771545410156, "blocks.5.feed_forward.w2": 0.586126983165741, "blocks.5.feed_forward.w3": 0.7435709834098816, "blocks.6.attention.w_q": 0.6473312377929688, "blocks.6.attention.w_k": 0.663818359375, "blocks.6.attention.w_v": 0.7236061096191406, "blocks.6.attention.w_out": 0.7195053100585938, "blocks.6.feed_forward.w1": 0.7344691157341003, "blocks.6.feed_forward.w2": 0.5746968984603882, "blocks.6.feed_forward.w3": 0.7342147827148438, "blocks.7.attention.w_q": 0.63140869140625, "blocks.7.attention.w_k": 0.6578559875488281, "blocks.7.attention.w_v": 0.7163009643554688, "blocks.7.attention.w_out": 0.7141761779785156, "blocks.7.feed_forward.w1": 0.7284774780273438, "blocks.7.feed_forward.w2": 0.5540313720703125, "blocks.7.feed_forward.w3": 0.7279612421989441, "blocks.8.attention.w_q": 0.6238365173339844, "blocks.8.attention.w_k": 0.6591758728027344, "blocks.8.attention.w_v": 0.7093353271484375, "blocks.8.attention.w_out": 0.7096977233886719, "blocks.8.feed_forward.w1": 0.7211736440658569, "blocks.8.feed_forward.w2": 0.5208753347396851, "blocks.8.feed_forward.w3": 0.7229270935058594}, "w_mask_frac": 0.13771921561823952} +{"step": 500, "changed_frac": 0.7178449386205429, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6759719848632812, "blocks.0.attention.w_k": 0.685455322265625, "blocks.0.attention.w_v": 0.7874526977539062, "blocks.0.attention.w_out": 0.7893638610839844, "blocks.0.feed_forward.w1": 0.7472013235092163, "blocks.0.feed_forward.w2": 0.6114603877067566, "blocks.0.feed_forward.w3": 0.7520586848258972, "blocks.1.attention.w_q": 0.6514129638671875, "blocks.1.attention.w_k": 0.6735115051269531, "blocks.1.attention.w_v": 0.7888221740722656, "blocks.1.attention.w_out": 0.7920265197753906, "blocks.1.feed_forward.w1": 0.7771720886230469, "blocks.1.feed_forward.w2": 0.6174468994140625, "blocks.1.feed_forward.w3": 0.7801069021224976, "blocks.2.attention.w_q": 0.6492500305175781, "blocks.2.attention.w_k": 0.6701850891113281, "blocks.2.attention.w_v": 0.7876701354980469, "blocks.2.attention.w_out": 0.7903175354003906, "blocks.2.feed_forward.w1": 0.7841631770133972, "blocks.2.feed_forward.w2": 0.6224441528320312, "blocks.2.feed_forward.w3": 0.7856267690658569, "blocks.3.attention.w_q": 0.6403427124023438, "blocks.3.attention.w_k": 0.6686286926269531, "blocks.3.attention.w_v": 0.7914810180664062, "blocks.3.attention.w_out": 0.7881431579589844, "blocks.3.feed_forward.w1": 0.7792841792106628, "blocks.3.feed_forward.w2": 0.6130701899528503, "blocks.3.feed_forward.w3": 0.7820116877555847, "blocks.4.attention.w_q": 0.6291732788085938, "blocks.4.attention.w_k": 0.6698722839355469, "blocks.4.attention.w_v": 0.7910003662109375, "blocks.4.attention.w_out": 0.7907829284667969, "blocks.4.feed_forward.w1": 0.7778142690658569, "blocks.4.feed_forward.w2": 0.6161651611328125, "blocks.4.feed_forward.w3": 0.7837295532226562, "blocks.5.attention.w_q": 0.6289215087890625, "blocks.5.attention.w_k": 0.6695976257324219, "blocks.5.attention.w_v": 0.7913360595703125, "blocks.5.attention.w_out": 0.7921943664550781, "blocks.5.feed_forward.w1": 0.7723681330680847, "blocks.5.feed_forward.w2": 0.6072362661361694, "blocks.5.feed_forward.w3": 0.7824389338493347, "blocks.6.attention.w_q": 0.6294937133789062, "blocks.6.attention.w_k": 0.6604690551757812, "blocks.6.attention.w_v": 0.795166015625, "blocks.6.attention.w_out": 0.7912483215332031, "blocks.6.feed_forward.w1": 0.7679977416992188, "blocks.6.feed_forward.w2": 0.6029040217399597, "blocks.6.feed_forward.w3": 0.7757924795150757, "blocks.7.attention.w_q": 0.6182632446289062, "blocks.7.attention.w_k": 0.6417732238769531, "blocks.7.attention.w_v": 0.7922554016113281, "blocks.7.attention.w_out": 0.7938652038574219, "blocks.7.feed_forward.w1": 0.7575620412826538, "blocks.7.feed_forward.w2": 0.5623054504394531, "blocks.7.feed_forward.w3": 0.7651202082633972, "blocks.8.attention.w_q": 0.6174964904785156, "blocks.8.attention.w_k": 0.639251708984375, "blocks.8.attention.w_v": 0.7943687438964844, "blocks.8.attention.w_out": 0.7915382385253906, "blocks.8.feed_forward.w1": 0.7588018178939819, "blocks.8.feed_forward.w2": 0.5915858149528503, "blocks.8.feed_forward.w3": 0.7693837881088257}, "w_mask_frac": 0.1511926942656183} +{"step": 600, "changed_frac": 0.7365138714130108, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.689849853515625, "blocks.0.attention.w_k": 0.7079887390136719, "blocks.0.attention.w_v": 0.8084678649902344, "blocks.0.attention.w_out": 0.8058929443359375, "blocks.0.feed_forward.w1": 0.7447624206542969, "blocks.0.feed_forward.w2": 0.6408323049545288, "blocks.0.feed_forward.w3": 0.7443021535873413, "blocks.1.attention.w_q": 0.6756362915039062, "blocks.1.attention.w_k": 0.7039413452148438, "blocks.1.attention.w_v": 0.8174591064453125, "blocks.1.attention.w_out": 0.818939208984375, "blocks.1.feed_forward.w1": 0.7826131582260132, "blocks.1.feed_forward.w2": 0.6395963430404663, "blocks.1.feed_forward.w3": 0.78515625, "blocks.2.attention.w_q": 0.6846199035644531, "blocks.2.attention.w_k": 0.7102203369140625, "blocks.2.attention.w_v": 0.8194465637207031, "blocks.2.attention.w_out": 0.8212203979492188, "blocks.2.feed_forward.w1": 0.7935587763786316, "blocks.2.feed_forward.w2": 0.6296526789665222, "blocks.2.feed_forward.w3": 0.7978655695915222, "blocks.3.attention.w_q": 0.6877861022949219, "blocks.3.attention.w_k": 0.7183647155761719, "blocks.3.attention.w_v": 0.8192062377929688, "blocks.3.attention.w_out": 0.8182106018066406, "blocks.3.feed_forward.w1": 0.7982190847396851, "blocks.3.feed_forward.w2": 0.6293716430664062, "blocks.3.feed_forward.w3": 0.8035430908203125, "blocks.4.attention.w_q": 0.6765975952148438, "blocks.4.attention.w_k": 0.7144317626953125, "blocks.4.attention.w_v": 0.8182411193847656, "blocks.4.attention.w_out": 0.8195610046386719, "blocks.4.feed_forward.w1": 0.7989031672477722, "blocks.4.feed_forward.w2": 0.606170654296875, "blocks.4.feed_forward.w3": 0.8048388361930847, "blocks.5.attention.w_q": 0.6808242797851562, "blocks.5.attention.w_k": 0.7052192687988281, "blocks.5.attention.w_v": 0.819091796875, "blocks.5.attention.w_out": 0.8188743591308594, "blocks.5.feed_forward.w1": 0.7972793579101562, "blocks.5.feed_forward.w2": 0.6101735830307007, "blocks.5.feed_forward.w3": 0.8065058588981628, "blocks.6.attention.w_q": 0.6785545349121094, "blocks.6.attention.w_k": 0.7095184326171875, "blocks.6.attention.w_v": 0.8219642639160156, "blocks.6.attention.w_out": 0.8187332153320312, "blocks.6.feed_forward.w1": 0.7958145141601562, "blocks.6.feed_forward.w2": 0.6097399592399597, "blocks.6.feed_forward.w3": 0.8036741018295288, "blocks.7.attention.w_q": 0.6682243347167969, "blocks.7.attention.w_k": 0.6878395080566406, "blocks.7.attention.w_v": 0.8218269348144531, "blocks.7.attention.w_out": 0.8194046020507812, "blocks.7.feed_forward.w1": 0.7744776606559753, "blocks.7.feed_forward.w2": 0.5364125967025757, "blocks.7.feed_forward.w3": 0.7829221487045288, "blocks.8.attention.w_q": 0.6711196899414062, "blocks.8.attention.w_k": 0.7069053649902344, "blocks.8.attention.w_v": 0.8206329345703125, "blocks.8.attention.w_out": 0.8182525634765625, "blocks.8.feed_forward.w1": 0.7914619445800781, "blocks.8.feed_forward.w2": 0.5494550466537476, "blocks.8.feed_forward.w3": 0.7990493774414062}, "w_mask_frac": 0.15053229339611837} +{"step": 700, "changed_frac": 0.7525803329598191, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7582511901855469, "blocks.0.attention.w_k": 0.7818756103515625, "blocks.0.attention.w_v": 0.8188514709472656, "blocks.0.attention.w_out": 0.8157501220703125, "blocks.0.feed_forward.w1": 0.7692591547966003, "blocks.0.feed_forward.w2": 0.6825294494628906, "blocks.0.feed_forward.w3": 0.7665519714355469, "blocks.1.attention.w_q": 0.7259292602539062, "blocks.1.attention.w_k": 0.7568473815917969, "blocks.1.attention.w_v": 0.8316268920898438, "blocks.1.attention.w_out": 0.8330154418945312, "blocks.1.feed_forward.w1": 0.8022994995117188, "blocks.1.feed_forward.w2": 0.6819076538085938, "blocks.1.feed_forward.w3": 0.8047701716423035, "blocks.2.attention.w_q": 0.732666015625, "blocks.2.attention.w_k": 0.7596397399902344, "blocks.2.attention.w_v": 0.8327560424804688, "blocks.2.attention.w_out": 0.8347396850585938, "blocks.2.feed_forward.w1": 0.8099174499511719, "blocks.2.feed_forward.w2": 0.6632283926010132, "blocks.2.feed_forward.w3": 0.8119125366210938, "blocks.3.attention.w_q": 0.7382049560546875, "blocks.3.attention.w_k": 0.7641944885253906, "blocks.3.attention.w_v": 0.8304862976074219, "blocks.3.attention.w_out": 0.8315620422363281, "blocks.3.feed_forward.w1": 0.8108558654785156, "blocks.3.feed_forward.w2": 0.6447245478630066, "blocks.3.feed_forward.w3": 0.8147773742675781, "blocks.4.attention.w_q": 0.7483100891113281, "blocks.4.attention.w_k": 0.7687950134277344, "blocks.4.attention.w_v": 0.8278579711914062, "blocks.4.attention.w_out": 0.8300857543945312, "blocks.4.feed_forward.w1": 0.8154335021972656, "blocks.4.feed_forward.w2": 0.6242536306381226, "blocks.4.feed_forward.w3": 0.818762481212616, "blocks.5.attention.w_q": 0.7450027465820312, "blocks.5.attention.w_k": 0.7720794677734375, "blocks.5.attention.w_v": 0.8276100158691406, "blocks.5.attention.w_out": 0.82843017578125, "blocks.5.feed_forward.w1": 0.8186912536621094, "blocks.5.feed_forward.w2": 0.5871162414550781, "blocks.5.feed_forward.w3": 0.8220507502555847, "blocks.6.attention.w_q": 0.7428932189941406, "blocks.6.attention.w_k": 0.7720909118652344, "blocks.6.attention.w_v": 0.826263427734375, "blocks.6.attention.w_out": 0.8240089416503906, "blocks.6.feed_forward.w1": 0.8077520132064819, "blocks.6.feed_forward.w2": 0.5395520925521851, "blocks.6.feed_forward.w3": 0.8114840388298035, "blocks.7.attention.w_q": 0.7286643981933594, "blocks.7.attention.w_k": 0.7549667358398438, "blocks.7.attention.w_v": 0.8264808654785156, "blocks.7.attention.w_out": 0.8223419189453125, "blocks.7.feed_forward.w1": 0.7960955500602722, "blocks.7.feed_forward.w2": 0.5196113586425781, "blocks.7.feed_forward.w3": 0.8062731623649597, "blocks.8.attention.w_q": 0.7209434509277344, "blocks.8.attention.w_k": 0.7624320983886719, "blocks.8.attention.w_v": 0.8230667114257812, "blocks.8.attention.w_out": 0.8186111450195312, "blocks.8.feed_forward.w1": 0.7947998046875, "blocks.8.feed_forward.w2": 0.44641751050949097, "blocks.8.feed_forward.w3": 0.8071619868278503}, "w_mask_frac": 0.15532997658110073} +{"step": 800, "changed_frac": 0.7638317336384047, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8004913330078125, "blocks.0.attention.w_k": 0.80987548828125, "blocks.0.attention.w_v": 0.8198013305664062, "blocks.0.attention.w_out": 0.815093994140625, "blocks.0.feed_forward.w1": 0.775604248046875, "blocks.0.feed_forward.w2": 0.6699931025505066, "blocks.0.feed_forward.w3": 0.773284912109375, "blocks.1.attention.w_q": 0.7537422180175781, "blocks.1.attention.w_k": 0.7824172973632812, "blocks.1.attention.w_v": 0.8333396911621094, "blocks.1.attention.w_out": 0.8360862731933594, "blocks.1.feed_forward.w1": 0.8098793029785156, "blocks.1.feed_forward.w2": 0.6894544363021851, "blocks.1.feed_forward.w3": 0.8111610412597656, "blocks.2.attention.w_q": 0.7495613098144531, "blocks.2.attention.w_k": 0.780517578125, "blocks.2.attention.w_v": 0.8350715637207031, "blocks.2.attention.w_out": 0.8382186889648438, "blocks.2.feed_forward.w1": 0.8178507685661316, "blocks.2.feed_forward.w2": 0.682745635509491, "blocks.2.feed_forward.w3": 0.8169873952865601, "blocks.3.attention.w_q": 0.7665252685546875, "blocks.3.attention.w_k": 0.7737884521484375, "blocks.3.attention.w_v": 0.8343391418457031, "blocks.3.attention.w_out": 0.8389320373535156, "blocks.3.feed_forward.w1": 0.8217570185661316, "blocks.3.feed_forward.w2": 0.6614608764648438, "blocks.3.feed_forward.w3": 0.8239568471908569, "blocks.4.attention.w_q": 0.7712440490722656, "blocks.4.attention.w_k": 0.7760162353515625, "blocks.4.attention.w_v": 0.8325881958007812, "blocks.4.attention.w_out": 0.8375511169433594, "blocks.4.feed_forward.w1": 0.8261579275131226, "blocks.4.feed_forward.w2": 0.6550712585449219, "blocks.4.feed_forward.w3": 0.8268508911132812, "blocks.5.attention.w_q": 0.7587203979492188, "blocks.5.attention.w_k": 0.784149169921875, "blocks.5.attention.w_v": 0.8316726684570312, "blocks.5.attention.w_out": 0.8331260681152344, "blocks.5.feed_forward.w1": 0.8283564448356628, "blocks.5.feed_forward.w2": 0.5792198181152344, "blocks.5.feed_forward.w3": 0.8299115896224976, "blocks.6.attention.w_q": 0.7677726745605469, "blocks.6.attention.w_k": 0.7886695861816406, "blocks.6.attention.w_v": 0.8308029174804688, "blocks.6.attention.w_out": 0.8294715881347656, "blocks.6.feed_forward.w1": 0.8162549734115601, "blocks.6.feed_forward.w2": 0.5344047546386719, "blocks.6.feed_forward.w3": 0.819098174571991, "blocks.7.attention.w_q": 0.7494544982910156, "blocks.7.attention.w_k": 0.7724990844726562, "blocks.7.attention.w_v": 0.8254165649414062, "blocks.7.attention.w_out": 0.8302879333496094, "blocks.7.feed_forward.w1": 0.815039336681366, "blocks.7.feed_forward.w2": 0.5546519160270691, "blocks.7.feed_forward.w3": 0.8199679255485535, "blocks.8.attention.w_q": 0.7534141540527344, "blocks.8.attention.w_k": 0.7810173034667969, "blocks.8.attention.w_v": 0.8252525329589844, "blocks.8.attention.w_out": 0.8284034729003906, "blocks.8.feed_forward.w1": 0.8071174621582031, "blocks.8.feed_forward.w2": 0.4820048213005066, "blocks.8.feed_forward.w3": 0.8160845637321472}, "w_mask_frac": 0.16265038025175405} +{"step": 900, "changed_frac": 0.7598348438230336, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8030242919921875, "blocks.0.attention.w_k": 0.8086662292480469, "blocks.0.attention.w_v": 0.80828857421875, "blocks.0.attention.w_out": 0.8005027770996094, "blocks.0.feed_forward.w1": 0.7670211791992188, "blocks.0.feed_forward.w2": 0.6267268061637878, "blocks.0.feed_forward.w3": 0.7633146047592163, "blocks.1.attention.w_q": 0.7739906311035156, "blocks.1.attention.w_k": 0.7885704040527344, "blocks.1.attention.w_v": 0.822845458984375, "blocks.1.attention.w_out": 0.8272171020507812, "blocks.1.feed_forward.w1": 0.802344024181366, "blocks.1.feed_forward.w2": 0.6618029475212097, "blocks.1.feed_forward.w3": 0.8022588491439819, "blocks.2.attention.w_q": 0.7650032043457031, "blocks.2.attention.w_k": 0.7766990661621094, "blocks.2.attention.w_v": 0.8274116516113281, "blocks.2.attention.w_out": 0.8319587707519531, "blocks.2.feed_forward.w1": 0.8126513361930847, "blocks.2.feed_forward.w2": 0.6743698120117188, "blocks.2.feed_forward.w3": 0.8114878535270691, "blocks.3.attention.w_q": 0.7782974243164062, "blocks.3.attention.w_k": 0.7751007080078125, "blocks.3.attention.w_v": 0.8269081115722656, "blocks.3.attention.w_out": 0.8328781127929688, "blocks.3.feed_forward.w1": 0.8172162771224976, "blocks.3.feed_forward.w2": 0.6513557434082031, "blocks.3.feed_forward.w3": 0.8177999258041382, "blocks.4.attention.w_q": 0.7744293212890625, "blocks.4.attention.w_k": 0.7731742858886719, "blocks.4.attention.w_v": 0.8271980285644531, "blocks.4.attention.w_out": 0.8315620422363281, "blocks.4.feed_forward.w1": 0.8206965327262878, "blocks.4.feed_forward.w2": 0.641021728515625, "blocks.4.feed_forward.w3": 0.8200963735580444, "blocks.5.attention.w_q": 0.7717094421386719, "blocks.5.attention.w_k": 0.7741508483886719, "blocks.5.attention.w_v": 0.82635498046875, "blocks.5.attention.w_out": 0.8284187316894531, "blocks.5.feed_forward.w1": 0.8205922842025757, "blocks.5.feed_forward.w2": 0.5849088430404663, "blocks.5.feed_forward.w3": 0.8212165832519531, "blocks.6.attention.w_q": 0.77862548828125, "blocks.6.attention.w_k": 0.782012939453125, "blocks.6.attention.w_v": 0.82464599609375, "blocks.6.attention.w_out": 0.8244590759277344, "blocks.6.feed_forward.w1": 0.8089396357536316, "blocks.6.feed_forward.w2": 0.5646375417709351, "blocks.6.feed_forward.w3": 0.8115514516830444, "blocks.7.attention.w_q": 0.733428955078125, "blocks.7.attention.w_k": 0.7345924377441406, "blocks.7.attention.w_v": 0.8201370239257812, "blocks.7.attention.w_out": 0.8222541809082031, "blocks.7.feed_forward.w1": 0.810614287853241, "blocks.7.feed_forward.w2": 0.5897051692008972, "blocks.7.feed_forward.w3": 0.815069854259491, "blocks.8.attention.w_q": 0.7480506896972656, "blocks.8.attention.w_k": 0.7564506530761719, "blocks.8.attention.w_v": 0.8194541931152344, "blocks.8.attention.w_out": 0.8210105895996094, "blocks.8.feed_forward.w1": 0.8044891357421875, "blocks.8.feed_forward.w2": 0.527862548828125, "blocks.8.feed_forward.w3": 0.8106473684310913}, "w_mask_frac": 0.16743283788872582} +{"step": 1000, "changed_frac": 0.7574209555601462, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8031806945800781, "blocks.0.attention.w_k": 0.8047599792480469, "blocks.0.attention.w_v": 0.8046646118164062, "blocks.0.attention.w_out": 0.7940711975097656, "blocks.0.feed_forward.w1": 0.7537829279899597, "blocks.0.feed_forward.w2": 0.6023992300033569, "blocks.0.feed_forward.w3": 0.7493896484375, "blocks.1.attention.w_q": 0.783477783203125, "blocks.1.attention.w_k": 0.7902412414550781, "blocks.1.attention.w_v": 0.8183326721191406, "blocks.1.attention.w_out": 0.8210296630859375, "blocks.1.feed_forward.w1": 0.7937850952148438, "blocks.1.feed_forward.w2": 0.646429717540741, "blocks.1.feed_forward.w3": 0.7937965393066406, "blocks.2.attention.w_q": 0.7659454345703125, "blocks.2.attention.w_k": 0.7742996215820312, "blocks.2.attention.w_v": 0.8193588256835938, "blocks.2.attention.w_out": 0.8255271911621094, "blocks.2.feed_forward.w1": 0.8057035207748413, "blocks.2.feed_forward.w2": 0.6572074890136719, "blocks.2.feed_forward.w3": 0.8058764338493347, "blocks.3.attention.w_q": 0.7883148193359375, "blocks.3.attention.w_k": 0.7775535583496094, "blocks.3.attention.w_v": 0.8189773559570312, "blocks.3.attention.w_out": 0.82403564453125, "blocks.3.feed_forward.w1": 0.8108012080192566, "blocks.3.feed_forward.w2": 0.6515655517578125, "blocks.3.feed_forward.w3": 0.8105303645133972, "blocks.4.attention.w_q": 0.8008041381835938, "blocks.4.attention.w_k": 0.7846221923828125, "blocks.4.attention.w_v": 0.8188667297363281, "blocks.4.attention.w_out": 0.8237533569335938, "blocks.4.feed_forward.w1": 0.8151588439941406, "blocks.4.feed_forward.w2": 0.6353251338005066, "blocks.4.feed_forward.w3": 0.8160476684570312, "blocks.5.attention.w_q": 0.7786636352539062, "blocks.5.attention.w_k": 0.77166748046875, "blocks.5.attention.w_v": 0.8180694580078125, "blocks.5.attention.w_out": 0.822601318359375, "blocks.5.feed_forward.w1": 0.8131523132324219, "blocks.5.feed_forward.w2": 0.6075999140739441, "blocks.5.feed_forward.w3": 0.8115882873535156, "blocks.6.attention.w_q": 0.7969627380371094, "blocks.6.attention.w_k": 0.7903289794921875, "blocks.6.attention.w_v": 0.8181915283203125, "blocks.6.attention.w_out": 0.8193473815917969, "blocks.6.feed_forward.w1": 0.804724395275116, "blocks.6.feed_forward.w2": 0.5911331176757812, "blocks.6.feed_forward.w3": 0.8032798767089844, "blocks.7.attention.w_q": 0.7220840454101562, "blocks.7.attention.w_k": 0.7161865234375, "blocks.7.attention.w_v": 0.8093070983886719, "blocks.7.attention.w_out": 0.81597900390625, "blocks.7.feed_forward.w1": 0.8050257563591003, "blocks.7.feed_forward.w2": 0.6318677663803101, "blocks.7.feed_forward.w3": 0.8087018728256226, "blocks.8.attention.w_q": 0.7286376953125, "blocks.8.attention.w_k": 0.7384567260742188, "blocks.8.attention.w_v": 0.8091468811035156, "blocks.8.attention.w_out": 0.8157119750976562, "blocks.8.feed_forward.w1": 0.803717315196991, "blocks.8.feed_forward.w2": 0.5692723989486694, "blocks.8.feed_forward.w3": 0.8038355708122253}, "w_mask_frac": 0.16559865319321299} +{"step": 1100, "changed_frac": 0.7487582345294137, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7978515625, "blocks.0.attention.w_k": 0.7932586669921875, "blocks.0.attention.w_v": 0.793670654296875, "blocks.0.attention.w_out": 0.7819671630859375, "blocks.0.feed_forward.w1": 0.7344297170639038, "blocks.0.feed_forward.w2": 0.5531184077262878, "blocks.0.feed_forward.w3": 0.7298228144645691, "blocks.1.attention.w_q": 0.7829933166503906, "blocks.1.attention.w_k": 0.7837982177734375, "blocks.1.attention.w_v": 0.8060302734375, "blocks.1.attention.w_out": 0.8131294250488281, "blocks.1.feed_forward.w1": 0.7818921804428101, "blocks.1.feed_forward.w2": 0.6085166931152344, "blocks.1.feed_forward.w3": 0.7803115844726562, "blocks.2.attention.w_q": 0.7626228332519531, "blocks.2.attention.w_k": 0.7683067321777344, "blocks.2.attention.w_v": 0.8093299865722656, "blocks.2.attention.w_out": 0.816070556640625, "blocks.2.feed_forward.w1": 0.7976595759391785, "blocks.2.feed_forward.w2": 0.6214231252670288, "blocks.2.feed_forward.w3": 0.7958425283432007, "blocks.3.attention.w_q": 0.7944068908691406, "blocks.3.attention.w_k": 0.7825813293457031, "blocks.3.attention.w_v": 0.8102073669433594, "blocks.3.attention.w_out": 0.8189620971679688, "blocks.3.feed_forward.w1": 0.8024063110351562, "blocks.3.feed_forward.w2": 0.6289991140365601, "blocks.3.feed_forward.w3": 0.8011945486068726, "blocks.4.attention.w_q": 0.8000259399414062, "blocks.4.attention.w_k": 0.7803421020507812, "blocks.4.attention.w_v": 0.812713623046875, "blocks.4.attention.w_out": 0.8157806396484375, "blocks.4.feed_forward.w1": 0.8095487356185913, "blocks.4.feed_forward.w2": 0.6203244924545288, "blocks.4.feed_forward.w3": 0.8087806701660156, "blocks.5.attention.w_q": 0.788818359375, "blocks.5.attention.w_k": 0.7715721130371094, "blocks.5.attention.w_v": 0.8122749328613281, "blocks.5.attention.w_out": 0.8135299682617188, "blocks.5.feed_forward.w1": 0.8096237182617188, "blocks.5.feed_forward.w2": 0.604498565196991, "blocks.5.feed_forward.w3": 0.8075053095817566, "blocks.6.attention.w_q": 0.7990951538085938, "blocks.6.attention.w_k": 0.7856636047363281, "blocks.6.attention.w_v": 0.8111724853515625, "blocks.6.attention.w_out": 0.8130378723144531, "blocks.6.feed_forward.w1": 0.8062973022460938, "blocks.6.feed_forward.w2": 0.5944480895996094, "blocks.6.feed_forward.w3": 0.8018112182617188, "blocks.7.attention.w_q": 0.7184486389160156, "blocks.7.attention.w_k": 0.7106170654296875, "blocks.7.attention.w_v": 0.8041419982910156, "blocks.7.attention.w_out": 0.8100318908691406, "blocks.7.feed_forward.w1": 0.8044726252555847, "blocks.7.feed_forward.w2": 0.6300086975097656, "blocks.7.feed_forward.w3": 0.8066381216049194, "blocks.8.attention.w_q": 0.7160835266113281, "blocks.8.attention.w_k": 0.7177848815917969, "blocks.8.attention.w_v": 0.8038177490234375, "blocks.8.attention.w_out": 0.8105926513671875, "blocks.8.feed_forward.w1": 0.7985814809799194, "blocks.8.feed_forward.w2": 0.5950469970703125, "blocks.8.feed_forward.w3": 0.7981249690055847}, "w_mask_frac": 0.16537394419185117} +{"step": 1200, "changed_frac": 0.7408023410373263, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7944145202636719, "blocks.0.attention.w_k": 0.7931175231933594, "blocks.0.attention.w_v": 0.7874259948730469, "blocks.0.attention.w_out": 0.773773193359375, "blocks.0.feed_forward.w1": 0.720916748046875, "blocks.0.feed_forward.w2": 0.5232989192008972, "blocks.0.feed_forward.w3": 0.7159106135368347, "blocks.1.attention.w_q": 0.7837982177734375, "blocks.1.attention.w_k": 0.7844047546386719, "blocks.1.attention.w_v": 0.8000602722167969, "blocks.1.attention.w_out": 0.8080215454101562, "blocks.1.feed_forward.w1": 0.7712974548339844, "blocks.1.feed_forward.w2": 0.5887934565544128, "blocks.1.feed_forward.w3": 0.7701797485351562, "blocks.2.attention.w_q": 0.7598533630371094, "blocks.2.attention.w_k": 0.7587852478027344, "blocks.2.attention.w_v": 0.8037834167480469, "blocks.2.attention.w_out": 0.8117027282714844, "blocks.2.feed_forward.w1": 0.7891807556152344, "blocks.2.feed_forward.w2": 0.6026293635368347, "blocks.2.feed_forward.w3": 0.7869669795036316, "blocks.3.attention.w_q": 0.7985115051269531, "blocks.3.attention.w_k": 0.7888755798339844, "blocks.3.attention.w_v": 0.8028068542480469, "blocks.3.attention.w_out": 0.8125419616699219, "blocks.3.feed_forward.w1": 0.7948888540267944, "blocks.3.feed_forward.w2": 0.6002489924430847, "blocks.3.feed_forward.w3": 0.7947807312011719, "blocks.4.attention.w_q": 0.8011817932128906, "blocks.4.attention.w_k": 0.7815513610839844, "blocks.4.attention.w_v": 0.8048667907714844, "blocks.4.attention.w_out": 0.8069114685058594, "blocks.4.feed_forward.w1": 0.8030523061752319, "blocks.4.feed_forward.w2": 0.6002477407455444, "blocks.4.feed_forward.w3": 0.8024444580078125, "blocks.5.attention.w_q": 0.7927780151367188, "blocks.5.attention.w_k": 0.7779693603515625, "blocks.5.attention.w_v": 0.8019180297851562, "blocks.5.attention.w_out": 0.8043251037597656, "blocks.5.feed_forward.w1": 0.8028272390365601, "blocks.5.feed_forward.w2": 0.5904884338378906, "blocks.5.feed_forward.w3": 0.7999827265739441, "blocks.6.attention.w_q": 0.7936973571777344, "blocks.6.attention.w_k": 0.7821464538574219, "blocks.6.attention.w_v": 0.8032760620117188, "blocks.6.attention.w_out": 0.8053474426269531, "blocks.6.feed_forward.w1": 0.8028488159179688, "blocks.6.feed_forward.w2": 0.588226318359375, "blocks.6.feed_forward.w3": 0.7980155944824219, "blocks.7.attention.w_q": 0.7123222351074219, "blocks.7.attention.w_k": 0.7090568542480469, "blocks.7.attention.w_v": 0.7919349670410156, "blocks.7.attention.w_out": 0.7986183166503906, "blocks.7.feed_forward.w1": 0.8029467463493347, "blocks.7.feed_forward.w2": 0.6168556213378906, "blocks.7.feed_forward.w3": 0.8036016225814819, "blocks.8.attention.w_q": 0.6967658996582031, "blocks.8.attention.w_k": 0.697998046875, "blocks.8.attention.w_v": 0.7952156066894531, "blocks.8.attention.w_out": 0.8011207580566406, "blocks.8.feed_forward.w1": 0.7995631098747253, "blocks.8.feed_forward.w2": 0.6163228750228882, "blocks.8.feed_forward.w3": 0.7978159785270691}, "w_mask_frac": 0.17526180214352077} +{"step": 1300, "changed_frac": 0.7346689925234542, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7864723205566406, "blocks.0.attention.w_k": 0.7824630737304688, "blocks.0.attention.w_v": 0.7814102172851562, "blocks.0.attention.w_out": 0.7668304443359375, "blocks.0.feed_forward.w1": 0.7161674499511719, "blocks.0.feed_forward.w2": 0.502264678478241, "blocks.0.feed_forward.w3": 0.7133445739746094, "blocks.1.attention.w_q": 0.788177490234375, "blocks.1.attention.w_k": 0.7834968566894531, "blocks.1.attention.w_v": 0.7970085144042969, "blocks.1.attention.w_out": 0.8015060424804688, "blocks.1.feed_forward.w1": 0.7674052119255066, "blocks.1.feed_forward.w2": 0.5683848261833191, "blocks.1.feed_forward.w3": 0.7668444514274597, "blocks.2.attention.w_q": 0.7634239196777344, "blocks.2.attention.w_k": 0.7621345520019531, "blocks.2.attention.w_v": 0.7966041564941406, "blocks.2.attention.w_out": 0.8035926818847656, "blocks.2.feed_forward.w1": 0.7840461730957031, "blocks.2.feed_forward.w2": 0.5843862295150757, "blocks.2.feed_forward.w3": 0.7827390432357788, "blocks.3.attention.w_q": 0.8017616271972656, "blocks.3.attention.w_k": 0.7953224182128906, "blocks.3.attention.w_v": 0.7976341247558594, "blocks.3.attention.w_out": 0.8043899536132812, "blocks.3.feed_forward.w1": 0.7900225520133972, "blocks.3.feed_forward.w2": 0.5776685476303101, "blocks.3.feed_forward.w3": 0.7894071340560913, "blocks.4.attention.w_q": 0.80474853515625, "blocks.4.attention.w_k": 0.7842445373535156, "blocks.4.attention.w_v": 0.7975616455078125, "blocks.4.attention.w_out": 0.7989349365234375, "blocks.4.feed_forward.w1": 0.7971916198730469, "blocks.4.feed_forward.w2": 0.5895652770996094, "blocks.4.feed_forward.w3": 0.7973442077636719, "blocks.5.attention.w_q": 0.7914772033691406, "blocks.5.attention.w_k": 0.7780799865722656, "blocks.5.attention.w_v": 0.7978401184082031, "blocks.5.attention.w_out": 0.7970504760742188, "blocks.5.feed_forward.w1": 0.7991231679916382, "blocks.5.feed_forward.w2": 0.58453369140625, "blocks.5.feed_forward.w3": 0.7966855764389038, "blocks.6.attention.w_q": 0.7876434326171875, "blocks.6.attention.w_k": 0.7751121520996094, "blocks.6.attention.w_v": 0.7980613708496094, "blocks.6.attention.w_out": 0.7999534606933594, "blocks.6.feed_forward.w1": 0.7998988032341003, "blocks.6.feed_forward.w2": 0.5806986689567566, "blocks.6.feed_forward.w3": 0.7944272756576538, "blocks.7.attention.w_q": 0.7110404968261719, "blocks.7.attention.w_k": 0.6967391967773438, "blocks.7.attention.w_v": 0.7855415344238281, "blocks.7.attention.w_out": 0.7839775085449219, "blocks.7.feed_forward.w1": 0.7983335256576538, "blocks.7.feed_forward.w2": 0.6102740168571472, "blocks.7.feed_forward.w3": 0.7992159724235535, "blocks.8.attention.w_q": 0.6910324096679688, "blocks.8.attention.w_k": 0.6825942993164062, "blocks.8.attention.w_v": 0.7873802185058594, "blocks.8.attention.w_out": 0.7930641174316406, "blocks.8.feed_forward.w1": 0.8000628352165222, "blocks.8.feed_forward.w2": 0.61614990234375, "blocks.8.feed_forward.w3": 0.794470489025116}, "w_mask_frac": 0.16333520043100047} +{"step": 1400, "changed_frac": 0.727810786320613, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7788619995117188, "blocks.0.attention.w_k": 0.7757339477539062, "blocks.0.attention.w_v": 0.7749710083007812, "blocks.0.attention.w_out": 0.7568626403808594, "blocks.0.feed_forward.w1": 0.7005653381347656, "blocks.0.feed_forward.w2": 0.47811001539230347, "blocks.0.feed_forward.w3": 0.6974881887435913, "blocks.1.attention.w_q": 0.7868728637695312, "blocks.1.attention.w_k": 0.7758903503417969, "blocks.1.attention.w_v": 0.7938461303710938, "blocks.1.attention.w_out": 0.7958602905273438, "blocks.1.feed_forward.w1": 0.7558428645133972, "blocks.1.feed_forward.w2": 0.5522283315658569, "blocks.1.feed_forward.w3": 0.7558199763298035, "blocks.2.attention.w_q": 0.7541999816894531, "blocks.2.attention.w_k": 0.74847412109375, "blocks.2.attention.w_v": 0.7919502258300781, "blocks.2.attention.w_out": 0.7965049743652344, "blocks.2.feed_forward.w1": 0.7765299677848816, "blocks.2.feed_forward.w2": 0.5689888000488281, "blocks.2.feed_forward.w3": 0.7754631042480469, "blocks.3.attention.w_q": 0.80035400390625, "blocks.3.attention.w_k": 0.7930679321289062, "blocks.3.attention.w_v": 0.7929763793945312, "blocks.3.attention.w_out": 0.8011283874511719, "blocks.3.feed_forward.w1": 0.7833290100097656, "blocks.3.feed_forward.w2": 0.5632985830307007, "blocks.3.feed_forward.w3": 0.7828330993652344, "blocks.4.attention.w_q": 0.7957611083984375, "blocks.4.attention.w_k": 0.7754631042480469, "blocks.4.attention.w_v": 0.7920341491699219, "blocks.4.attention.w_out": 0.7959365844726562, "blocks.4.feed_forward.w1": 0.7928492426872253, "blocks.4.feed_forward.w2": 0.5754725337028503, "blocks.4.feed_forward.w3": 0.7916831970214844, "blocks.5.attention.w_q": 0.7931327819824219, "blocks.5.attention.w_k": 0.7740402221679688, "blocks.5.attention.w_v": 0.7925682067871094, "blocks.5.attention.w_out": 0.7929153442382812, "blocks.5.feed_forward.w1": 0.7933731079101562, "blocks.5.feed_forward.w2": 0.5809440612792969, "blocks.5.feed_forward.w3": 0.7929713129997253, "blocks.6.attention.w_q": 0.7820701599121094, "blocks.6.attention.w_k": 0.7667007446289062, "blocks.6.attention.w_v": 0.7903366088867188, "blocks.6.attention.w_out": 0.7937698364257812, "blocks.6.feed_forward.w1": 0.7938575744628906, "blocks.6.feed_forward.w2": 0.5834465026855469, "blocks.6.feed_forward.w3": 0.7860502004623413, "blocks.7.attention.w_q": 0.713409423828125, "blocks.7.attention.w_k": 0.7077980041503906, "blocks.7.attention.w_v": 0.7829399108886719, "blocks.7.attention.w_out": 0.7732048034667969, "blocks.7.feed_forward.w1": 0.7883148193359375, "blocks.7.feed_forward.w2": 0.6221567988395691, "blocks.7.feed_forward.w3": 0.7893562316894531, "blocks.8.attention.w_q": 0.6964378356933594, "blocks.8.attention.w_k": 0.6795654296875, "blocks.8.attention.w_v": 0.7863883972167969, "blocks.8.attention.w_out": 0.7830772399902344, "blocks.8.feed_forward.w1": 0.7898687124252319, "blocks.8.feed_forward.w2": 0.6322886347770691, "blocks.8.feed_forward.w3": 0.7864558100700378}, "w_mask_frac": 0.17357955274418888} +{"step": 1500, "changed_frac": 0.7178446125780416, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7763671875, "blocks.0.attention.w_k": 0.7742156982421875, "blocks.0.attention.w_v": 0.7683830261230469, "blocks.0.attention.w_out": 0.7487640380859375, "blocks.0.feed_forward.w1": 0.6870638728141785, "blocks.0.feed_forward.w2": 0.4637502133846283, "blocks.0.feed_forward.w3": 0.6849416494369507, "blocks.1.attention.w_q": 0.7865142822265625, "blocks.1.attention.w_k": 0.7745590209960938, "blocks.1.attention.w_v": 0.7836685180664062, "blocks.1.attention.w_out": 0.7886466979980469, "blocks.1.feed_forward.w1": 0.7466773986816406, "blocks.1.feed_forward.w2": 0.5367012023925781, "blocks.1.feed_forward.w3": 0.7454261779785156, "blocks.2.attention.w_q": 0.7536277770996094, "blocks.2.attention.w_k": 0.7430610656738281, "blocks.2.attention.w_v": 0.7837333679199219, "blocks.2.attention.w_out": 0.7871322631835938, "blocks.2.feed_forward.w1": 0.7700347900390625, "blocks.2.feed_forward.w2": 0.5540174245834351, "blocks.2.feed_forward.w3": 0.7679647207260132, "blocks.3.attention.w_q": 0.7924652099609375, "blocks.3.attention.w_k": 0.7855720520019531, "blocks.3.attention.w_v": 0.7828369140625, "blocks.3.attention.w_out": 0.7928657531738281, "blocks.3.feed_forward.w1": 0.7756284475326538, "blocks.3.feed_forward.w2": 0.5500971674919128, "blocks.3.feed_forward.w3": 0.7744725942611694, "blocks.4.attention.w_q": 0.7909011840820312, "blocks.4.attention.w_k": 0.7717437744140625, "blocks.4.attention.w_v": 0.7846603393554688, "blocks.4.attention.w_out": 0.7866134643554688, "blocks.4.feed_forward.w1": 0.7856013178825378, "blocks.4.feed_forward.w2": 0.5715802907943726, "blocks.4.feed_forward.w3": 0.7849540710449219, "blocks.5.attention.w_q": 0.7865982055664062, "blocks.5.attention.w_k": 0.7644920349121094, "blocks.5.attention.w_v": 0.7820777893066406, "blocks.5.attention.w_out": 0.78411865234375, "blocks.5.feed_forward.w1": 0.7851511836051941, "blocks.5.feed_forward.w2": 0.5741996765136719, "blocks.5.feed_forward.w3": 0.7845319509506226, "blocks.6.attention.w_q": 0.7778053283691406, "blocks.6.attention.w_k": 0.7535972595214844, "blocks.6.attention.w_v": 0.7788124084472656, "blocks.6.attention.w_out": 0.779083251953125, "blocks.6.feed_forward.w1": 0.7794151306152344, "blocks.6.feed_forward.w2": 0.5626691579818726, "blocks.6.feed_forward.w3": 0.772925078868866, "blocks.7.attention.w_q": 0.70703125, "blocks.7.attention.w_k": 0.6984596252441406, "blocks.7.attention.w_v": 0.7732124328613281, "blocks.7.attention.w_out": 0.7510948181152344, "blocks.7.feed_forward.w1": 0.782958984375, "blocks.7.feed_forward.w2": 0.6120758056640625, "blocks.7.feed_forward.w3": 0.7801259756088257, "blocks.8.attention.w_q": 0.6775283813476562, "blocks.8.attention.w_k": 0.6582794189453125, "blocks.8.attention.w_v": 0.7714309692382812, "blocks.8.attention.w_out": 0.7669639587402344, "blocks.8.feed_forward.w1": 0.7818654775619507, "blocks.8.feed_forward.w2": 0.6115939021110535, "blocks.8.feed_forward.w3": 0.7805442810058594}, "w_mask_frac": 0.1819786372093054} +{"step": 1600, "changed_frac": 0.7154012174687834, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7688255310058594, "blocks.0.attention.w_k": 0.7665176391601562, "blocks.0.attention.w_v": 0.7625312805175781, "blocks.0.attention.w_out": 0.7413978576660156, "blocks.0.feed_forward.w1": 0.6801732778549194, "blocks.0.feed_forward.w2": 0.45503872632980347, "blocks.0.feed_forward.w3": 0.6777992248535156, "blocks.1.attention.w_q": 0.7875099182128906, "blocks.1.attention.w_k": 0.7743568420410156, "blocks.1.attention.w_v": 0.7785606384277344, "blocks.1.attention.w_out": 0.7816352844238281, "blocks.1.feed_forward.w1": 0.7434730529785156, "blocks.1.feed_forward.w2": 0.5302340388298035, "blocks.1.feed_forward.w3": 0.7420578002929688, "blocks.2.attention.w_q": 0.7521781921386719, "blocks.2.attention.w_k": 0.7397041320800781, "blocks.2.attention.w_v": 0.7749061584472656, "blocks.2.attention.w_out": 0.7795829772949219, "blocks.2.feed_forward.w1": 0.7654559016227722, "blocks.2.feed_forward.w2": 0.54840087890625, "blocks.2.feed_forward.w3": 0.7630907893180847, "blocks.3.attention.w_q": 0.79229736328125, "blocks.3.attention.w_k": 0.7831916809082031, "blocks.3.attention.w_v": 0.777191162109375, "blocks.3.attention.w_out": 0.7837715148925781, "blocks.3.feed_forward.w1": 0.7708905935287476, "blocks.3.feed_forward.w2": 0.5483907461166382, "blocks.3.feed_forward.w3": 0.7685407400131226, "blocks.4.attention.w_q": 0.7907295227050781, "blocks.4.attention.w_k": 0.7717208862304688, "blocks.4.attention.w_v": 0.7752189636230469, "blocks.4.attention.w_out": 0.7789497375488281, "blocks.4.feed_forward.w1": 0.7824783325195312, "blocks.4.feed_forward.w2": 0.5640271902084351, "blocks.4.feed_forward.w3": 0.7811660766601562, "blocks.5.attention.w_q": 0.7828178405761719, "blocks.5.attention.w_k": 0.76202392578125, "blocks.5.attention.w_v": 0.7757072448730469, "blocks.5.attention.w_out": 0.7740440368652344, "blocks.5.feed_forward.w1": 0.7847518920898438, "blocks.5.feed_forward.w2": 0.5641288757324219, "blocks.5.feed_forward.w3": 0.7838045954704285, "blocks.6.attention.w_q": 0.7800788879394531, "blocks.6.attention.w_k": 0.7526321411132812, "blocks.6.attention.w_v": 0.7724952697753906, "blocks.6.attention.w_out": 0.7713165283203125, "blocks.6.feed_forward.w1": 0.7828394770622253, "blocks.6.feed_forward.w2": 0.5747630000114441, "blocks.6.feed_forward.w3": 0.7744611501693726, "blocks.7.attention.w_q": 0.7104301452636719, "blocks.7.attention.w_k": 0.7071495056152344, "blocks.7.attention.w_v": 0.7686119079589844, "blocks.7.attention.w_out": 0.74224853515625, "blocks.7.feed_forward.w1": 0.7847518920898438, "blocks.7.feed_forward.w2": 0.6156654357910156, "blocks.7.feed_forward.w3": 0.7822520136833191, "blocks.8.attention.w_q": 0.6849594116210938, "blocks.8.attention.w_k": 0.6632919311523438, "blocks.8.attention.w_v": 0.7700653076171875, "blocks.8.attention.w_out": 0.7590789794921875, "blocks.8.feed_forward.w1": 0.785308837890625, "blocks.8.feed_forward.w2": 0.6164106130599976, "blocks.8.feed_forward.w3": 0.7843831777572632}, "w_mask_frac": 0.1710828834364557} +{"step": 1700, "changed_frac": 0.7083747407310029, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7653312683105469, "blocks.0.attention.w_k": 0.7619400024414062, "blocks.0.attention.w_v": 0.7592658996582031, "blocks.0.attention.w_out": 0.7349472045898438, "blocks.0.feed_forward.w1": 0.6686681509017944, "blocks.0.feed_forward.w2": 0.43512091040611267, "blocks.0.feed_forward.w3": 0.6677729487419128, "blocks.1.attention.w_q": 0.7825546264648438, "blocks.1.attention.w_k": 0.7694015502929688, "blocks.1.attention.w_v": 0.7718124389648438, "blocks.1.attention.w_out": 0.779205322265625, "blocks.1.feed_forward.w1": 0.7319450378417969, "blocks.1.feed_forward.w2": 0.5155817866325378, "blocks.1.feed_forward.w3": 0.730841338634491, "blocks.2.attention.w_q": 0.7467384338378906, "blocks.2.attention.w_k": 0.7363014221191406, "blocks.2.attention.w_v": 0.7708930969238281, "blocks.2.attention.w_out": 0.7718429565429688, "blocks.2.feed_forward.w1": 0.7589353322982788, "blocks.2.feed_forward.w2": 0.5342305898666382, "blocks.2.feed_forward.w3": 0.7560030817985535, "blocks.3.attention.w_q": 0.7886199951171875, "blocks.3.attention.w_k": 0.7743453979492188, "blocks.3.attention.w_v": 0.7685050964355469, "blocks.3.attention.w_out": 0.7784385681152344, "blocks.3.feed_forward.w1": 0.7653121948242188, "blocks.3.feed_forward.w2": 0.5363349914550781, "blocks.3.feed_forward.w3": 0.7633056640625, "blocks.4.attention.w_q": 0.7860832214355469, "blocks.4.attention.w_k": 0.7661972045898438, "blocks.4.attention.w_v": 0.7687225341796875, "blocks.4.attention.w_out": 0.7733917236328125, "blocks.4.feed_forward.w1": 0.7775179743766785, "blocks.4.feed_forward.w2": 0.5545387268066406, "blocks.4.feed_forward.w3": 0.7750422358512878, "blocks.5.attention.w_q": 0.7859382629394531, "blocks.5.attention.w_k": 0.7642326354980469, "blocks.5.attention.w_v": 0.7671279907226562, "blocks.5.attention.w_out": 0.7665328979492188, "blocks.5.feed_forward.w1": 0.7800840139389038, "blocks.5.feed_forward.w2": 0.5574455261230469, "blocks.5.feed_forward.w3": 0.7785695791244507, "blocks.6.attention.w_q": 0.7782325744628906, "blocks.6.attention.w_k": 0.7525062561035156, "blocks.6.attention.w_v": 0.7640113830566406, "blocks.6.attention.w_out": 0.76165771484375, "blocks.6.feed_forward.w1": 0.7788709402084351, "blocks.6.feed_forward.w2": 0.5541229248046875, "blocks.6.feed_forward.w3": 0.7745577692985535, "blocks.7.attention.w_q": 0.7066688537597656, "blocks.7.attention.w_k": 0.6911392211914062, "blocks.7.attention.w_v": 0.7623863220214844, "blocks.7.attention.w_out": 0.7326469421386719, "blocks.7.feed_forward.w1": 0.7825088500976562, "blocks.7.feed_forward.w2": 0.6057586669921875, "blocks.7.feed_forward.w3": 0.7778753042221069, "blocks.8.attention.w_q": 0.6928939819335938, "blocks.8.attention.w_k": 0.6657791137695312, "blocks.8.attention.w_v": 0.76409912109375, "blocks.8.attention.w_out": 0.7486000061035156, "blocks.8.feed_forward.w1": 0.7845535278320312, "blocks.8.feed_forward.w2": 0.6107279658317566, "blocks.8.feed_forward.w3": 0.7840589284896851}, "w_mask_frac": 0.168684841093854} +{"step": 1800, "changed_frac": 0.6996556143475394, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.76019287109375, "blocks.0.attention.w_k": 0.7573738098144531, "blocks.0.attention.w_v": 0.7530097961425781, "blocks.0.attention.w_out": 0.7281112670898438, "blocks.0.feed_forward.w1": 0.6607310175895691, "blocks.0.feed_forward.w2": 0.4290924072265625, "blocks.0.feed_forward.w3": 0.6592063903808594, "blocks.1.attention.w_q": 0.7785301208496094, "blocks.1.attention.w_k": 0.7636566162109375, "blocks.1.attention.w_v": 0.7671775817871094, "blocks.1.attention.w_out": 0.7736549377441406, "blocks.1.feed_forward.w1": 0.7252897024154663, "blocks.1.feed_forward.w2": 0.5057868957519531, "blocks.1.feed_forward.w3": 0.7240371704101562, "blocks.2.attention.w_q": 0.7483062744140625, "blocks.2.attention.w_k": 0.7367820739746094, "blocks.2.attention.w_v": 0.7649879455566406, "blocks.2.attention.w_out": 0.7659034729003906, "blocks.2.feed_forward.w1": 0.7513822317123413, "blocks.2.feed_forward.w2": 0.5247396230697632, "blocks.2.feed_forward.w3": 0.7501640319824219, "blocks.3.attention.w_q": 0.7865409851074219, "blocks.3.attention.w_k": 0.7735939025878906, "blocks.3.attention.w_v": 0.7626304626464844, "blocks.3.attention.w_out": 0.7720985412597656, "blocks.3.feed_forward.w1": 0.7592061758041382, "blocks.3.feed_forward.w2": 0.5286039113998413, "blocks.3.feed_forward.w3": 0.7563298940658569, "blocks.4.attention.w_q": 0.7870903015136719, "blocks.4.attention.w_k": 0.7629241943359375, "blocks.4.attention.w_v": 0.760589599609375, "blocks.4.attention.w_out": 0.7641677856445312, "blocks.4.feed_forward.w1": 0.7707939147949219, "blocks.4.feed_forward.w2": 0.5498988032341003, "blocks.4.feed_forward.w3": 0.7671534419059753, "blocks.5.attention.w_q": 0.7855720520019531, "blocks.5.attention.w_k": 0.7613677978515625, "blocks.5.attention.w_v": 0.7593269348144531, "blocks.5.attention.w_out": 0.7602386474609375, "blocks.5.feed_forward.w1": 0.7719027400016785, "blocks.5.feed_forward.w2": 0.5486170649528503, "blocks.5.feed_forward.w3": 0.7711741328239441, "blocks.6.attention.w_q": 0.7786598205566406, "blocks.6.attention.w_k": 0.7467689514160156, "blocks.6.attention.w_v": 0.757476806640625, "blocks.6.attention.w_out": 0.7520904541015625, "blocks.6.feed_forward.w1": 0.7649345397949219, "blocks.6.feed_forward.w2": 0.5545845031738281, "blocks.6.feed_forward.w3": 0.7555872797966003, "blocks.7.attention.w_q": 0.7142982482910156, "blocks.7.attention.w_k": 0.6996650695800781, "blocks.7.attention.w_v": 0.7542724609375, "blocks.7.attention.w_out": 0.7208633422851562, "blocks.7.feed_forward.w1": 0.7655500173568726, "blocks.7.feed_forward.w2": 0.5800895690917969, "blocks.7.feed_forward.w3": 0.7620226740837097, "blocks.8.attention.w_q": 0.6982803344726562, "blocks.8.attention.w_k": 0.672576904296875, "blocks.8.attention.w_v": 0.7609748840332031, "blocks.8.attention.w_out": 0.7331657409667969, "blocks.8.feed_forward.w1": 0.7637799978256226, "blocks.8.feed_forward.w2": 0.5779864192008972, "blocks.8.feed_forward.w3": 0.7669512629508972}, "w_mask_frac": 0.16452682132904345} +{"step": 1900, "changed_frac": 0.6947157118055556, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7526016235351562, "blocks.0.attention.w_k": 0.7518768310546875, "blocks.0.attention.w_v": 0.7452545166015625, "blocks.0.attention.w_out": 0.7172660827636719, "blocks.0.feed_forward.w1": 0.6427561640739441, "blocks.0.feed_forward.w2": 0.41947048902511597, "blocks.0.feed_forward.w3": 0.6431414484977722, "blocks.1.attention.w_q": 0.7732887268066406, "blocks.1.attention.w_k": 0.7615585327148438, "blocks.1.attention.w_v": 0.7561874389648438, "blocks.1.attention.w_out": 0.7660179138183594, "blocks.1.feed_forward.w1": 0.715118408203125, "blocks.1.feed_forward.w2": 0.4997138977050781, "blocks.1.feed_forward.w3": 0.7128779292106628, "blocks.2.attention.w_q": 0.7416038513183594, "blocks.2.attention.w_k": 0.72930908203125, "blocks.2.attention.w_v": 0.7552719116210938, "blocks.2.attention.w_out": 0.757659912109375, "blocks.2.feed_forward.w1": 0.7420234680175781, "blocks.2.feed_forward.w2": 0.5203196406364441, "blocks.2.feed_forward.w3": 0.7412109375, "blocks.3.attention.w_q": 0.7800712585449219, "blocks.3.attention.w_k": 0.7696723937988281, "blocks.3.attention.w_v": 0.7525787353515625, "blocks.3.attention.w_out": 0.7646064758300781, "blocks.3.feed_forward.w1": 0.7517623901367188, "blocks.3.feed_forward.w2": 0.5209935903549194, "blocks.3.feed_forward.w3": 0.7490921020507812, "blocks.4.attention.w_q": 0.779510498046875, "blocks.4.attention.w_k": 0.7551651000976562, "blocks.4.attention.w_v": 0.7510795593261719, "blocks.4.attention.w_out": 0.7578773498535156, "blocks.4.feed_forward.w1": 0.7647387385368347, "blocks.4.feed_forward.w2": 0.545721709728241, "blocks.4.feed_forward.w3": 0.7624308466911316, "blocks.5.attention.w_q": 0.781341552734375, "blocks.5.attention.w_k": 0.7560768127441406, "blocks.5.attention.w_v": 0.7482414245605469, "blocks.5.attention.w_out": 0.7515602111816406, "blocks.5.feed_forward.w1": 0.7664146423339844, "blocks.5.feed_forward.w2": 0.5416730642318726, "blocks.5.feed_forward.w3": 0.7662760615348816, "blocks.6.attention.w_q": 0.7779960632324219, "blocks.6.attention.w_k": 0.7478599548339844, "blocks.6.attention.w_v": 0.7474288940429688, "blocks.6.attention.w_out": 0.7404632568359375, "blocks.6.feed_forward.w1": 0.7650007009506226, "blocks.6.feed_forward.w2": 0.5563901662826538, "blocks.6.feed_forward.w3": 0.7556928396224976, "blocks.7.attention.w_q": 0.7122306823730469, "blocks.7.attention.w_k": 0.6936569213867188, "blocks.7.attention.w_v": 0.743499755859375, "blocks.7.attention.w_out": 0.7095832824707031, "blocks.7.feed_forward.w1": 0.7681083679199219, "blocks.7.feed_forward.w2": 0.5839576721191406, "blocks.7.feed_forward.w3": 0.7621841430664062, "blocks.8.attention.w_q": 0.7058181762695312, "blocks.8.attention.w_k": 0.6813240051269531, "blocks.8.attention.w_v": 0.7520942687988281, "blocks.8.attention.w_out": 0.7226982116699219, "blocks.8.feed_forward.w1": 0.7732416987419128, "blocks.8.feed_forward.w2": 0.5892779231071472, "blocks.8.feed_forward.w3": 0.770880401134491}, "w_mask_frac": 0.15819977000992522} +{"step": 2000, "changed_frac": 0.6844057914538261, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7501411437988281, "blocks.0.attention.w_k": 0.7500495910644531, "blocks.0.attention.w_v": 0.74151611328125, "blocks.0.attention.w_out": 0.7078628540039062, "blocks.0.feed_forward.w1": 0.6303138732910156, "blocks.0.feed_forward.w2": 0.40607962012290955, "blocks.0.feed_forward.w3": 0.6301180720329285, "blocks.1.attention.w_q": 0.7664070129394531, "blocks.1.attention.w_k": 0.7537689208984375, "blocks.1.attention.w_v": 0.7481231689453125, "blocks.1.attention.w_out": 0.7598953247070312, "blocks.1.feed_forward.w1": 0.7032267451286316, "blocks.1.feed_forward.w2": 0.48698553442955017, "blocks.1.feed_forward.w3": 0.7000478506088257, "blocks.2.attention.w_q": 0.7373390197753906, "blocks.2.attention.w_k": 0.7242012023925781, "blocks.2.attention.w_v": 0.7473564147949219, "blocks.2.attention.w_out": 0.7510452270507812, "blocks.2.feed_forward.w1": 0.7341041564941406, "blocks.2.feed_forward.w2": 0.5056241750717163, "blocks.2.feed_forward.w3": 0.7316678762435913, "blocks.3.attention.w_q": 0.7751312255859375, "blocks.3.attention.w_k": 0.7630767822265625, "blocks.3.attention.w_v": 0.7418479919433594, "blocks.3.attention.w_out": 0.7561912536621094, "blocks.3.feed_forward.w1": 0.7431475520133972, "blocks.3.feed_forward.w2": 0.5100721120834351, "blocks.3.feed_forward.w3": 0.7405713796615601, "blocks.4.attention.w_q": 0.7756500244140625, "blocks.4.attention.w_k": 0.7546539306640625, "blocks.4.attention.w_v": 0.7403068542480469, "blocks.4.attention.w_out": 0.7475204467773438, "blocks.4.feed_forward.w1": 0.7564430236816406, "blocks.4.feed_forward.w2": 0.5355581045150757, "blocks.4.feed_forward.w3": 0.7535324096679688, "blocks.5.attention.w_q": 0.7753448486328125, "blocks.5.attention.w_k": 0.7479515075683594, "blocks.5.attention.w_v": 0.7371749877929688, "blocks.5.attention.w_out": 0.7418098449707031, "blocks.5.feed_forward.w1": 0.7618751525878906, "blocks.5.feed_forward.w2": 0.5291888117790222, "blocks.5.feed_forward.w3": 0.7599424123764038, "blocks.6.attention.w_q": 0.7704658508300781, "blocks.6.attention.w_k": 0.7368965148925781, "blocks.6.attention.w_v": 0.7335281372070312, "blocks.6.attention.w_out": 0.7267112731933594, "blocks.6.feed_forward.w1": 0.7578582763671875, "blocks.6.feed_forward.w2": 0.5313352346420288, "blocks.6.feed_forward.w3": 0.7453778982162476, "blocks.7.attention.w_q": 0.7117691040039062, "blocks.7.attention.w_k": 0.6869087219238281, "blocks.7.attention.w_v": 0.7314529418945312, "blocks.7.attention.w_out": 0.6938858032226562, "blocks.7.feed_forward.w1": 0.7593562006950378, "blocks.7.feed_forward.w2": 0.5702298879623413, "blocks.7.feed_forward.w3": 0.7539724111557007, "blocks.8.attention.w_q": 0.6942558288574219, "blocks.8.attention.w_k": 0.6648712158203125, "blocks.8.attention.w_v": 0.7373313903808594, "blocks.8.attention.w_out": 0.7084083557128906, "blocks.8.feed_forward.w1": 0.763403594493866, "blocks.8.feed_forward.w2": 0.5674235224723816, "blocks.8.feed_forward.w3": 0.7607536315917969}, "w_mask_frac": 0.17877096549058571} +{"step": 2100, "changed_frac": 0.6804434947478466, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.745513916015625, "blocks.0.attention.w_k": 0.7423095703125, "blocks.0.attention.w_v": 0.737030029296875, "blocks.0.attention.w_out": 0.7062110900878906, "blocks.0.feed_forward.w1": 0.6285934448242188, "blocks.0.feed_forward.w2": 0.40295156836509705, "blocks.0.feed_forward.w3": 0.6279500722885132, "blocks.1.attention.w_q": 0.7646713256835938, "blocks.1.attention.w_k": 0.7553825378417969, "blocks.1.attention.w_v": 0.7476539611816406, "blocks.1.attention.w_out": 0.7576065063476562, "blocks.1.feed_forward.w1": 0.7007331848144531, "blocks.1.feed_forward.w2": 0.4866434931755066, "blocks.1.feed_forward.w3": 0.7008895874023438, "blocks.2.attention.w_q": 0.7400779724121094, "blocks.2.attention.w_k": 0.7253189086914062, "blocks.2.attention.w_v": 0.7440223693847656, "blocks.2.attention.w_out": 0.7467117309570312, "blocks.2.feed_forward.w1": 0.7312914729118347, "blocks.2.feed_forward.w2": 0.5058683156967163, "blocks.2.feed_forward.w3": 0.7287864685058594, "blocks.3.attention.w_q": 0.7736015319824219, "blocks.3.attention.w_k": 0.7622489929199219, "blocks.3.attention.w_v": 0.7408866882324219, "blocks.3.attention.w_out": 0.7561798095703125, "blocks.3.feed_forward.w1": 0.7388585805892944, "blocks.3.feed_forward.w2": 0.5121473073959351, "blocks.3.feed_forward.w3": 0.7368596792221069, "blocks.4.attention.w_q": 0.77337646484375, "blocks.4.attention.w_k": 0.7520866394042969, "blocks.4.attention.w_v": 0.7368583679199219, "blocks.4.attention.w_out": 0.7446212768554688, "blocks.4.feed_forward.w1": 0.7516670227050781, "blocks.4.feed_forward.w2": 0.5393409729003906, "blocks.4.feed_forward.w3": 0.7501691579818726, "blocks.5.attention.w_q": 0.7730712890625, "blocks.5.attention.w_k": 0.7488861083984375, "blocks.5.attention.w_v": 0.7312393188476562, "blocks.5.attention.w_out": 0.7414474487304688, "blocks.5.feed_forward.w1": 0.7565282583236694, "blocks.5.feed_forward.w2": 0.5266571044921875, "blocks.5.feed_forward.w3": 0.7536773681640625, "blocks.6.attention.w_q": 0.7685127258300781, "blocks.6.attention.w_k": 0.7419662475585938, "blocks.6.attention.w_v": 0.7308311462402344, "blocks.6.attention.w_out": 0.7253265380859375, "blocks.6.feed_forward.w1": 0.7481359243392944, "blocks.6.feed_forward.w2": 0.5223287343978882, "blocks.6.feed_forward.w3": 0.7372322082519531, "blocks.7.attention.w_q": 0.7059249877929688, "blocks.7.attention.w_k": 0.6866035461425781, "blocks.7.attention.w_v": 0.7285919189453125, "blocks.7.attention.w_out": 0.6908035278320312, "blocks.7.feed_forward.w1": 0.754797637462616, "blocks.7.feed_forward.w2": 0.5584869384765625, "blocks.7.feed_forward.w3": 0.7503318786621094, "blocks.8.attention.w_q": 0.6915435791015625, "blocks.8.attention.w_k": 0.6621208190917969, "blocks.8.attention.w_v": 0.7362899780273438, "blocks.8.attention.w_out": 0.7003135681152344, "blocks.8.feed_forward.w1": 0.7522265315055847, "blocks.8.feed_forward.w2": 0.5455564260482788, "blocks.8.feed_forward.w3": 0.7499732971191406}, "w_mask_frac": 0.16742912049476916} +{"step": 2200, "changed_frac": 0.6745511242467114, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7433242797851562, "blocks.0.attention.w_k": 0.7425422668457031, "blocks.0.attention.w_v": 0.7339973449707031, "blocks.0.attention.w_out": 0.6970100402832031, "blocks.0.feed_forward.w1": 0.6163381338119507, "blocks.0.feed_forward.w2": 0.39927801489830017, "blocks.0.feed_forward.w3": 0.6168950796127319, "blocks.1.attention.w_q": 0.7626190185546875, "blocks.1.attention.w_k": 0.7517738342285156, "blocks.1.attention.w_v": 0.7410392761230469, "blocks.1.attention.w_out": 0.7523841857910156, "blocks.1.feed_forward.w1": 0.6915448904037476, "blocks.1.feed_forward.w2": 0.4815788269042969, "blocks.1.feed_forward.w3": 0.6913299560546875, "blocks.2.attention.w_q": 0.7387733459472656, "blocks.2.attention.w_k": 0.7264442443847656, "blocks.2.attention.w_v": 0.7368316650390625, "blocks.2.attention.w_out": 0.7401008605957031, "blocks.2.feed_forward.w1": 0.7229563593864441, "blocks.2.feed_forward.w2": 0.5005811452865601, "blocks.2.feed_forward.w3": 0.7213045954704285, "blocks.3.attention.w_q": 0.770111083984375, "blocks.3.attention.w_k": 0.7571868896484375, "blocks.3.attention.w_v": 0.7308921813964844, "blocks.3.attention.w_out": 0.7462196350097656, "blocks.3.feed_forward.w1": 0.7301063537597656, "blocks.3.feed_forward.w2": 0.5074679255485535, "blocks.3.feed_forward.w3": 0.7294273376464844, "blocks.4.attention.w_q": 0.769287109375, "blocks.4.attention.w_k": 0.7478523254394531, "blocks.4.attention.w_v": 0.7267303466796875, "blocks.4.attention.w_out": 0.7350082397460938, "blocks.4.feed_forward.w1": 0.7468236684799194, "blocks.4.feed_forward.w2": 0.5357284545898438, "blocks.4.feed_forward.w3": 0.7430928945541382, "blocks.5.attention.w_q": 0.7685966491699219, "blocks.5.attention.w_k": 0.7432098388671875, "blocks.5.attention.w_v": 0.7225532531738281, "blocks.5.attention.w_out": 0.7292289733886719, "blocks.5.feed_forward.w1": 0.7496694326400757, "blocks.5.feed_forward.w2": 0.520422637462616, "blocks.5.feed_forward.w3": 0.7491188049316406, "blocks.6.attention.w_q": 0.7625923156738281, "blocks.6.attention.w_k": 0.7296409606933594, "blocks.6.attention.w_v": 0.7178726196289062, "blocks.6.attention.w_out": 0.7114105224609375, "blocks.6.feed_forward.w1": 0.7427787780761719, "blocks.6.feed_forward.w2": 0.5139707326889038, "blocks.6.feed_forward.w3": 0.7311961054801941, "blocks.7.attention.w_q": 0.7012138366699219, "blocks.7.attention.w_k": 0.6812362670898438, "blocks.7.attention.w_v": 0.7175750732421875, "blocks.7.attention.w_out": 0.6731529235839844, "blocks.7.feed_forward.w1": 0.7553800344467163, "blocks.7.feed_forward.w2": 0.5545972585678101, "blocks.7.feed_forward.w3": 0.7502848505973816, "blocks.8.attention.w_q": 0.6892738342285156, "blocks.8.attention.w_k": 0.668914794921875, "blocks.8.attention.w_v": 0.7257423400878906, "blocks.8.attention.w_out": 0.6855545043945312, "blocks.8.feed_forward.w1": 0.7516721487045288, "blocks.8.feed_forward.w2": 0.5445607900619507, "blocks.8.feed_forward.w3": 0.7500903010368347}, "w_mask_frac": 0.17059306965933907} +{"step": 2300, "changed_frac": 0.6653992090469751, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7322044372558594, "blocks.0.attention.w_k": 0.73162841796875, "blocks.0.attention.w_v": 0.7252349853515625, "blocks.0.attention.w_out": 0.688751220703125, "blocks.0.feed_forward.w1": 0.6038030385971069, "blocks.0.feed_forward.w2": 0.3877767026424408, "blocks.0.feed_forward.w3": 0.6035258173942566, "blocks.1.attention.w_q": 0.7559585571289062, "blocks.1.attention.w_k": 0.7462997436523438, "blocks.1.attention.w_v": 0.7332649230957031, "blocks.1.attention.w_out": 0.7440223693847656, "blocks.1.feed_forward.w1": 0.6806564331054688, "blocks.1.feed_forward.w2": 0.4732246398925781, "blocks.1.feed_forward.w3": 0.6810659170150757, "blocks.2.attention.w_q": 0.7334518432617188, "blocks.2.attention.w_k": 0.7168006896972656, "blocks.2.attention.w_v": 0.7268218994140625, "blocks.2.attention.w_out": 0.7337265014648438, "blocks.2.feed_forward.w1": 0.7131093740463257, "blocks.2.feed_forward.w2": 0.4921976923942566, "blocks.2.feed_forward.w3": 0.7112630605697632, "blocks.3.attention.w_q": 0.7648506164550781, "blocks.3.attention.w_k": 0.7523727416992188, "blocks.3.attention.w_v": 0.7224197387695312, "blocks.3.attention.w_out": 0.7402305603027344, "blocks.3.feed_forward.w1": 0.7214711904525757, "blocks.3.feed_forward.w2": 0.4994290769100189, "blocks.3.feed_forward.w3": 0.7195142507553101, "blocks.4.attention.w_q": 0.7640457153320312, "blocks.4.attention.w_k": 0.7406959533691406, "blocks.4.attention.w_v": 0.7171859741210938, "blocks.4.attention.w_out": 0.7283973693847656, "blocks.4.feed_forward.w1": 0.7365748286247253, "blocks.4.feed_forward.w2": 0.5246455073356628, "blocks.4.feed_forward.w3": 0.7335497736930847, "blocks.5.attention.w_q": 0.7637138366699219, "blocks.5.attention.w_k": 0.733795166015625, "blocks.5.attention.w_v": 0.7091789245605469, "blocks.5.attention.w_out": 0.7227821350097656, "blocks.5.feed_forward.w1": 0.7422460317611694, "blocks.5.feed_forward.w2": 0.5059661865234375, "blocks.5.feed_forward.w3": 0.7400550842285156, "blocks.6.attention.w_q": 0.7560195922851562, "blocks.6.attention.w_k": 0.7222251892089844, "blocks.6.attention.w_v": 0.7093963623046875, "blocks.6.attention.w_out": 0.7017631530761719, "blocks.6.feed_forward.w1": 0.7352231740951538, "blocks.6.feed_forward.w2": 0.505652129650116, "blocks.6.feed_forward.w3": 0.7249094843864441, "blocks.7.attention.w_q": 0.6851081848144531, "blocks.7.attention.w_k": 0.6698989868164062, "blocks.7.attention.w_v": 0.7123794555664062, "blocks.7.attention.w_out": 0.6679649353027344, "blocks.7.feed_forward.w1": 0.7463697195053101, "blocks.7.feed_forward.w2": 0.5435142517089844, "blocks.7.feed_forward.w3": 0.7403640747070312, "blocks.8.attention.w_q": 0.6796493530273438, "blocks.8.attention.w_k": 0.655792236328125, "blocks.8.attention.w_v": 0.71881103515625, "blocks.8.attention.w_out": 0.6747207641601562, "blocks.8.feed_forward.w1": 0.7490895986557007, "blocks.8.feed_forward.w2": 0.5296770930290222, "blocks.8.feed_forward.w3": 0.7451744079589844}, "w_mask_frac": 0.16761072629537338} +{"step": 2400, "changed_frac": 0.6608290713057559, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7318496704101562, "blocks.0.attention.w_k": 0.7269439697265625, "blocks.0.attention.w_v": 0.7185859680175781, "blocks.0.attention.w_out": 0.6806640625, "blocks.0.feed_forward.w1": 0.5917396545410156, "blocks.0.feed_forward.w2": 0.38977178931236267, "blocks.0.feed_forward.w3": 0.5928153991699219, "blocks.1.attention.w_q": 0.7502822875976562, "blocks.1.attention.w_k": 0.7400093078613281, "blocks.1.attention.w_v": 0.7228927612304688, "blocks.1.attention.w_out": 0.7378578186035156, "blocks.1.feed_forward.w1": 0.6733385920524597, "blocks.1.feed_forward.w2": 0.47220611572265625, "blocks.1.feed_forward.w3": 0.6729393005371094, "blocks.2.attention.w_q": 0.7258567810058594, "blocks.2.attention.w_k": 0.7078666687011719, "blocks.2.attention.w_v": 0.7199287414550781, "blocks.2.attention.w_out": 0.7261581420898438, "blocks.2.feed_forward.w1": 0.708563506603241, "blocks.2.feed_forward.w2": 0.48992666602134705, "blocks.2.feed_forward.w3": 0.7065328359603882, "blocks.3.attention.w_q": 0.7618255615234375, "blocks.3.attention.w_k": 0.7467689514160156, "blocks.3.attention.w_v": 0.7142486572265625, "blocks.3.attention.w_out": 0.7357139587402344, "blocks.3.feed_forward.w1": 0.7165171504020691, "blocks.3.feed_forward.w2": 0.4974047541618347, "blocks.3.feed_forward.w3": 0.7151629328727722, "blocks.4.attention.w_q": 0.7595176696777344, "blocks.4.attention.w_k": 0.735137939453125, "blocks.4.attention.w_v": 0.7079010009765625, "blocks.4.attention.w_out": 0.7190055847167969, "blocks.4.feed_forward.w1": 0.7340494990348816, "blocks.4.feed_forward.w2": 0.5215644836425781, "blocks.4.feed_forward.w3": 0.7287470698356628, "blocks.5.attention.w_q": 0.7613410949707031, "blocks.5.attention.w_k": 0.72918701171875, "blocks.5.attention.w_v": 0.7013053894042969, "blocks.5.attention.w_out": 0.7130126953125, "blocks.5.feed_forward.w1": 0.7389386892318726, "blocks.5.feed_forward.w2": 0.5051714777946472, "blocks.5.feed_forward.w3": 0.7375386953353882, "blocks.6.attention.w_q": 0.7543869018554688, "blocks.6.attention.w_k": 0.7204856872558594, "blocks.6.attention.w_v": 0.6995124816894531, "blocks.6.attention.w_out": 0.6951942443847656, "blocks.6.feed_forward.w1": 0.7343991994857788, "blocks.6.feed_forward.w2": 0.4943949580192566, "blocks.6.feed_forward.w3": 0.7275962829589844, "blocks.7.attention.w_q": 0.6962928771972656, "blocks.7.attention.w_k": 0.6746864318847656, "blocks.7.attention.w_v": 0.70013427734375, "blocks.7.attention.w_out": 0.6567497253417969, "blocks.7.feed_forward.w1": 0.7427660822868347, "blocks.7.feed_forward.w2": 0.5521672964096069, "blocks.7.feed_forward.w3": 0.7361335754394531, "blocks.8.attention.w_q": 0.6836395263671875, "blocks.8.attention.w_k": 0.6521682739257812, "blocks.8.attention.w_v": 0.7106246948242188, "blocks.8.attention.w_out": 0.6619529724121094, "blocks.8.feed_forward.w1": 0.7400843501091003, "blocks.8.feed_forward.w2": 0.5253995656967163, "blocks.8.feed_forward.w3": 0.7332344055175781}, "w_mask_frac": 0.16779282154181066} +{"step": 2500, "changed_frac": 0.6549489926069211, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7287712097167969, "blocks.0.attention.w_k": 0.7288703918457031, "blocks.0.attention.w_v": 0.7149505615234375, "blocks.0.attention.w_out": 0.6751632690429688, "blocks.0.feed_forward.w1": 0.5835952758789062, "blocks.0.feed_forward.w2": 0.3788897395133972, "blocks.0.feed_forward.w3": 0.5843366384506226, "blocks.1.attention.w_q": 0.7480735778808594, "blocks.1.attention.w_k": 0.73468017578125, "blocks.1.attention.w_v": 0.7165603637695312, "blocks.1.attention.w_out": 0.73431396484375, "blocks.1.feed_forward.w1": 0.6660817861557007, "blocks.1.feed_forward.w2": 0.4625282287597656, "blocks.1.feed_forward.w3": 0.6636645197868347, "blocks.2.attention.w_q": 0.7288589477539062, "blocks.2.attention.w_k": 0.70806884765625, "blocks.2.attention.w_v": 0.7127876281738281, "blocks.2.attention.w_out": 0.7178802490234375, "blocks.2.feed_forward.w1": 0.6984545588493347, "blocks.2.feed_forward.w2": 0.48231762647628784, "blocks.2.feed_forward.w3": 0.6969007253646851, "blocks.3.attention.w_q": 0.7565231323242188, "blocks.3.attention.w_k": 0.7407302856445312, "blocks.3.attention.w_v": 0.7039527893066406, "blocks.3.attention.w_out": 0.7227020263671875, "blocks.3.feed_forward.w1": 0.7076263427734375, "blocks.3.feed_forward.w2": 0.49341583251953125, "blocks.3.feed_forward.w3": 0.705877959728241, "blocks.4.attention.w_q": 0.7574882507324219, "blocks.4.attention.w_k": 0.7303962707519531, "blocks.4.attention.w_v": 0.6988182067871094, "blocks.4.attention.w_out": 0.7079124450683594, "blocks.4.feed_forward.w1": 0.7264747619628906, "blocks.4.feed_forward.w2": 0.5219587087631226, "blocks.4.feed_forward.w3": 0.7231585383415222, "blocks.5.attention.w_q": 0.7577781677246094, "blocks.5.attention.w_k": 0.7281227111816406, "blocks.5.attention.w_v": 0.6937141418457031, "blocks.5.attention.w_out": 0.7062339782714844, "blocks.5.feed_forward.w1": 0.7303441762924194, "blocks.5.feed_forward.w2": 0.4999885559082031, "blocks.5.feed_forward.w3": 0.7283401489257812, "blocks.6.attention.w_q": 0.75537109375, "blocks.6.attention.w_k": 0.7190017700195312, "blocks.6.attention.w_v": 0.6918601989746094, "blocks.6.attention.w_out": 0.6846237182617188, "blocks.6.feed_forward.w1": 0.7259712219238281, "blocks.6.feed_forward.w2": 0.50445556640625, "blocks.6.feed_forward.w3": 0.7170817255973816, "blocks.7.attention.w_q": 0.6917991638183594, "blocks.7.attention.w_k": 0.6676254272460938, "blocks.7.attention.w_v": 0.6977043151855469, "blocks.7.attention.w_out": 0.6482925415039062, "blocks.7.feed_forward.w1": 0.73779296875, "blocks.7.feed_forward.w2": 0.5436922907829285, "blocks.7.feed_forward.w3": 0.7308985590934753, "blocks.8.attention.w_q": 0.6920204162597656, "blocks.8.attention.w_k": 0.6685752868652344, "blocks.8.attention.w_v": 0.7058792114257812, "blocks.8.attention.w_out": 0.6538848876953125, "blocks.8.feed_forward.w1": 0.7364146113395691, "blocks.8.feed_forward.w2": 0.5194333791732788, "blocks.8.feed_forward.w3": 0.729986846446991}, "w_mask_frac": 0.16811785318403163} +{"step": 2600, "changed_frac": 0.646280500623915, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7192192077636719, "blocks.0.attention.w_k": 0.7168769836425781, "blocks.0.attention.w_v": 0.7059211730957031, "blocks.0.attention.w_out": 0.6660079956054688, "blocks.0.feed_forward.w1": 0.5686849355697632, "blocks.0.feed_forward.w2": 0.3738670349121094, "blocks.0.feed_forward.w3": 0.5710309743881226, "blocks.1.attention.w_q": 0.7429542541503906, "blocks.1.attention.w_k": 0.7267799377441406, "blocks.1.attention.w_v": 0.7084808349609375, "blocks.1.attention.w_out": 0.7265548706054688, "blocks.1.feed_forward.w1": 0.6541265249252319, "blocks.1.feed_forward.w2": 0.45630645751953125, "blocks.1.feed_forward.w3": 0.6527875661849976, "blocks.2.attention.w_q": 0.7206153869628906, "blocks.2.attention.w_k": 0.7052764892578125, "blocks.2.attention.w_v": 0.7033424377441406, "blocks.2.attention.w_out": 0.7116813659667969, "blocks.2.feed_forward.w1": 0.6887868642807007, "blocks.2.feed_forward.w2": 0.4781939387321472, "blocks.2.feed_forward.w3": 0.68719482421875, "blocks.3.attention.w_q": 0.7470359802246094, "blocks.3.attention.w_k": 0.7320671081542969, "blocks.3.attention.w_v": 0.6956100463867188, "blocks.3.attention.w_out": 0.7173271179199219, "blocks.3.feed_forward.w1": 0.6987025141716003, "blocks.3.feed_forward.w2": 0.4864921569824219, "blocks.3.feed_forward.w3": 0.6962051391601562, "blocks.4.attention.w_q": 0.748504638671875, "blocks.4.attention.w_k": 0.7240104675292969, "blocks.4.attention.w_v": 0.6892890930175781, "blocks.4.attention.w_out": 0.6988563537597656, "blocks.4.feed_forward.w1": 0.7201919555664062, "blocks.4.feed_forward.w2": 0.5147489309310913, "blocks.4.feed_forward.w3": 0.7154490351676941, "blocks.5.attention.w_q": 0.7507858276367188, "blocks.5.attention.w_k": 0.7130546569824219, "blocks.5.attention.w_v": 0.6805267333984375, "blocks.5.attention.w_out": 0.69586181640625, "blocks.5.feed_forward.w1": 0.7240626215934753, "blocks.5.feed_forward.w2": 0.4940757751464844, "blocks.5.feed_forward.w3": 0.7220268249511719, "blocks.6.attention.w_q": 0.7457847595214844, "blocks.6.attention.w_k": 0.708282470703125, "blocks.6.attention.w_v": 0.6817893981933594, "blocks.6.attention.w_out": 0.6742401123046875, "blocks.6.feed_forward.w1": 0.7217203974723816, "blocks.6.feed_forward.w2": 0.4883003234863281, "blocks.6.feed_forward.w3": 0.7136090993881226, "blocks.7.attention.w_q": 0.6911392211914062, "blocks.7.attention.w_k": 0.6598854064941406, "blocks.7.attention.w_v": 0.6831245422363281, "blocks.7.attention.w_out": 0.6329765319824219, "blocks.7.feed_forward.w1": 0.7300898432731628, "blocks.7.feed_forward.w2": 0.5270716547966003, "blocks.7.feed_forward.w3": 0.7237027883529663, "blocks.8.attention.w_q": 0.6684112548828125, "blocks.8.attention.w_k": 0.6345405578613281, "blocks.8.attention.w_v": 0.6912765502929688, "blocks.8.attention.w_out": 0.6425132751464844, "blocks.8.feed_forward.w1": 0.7305908203125, "blocks.8.feed_forward.w2": 0.5221850275993347, "blocks.8.feed_forward.w3": 0.7245343923568726}, "w_mask_frac": 0.17229302316649348} +{"step": 2700, "changed_frac": 0.63966796744583, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6966934204101562, "blocks.0.attention.w_k": 0.7031822204589844, "blocks.0.attention.w_v": 0.702484130859375, "blocks.0.attention.w_out": 0.6567726135253906, "blocks.0.feed_forward.w1": 0.5390256643295288, "blocks.0.feed_forward.w2": 0.36915844678878784, "blocks.0.feed_forward.w3": 0.5513877868652344, "blocks.1.attention.w_q": 0.7355461120605469, "blocks.1.attention.w_k": 0.7229347229003906, "blocks.1.attention.w_v": 0.7010231018066406, "blocks.1.attention.w_out": 0.72027587890625, "blocks.1.feed_forward.w1": 0.6402257680892944, "blocks.1.feed_forward.w2": 0.4502042233943939, "blocks.1.feed_forward.w3": 0.6385332942008972, "blocks.2.attention.w_q": 0.7171516418457031, "blocks.2.attention.w_k": 0.6990318298339844, "blocks.2.attention.w_v": 0.6983909606933594, "blocks.2.attention.w_out": 0.7089767456054688, "blocks.2.feed_forward.w1": 0.6800003051757812, "blocks.2.feed_forward.w2": 0.4761441648006439, "blocks.2.feed_forward.w3": 0.6776847839355469, "blocks.3.attention.w_q": 0.7491531372070312, "blocks.3.attention.w_k": 0.7304763793945312, "blocks.3.attention.w_v": 0.6891136169433594, "blocks.3.attention.w_out": 0.7126045227050781, "blocks.3.feed_forward.w1": 0.6898473501205444, "blocks.3.feed_forward.w2": 0.4833869934082031, "blocks.3.feed_forward.w3": 0.6882807612419128, "blocks.4.attention.w_q": 0.74554443359375, "blocks.4.attention.w_k": 0.7182426452636719, "blocks.4.attention.w_v": 0.6786422729492188, "blocks.4.attention.w_out": 0.6929283142089844, "blocks.4.feed_forward.w1": 0.7124099731445312, "blocks.4.feed_forward.w2": 0.5145823359489441, "blocks.4.feed_forward.w3": 0.7073911428451538, "blocks.5.attention.w_q": 0.7505111694335938, "blocks.5.attention.w_k": 0.7150077819824219, "blocks.5.attention.w_v": 0.6719322204589844, "blocks.5.attention.w_out": 0.6898994445800781, "blocks.5.feed_forward.w1": 0.7178904414176941, "blocks.5.feed_forward.w2": 0.4934743344783783, "blocks.5.feed_forward.w3": 0.7157033681869507, "blocks.6.attention.w_q": 0.7409439086914062, "blocks.6.attention.w_k": 0.7041816711425781, "blocks.6.attention.w_v": 0.6723365783691406, "blocks.6.attention.w_out": 0.66748046875, "blocks.6.feed_forward.w1": 0.7209256887435913, "blocks.6.feed_forward.w2": 0.4992167353630066, "blocks.6.feed_forward.w3": 0.7109311819076538, "blocks.7.attention.w_q": 0.6656227111816406, "blocks.7.attention.w_k": 0.6466712951660156, "blocks.7.attention.w_v": 0.67950439453125, "blocks.7.attention.w_out": 0.6268043518066406, "blocks.7.feed_forward.w1": 0.7226511836051941, "blocks.7.feed_forward.w2": 0.5282465815544128, "blocks.7.feed_forward.w3": 0.7154287099838257, "blocks.8.attention.w_q": 0.6761970520019531, "blocks.8.attention.w_k": 0.6470413208007812, "blocks.8.attention.w_v": 0.6859321594238281, "blocks.8.attention.w_out": 0.6316604614257812, "blocks.8.feed_forward.w1": 0.7241986989974976, "blocks.8.feed_forward.w2": 0.5105514526367188, "blocks.8.feed_forward.w3": 0.7192713618278503}, "w_mask_frac": 0.17046855351863763} +{"step": 2800, "changed_frac": 0.6318708892561432, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7148094177246094, "blocks.0.attention.w_k": 0.7210273742675781, "blocks.0.attention.w_v": 0.7010574340820312, "blocks.0.attention.w_out": 0.654022216796875, "blocks.0.feed_forward.w1": 0.5434583425521851, "blocks.0.feed_forward.w2": 0.3633524775505066, "blocks.0.feed_forward.w3": 0.5517489314079285, "blocks.1.attention.w_q": 0.7312278747558594, "blocks.1.attention.w_k": 0.7196922302246094, "blocks.1.attention.w_v": 0.6974678039550781, "blocks.1.attention.w_out": 0.7172737121582031, "blocks.1.feed_forward.w1": 0.6380195617675781, "blocks.1.feed_forward.w2": 0.4437255859375, "blocks.1.feed_forward.w3": 0.6359113454818726, "blocks.2.attention.w_q": 0.7114295959472656, "blocks.2.attention.w_k": 0.6909866333007812, "blocks.2.attention.w_v": 0.6879615783691406, "blocks.2.attention.w_out": 0.7005538940429688, "blocks.2.feed_forward.w1": 0.6764081716537476, "blocks.2.feed_forward.w2": 0.46868896484375, "blocks.2.feed_forward.w3": 0.6741358637809753, "blocks.3.attention.w_q": 0.73968505859375, "blocks.3.attention.w_k": 0.7227516174316406, "blocks.3.attention.w_v": 0.6777572631835938, "blocks.3.attention.w_out": 0.7077064514160156, "blocks.3.feed_forward.w1": 0.6825332641601562, "blocks.3.feed_forward.w2": 0.47717031836509705, "blocks.3.feed_forward.w3": 0.6793429255485535, "blocks.4.attention.w_q": 0.7393379211425781, "blocks.4.attention.w_k": 0.7124366760253906, "blocks.4.attention.w_v": 0.6680450439453125, "blocks.4.attention.w_out": 0.6843795776367188, "blocks.4.feed_forward.w1": 0.7073822021484375, "blocks.4.feed_forward.w2": 0.50390625, "blocks.4.feed_forward.w3": 0.70184326171875, "blocks.5.attention.w_q": 0.7436447143554688, "blocks.5.attention.w_k": 0.7085075378417969, "blocks.5.attention.w_v": 0.6608963012695312, "blocks.5.attention.w_out": 0.6786270141601562, "blocks.5.feed_forward.w1": 0.7127062678337097, "blocks.5.feed_forward.w2": 0.4769783020019531, "blocks.5.feed_forward.w3": 0.7093658447265625, "blocks.6.attention.w_q": 0.7355880737304688, "blocks.6.attention.w_k": 0.6990165710449219, "blocks.6.attention.w_v": 0.6597328186035156, "blocks.6.attention.w_out": 0.6564178466796875, "blocks.6.feed_forward.w1": 0.7027053833007812, "blocks.6.feed_forward.w2": 0.4641520380973816, "blocks.6.feed_forward.w3": 0.6926841735839844, "blocks.7.attention.w_q": 0.6827926635742188, "blocks.7.attention.w_k": 0.6534652709960938, "blocks.7.attention.w_v": 0.6645545959472656, "blocks.7.attention.w_out": 0.6107711791992188, "blocks.7.feed_forward.w1": 0.71685791015625, "blocks.7.feed_forward.w2": 0.5208944082260132, "blocks.7.feed_forward.w3": 0.708685576915741, "blocks.8.attention.w_q": 0.6714248657226562, "blocks.8.attention.w_k": 0.6397705078125, "blocks.8.attention.w_v": 0.6732025146484375, "blocks.8.attention.w_out": 0.6153640747070312, "blocks.8.feed_forward.w1": 0.7138214111328125, "blocks.8.feed_forward.w2": 0.48417919874191284, "blocks.8.feed_forward.w3": 0.7078450918197632}, "w_mask_frac": 0.1719911390135431} +{"step": 2900, "changed_frac": 0.6247609130337707, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6976470947265625, "blocks.0.attention.w_k": 0.70458984375, "blocks.0.attention.w_v": 0.6878662109375, "blocks.0.attention.w_out": 0.6401252746582031, "blocks.0.feed_forward.w1": 0.5343704223632812, "blocks.0.feed_forward.w2": 0.3631375730037689, "blocks.0.feed_forward.w3": 0.5372581481933594, "blocks.1.attention.w_q": 0.7211647033691406, "blocks.1.attention.w_k": 0.708160400390625, "blocks.1.attention.w_v": 0.6853103637695312, "blocks.1.attention.w_out": 0.7074661254882812, "blocks.1.feed_forward.w1": 0.6268692016601562, "blocks.1.feed_forward.w2": 0.43811291456222534, "blocks.1.feed_forward.w3": 0.6253103017807007, "blocks.2.attention.w_q": 0.7071723937988281, "blocks.2.attention.w_k": 0.6893081665039062, "blocks.2.attention.w_v": 0.676666259765625, "blocks.2.attention.w_out": 0.692901611328125, "blocks.2.feed_forward.w1": 0.6667913198471069, "blocks.2.feed_forward.w2": 0.4642168879508972, "blocks.2.feed_forward.w3": 0.6643816828727722, "blocks.3.attention.w_q": 0.7356758117675781, "blocks.3.attention.w_k": 0.7144317626953125, "blocks.3.attention.w_v": 0.6670722961425781, "blocks.3.attention.w_out": 0.6951904296875, "blocks.3.feed_forward.w1": 0.6724179983139038, "blocks.3.feed_forward.w2": 0.4726142883300781, "blocks.3.feed_forward.w3": 0.6708285212516785, "blocks.4.attention.w_q": 0.7355575561523438, "blocks.4.attention.w_k": 0.7074508666992188, "blocks.4.attention.w_v": 0.6559715270996094, "blocks.4.attention.w_out": 0.6732711791992188, "blocks.4.feed_forward.w1": 0.6971664428710938, "blocks.4.feed_forward.w2": 0.5017814636230469, "blocks.4.feed_forward.w3": 0.6933123469352722, "blocks.5.attention.w_q": 0.7346076965332031, "blocks.5.attention.w_k": 0.6986846923828125, "blocks.5.attention.w_v": 0.6472129821777344, "blocks.5.attention.w_out": 0.667083740234375, "blocks.5.feed_forward.w1": 0.7051976919174194, "blocks.5.feed_forward.w2": 0.47856396436691284, "blocks.5.feed_forward.w3": 0.7029495239257812, "blocks.6.attention.w_q": 0.7292518615722656, "blocks.6.attention.w_k": 0.6852874755859375, "blocks.6.attention.w_v": 0.6485176086425781, "blocks.6.attention.w_out": 0.6445426940917969, "blocks.6.feed_forward.w1": 0.7014452815055847, "blocks.6.feed_forward.w2": 0.47001904249191284, "blocks.6.feed_forward.w3": 0.6877161860466003, "blocks.7.attention.w_q": 0.6635627746582031, "blocks.7.attention.w_k": 0.6381034851074219, "blocks.7.attention.w_v": 0.6504287719726562, "blocks.7.attention.w_out": 0.6071510314941406, "blocks.7.feed_forward.w1": 0.7088521718978882, "blocks.7.feed_forward.w2": 0.5251197814941406, "blocks.7.feed_forward.w3": 0.7019577026367188, "blocks.8.attention.w_q": 0.6611442565917969, "blocks.8.attention.w_k": 0.6306114196777344, "blocks.8.attention.w_v": 0.6574020385742188, "blocks.8.attention.w_out": 0.6008415222167969, "blocks.8.feed_forward.w1": 0.7083448171615601, "blocks.8.feed_forward.w2": 0.4885063171386719, "blocks.8.feed_forward.w3": 0.7026227712631226}, "w_mask_frac": 0.17615724558758938} +{"step": 3000, "changed_frac": 0.6206874684390858, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6924591064453125, "blocks.0.attention.w_k": 0.6975212097167969, "blocks.0.attention.w_v": 0.6824760437011719, "blocks.0.attention.w_out": 0.6341476440429688, "blocks.0.feed_forward.w1": 0.5256919860839844, "blocks.0.feed_forward.w2": 0.3558349609375, "blocks.0.feed_forward.w3": 0.5298576354980469, "blocks.1.attention.w_q": 0.7193183898925781, "blocks.1.attention.w_k": 0.7055892944335938, "blocks.1.attention.w_v": 0.6777839660644531, "blocks.1.attention.w_out": 0.7004127502441406, "blocks.1.feed_forward.w1": 0.6199582815170288, "blocks.1.feed_forward.w2": 0.43465933203697205, "blocks.1.feed_forward.w3": 0.6181501150131226, "blocks.2.attention.w_q": 0.7092666625976562, "blocks.2.attention.w_k": 0.6843528747558594, "blocks.2.attention.w_v": 0.6676673889160156, "blocks.2.attention.w_out": 0.6879310607910156, "blocks.2.feed_forward.w1": 0.6588491201400757, "blocks.2.feed_forward.w2": 0.46245449781417847, "blocks.2.feed_forward.w3": 0.6570905447006226, "blocks.3.attention.w_q": 0.7304191589355469, "blocks.3.attention.w_k": 0.7121086120605469, "blocks.3.attention.w_v": 0.6591758728027344, "blocks.3.attention.w_out": 0.6904678344726562, "blocks.3.feed_forward.w1": 0.667144775390625, "blocks.3.feed_forward.w2": 0.4716796875, "blocks.3.feed_forward.w3": 0.6650721430778503, "blocks.4.attention.w_q": 0.7317276000976562, "blocks.4.attention.w_k": 0.70263671875, "blocks.4.attention.w_v": 0.6469917297363281, "blocks.4.attention.w_out": 0.6611747741699219, "blocks.4.feed_forward.w1": 0.6938387751579285, "blocks.4.feed_forward.w2": 0.5010923147201538, "blocks.4.feed_forward.w3": 0.690375030040741, "blocks.5.attention.w_q": 0.7321128845214844, "blocks.5.attention.w_k": 0.6981315612792969, "blocks.5.attention.w_v": 0.6391372680664062, "blocks.5.attention.w_out": 0.6642608642578125, "blocks.5.feed_forward.w1": 0.7026761770248413, "blocks.5.feed_forward.w2": 0.47256216406822205, "blocks.5.feed_forward.w3": 0.6992098689079285, "blocks.6.attention.w_q": 0.7300605773925781, "blocks.6.attention.w_k": 0.6921424865722656, "blocks.6.attention.w_v": 0.6429443359375, "blocks.6.attention.w_out": 0.6414375305175781, "blocks.6.feed_forward.w1": 0.6969070434570312, "blocks.6.feed_forward.w2": 0.46096548438072205, "blocks.6.feed_forward.w3": 0.6888517141342163, "blocks.7.attention.w_q": 0.6696281433105469, "blocks.7.attention.w_k": 0.6401596069335938, "blocks.7.attention.w_v": 0.652740478515625, "blocks.7.attention.w_out": 0.5977745056152344, "blocks.7.feed_forward.w1": 0.7060534358024597, "blocks.7.feed_forward.w2": 0.5173301696777344, "blocks.7.feed_forward.w3": 0.6999092102050781, "blocks.8.attention.w_q": 0.6669769287109375, "blocks.8.attention.w_k": 0.6334648132324219, "blocks.8.attention.w_v": 0.6586723327636719, "blocks.8.attention.w_out": 0.5962944030761719, "blocks.8.feed_forward.w1": 0.7073923945426941, "blocks.8.feed_forward.w2": 0.48494720458984375, "blocks.8.feed_forward.w3": 0.7017351984977722}, "w_mask_frac": 0.16808955263123554} +{"step": 3100, "changed_frac": 0.611435295170189, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6846809387207031, "blocks.0.attention.w_k": 0.6917800903320312, "blocks.0.attention.w_v": 0.6776046752929688, "blocks.0.attention.w_out": 0.6260528564453125, "blocks.0.feed_forward.w1": 0.5150489807128906, "blocks.0.feed_forward.w2": 0.3492380976676941, "blocks.0.feed_forward.w3": 0.5188013911247253, "blocks.1.attention.w_q": 0.7139320373535156, "blocks.1.attention.w_k": 0.702789306640625, "blocks.1.attention.w_v": 0.6717033386230469, "blocks.1.attention.w_out": 0.6955223083496094, "blocks.1.feed_forward.w1": 0.6084429621696472, "blocks.1.feed_forward.w2": 0.42813238501548767, "blocks.1.feed_forward.w3": 0.608001708984375, "blocks.2.attention.w_q": 0.6967239379882812, "blocks.2.attention.w_k": 0.6768074035644531, "blocks.2.attention.w_v": 0.6610641479492188, "blocks.2.attention.w_out": 0.6808891296386719, "blocks.2.feed_forward.w1": 0.6484705805778503, "blocks.2.feed_forward.w2": 0.4530601501464844, "blocks.2.feed_forward.w3": 0.6466420888900757, "blocks.3.attention.w_q": 0.7210350036621094, "blocks.3.attention.w_k": 0.7029914855957031, "blocks.3.attention.w_v": 0.6485557556152344, "blocks.3.attention.w_out": 0.6844329833984375, "blocks.3.feed_forward.w1": 0.6550559997558594, "blocks.3.feed_forward.w2": 0.46141308546066284, "blocks.3.feed_forward.w3": 0.6521657705307007, "blocks.4.attention.w_q": 0.7233810424804688, "blocks.4.attention.w_k": 0.6929092407226562, "blocks.4.attention.w_v": 0.6371994018554688, "blocks.4.attention.w_out": 0.652801513671875, "blocks.4.feed_forward.w1": 0.6843249201774597, "blocks.4.feed_forward.w2": 0.4932645261287689, "blocks.4.feed_forward.w3": 0.6796659231185913, "blocks.5.attention.w_q": 0.7259635925292969, "blocks.5.attention.w_k": 0.6870994567871094, "blocks.5.attention.w_v": 0.628875732421875, "blocks.5.attention.w_out": 0.6502151489257812, "blocks.5.feed_forward.w1": 0.6918296813964844, "blocks.5.feed_forward.w2": 0.4667243957519531, "blocks.5.feed_forward.w3": 0.6871070861816406, "blocks.6.attention.w_q": 0.7182044982910156, "blocks.6.attention.w_k": 0.6785888671875, "blocks.6.attention.w_v": 0.6297111511230469, "blocks.6.attention.w_out": 0.6268196105957031, "blocks.6.feed_forward.w1": 0.6897176504135132, "blocks.6.feed_forward.w2": 0.4673144221305847, "blocks.6.feed_forward.w3": 0.678290069103241, "blocks.7.attention.w_q": 0.6509819030761719, "blocks.7.attention.w_k": 0.6199111938476562, "blocks.7.attention.w_v": 0.6339454650878906, "blocks.7.attention.w_out": 0.5825729370117188, "blocks.7.feed_forward.w1": 0.6957206726074219, "blocks.7.feed_forward.w2": 0.5171470642089844, "blocks.7.feed_forward.w3": 0.6888847351074219, "blocks.8.attention.w_q": 0.6558570861816406, "blocks.8.attention.w_k": 0.6222000122070312, "blocks.8.attention.w_v": 0.6485443115234375, "blocks.8.attention.w_out": 0.5875396728515625, "blocks.8.feed_forward.w1": 0.6979866027832031, "blocks.8.feed_forward.w2": 0.4745890498161316, "blocks.8.feed_forward.w3": 0.6923078298568726}, "w_mask_frac": 0.17489441694357458} +{"step": 3200, "changed_frac": 0.604981446877504, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6796951293945312, "blocks.0.attention.w_k": 0.6861953735351562, "blocks.0.attention.w_v": 0.6704330444335938, "blocks.0.attention.w_out": 0.6141586303710938, "blocks.0.feed_forward.w1": 0.5032450556755066, "blocks.0.feed_forward.w2": 0.34361523389816284, "blocks.0.feed_forward.w3": 0.506964385509491, "blocks.1.attention.w_q": 0.7033729553222656, "blocks.1.attention.w_k": 0.6927947998046875, "blocks.1.attention.w_v": 0.6600379943847656, "blocks.1.attention.w_out": 0.6902580261230469, "blocks.1.feed_forward.w1": 0.5970052480697632, "blocks.1.feed_forward.w2": 0.42061489820480347, "blocks.1.feed_forward.w3": 0.5967458486557007, "blocks.2.attention.w_q": 0.6879234313964844, "blocks.2.attention.w_k": 0.6649703979492188, "blocks.2.attention.w_v": 0.646728515625, "blocks.2.attention.w_out": 0.6731109619140625, "blocks.2.feed_forward.w1": 0.6411031484603882, "blocks.2.feed_forward.w2": 0.44549816846847534, "blocks.2.feed_forward.w3": 0.6393661499023438, "blocks.3.attention.w_q": 0.7201957702636719, "blocks.3.attention.w_k": 0.6966094970703125, "blocks.3.attention.w_v": 0.6328468322753906, "blocks.3.attention.w_out": 0.6748390197753906, "blocks.3.feed_forward.w1": 0.6468353271484375, "blocks.3.feed_forward.w2": 0.4583117365837097, "blocks.3.feed_forward.w3": 0.6451225280761719, "blocks.4.attention.w_q": 0.7162399291992188, "blocks.4.attention.w_k": 0.6861534118652344, "blocks.4.attention.w_v": 0.6182937622070312, "blocks.4.attention.w_out": 0.6474113464355469, "blocks.4.feed_forward.w1": 0.677734375, "blocks.4.feed_forward.w2": 0.4905942380428314, "blocks.4.feed_forward.w3": 0.6727396845817566, "blocks.5.attention.w_q": 0.7161598205566406, "blocks.5.attention.w_k": 0.6793746948242188, "blocks.5.attention.w_v": 0.6111335754394531, "blocks.5.attention.w_out": 0.6456222534179688, "blocks.5.feed_forward.w1": 0.6873957514762878, "blocks.5.feed_forward.w2": 0.45909246802330017, "blocks.5.feed_forward.w3": 0.6853739619255066, "blocks.6.attention.w_q": 0.7108306884765625, "blocks.6.attention.w_k": 0.6686058044433594, "blocks.6.attention.w_v": 0.61590576171875, "blocks.6.attention.w_out": 0.6222953796386719, "blocks.6.feed_forward.w1": 0.6908760070800781, "blocks.6.feed_forward.w2": 0.4617462158203125, "blocks.6.feed_forward.w3": 0.6793696284294128, "blocks.7.attention.w_q": 0.6428413391113281, "blocks.7.attention.w_k": 0.6240882873535156, "blocks.7.attention.w_v": 0.6245956420898438, "blocks.7.attention.w_out": 0.5738372802734375, "blocks.7.feed_forward.w1": 0.6896960139274597, "blocks.7.feed_forward.w2": 0.5132077932357788, "blocks.7.feed_forward.w3": 0.6843541860580444, "blocks.8.attention.w_q": 0.6526947021484375, "blocks.8.attention.w_k": 0.6276664733886719, "blocks.8.attention.w_v": 0.63348388671875, "blocks.8.attention.w_out": 0.5789909362792969, "blocks.8.feed_forward.w1": 0.6942164301872253, "blocks.8.feed_forward.w2": 0.47943753004074097, "blocks.8.feed_forward.w3": 0.6872164607048035}, "w_mask_frac": 0.17606735560629103} +{"step": 3300, "changed_frac": 0.5974512996836605, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6751670837402344, "blocks.0.attention.w_k": 0.6823692321777344, "blocks.0.attention.w_v": 0.6660614013671875, "blocks.0.attention.w_out": 0.6092529296875, "blocks.0.feed_forward.w1": 0.49967193603515625, "blocks.0.feed_forward.w2": 0.3429858088493347, "blocks.0.feed_forward.w3": 0.5032476186752319, "blocks.1.attention.w_q": 0.7016716003417969, "blocks.1.attention.w_k": 0.6900711059570312, "blocks.1.attention.w_v": 0.6537857055664062, "blocks.1.attention.w_out": 0.6840629577636719, "blocks.1.feed_forward.w1": 0.5899836421012878, "blocks.1.feed_forward.w2": 0.41900634765625, "blocks.1.feed_forward.w3": 0.5914179682731628, "blocks.2.attention.w_q": 0.6857337951660156, "blocks.2.attention.w_k": 0.669769287109375, "blocks.2.attention.w_v": 0.6431045532226562, "blocks.2.attention.w_out": 0.6667900085449219, "blocks.2.feed_forward.w1": 0.6321246027946472, "blocks.2.feed_forward.w2": 0.4427782893180847, "blocks.2.feed_forward.w3": 0.6307119131088257, "blocks.3.attention.w_q": 0.7133560180664062, "blocks.3.attention.w_k": 0.6917457580566406, "blocks.3.attention.w_v": 0.6288375854492188, "blocks.3.attention.w_out": 0.6691703796386719, "blocks.3.feed_forward.w1": 0.6374588012695312, "blocks.3.feed_forward.w2": 0.451904296875, "blocks.3.feed_forward.w3": 0.6349424123764038, "blocks.4.attention.w_q": 0.71063232421875, "blocks.4.attention.w_k": 0.6803627014160156, "blocks.4.attention.w_v": 0.6142387390136719, "blocks.4.attention.w_out": 0.6353912353515625, "blocks.4.feed_forward.w1": 0.6694539785385132, "blocks.4.feed_forward.w2": 0.48373669385910034, "blocks.4.feed_forward.w3": 0.6664136648178101, "blocks.5.attention.w_q": 0.7126922607421875, "blocks.5.attention.w_k": 0.6774368286132812, "blocks.5.attention.w_v": 0.6064910888671875, "blocks.5.attention.w_out": 0.6336212158203125, "blocks.5.feed_forward.w1": 0.6792920827865601, "blocks.5.feed_forward.w2": 0.4509328305721283, "blocks.5.feed_forward.w3": 0.6754888296127319, "blocks.6.attention.w_q": 0.7082252502441406, "blocks.6.attention.w_k": 0.6721153259277344, "blocks.6.attention.w_v": 0.61029052734375, "blocks.6.attention.w_out": 0.6108207702636719, "blocks.6.feed_forward.w1": 0.6788038015365601, "blocks.6.feed_forward.w2": 0.45308560132980347, "blocks.6.feed_forward.w3": 0.6634992361068726, "blocks.7.attention.w_q": 0.6332969665527344, "blocks.7.attention.w_k": 0.617279052734375, "blocks.7.attention.w_v": 0.6142501831054688, "blocks.7.attention.w_out": 0.5589523315429688, "blocks.7.feed_forward.w1": 0.6809438467025757, "blocks.7.feed_forward.w2": 0.5009244680404663, "blocks.7.feed_forward.w3": 0.6744436025619507, "blocks.8.attention.w_q": 0.6462211608886719, "blocks.8.attention.w_k": 0.6126899719238281, "blocks.8.attention.w_v": 0.62506103515625, "blocks.8.attention.w_out": 0.5617141723632812, "blocks.8.feed_forward.w1": 0.6817703247070312, "blocks.8.feed_forward.w2": 0.46848297119140625, "blocks.8.feed_forward.w3": 0.6728515625}, "w_mask_frac": 0.16376101123726267} +{"step": 3400, "changed_frac": 0.5865049117650741, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6641464233398438, "blocks.0.attention.w_k": 0.6693611145019531, "blocks.0.attention.w_v": 0.6544418334960938, "blocks.0.attention.w_out": 0.5974960327148438, "blocks.0.feed_forward.w1": 0.4871661067008972, "blocks.0.feed_forward.w2": 0.33094534277915955, "blocks.0.feed_forward.w3": 0.4915059506893158, "blocks.1.attention.w_q": 0.6930923461914062, "blocks.1.attention.w_k": 0.6779632568359375, "blocks.1.attention.w_v": 0.6449775695800781, "blocks.1.attention.w_out": 0.6752815246582031, "blocks.1.feed_forward.w1": 0.5818227529525757, "blocks.1.feed_forward.w2": 0.406219482421875, "blocks.1.feed_forward.w3": 0.5818748474121094, "blocks.2.attention.w_q": 0.6789016723632812, "blocks.2.attention.w_k": 0.6603851318359375, "blocks.2.attention.w_v": 0.6273536682128906, "blocks.2.attention.w_out": 0.6566200256347656, "blocks.2.feed_forward.w1": 0.6243502497673035, "blocks.2.feed_forward.w2": 0.4299151301383972, "blocks.2.feed_forward.w3": 0.6235491633415222, "blocks.3.attention.w_q": 0.7078323364257812, "blocks.3.attention.w_k": 0.6823234558105469, "blocks.3.attention.w_v": 0.6065902709960938, "blocks.3.attention.w_out": 0.65338134765625, "blocks.3.feed_forward.w1": 0.6296005249023438, "blocks.3.feed_forward.w2": 0.4417928159236908, "blocks.3.feed_forward.w3": 0.6270269155502319, "blocks.4.attention.w_q": 0.7090797424316406, "blocks.4.attention.w_k": 0.6707000732421875, "blocks.4.attention.w_v": 0.5928421020507812, "blocks.4.attention.w_out": 0.618865966796875, "blocks.4.feed_forward.w1": 0.6616948843002319, "blocks.4.feed_forward.w2": 0.4712015986442566, "blocks.4.feed_forward.w3": 0.6575953364372253, "blocks.5.attention.w_q": 0.7039871215820312, "blocks.5.attention.w_k": 0.6625518798828125, "blocks.5.attention.w_v": 0.577484130859375, "blocks.5.attention.w_out": 0.6187667846679688, "blocks.5.feed_forward.w1": 0.6707407832145691, "blocks.5.feed_forward.w2": 0.43361157178878784, "blocks.5.feed_forward.w3": 0.6666425466537476, "blocks.6.attention.w_q": 0.6970252990722656, "blocks.6.attention.w_k": 0.6511344909667969, "blocks.6.attention.w_v": 0.5830192565917969, "blocks.6.attention.w_out": 0.5928573608398438, "blocks.6.feed_forward.w1": 0.6686198115348816, "blocks.6.feed_forward.w2": 0.4304008483886719, "blocks.6.feed_forward.w3": 0.6537717580795288, "blocks.7.attention.w_q": 0.6413993835449219, "blocks.7.attention.w_k": 0.6136207580566406, "blocks.7.attention.w_v": 0.599029541015625, "blocks.7.attention.w_out": 0.5312957763671875, "blocks.7.feed_forward.w1": 0.6746991872787476, "blocks.7.feed_forward.w2": 0.48979949951171875, "blocks.7.feed_forward.w3": 0.6678237915039062, "blocks.8.attention.w_q": 0.6489944458007812, "blocks.8.attention.w_k": 0.6160087585449219, "blocks.8.attention.w_v": 0.6114540100097656, "blocks.8.attention.w_out": 0.5382156372070312, "blocks.8.feed_forward.w1": 0.6745681762695312, "blocks.8.feed_forward.w2": 0.45262908935546875, "blocks.8.feed_forward.w3": 0.6679636836051941}, "w_mask_frac": 0.16650166056859186} +{"step": 3500, "changed_frac": 0.578205793331831, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6553802490234375, "blocks.0.attention.w_k": 0.6639289855957031, "blocks.0.attention.w_v": 0.6516799926757812, "blocks.0.attention.w_out": 0.5921440124511719, "blocks.0.feed_forward.w1": 0.4790763854980469, "blocks.0.feed_forward.w2": 0.3328094482421875, "blocks.0.feed_forward.w3": 0.48330816626548767, "blocks.1.attention.w_q": 0.6827316284179688, "blocks.1.attention.w_k": 0.6744842529296875, "blocks.1.attention.w_v": 0.63238525390625, "blocks.1.attention.w_out": 0.6685447692871094, "blocks.1.feed_forward.w1": 0.5738143920898438, "blocks.1.feed_forward.w2": 0.4037971496582031, "blocks.1.feed_forward.w3": 0.5760511159896851, "blocks.2.attention.w_q": 0.6691207885742188, "blocks.2.attention.w_k": 0.6451530456542969, "blocks.2.attention.w_v": 0.6212730407714844, "blocks.2.attention.w_out": 0.6516265869140625, "blocks.2.feed_forward.w1": 0.6153831481933594, "blocks.2.feed_forward.w2": 0.43040594458580017, "blocks.2.feed_forward.w3": 0.6138076782226562, "blocks.3.attention.w_q": 0.6957473754882812, "blocks.3.attention.w_k": 0.6727790832519531, "blocks.3.attention.w_v": 0.6014747619628906, "blocks.3.attention.w_out": 0.6497344970703125, "blocks.3.feed_forward.w1": 0.6200294494628906, "blocks.3.feed_forward.w2": 0.4372673034667969, "blocks.3.feed_forward.w3": 0.6175715327262878, "blocks.4.attention.w_q": 0.6950874328613281, "blocks.4.attention.w_k": 0.6634178161621094, "blocks.4.attention.w_v": 0.5836601257324219, "blocks.4.attention.w_out": 0.61309814453125, "blocks.4.feed_forward.w1": 0.6520563960075378, "blocks.4.feed_forward.w2": 0.4675750732421875, "blocks.4.feed_forward.w3": 0.6485812067985535, "blocks.5.attention.w_q": 0.6948318481445312, "blocks.5.attention.w_k": 0.6509742736816406, "blocks.5.attention.w_v": 0.5751991271972656, "blocks.5.attention.w_out": 0.6119117736816406, "blocks.5.feed_forward.w1": 0.6618461608886719, "blocks.5.feed_forward.w2": 0.4286766052246094, "blocks.5.feed_forward.w3": 0.6587003469467163, "blocks.6.attention.w_q": 0.6888427734375, "blocks.6.attention.w_k": 0.6406364440917969, "blocks.6.attention.w_v": 0.5799331665039062, "blocks.6.attention.w_out": 0.5863380432128906, "blocks.6.feed_forward.w1": 0.6542028188705444, "blocks.6.feed_forward.w2": 0.43165716528892517, "blocks.6.feed_forward.w3": 0.6395543813705444, "blocks.7.attention.w_q": 0.6181755065917969, "blocks.7.attention.w_k": 0.5861129760742188, "blocks.7.attention.w_v": 0.5885429382324219, "blocks.7.attention.w_out": 0.5267333984375, "blocks.7.feed_forward.w1": 0.6660906672477722, "blocks.7.feed_forward.w2": 0.4795417785644531, "blocks.7.feed_forward.w3": 0.6589330434799194, "blocks.8.attention.w_q": 0.6165771484375, "blocks.8.attention.w_k": 0.5790557861328125, "blocks.8.attention.w_v": 0.599395751953125, "blocks.8.attention.w_out": 0.528961181640625, "blocks.8.feed_forward.w1": 0.6666603088378906, "blocks.8.feed_forward.w2": 0.44276174902915955, "blocks.8.feed_forward.w3": 0.6579755544662476}, "w_mask_frac": 0.16487917367719176} +{"step": 3600, "changed_frac": 0.5672041412092682, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6407051086425781, "blocks.0.attention.w_k": 0.6524734497070312, "blocks.0.attention.w_v": 0.6401214599609375, "blocks.0.attention.w_out": 0.578155517578125, "blocks.0.feed_forward.w1": 0.4612872004508972, "blocks.0.feed_forward.w2": 0.3220176696777344, "blocks.0.feed_forward.w3": 0.4679463803768158, "blocks.1.attention.w_q": 0.6726722717285156, "blocks.1.attention.w_k": 0.6624984741210938, "blocks.1.attention.w_v": 0.6157646179199219, "blocks.1.attention.w_out": 0.6592559814453125, "blocks.1.feed_forward.w1": 0.5584602355957031, "blocks.1.feed_forward.w2": 0.39399465918540955, "blocks.1.feed_forward.w3": 0.5582148432731628, "blocks.2.attention.w_q": 0.66448974609375, "blocks.2.attention.w_k": 0.643218994140625, "blocks.2.attention.w_v": 0.6044082641601562, "blocks.2.attention.w_out": 0.6385383605957031, "blocks.2.feed_forward.w1": 0.6028417348861694, "blocks.2.feed_forward.w2": 0.41908901929855347, "blocks.2.feed_forward.w3": 0.6003469228744507, "blocks.3.attention.w_q": 0.6913909912109375, "blocks.3.attention.w_k": 0.6660308837890625, "blocks.3.attention.w_v": 0.5816421508789062, "blocks.3.attention.w_out": 0.6356620788574219, "blocks.3.feed_forward.w1": 0.6092453002929688, "blocks.3.feed_forward.w2": 0.4293925166130066, "blocks.3.feed_forward.w3": 0.6071510314941406, "blocks.4.attention.w_q": 0.6904144287109375, "blocks.4.attention.w_k": 0.6528053283691406, "blocks.4.attention.w_v": 0.56329345703125, "blocks.4.attention.w_out": 0.5945663452148438, "blocks.4.feed_forward.w1": 0.6432723999023438, "blocks.4.feed_forward.w2": 0.45906704664230347, "blocks.4.feed_forward.w3": 0.6410535573959351, "blocks.5.attention.w_q": 0.6850814819335938, "blocks.5.attention.w_k": 0.642303466796875, "blocks.5.attention.w_v": 0.5538749694824219, "blocks.5.attention.w_out": 0.593841552734375, "blocks.5.feed_forward.w1": 0.6520678400993347, "blocks.5.feed_forward.w2": 0.42269009351730347, "blocks.5.feed_forward.w3": 0.6485532522201538, "blocks.6.attention.w_q": 0.6781578063964844, "blocks.6.attention.w_k": 0.6322059631347656, "blocks.6.attention.w_v": 0.5551071166992188, "blocks.6.attention.w_out": 0.5700569152832031, "blocks.6.feed_forward.w1": 0.6509437561035156, "blocks.6.feed_forward.w2": 0.4163970947265625, "blocks.6.feed_forward.w3": 0.6353034973144531, "blocks.7.attention.w_q": 0.6152076721191406, "blocks.7.attention.w_k": 0.5856170654296875, "blocks.7.attention.w_v": 0.5651969909667969, "blocks.7.attention.w_out": 0.5098304748535156, "blocks.7.feed_forward.w1": 0.6572850942611694, "blocks.7.feed_forward.w2": 0.4719429016113281, "blocks.7.feed_forward.w3": 0.650030791759491, "blocks.8.attention.w_q": 0.6159477233886719, "blocks.8.attention.w_k": 0.5813179016113281, "blocks.8.attention.w_v": 0.5751457214355469, "blocks.8.attention.w_out": 0.5093040466308594, "blocks.8.feed_forward.w1": 0.6566556692123413, "blocks.8.feed_forward.w2": 0.4316609799861908, "blocks.8.feed_forward.w3": 0.6486155390739441}, "w_mask_frac": 0.1732416766194197} +{"step": 3700, "changed_frac": 0.5603753766442976, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6366691589355469, "blocks.0.attention.w_k": 0.645782470703125, "blocks.0.attention.w_v": 0.6301116943359375, "blocks.0.attention.w_out": 0.5691947937011719, "blocks.0.feed_forward.w1": 0.45347851514816284, "blocks.0.feed_forward.w2": 0.3182474970817566, "blocks.0.feed_forward.w3": 0.46183905005455017, "blocks.1.attention.w_q": 0.6628570556640625, "blocks.1.attention.w_k": 0.6541366577148438, "blocks.1.attention.w_v": 0.6070404052734375, "blocks.1.attention.w_out": 0.6493034362792969, "blocks.1.feed_forward.w1": 0.5490099787712097, "blocks.1.feed_forward.w2": 0.38901394605636597, "blocks.1.feed_forward.w3": 0.5500564575195312, "blocks.2.attention.w_q": 0.6546745300292969, "blocks.2.attention.w_k": 0.6323204040527344, "blocks.2.attention.w_v": 0.5939064025878906, "blocks.2.attention.w_out": 0.6306877136230469, "blocks.2.feed_forward.w1": 0.5944125056266785, "blocks.2.feed_forward.w2": 0.4105033874511719, "blocks.2.feed_forward.w3": 0.5903956294059753, "blocks.3.attention.w_q": 0.6814804077148438, "blocks.3.attention.w_k": 0.6533927917480469, "blocks.3.attention.w_v": 0.5671768188476562, "blocks.3.attention.w_out": 0.6273612976074219, "blocks.3.feed_forward.w1": 0.5986582636833191, "blocks.3.feed_forward.w2": 0.4233029782772064, "blocks.3.feed_forward.w3": 0.5971641540527344, "blocks.4.attention.w_q": 0.6787986755371094, "blocks.4.attention.w_k": 0.6445465087890625, "blocks.4.attention.w_v": 0.54742431640625, "blocks.4.attention.w_out": 0.5841331481933594, "blocks.4.feed_forward.w1": 0.6363143920898438, "blocks.4.feed_forward.w2": 0.4565061032772064, "blocks.4.feed_forward.w3": 0.6319491267204285, "blocks.5.attention.w_q": 0.6774520874023438, "blocks.5.attention.w_k": 0.6364669799804688, "blocks.5.attention.w_v": 0.5433540344238281, "blocks.5.attention.w_out": 0.5866279602050781, "blocks.5.feed_forward.w1": 0.6431986689567566, "blocks.5.feed_forward.w2": 0.4197642207145691, "blocks.5.feed_forward.w3": 0.6380996704101562, "blocks.6.attention.w_q": 0.67364501953125, "blocks.6.attention.w_k": 0.6307563781738281, "blocks.6.attention.w_v": 0.5457382202148438, "blocks.6.attention.w_out": 0.5603256225585938, "blocks.6.feed_forward.w1": 0.6510276794433594, "blocks.6.feed_forward.w2": 0.4188652038574219, "blocks.6.feed_forward.w3": 0.6377195119857788, "blocks.7.attention.w_q": 0.6119232177734375, "blocks.7.attention.w_k": 0.5863876342773438, "blocks.7.attention.w_v": 0.556854248046875, "blocks.7.attention.w_out": 0.5052375793457031, "blocks.7.feed_forward.w1": 0.6488024592399597, "blocks.7.feed_forward.w2": 0.46560415625572205, "blocks.7.feed_forward.w3": 0.6383641958236694, "blocks.8.attention.w_q": 0.6200065612792969, "blocks.8.attention.w_k": 0.5958061218261719, "blocks.8.attention.w_v": 0.5672798156738281, "blocks.8.attention.w_out": 0.5037345886230469, "blocks.8.feed_forward.w1": 0.6473503112792969, "blocks.8.feed_forward.w2": 0.4258359372615814, "blocks.8.feed_forward.w3": 0.6416244506835938}, "w_mask_frac": 0.16390825183982524} +{"step": 3800, "changed_frac": 0.5476599473219651, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6236000061035156, "blocks.0.attention.w_k": 0.63519287109375, "blocks.0.attention.w_v": 0.6251640319824219, "blocks.0.attention.w_out": 0.5556488037109375, "blocks.0.feed_forward.w1": 0.4445126950740814, "blocks.0.feed_forward.w2": 0.31260427832603455, "blocks.0.feed_forward.w3": 0.45007580518722534, "blocks.1.attention.w_q": 0.6581535339355469, "blocks.1.attention.w_k": 0.6460189819335938, "blocks.1.attention.w_v": 0.5975494384765625, "blocks.1.attention.w_out": 0.6402053833007812, "blocks.1.feed_forward.w1": 0.5386072993278503, "blocks.1.feed_forward.w2": 0.3798103332519531, "blocks.1.feed_forward.w3": 0.5409126281738281, "blocks.2.attention.w_q": 0.6374549865722656, "blocks.2.attention.w_k": 0.6125144958496094, "blocks.2.attention.w_v": 0.5813980102539062, "blocks.2.attention.w_out": 0.6224098205566406, "blocks.2.feed_forward.w1": 0.5817235708236694, "blocks.2.feed_forward.w2": 0.4026743769645691, "blocks.2.feed_forward.w3": 0.5796585083007812, "blocks.3.attention.w_q": 0.6749000549316406, "blocks.3.attention.w_k": 0.6510238647460938, "blocks.3.attention.w_v": 0.55523681640625, "blocks.3.attention.w_out": 0.61279296875, "blocks.3.feed_forward.w1": 0.5848731994628906, "blocks.3.feed_forward.w2": 0.4128354489803314, "blocks.3.feed_forward.w3": 0.5832672119140625, "blocks.4.attention.w_q": 0.6741676330566406, "blocks.4.attention.w_k": 0.6405410766601562, "blocks.4.attention.w_v": 0.5318679809570312, "blocks.4.attention.w_out": 0.5676689147949219, "blocks.4.feed_forward.w1": 0.6234792470932007, "blocks.4.feed_forward.w2": 0.4454142451286316, "blocks.4.feed_forward.w3": 0.6184806823730469, "blocks.5.attention.w_q": 0.6724205017089844, "blocks.5.attention.w_k": 0.6275749206542969, "blocks.5.attention.w_v": 0.5226364135742188, "blocks.5.attention.w_out": 0.5697174072265625, "blocks.5.feed_forward.w1": 0.6299489736557007, "blocks.5.feed_forward.w2": 0.40137991309165955, "blocks.5.feed_forward.w3": 0.6248003840446472, "blocks.6.attention.w_q": 0.6639137268066406, "blocks.6.attention.w_k": 0.6186447143554688, "blocks.6.attention.w_v": 0.5323104858398438, "blocks.6.attention.w_out": 0.5455551147460938, "blocks.6.feed_forward.w1": 0.6279513239860535, "blocks.6.feed_forward.w2": 0.3956565856933594, "blocks.6.feed_forward.w3": 0.6146138906478882, "blocks.7.attention.w_q": 0.59478759765625, "blocks.7.attention.w_k": 0.5726242065429688, "blocks.7.attention.w_v": 0.5496673583984375, "blocks.7.attention.w_out": 0.4799766540527344, "blocks.7.feed_forward.w1": 0.6380246877670288, "blocks.7.feed_forward.w2": 0.4508247375488281, "blocks.7.feed_forward.w3": 0.6286290884017944, "blocks.8.attention.w_q": 0.6093788146972656, "blocks.8.attention.w_k": 0.5786285400390625, "blocks.8.attention.w_v": 0.5621414184570312, "blocks.8.attention.w_out": 0.48082733154296875, "blocks.8.feed_forward.w1": 0.6354827880859375, "blocks.8.feed_forward.w2": 0.4101409912109375, "blocks.8.feed_forward.w3": 0.6275838613510132}, "w_mask_frac": 0.16889666956968796} +{"step": 3900, "changed_frac": 0.5375201233431824, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6174354553222656, "blocks.0.attention.w_k": 0.6301651000976562, "blocks.0.attention.w_v": 0.6118850708007812, "blocks.0.attention.w_out": 0.5461311340332031, "blocks.0.feed_forward.w1": 0.4325510859489441, "blocks.0.feed_forward.w2": 0.3074073791503906, "blocks.0.feed_forward.w3": 0.44144314527511597, "blocks.1.attention.w_q": 0.6513252258300781, "blocks.1.attention.w_k": 0.6400642395019531, "blocks.1.attention.w_v": 0.5852279663085938, "blocks.1.attention.w_out": 0.6322975158691406, "blocks.1.feed_forward.w1": 0.5304819941520691, "blocks.1.feed_forward.w2": 0.3726437985897064, "blocks.1.feed_forward.w3": 0.5308761596679688, "blocks.2.attention.w_q": 0.6371879577636719, "blocks.2.attention.w_k": 0.6176261901855469, "blocks.2.attention.w_v": 0.568878173828125, "blocks.2.attention.w_out": 0.610809326171875, "blocks.2.feed_forward.w1": 0.5730870962142944, "blocks.2.feed_forward.w2": 0.3929290771484375, "blocks.2.feed_forward.w3": 0.5719680786132812, "blocks.3.attention.w_q": 0.6609840393066406, "blocks.3.attention.w_k": 0.6322822570800781, "blocks.3.attention.w_v": 0.537567138671875, "blocks.3.attention.w_out": 0.6004753112792969, "blocks.3.feed_forward.w1": 0.578155517578125, "blocks.3.feed_forward.w2": 0.40453213453292847, "blocks.3.feed_forward.w3": 0.5764364004135132, "blocks.4.attention.w_q": 0.6659393310546875, "blocks.4.attention.w_k": 0.6273040771484375, "blocks.4.attention.w_v": 0.5178298950195312, "blocks.4.attention.w_out": 0.5551338195800781, "blocks.4.feed_forward.w1": 0.6153170466423035, "blocks.4.feed_forward.w2": 0.4316368103027344, "blocks.4.feed_forward.w3": 0.61065673828125, "blocks.5.attention.w_q": 0.6602058410644531, "blocks.5.attention.w_k": 0.6202545166015625, "blocks.5.attention.w_v": 0.5118331909179688, "blocks.5.attention.w_out": 0.5573959350585938, "blocks.5.feed_forward.w1": 0.6266886591911316, "blocks.5.feed_forward.w2": 0.3886248469352722, "blocks.5.feed_forward.w3": 0.6220652461051941, "blocks.6.attention.w_q": 0.6539421081542969, "blocks.6.attention.w_k": 0.6082916259765625, "blocks.6.attention.w_v": 0.5166130065917969, "blocks.6.attention.w_out": 0.529327392578125, "blocks.6.feed_forward.w1": 0.620062530040741, "blocks.6.feed_forward.w2": 0.367950439453125, "blocks.6.feed_forward.w3": 0.6068115234375, "blocks.7.attention.w_q": 0.5822410583496094, "blocks.7.attention.w_k": 0.557952880859375, "blocks.7.attention.w_v": 0.5263404846191406, "blocks.7.attention.w_out": 0.4619789123535156, "blocks.7.feed_forward.w1": 0.6321920156478882, "blocks.7.feed_forward.w2": 0.43700408935546875, "blocks.7.feed_forward.w3": 0.6221758723258972, "blocks.8.attention.w_q": 0.5927505493164062, "blocks.8.attention.w_k": 0.5628166198730469, "blocks.8.attention.w_v": 0.5350990295410156, "blocks.8.attention.w_out": 0.4615898132324219, "blocks.8.feed_forward.w1": 0.6296247243881226, "blocks.8.feed_forward.w2": 0.3925425410270691, "blocks.8.feed_forward.w3": 0.6190261840820312}, "w_mask_frac": 0.1738788609703382} +{"step": 4000, "changed_frac": 0.5295320494562132, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6038818359375, "blocks.0.attention.w_k": 0.6186676025390625, "blocks.0.attention.w_v": 0.60247802734375, "blocks.0.attention.w_out": 0.5338363647460938, "blocks.0.feed_forward.w1": 0.4230461120605469, "blocks.0.feed_forward.w2": 0.2985623776912689, "blocks.0.feed_forward.w3": 0.4304491877555847, "blocks.1.attention.w_q": 0.6374053955078125, "blocks.1.attention.w_k": 0.6222038269042969, "blocks.1.attention.w_v": 0.571929931640625, "blocks.1.attention.w_out": 0.6205329895019531, "blocks.1.feed_forward.w1": 0.5172653198242188, "blocks.1.feed_forward.w2": 0.36589813232421875, "blocks.1.feed_forward.w3": 0.517553985118866, "blocks.2.attention.w_q": 0.624786376953125, "blocks.2.attention.w_k": 0.5984382629394531, "blocks.2.attention.w_v": 0.5504112243652344, "blocks.2.attention.w_out": 0.5994415283203125, "blocks.2.feed_forward.w1": 0.5662841796875, "blocks.2.feed_forward.w2": 0.38871002197265625, "blocks.2.feed_forward.w3": 0.5632337331771851, "blocks.3.attention.w_q": 0.6528663635253906, "blocks.3.attention.w_k": 0.6224784851074219, "blocks.3.attention.w_v": 0.5194625854492188, "blocks.3.attention.w_out": 0.5875091552734375, "blocks.3.feed_forward.w1": 0.5677515864372253, "blocks.3.feed_forward.w2": 0.40116629004478455, "blocks.3.feed_forward.w3": 0.5640945434570312, "blocks.4.attention.w_q": 0.6571464538574219, "blocks.4.attention.w_k": 0.617431640625, "blocks.4.attention.w_v": 0.498077392578125, "blocks.4.attention.w_out": 0.5384941101074219, "blocks.4.feed_forward.w1": 0.606903076171875, "blocks.4.feed_forward.w2": 0.4272601008415222, "blocks.4.feed_forward.w3": 0.6010398864746094, "blocks.5.attention.w_q": 0.6526298522949219, "blocks.5.attention.w_k": 0.6092338562011719, "blocks.5.attention.w_v": 0.49164581298828125, "blocks.5.attention.w_out": 0.5408592224121094, "blocks.5.feed_forward.w1": 0.6151555776596069, "blocks.5.feed_forward.w2": 0.38905590772628784, "blocks.5.feed_forward.w3": 0.6100082397460938, "blocks.6.attention.w_q": 0.6430816650390625, "blocks.6.attention.w_k": 0.6002845764160156, "blocks.6.attention.w_v": 0.49953460693359375, "blocks.6.attention.w_out": 0.5197525024414062, "blocks.6.feed_forward.w1": 0.6204656362533569, "blocks.6.feed_forward.w2": 0.38354620337486267, "blocks.6.feed_forward.w3": 0.6046422719955444, "blocks.7.attention.w_q": 0.5852470397949219, "blocks.7.attention.w_k": 0.5612945556640625, "blocks.7.attention.w_v": 0.5104408264160156, "blocks.7.attention.w_out": 0.4581336975097656, "blocks.7.feed_forward.w1": 0.6219622492790222, "blocks.7.feed_forward.w2": 0.4315427243709564, "blocks.7.feed_forward.w3": 0.6152153015136719, "blocks.8.attention.w_q": 0.58319091796875, "blocks.8.attention.w_k": 0.5604629516601562, "blocks.8.attention.w_v": 0.5182647705078125, "blocks.8.attention.w_out": 0.4539985656738281, "blocks.8.feed_forward.w1": 0.6236928701400757, "blocks.8.feed_forward.w2": 0.3935483396053314, "blocks.8.feed_forward.w3": 0.6151847839355469}, "w_mask_frac": 0.16869445826508042} +{"step": 4100, "changed_frac": 0.5207261305588943, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5872077941894531, "blocks.0.attention.w_k": 0.6060638427734375, "blocks.0.attention.w_v": 0.591705322265625, "blocks.0.attention.w_out": 0.52178955078125, "blocks.0.feed_forward.w1": 0.41419219970703125, "blocks.0.feed_forward.w2": 0.2934557795524597, "blocks.0.feed_forward.w3": 0.42227429151535034, "blocks.1.attention.w_q": 0.6256790161132812, "blocks.1.attention.w_k": 0.6166954040527344, "blocks.1.attention.w_v": 0.5591316223144531, "blocks.1.attention.w_out": 0.6099205017089844, "blocks.1.feed_forward.w1": 0.5060933828353882, "blocks.1.feed_forward.w2": 0.3597831726074219, "blocks.1.feed_forward.w3": 0.5063947439193726, "blocks.2.attention.w_q": 0.6096420288085938, "blocks.2.attention.w_k": 0.5822296142578125, "blocks.2.attention.w_v": 0.5395011901855469, "blocks.2.attention.w_out": 0.5930557250976562, "blocks.2.feed_forward.w1": 0.5540034174919128, "blocks.2.feed_forward.w2": 0.38380178809165955, "blocks.2.feed_forward.w3": 0.5506464838981628, "blocks.3.attention.w_q": 0.6412849426269531, "blocks.3.attention.w_k": 0.6146049499511719, "blocks.3.attention.w_v": 0.5036544799804688, "blocks.3.attention.w_out": 0.5779342651367188, "blocks.3.feed_forward.w1": 0.5565465688705444, "blocks.3.feed_forward.w2": 0.3925463557243347, "blocks.3.feed_forward.w3": 0.5539144277572632, "blocks.4.attention.w_q": 0.6424789428710938, "blocks.4.attention.w_k": 0.6042556762695312, "blocks.4.attention.w_v": 0.48226165771484375, "blocks.4.attention.w_out": 0.5290946960449219, "blocks.4.feed_forward.w1": 0.5963249206542969, "blocks.4.feed_forward.w2": 0.4273719787597656, "blocks.4.feed_forward.w3": 0.5916265249252319, "blocks.5.attention.w_q": 0.6382942199707031, "blocks.5.attention.w_k": 0.5977668762207031, "blocks.5.attention.w_v": 0.4768791198730469, "blocks.5.attention.w_out": 0.5332908630371094, "blocks.5.feed_forward.w1": 0.6090215444564819, "blocks.5.feed_forward.w2": 0.39816030859947205, "blocks.5.feed_forward.w3": 0.6045964956283569, "blocks.6.attention.w_q": 0.6313209533691406, "blocks.6.attention.w_k": 0.5848121643066406, "blocks.6.attention.w_v": 0.48377227783203125, "blocks.6.attention.w_out": 0.5114593505859375, "blocks.6.feed_forward.w1": 0.6170858144760132, "blocks.6.feed_forward.w2": 0.3922055661678314, "blocks.6.feed_forward.w3": 0.6075515747070312, "blocks.7.attention.w_q": 0.56060791015625, "blocks.7.attention.w_k": 0.5392417907714844, "blocks.7.attention.w_v": 0.49394989013671875, "blocks.7.attention.w_out": 0.4487419128417969, "blocks.7.feed_forward.w1": 0.6135520935058594, "blocks.7.feed_forward.w2": 0.42397817969322205, "blocks.7.feed_forward.w3": 0.6036733388900757, "blocks.8.attention.w_q": 0.5623092651367188, "blocks.8.attention.w_k": 0.5331993103027344, "blocks.8.attention.w_v": 0.5003547668457031, "blocks.8.attention.w_out": 0.44438934326171875, "blocks.8.feed_forward.w1": 0.6103744506835938, "blocks.8.feed_forward.w2": 0.3910942077636719, "blocks.8.feed_forward.w3": 0.6018562316894531}, "w_mask_frac": 0.16510551244529903} +{"step": 4200, "changed_frac": 0.5079322880149907, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5746498107910156, "blocks.0.attention.w_k": 0.5935478210449219, "blocks.0.attention.w_v": 0.5767364501953125, "blocks.0.attention.w_out": 0.5095634460449219, "blocks.0.feed_forward.w1": 0.398712158203125, "blocks.0.feed_forward.w2": 0.28664273023605347, "blocks.0.feed_forward.w3": 0.4092051386833191, "blocks.1.attention.w_q": 0.6125869750976562, "blocks.1.attention.w_k": 0.5999298095703125, "blocks.1.attention.w_v": 0.5429878234863281, "blocks.1.attention.w_out": 0.5987930297851562, "blocks.1.feed_forward.w1": 0.49438223242759705, "blocks.1.feed_forward.w2": 0.3517723083496094, "blocks.1.feed_forward.w3": 0.49447378516197205, "blocks.2.attention.w_q": 0.6039161682128906, "blocks.2.attention.w_k": 0.5810050964355469, "blocks.2.attention.w_v": 0.5232620239257812, "blocks.2.attention.w_out": 0.5824317932128906, "blocks.2.feed_forward.w1": 0.5402476191520691, "blocks.2.feed_forward.w2": 0.3732109069824219, "blocks.2.feed_forward.w3": 0.5383822321891785, "blocks.3.attention.w_q": 0.6321144104003906, "blocks.3.attention.w_k": 0.6033668518066406, "blocks.3.attention.w_v": 0.49109649658203125, "blocks.3.attention.w_out": 0.5671958923339844, "blocks.3.feed_forward.w1": 0.5430514216423035, "blocks.3.feed_forward.w2": 0.38270315527915955, "blocks.3.feed_forward.w3": 0.5401255488395691, "blocks.4.attention.w_q": 0.6356620788574219, "blocks.4.attention.w_k": 0.5921669006347656, "blocks.4.attention.w_v": 0.4697914123535156, "blocks.4.attention.w_out": 0.5114479064941406, "blocks.4.feed_forward.w1": 0.5814450979232788, "blocks.4.feed_forward.w2": 0.4151344299316406, "blocks.4.feed_forward.w3": 0.5761553645133972, "blocks.5.attention.w_q": 0.6289482116699219, "blocks.5.attention.w_k": 0.5802116394042969, "blocks.5.attention.w_v": 0.4564399719238281, "blocks.5.attention.w_out": 0.5136909484863281, "blocks.5.feed_forward.w1": 0.5894966125488281, "blocks.5.feed_forward.w2": 0.3854306638240814, "blocks.5.feed_forward.w3": 0.5848376154899597, "blocks.6.attention.w_q": 0.6244621276855469, "blocks.6.attention.w_k": 0.5759086608886719, "blocks.6.attention.w_v": 0.4667472839355469, "blocks.6.attention.w_out": 0.4958381652832031, "blocks.6.feed_forward.w1": 0.6041641235351562, "blocks.6.feed_forward.w2": 0.38747280836105347, "blocks.6.feed_forward.w3": 0.5950584411621094, "blocks.7.attention.w_q": 0.5603523254394531, "blocks.7.attention.w_k": 0.5320701599121094, "blocks.7.attention.w_v": 0.47644805908203125, "blocks.7.attention.w_out": 0.430084228515625, "blocks.7.feed_forward.w1": 0.5973600149154663, "blocks.7.feed_forward.w2": 0.40843456983566284, "blocks.7.feed_forward.w3": 0.5893058776855469, "blocks.8.attention.w_q": 0.5631256103515625, "blocks.8.attention.w_k": 0.5308494567871094, "blocks.8.attention.w_v": 0.4842414855957031, "blocks.8.attention.w_out": 0.4249420166015625, "blocks.8.feed_forward.w1": 0.5972201228141785, "blocks.8.feed_forward.w2": 0.3761253356933594, "blocks.8.feed_forward.w3": 0.5866050720214844}, "w_mask_frac": 0.1633495466194601} +{"step": 4300, "changed_frac": 0.4932108495989416, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5611152648925781, "blocks.0.attention.w_k": 0.5811347961425781, "blocks.0.attention.w_v": 0.5646018981933594, "blocks.0.attention.w_out": 0.49213409423828125, "blocks.0.feed_forward.w1": 0.38831329345703125, "blocks.0.feed_forward.w2": 0.2748908996582031, "blocks.0.feed_forward.w3": 0.3954099118709564, "blocks.1.attention.w_q": 0.6014060974121094, "blocks.1.attention.w_k": 0.5859375, "blocks.1.attention.w_v": 0.5248794555664062, "blocks.1.attention.w_out": 0.5833511352539062, "blocks.1.feed_forward.w1": 0.4807891845703125, "blocks.1.feed_forward.w2": 0.3375511169433594, "blocks.1.feed_forward.w3": 0.48148345947265625, "blocks.2.attention.w_q": 0.5940666198730469, "blocks.2.attention.w_k": 0.5679512023925781, "blocks.2.attention.w_v": 0.5027694702148438, "blocks.2.attention.w_out": 0.5684318542480469, "blocks.2.feed_forward.w1": 0.5297648310661316, "blocks.2.feed_forward.w2": 0.3618431091308594, "blocks.2.feed_forward.w3": 0.5272229909896851, "blocks.3.attention.w_q": 0.6220626831054688, "blocks.3.attention.w_k": 0.5896034240722656, "blocks.3.attention.w_v": 0.4665565490722656, "blocks.3.attention.w_out": 0.5517807006835938, "blocks.3.feed_forward.w1": 0.5311101675033569, "blocks.3.feed_forward.w2": 0.36933135986328125, "blocks.3.feed_forward.w3": 0.5272763967514038, "blocks.4.attention.w_q": 0.6196937561035156, "blocks.4.attention.w_k": 0.5726737976074219, "blocks.4.attention.w_v": 0.4417724609375, "blocks.4.attention.w_out": 0.4912681579589844, "blocks.4.feed_forward.w1": 0.5686658620834351, "blocks.4.feed_forward.w2": 0.3978004455566406, "blocks.4.feed_forward.w3": 0.5624593496322632, "blocks.5.attention.w_q": 0.6209220886230469, "blocks.5.attention.w_k": 0.5668296813964844, "blocks.5.attention.w_v": 0.433502197265625, "blocks.5.attention.w_out": 0.4996452331542969, "blocks.5.feed_forward.w1": 0.5804736018180847, "blocks.5.feed_forward.w2": 0.36164984107017517, "blocks.5.feed_forward.w3": 0.5754216909408569, "blocks.6.attention.w_q": 0.6118621826171875, "blocks.6.attention.w_k": 0.5630645751953125, "blocks.6.attention.w_v": 0.4462242126464844, "blocks.6.attention.w_out": 0.4800834655761719, "blocks.6.feed_forward.w1": 0.5874583125114441, "blocks.6.feed_forward.w2": 0.3575859069824219, "blocks.6.feed_forward.w3": 0.5817451477050781, "blocks.7.attention.w_q": 0.5473136901855469, "blocks.7.attention.w_k": 0.5143699645996094, "blocks.7.attention.w_v": 0.4586639404296875, "blocks.7.attention.w_out": 0.41490936279296875, "blocks.7.feed_forward.w1": 0.5845121145248413, "blocks.7.feed_forward.w2": 0.3934071958065033, "blocks.7.feed_forward.w3": 0.5742683410644531, "blocks.8.attention.w_q": 0.5482635498046875, "blocks.8.attention.w_k": 0.5134010314941406, "blocks.8.attention.w_v": 0.4639434814453125, "blocks.8.attention.w_out": 0.40496826171875, "blocks.8.feed_forward.w1": 0.5835101008415222, "blocks.8.feed_forward.w2": 0.3587226867675781, "blocks.8.feed_forward.w3": 0.5721702575683594}, "w_mask_frac": 0.160864612676649} +{"step": 4400, "changed_frac": 0.4796915624895666, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5482139587402344, "blocks.0.attention.w_k": 0.569915771484375, "blocks.0.attention.w_v": 0.5534477233886719, "blocks.0.attention.w_out": 0.4771881103515625, "blocks.0.feed_forward.w1": 0.38244757056236267, "blocks.0.feed_forward.w2": 0.26637396216392517, "blocks.0.feed_forward.w3": 0.3899345397949219, "blocks.1.attention.w_q": 0.58697509765625, "blocks.1.attention.w_k": 0.5792503356933594, "blocks.1.attention.w_v": 0.5064582824707031, "blocks.1.attention.w_out": 0.5706329345703125, "blocks.1.feed_forward.w1": 0.47308477759361267, "blocks.1.feed_forward.w2": 0.32697296142578125, "blocks.1.feed_forward.w3": 0.47271856665611267, "blocks.2.attention.w_q": 0.5776252746582031, "blocks.2.attention.w_k": 0.5553359985351562, "blocks.2.attention.w_v": 0.4774932861328125, "blocks.2.attention.w_out": 0.555877685546875, "blocks.2.feed_forward.w1": 0.5200182795524597, "blocks.2.feed_forward.w2": 0.3485603332519531, "blocks.2.feed_forward.w3": 0.5166435241699219, "blocks.3.attention.w_q": 0.6041221618652344, "blocks.3.attention.w_k": 0.5683631896972656, "blocks.3.attention.w_v": 0.4326362609863281, "blocks.3.attention.w_out": 0.5310478210449219, "blocks.3.feed_forward.w1": 0.5184402465820312, "blocks.3.feed_forward.w2": 0.35856884717941284, "blocks.3.feed_forward.w3": 0.5162391662597656, "blocks.4.attention.w_q": 0.6070022583007812, "blocks.4.attention.w_k": 0.5647697448730469, "blocks.4.attention.w_v": 0.4097480773925781, "blocks.4.attention.w_out": 0.4737129211425781, "blocks.4.feed_forward.w1": 0.5597457885742188, "blocks.4.feed_forward.w2": 0.3850046992301941, "blocks.4.feed_forward.w3": 0.5535303950309753, "blocks.5.attention.w_q": 0.6065902709960938, "blocks.5.attention.w_k": 0.5565299987792969, "blocks.5.attention.w_v": 0.4037895202636719, "blocks.5.attention.w_out": 0.4784088134765625, "blocks.5.feed_forward.w1": 0.5730502009391785, "blocks.5.feed_forward.w2": 0.34558361768722534, "blocks.5.feed_forward.w3": 0.5670013427734375, "blocks.6.attention.w_q": 0.5991783142089844, "blocks.6.attention.w_k": 0.5400428771972656, "blocks.6.attention.w_v": 0.4173126220703125, "blocks.6.attention.w_out": 0.4566154479980469, "blocks.6.feed_forward.w1": 0.5764173269271851, "blocks.6.feed_forward.w2": 0.33657073974609375, "blocks.6.feed_forward.w3": 0.5665563344955444, "blocks.7.attention.w_q": 0.5287361145019531, "blocks.7.attention.w_k": 0.495025634765625, "blocks.7.attention.w_v": 0.4347724914550781, "blocks.7.attention.w_out": 0.3847312927246094, "blocks.7.feed_forward.w1": 0.5741310119628906, "blocks.7.feed_forward.w2": 0.3785285949707031, "blocks.7.feed_forward.w3": 0.5624160766601562, "blocks.8.attention.w_q": 0.5413894653320312, "blocks.8.attention.w_k": 0.5044326782226562, "blocks.8.attention.w_v": 0.4457664489746094, "blocks.8.attention.w_out": 0.3760986328125, "blocks.8.feed_forward.w1": 0.5684674978256226, "blocks.8.feed_forward.w2": 0.34013494849205017, "blocks.8.feed_forward.w3": 0.5577507019042969}, "w_mask_frac": 0.1608410391670007} +{"step": 4500, "changed_frac": 0.4682562412359776, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.536285400390625, "blocks.0.attention.w_k": 0.5591850280761719, "blocks.0.attention.w_v": 0.5449256896972656, "blocks.0.attention.w_out": 0.46305084228515625, "blocks.0.feed_forward.w1": 0.3684234619140625, "blocks.0.feed_forward.w2": 0.25993603467941284, "blocks.0.feed_forward.w3": 0.379913330078125, "blocks.1.attention.w_q": 0.5742378234863281, "blocks.1.attention.w_k": 0.565093994140625, "blocks.1.attention.w_v": 0.4929389953613281, "blocks.1.attention.w_out": 0.5630416870117188, "blocks.1.feed_forward.w1": 0.4620564877986908, "blocks.1.feed_forward.w2": 0.31852468848228455, "blocks.1.feed_forward.w3": 0.46235403418540955, "blocks.2.attention.w_q": 0.5729560852050781, "blocks.2.attention.w_k": 0.5443878173828125, "blocks.2.attention.w_v": 0.4625740051269531, "blocks.2.attention.w_out": 0.5461883544921875, "blocks.2.feed_forward.w1": 0.5080134272575378, "blocks.2.feed_forward.w2": 0.3387413024902344, "blocks.2.feed_forward.w3": 0.5045980215072632, "blocks.3.attention.w_q": 0.5933799743652344, "blocks.3.attention.w_k": 0.55584716796875, "blocks.3.attention.w_v": 0.4094352722167969, "blocks.3.attention.w_out": 0.5140533447265625, "blocks.3.feed_forward.w1": 0.5076065063476562, "blocks.3.feed_forward.w2": 0.34895962476730347, "blocks.3.feed_forward.w3": 0.5071576833724976, "blocks.4.attention.w_q": 0.5996246337890625, "blocks.4.attention.w_k": 0.5517082214355469, "blocks.4.attention.w_v": 0.38574981689453125, "blocks.4.attention.w_out": 0.4577445983886719, "blocks.4.feed_forward.w1": 0.5472068786621094, "blocks.4.feed_forward.w2": 0.37567394971847534, "blocks.4.feed_forward.w3": 0.5400060415267944, "blocks.5.attention.w_q": 0.5919418334960938, "blocks.5.attention.w_k": 0.5404548645019531, "blocks.5.attention.w_v": 0.387725830078125, "blocks.5.attention.w_out": 0.4613380432128906, "blocks.5.feed_forward.w1": 0.5587107539176941, "blocks.5.feed_forward.w2": 0.33894985914230347, "blocks.5.feed_forward.w3": 0.5528755187988281, "blocks.6.attention.w_q": 0.5819740295410156, "blocks.6.attention.w_k": 0.5236854553222656, "blocks.6.attention.w_v": 0.3952140808105469, "blocks.6.attention.w_out": 0.4420623779296875, "blocks.6.feed_forward.w1": 0.56622314453125, "blocks.6.feed_forward.w2": 0.32880911231040955, "blocks.6.feed_forward.w3": 0.5537948608398438, "blocks.7.attention.w_q": 0.51348876953125, "blocks.7.attention.w_k": 0.48963165283203125, "blocks.7.attention.w_v": 0.4121551513671875, "blocks.7.attention.w_out": 0.37091827392578125, "blocks.7.feed_forward.w1": 0.5675519704818726, "blocks.7.feed_forward.w2": 0.36430105566978455, "blocks.7.feed_forward.w3": 0.5558828115463257, "blocks.8.attention.w_q": 0.5190658569335938, "blocks.8.attention.w_k": 0.4934654235839844, "blocks.8.attention.w_v": 0.4231681823730469, "blocks.8.attention.w_out": 0.36631011962890625, "blocks.8.feed_forward.w1": 0.5633837580680847, "blocks.8.feed_forward.w2": 0.3287709653377533, "blocks.8.feed_forward.w3": 0.5518989562988281}, "w_mask_frac": 0.16644691809629783} +{"step": 4600, "changed_frac": 0.4563578744219918, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5142135620117188, "blocks.0.attention.w_k": 0.5432014465332031, "blocks.0.attention.w_v": 0.5265579223632812, "blocks.0.attention.w_out": 0.44891357421875, "blocks.0.feed_forward.w1": 0.35942333936691284, "blocks.0.feed_forward.w2": 0.2538096308708191, "blocks.0.feed_forward.w3": 0.3690999448299408, "blocks.1.attention.w_q": 0.5581092834472656, "blocks.1.attention.w_k": 0.5445022583007812, "blocks.1.attention.w_v": 0.4752235412597656, "blocks.1.attention.w_out": 0.5467605590820312, "blocks.1.feed_forward.w1": 0.44707998633384705, "blocks.1.feed_forward.w2": 0.3085416257381439, "blocks.1.feed_forward.w3": 0.44806545972824097, "blocks.2.attention.w_q": 0.5426750183105469, "blocks.2.attention.w_k": 0.5146942138671875, "blocks.2.attention.w_v": 0.4425926208496094, "blocks.2.attention.w_out": 0.528076171875, "blocks.2.feed_forward.w1": 0.49349087476730347, "blocks.2.feed_forward.w2": 0.3302040100097656, "blocks.2.feed_forward.w3": 0.4900004267692566, "blocks.3.attention.w_q": 0.5734329223632812, "blocks.3.attention.w_k": 0.5382347106933594, "blocks.3.attention.w_v": 0.3946571350097656, "blocks.3.attention.w_out": 0.50201416015625, "blocks.3.feed_forward.w1": 0.493896484375, "blocks.3.feed_forward.w2": 0.33792877197265625, "blocks.3.feed_forward.w3": 0.49206671118736267, "blocks.4.attention.w_q": 0.578369140625, "blocks.4.attention.w_k": 0.5343055725097656, "blocks.4.attention.w_v": 0.3666725158691406, "blocks.4.attention.w_out": 0.4408531188964844, "blocks.4.feed_forward.w1": 0.5366338491439819, "blocks.4.feed_forward.w2": 0.3675575256347656, "blocks.4.feed_forward.w3": 0.5294622182846069, "blocks.5.attention.w_q": 0.5743980407714844, "blocks.5.attention.w_k": 0.5277824401855469, "blocks.5.attention.w_v": 0.3723487854003906, "blocks.5.attention.w_out": 0.4477043151855469, "blocks.5.feed_forward.w1": 0.5478134155273438, "blocks.5.feed_forward.w2": 0.3396085202693939, "blocks.5.feed_forward.w3": 0.5411847829818726, "blocks.6.attention.w_q": 0.5678062438964844, "blocks.6.attention.w_k": 0.5141181945800781, "blocks.6.attention.w_v": 0.38246917724609375, "blocks.6.attention.w_out": 0.42778778076171875, "blocks.6.feed_forward.w1": 0.5543861389160156, "blocks.6.feed_forward.w2": 0.33952587842941284, "blocks.6.feed_forward.w3": 0.5442746877670288, "blocks.7.attention.w_q": 0.49994659423828125, "blocks.7.attention.w_k": 0.47995758056640625, "blocks.7.attention.w_v": 0.4076118469238281, "blocks.7.attention.w_out": 0.35848236083984375, "blocks.7.feed_forward.w1": 0.5564143061637878, "blocks.7.feed_forward.w2": 0.3554624021053314, "blocks.7.feed_forward.w3": 0.5444793701171875, "blocks.8.attention.w_q": 0.5003089904785156, "blocks.8.attention.w_k": 0.4678955078125, "blocks.8.attention.w_v": 0.408538818359375, "blocks.8.attention.w_out": 0.35369110107421875, "blocks.8.feed_forward.w1": 0.5539602041244507, "blocks.8.feed_forward.w2": 0.3209342956542969, "blocks.8.feed_forward.w3": 0.5410169363021851}, "w_mask_frac": 0.16605452448129654} +{"step": 4700, "changed_frac": 0.44371922810872394, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4980316162109375, "blocks.0.attention.w_k": 0.5276756286621094, "blocks.0.attention.w_v": 0.5126419067382812, "blocks.0.attention.w_out": 0.4305267333984375, "blocks.0.feed_forward.w1": 0.34781646728515625, "blocks.0.feed_forward.w2": 0.24712245166301727, "blocks.0.feed_forward.w3": 0.3593495786190033, "blocks.1.attention.w_q": 0.5410919189453125, "blocks.1.attention.w_k": 0.5357589721679688, "blocks.1.attention.w_v": 0.4579811096191406, "blocks.1.attention.w_out": 0.5350112915039062, "blocks.1.feed_forward.w1": 0.4374491572380066, "blocks.1.feed_forward.w2": 0.30254873633384705, "blocks.1.feed_forward.w3": 0.4365946650505066, "blocks.2.attention.w_q": 0.5297660827636719, "blocks.2.attention.w_k": 0.5060615539550781, "blocks.2.attention.w_v": 0.42357635498046875, "blocks.2.attention.w_out": 0.5151329040527344, "blocks.2.feed_forward.w1": 0.4842783808708191, "blocks.2.feed_forward.w2": 0.31912359595298767, "blocks.2.feed_forward.w3": 0.48193106055259705, "blocks.3.attention.w_q": 0.5572433471679688, "blocks.3.attention.w_k": 0.5272560119628906, "blocks.3.attention.w_v": 0.3796348571777344, "blocks.3.attention.w_out": 0.4875030517578125, "blocks.3.feed_forward.w1": 0.48544567823410034, "blocks.3.feed_forward.w2": 0.3290151059627533, "blocks.3.feed_forward.w3": 0.48081207275390625, "blocks.4.attention.w_q": 0.5572662353515625, "blocks.4.attention.w_k": 0.5137062072753906, "blocks.4.attention.w_v": 0.3487548828125, "blocks.4.attention.w_out": 0.4250221252441406, "blocks.4.feed_forward.w1": 0.5281600952148438, "blocks.4.feed_forward.w2": 0.3558553159236908, "blocks.4.feed_forward.w3": 0.5204760432243347, "blocks.5.attention.w_q": 0.5589065551757812, "blocks.5.attention.w_k": 0.5065460205078125, "blocks.5.attention.w_v": 0.34769439697265625, "blocks.5.attention.w_out": 0.4319305419921875, "blocks.5.feed_forward.w1": 0.5357145071029663, "blocks.5.feed_forward.w2": 0.3285001218318939, "blocks.5.feed_forward.w3": 0.5277125239372253, "blocks.6.attention.w_q": 0.5476226806640625, "blocks.6.attention.w_k": 0.49549102783203125, "blocks.6.attention.w_v": 0.3564949035644531, "blocks.6.attention.w_out": 0.4118232727050781, "blocks.6.feed_forward.w1": 0.5469030141830444, "blocks.6.feed_forward.w2": 0.33624395728111267, "blocks.6.feed_forward.w3": 0.5389569997787476, "blocks.7.attention.w_q": 0.4895057678222656, "blocks.7.attention.w_k": 0.4624443054199219, "blocks.7.attention.w_v": 0.3704795837402344, "blocks.7.attention.w_out": 0.3443260192871094, "blocks.7.feed_forward.w1": 0.5424474477767944, "blocks.7.feed_forward.w2": 0.340545654296875, "blocks.7.feed_forward.w3": 0.5290718078613281, "blocks.8.attention.w_q": 0.4843330383300781, "blocks.8.attention.w_k": 0.45728302001953125, "blocks.8.attention.w_v": 0.3697547912597656, "blocks.8.attention.w_out": 0.33255767822265625, "blocks.8.feed_forward.w1": 0.5377604365348816, "blocks.8.feed_forward.w2": 0.307373046875, "blocks.8.feed_forward.w3": 0.5255635976791382}, "w_mask_frac": 0.16768405248976162} +{"step": 4800, "changed_frac": 0.4310088198409121, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4912300109863281, "blocks.0.attention.w_k": 0.5210113525390625, "blocks.0.attention.w_v": 0.5009574890136719, "blocks.0.attention.w_out": 0.4213829040527344, "blocks.0.feed_forward.w1": 0.3416341245174408, "blocks.0.feed_forward.w2": 0.24254608154296875, "blocks.0.feed_forward.w3": 0.3529459834098816, "blocks.1.attention.w_q": 0.5298004150390625, "blocks.1.attention.w_k": 0.5224800109863281, "blocks.1.attention.w_v": 0.4410057067871094, "blocks.1.attention.w_out": 0.5216484069824219, "blocks.1.feed_forward.w1": 0.42794927954673767, "blocks.1.feed_forward.w2": 0.29400762915611267, "blocks.1.feed_forward.w3": 0.4271634519100189, "blocks.2.attention.w_q": 0.5185012817382812, "blocks.2.attention.w_k": 0.49044036865234375, "blocks.2.attention.w_v": 0.4059715270996094, "blocks.2.attention.w_out": 0.497406005859375, "blocks.2.feed_forward.w1": 0.4741503596305847, "blocks.2.feed_forward.w2": 0.31060028076171875, "blocks.2.feed_forward.w3": 0.47111639380455017, "blocks.3.attention.w_q": 0.5474624633789062, "blocks.3.attention.w_k": 0.5192718505859375, "blocks.3.attention.w_v": 0.3553428649902344, "blocks.3.attention.w_out": 0.4693870544433594, "blocks.3.feed_forward.w1": 0.47200649976730347, "blocks.3.feed_forward.w2": 0.32035955786705017, "blocks.3.feed_forward.w3": 0.46929678320884705, "blocks.4.attention.w_q": 0.5512924194335938, "blocks.4.attention.w_k": 0.4997825622558594, "blocks.4.attention.w_v": 0.3318672180175781, "blocks.4.attention.w_out": 0.4054603576660156, "blocks.4.feed_forward.w1": 0.5181350708007812, "blocks.4.feed_forward.w2": 0.34602993726730347, "blocks.4.feed_forward.w3": 0.5086301565170288, "blocks.5.attention.w_q": 0.5462188720703125, "blocks.5.attention.w_k": 0.494140625, "blocks.5.attention.w_v": 0.3249664306640625, "blocks.5.attention.w_out": 0.41127777099609375, "blocks.5.feed_forward.w1": 0.5229644775390625, "blocks.5.feed_forward.w2": 0.3163299560546875, "blocks.5.feed_forward.w3": 0.5169703364372253, "blocks.6.attention.w_q": 0.5399894714355469, "blocks.6.attention.w_k": 0.4865875244140625, "blocks.6.attention.w_v": 0.33860015869140625, "blocks.6.attention.w_out": 0.39373016357421875, "blocks.6.feed_forward.w1": 0.5316938161849976, "blocks.6.feed_forward.w2": 0.3167381286621094, "blocks.6.feed_forward.w3": 0.5247688293457031, "blocks.7.attention.w_q": 0.4625434875488281, "blocks.7.attention.w_k": 0.4318351745605469, "blocks.7.attention.w_v": 0.3518867492675781, "blocks.7.attention.w_out": 0.3246040344238281, "blocks.7.feed_forward.w1": 0.5284246206283569, "blocks.7.feed_forward.w2": 0.32553738355636597, "blocks.7.feed_forward.w3": 0.5154609680175781, "blocks.8.attention.w_q": 0.45761871337890625, "blocks.8.attention.w_k": 0.4382820129394531, "blocks.8.attention.w_v": 0.3562812805175781, "blocks.8.attention.w_out": 0.3145751953125, "blocks.8.feed_forward.w1": 0.5242729187011719, "blocks.8.feed_forward.w2": 0.29339599609375, "blocks.8.feed_forward.w3": 0.5112686157226562}, "w_mask_frac": 0.17343697493147647} +{"step": 4900, "changed_frac": 0.41781912909613717, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4687919616699219, "blocks.0.attention.w_k": 0.5032882690429688, "blocks.0.attention.w_v": 0.480560302734375, "blocks.0.attention.w_out": 0.4033355712890625, "blocks.0.feed_forward.w1": 0.33130520582199097, "blocks.0.feed_forward.w2": 0.23133596777915955, "blocks.0.feed_forward.w3": 0.34015148878097534, "blocks.1.attention.w_q": 0.5127944946289062, "blocks.1.attention.w_k": 0.5075569152832031, "blocks.1.attention.w_v": 0.4198455810546875, "blocks.1.attention.w_out": 0.5037956237792969, "blocks.1.feed_forward.w1": 0.41641998291015625, "blocks.1.feed_forward.w2": 0.28265380859375, "blocks.1.feed_forward.w3": 0.4153544306755066, "blocks.2.attention.w_q": 0.5100021362304688, "blocks.2.attention.w_k": 0.4848480224609375, "blocks.2.attention.w_v": 0.3812217712402344, "blocks.2.attention.w_out": 0.4825706481933594, "blocks.2.feed_forward.w1": 0.46528372168540955, "blocks.2.feed_forward.w2": 0.2999725341796875, "blocks.2.feed_forward.w3": 0.4599418640136719, "blocks.3.attention.w_q": 0.5366020202636719, "blocks.3.attention.w_k": 0.5024185180664062, "blocks.3.attention.w_v": 0.3300018310546875, "blocks.3.attention.w_out": 0.4473991394042969, "blocks.3.feed_forward.w1": 0.46137112379074097, "blocks.3.feed_forward.w2": 0.3069203794002533, "blocks.3.feed_forward.w3": 0.45699819922447205, "blocks.4.attention.w_q": 0.538055419921875, "blocks.4.attention.w_k": 0.4835777282714844, "blocks.4.attention.w_v": 0.2983131408691406, "blocks.4.attention.w_out": 0.38196563720703125, "blocks.4.feed_forward.w1": 0.5081723928451538, "blocks.4.feed_forward.w2": 0.3344459533691406, "blocks.4.feed_forward.w3": 0.5028038024902344, "blocks.5.attention.w_q": 0.5347785949707031, "blocks.5.attention.w_k": 0.4834480285644531, "blocks.5.attention.w_v": 0.3030891418457031, "blocks.5.attention.w_out": 0.39263153076171875, "blocks.5.feed_forward.w1": 0.5174140930175781, "blocks.5.feed_forward.w2": 0.2998288571834564, "blocks.5.feed_forward.w3": 0.5090154409408569, "blocks.6.attention.w_q": 0.5280418395996094, "blocks.6.attention.w_k": 0.4682273864746094, "blocks.6.attention.w_v": 0.323516845703125, "blocks.6.attention.w_out": 0.3765716552734375, "blocks.6.feed_forward.w1": 0.5244369506835938, "blocks.6.feed_forward.w2": 0.2942797541618347, "blocks.6.feed_forward.w3": 0.5151087641716003, "blocks.7.attention.w_q": 0.4618034362792969, "blocks.7.attention.w_k": 0.4266014099121094, "blocks.7.attention.w_v": 0.3304939270019531, "blocks.7.attention.w_out": 0.3144035339355469, "blocks.7.feed_forward.w1": 0.5136973261833191, "blocks.7.feed_forward.w2": 0.31279247999191284, "blocks.7.feed_forward.w3": 0.5021222829818726, "blocks.8.attention.w_q": 0.451690673828125, "blocks.8.attention.w_k": 0.4182777404785156, "blocks.8.attention.w_v": 0.33856964111328125, "blocks.8.attention.w_out": 0.2981719970703125, "blocks.8.feed_forward.w1": 0.5082944631576538, "blocks.8.feed_forward.w2": 0.2789421081542969, "blocks.8.feed_forward.w3": 0.496795654296875}, "w_mask_frac": 0.1680733145047457} +{"step": 5000, "changed_frac": 0.40102503849909854, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.44568634033203125, "blocks.0.attention.w_k": 0.48482513427734375, "blocks.0.attention.w_v": 0.4631309509277344, "blocks.0.attention.w_out": 0.3820343017578125, "blocks.0.feed_forward.w1": 0.3169148862361908, "blocks.0.feed_forward.w2": 0.21949386596679688, "blocks.0.feed_forward.w3": 0.32671231031417847, "blocks.1.attention.w_q": 0.4940605163574219, "blocks.1.attention.w_k": 0.48354339599609375, "blocks.1.attention.w_v": 0.3977813720703125, "blocks.1.attention.w_out": 0.4899444580078125, "blocks.1.feed_forward.w1": 0.4005076289176941, "blocks.1.feed_forward.w2": 0.2672678828239441, "blocks.1.feed_forward.w3": 0.40078991651535034, "blocks.2.attention.w_q": 0.48802947998046875, "blocks.2.attention.w_k": 0.4602851867675781, "blocks.2.attention.w_v": 0.3577995300292969, "blocks.2.attention.w_out": 0.4640464782714844, "blocks.2.feed_forward.w1": 0.44755810499191284, "blocks.2.feed_forward.w2": 0.2839902341365814, "blocks.2.feed_forward.w3": 0.4439226984977722, "blocks.3.attention.w_q": 0.512786865234375, "blocks.3.attention.w_k": 0.475250244140625, "blocks.3.attention.w_v": 0.3022193908691406, "blocks.3.attention.w_out": 0.4278678894042969, "blocks.3.feed_forward.w1": 0.4467887878417969, "blocks.3.feed_forward.w2": 0.29580435156822205, "blocks.3.feed_forward.w3": 0.44461822509765625, "blocks.4.attention.w_q": 0.5212326049804688, "blocks.4.attention.w_k": 0.461273193359375, "blocks.4.attention.w_v": 0.27004241943359375, "blocks.4.attention.w_out": 0.36185455322265625, "blocks.4.feed_forward.w1": 0.4880409240722656, "blocks.4.feed_forward.w2": 0.31932196021080017, "blocks.4.feed_forward.w3": 0.4833030700683594, "blocks.5.attention.w_q": 0.5086555480957031, "blocks.5.attention.w_k": 0.45616912841796875, "blocks.5.attention.w_v": 0.2777862548828125, "blocks.5.attention.w_out": 0.3692512512207031, "blocks.5.feed_forward.w1": 0.4958127439022064, "blocks.5.feed_forward.w2": 0.2936147153377533, "blocks.5.feed_forward.w3": 0.4896494746208191, "blocks.6.attention.w_q": 0.5024604797363281, "blocks.6.attention.w_k": 0.44583892822265625, "blocks.6.attention.w_v": 0.29496002197265625, "blocks.6.attention.w_out": 0.3563957214355469, "blocks.6.feed_forward.w1": 0.506750762462616, "blocks.6.feed_forward.w2": 0.29950588941574097, "blocks.6.feed_forward.w3": 0.5000495910644531, "blocks.7.attention.w_q": 0.4488334655761719, "blocks.7.attention.w_k": 0.41994476318359375, "blocks.7.attention.w_v": 0.3077507019042969, "blocks.7.attention.w_out": 0.2957000732421875, "blocks.7.feed_forward.w1": 0.49771374464035034, "blocks.7.feed_forward.w2": 0.2923876643180847, "blocks.7.feed_forward.w3": 0.4852447509765625, "blocks.8.attention.w_q": 0.43798828125, "blocks.8.attention.w_k": 0.4028205871582031, "blocks.8.attention.w_v": 0.32596588134765625, "blocks.8.attention.w_out": 0.2849845886230469, "blocks.8.feed_forward.w1": 0.49103420972824097, "blocks.8.feed_forward.w2": 0.2664756774902344, "blocks.8.feed_forward.w3": 0.4769694209098816}, "w_mask_frac": 0.1645541747029011} +{"step": 5100, "changed_frac": 0.3820459414751102, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4290275573730469, "blocks.0.attention.w_k": 0.47320556640625, "blocks.0.attention.w_v": 0.4479484558105469, "blocks.0.attention.w_out": 0.36347198486328125, "blocks.0.feed_forward.w1": 0.3098093867301941, "blocks.0.feed_forward.w2": 0.20966212451457977, "blocks.0.feed_forward.w3": 0.3198331296443939, "blocks.1.attention.w_q": 0.4706153869628906, "blocks.1.attention.w_k": 0.4689292907714844, "blocks.1.attention.w_v": 0.3743629455566406, "blocks.1.attention.w_out": 0.4699363708496094, "blocks.1.feed_forward.w1": 0.3888041377067566, "blocks.1.feed_forward.w2": 0.2569872736930847, "blocks.1.feed_forward.w3": 0.38828277587890625, "blocks.2.attention.w_q": 0.46661376953125, "blocks.2.attention.w_k": 0.4427299499511719, "blocks.2.attention.w_v": 0.3351783752441406, "blocks.2.attention.w_out": 0.4454154968261719, "blocks.2.feed_forward.w1": 0.43471020460128784, "blocks.2.feed_forward.w2": 0.2700691223144531, "blocks.2.feed_forward.w3": 0.4286842346191406, "blocks.3.attention.w_q": 0.4917640686035156, "blocks.3.attention.w_k": 0.45394134521484375, "blocks.3.attention.w_v": 0.2763481140136719, "blocks.3.attention.w_out": 0.4070243835449219, "blocks.3.feed_forward.w1": 0.4313494563102722, "blocks.3.feed_forward.w2": 0.280670166015625, "blocks.3.feed_forward.w3": 0.42805609107017517, "blocks.4.attention.w_q": 0.496185302734375, "blocks.4.attention.w_k": 0.4467506408691406, "blocks.4.attention.w_v": 0.24953079223632812, "blocks.4.attention.w_out": 0.3364143371582031, "blocks.4.feed_forward.w1": 0.4702644348144531, "blocks.4.feed_forward.w2": 0.30204010009765625, "blocks.4.feed_forward.w3": 0.465789794921875, "blocks.5.attention.w_q": 0.486236572265625, "blocks.5.attention.w_k": 0.4276008605957031, "blocks.5.attention.w_v": 0.2503776550292969, "blocks.5.attention.w_out": 0.3447456359863281, "blocks.5.feed_forward.w1": 0.47622808814048767, "blocks.5.feed_forward.w2": 0.27261099219322205, "blocks.5.feed_forward.w3": 0.468658447265625, "blocks.6.attention.w_q": 0.48278045654296875, "blocks.6.attention.w_k": 0.43430328369140625, "blocks.6.attention.w_v": 0.2697486877441406, "blocks.6.attention.w_out": 0.328277587890625, "blocks.6.feed_forward.w1": 0.48535284399986267, "blocks.6.feed_forward.w2": 0.26421231031417847, "blocks.6.feed_forward.w3": 0.47632092237472534, "blocks.7.attention.w_q": 0.4115257263183594, "blocks.7.attention.w_k": 0.3857460021972656, "blocks.7.attention.w_v": 0.2853050231933594, "blocks.7.attention.w_out": 0.2658195495605469, "blocks.7.feed_forward.w1": 0.47884368896484375, "blocks.7.feed_forward.w2": 0.2762298583984375, "blocks.7.feed_forward.w3": 0.4650166928768158, "blocks.8.attention.w_q": 0.4040565490722656, "blocks.8.attention.w_k": 0.37172698974609375, "blocks.8.attention.w_v": 0.2873268127441406, "blocks.8.attention.w_out": 0.2556877136230469, "blocks.8.feed_forward.w1": 0.47084683179855347, "blocks.8.feed_forward.w2": 0.24455007910728455, "blocks.8.feed_forward.w3": 0.45702236890792847}, "w_mask_frac": 0.16752083650511554} +{"step": 5200, "changed_frac": 0.3691211602626703, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.41536712646484375, "blocks.0.attention.w_k": 0.4600944519042969, "blocks.0.attention.w_v": 0.4278411865234375, "blocks.0.attention.w_out": 0.3495597839355469, "blocks.0.feed_forward.w1": 0.30137890577316284, "blocks.0.feed_forward.w2": 0.20268504321575165, "blocks.0.feed_forward.w3": 0.312530517578125, "blocks.1.attention.w_q": 0.4517364501953125, "blocks.1.attention.w_k": 0.4560813903808594, "blocks.1.attention.w_v": 0.35923004150390625, "blocks.1.attention.w_out": 0.45941162109375, "blocks.1.feed_forward.w1": 0.37831372022628784, "blocks.1.feed_forward.w2": 0.2478790283203125, "blocks.1.feed_forward.w3": 0.37653860449790955, "blocks.2.attention.w_q": 0.447845458984375, "blocks.2.attention.w_k": 0.4318962097167969, "blocks.2.attention.w_v": 0.31494140625, "blocks.2.attention.w_out": 0.42972564697265625, "blocks.2.feed_forward.w1": 0.4197540283203125, "blocks.2.feed_forward.w2": 0.2588233947753906, "blocks.2.feed_forward.w3": 0.4161580502986908, "blocks.3.attention.w_q": 0.472320556640625, "blocks.3.attention.w_k": 0.4384918212890625, "blocks.3.attention.w_v": 0.2565803527832031, "blocks.3.attention.w_out": 0.3862113952636719, "blocks.3.feed_forward.w1": 0.4213002622127533, "blocks.3.feed_forward.w2": 0.2690175473690033, "blocks.3.feed_forward.w3": 0.41695404052734375, "blocks.4.attention.w_q": 0.4726448059082031, "blocks.4.attention.w_k": 0.4292144775390625, "blocks.4.attention.w_v": 0.23202133178710938, "blocks.4.attention.w_out": 0.3214073181152344, "blocks.4.feed_forward.w1": 0.4594612121582031, "blocks.4.feed_forward.w2": 0.28790283203125, "blocks.4.feed_forward.w3": 0.4531517028808594, "blocks.5.attention.w_q": 0.4598121643066406, "blocks.5.attention.w_k": 0.40895843505859375, "blocks.5.attention.w_v": 0.22739410400390625, "blocks.5.attention.w_out": 0.3271217346191406, "blocks.5.feed_forward.w1": 0.46715039014816284, "blocks.5.feed_forward.w2": 0.26513928174972534, "blocks.5.feed_forward.w3": 0.45937857031822205, "blocks.6.attention.w_q": 0.46506500244140625, "blocks.6.attention.w_k": 0.4106559753417969, "blocks.6.attention.w_v": 0.24682235717773438, "blocks.6.attention.w_out": 0.31633758544921875, "blocks.6.feed_forward.w1": 0.47272491455078125, "blocks.6.feed_forward.w2": 0.26324591040611267, "blocks.6.feed_forward.w3": 0.46691131591796875, "blocks.7.attention.w_q": 0.39777374267578125, "blocks.7.attention.w_k": 0.3680686950683594, "blocks.7.attention.w_v": 0.2616844177246094, "blocks.7.attention.w_out": 0.25611114501953125, "blocks.7.feed_forward.w1": 0.4640617370605469, "blocks.7.feed_forward.w2": 0.26392364501953125, "blocks.7.feed_forward.w3": 0.4510091245174408, "blocks.8.attention.w_q": 0.3943367004394531, "blocks.8.attention.w_k": 0.3522071838378906, "blocks.8.attention.w_v": 0.2588996887207031, "blocks.8.attention.w_out": 0.23941802978515625, "blocks.8.feed_forward.w1": 0.4563738703727722, "blocks.8.feed_forward.w2": 0.23378881812095642, "blocks.8.feed_forward.w3": 0.44240570068359375}, "w_mask_frac": 0.17005594692423812} +{"step": 5300, "changed_frac": 0.35567715636685365, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.39066314697265625, "blocks.0.attention.w_k": 0.4430694580078125, "blocks.0.attention.w_v": 0.41053009033203125, "blocks.0.attention.w_out": 0.3327293395996094, "blocks.0.feed_forward.w1": 0.29027432203292847, "blocks.0.feed_forward.w2": 0.1940460205078125, "blocks.0.feed_forward.w3": 0.30133819580078125, "blocks.1.attention.w_q": 0.4358673095703125, "blocks.1.attention.w_k": 0.44194793701171875, "blocks.1.attention.w_v": 0.3345794677734375, "blocks.1.attention.w_out": 0.4405403137207031, "blocks.1.feed_forward.w1": 0.365234375, "blocks.1.feed_forward.w2": 0.23910140991210938, "blocks.1.feed_forward.w3": 0.3636283874511719, "blocks.2.attention.w_q": 0.433074951171875, "blocks.2.attention.w_k": 0.419769287109375, "blocks.2.attention.w_v": 0.2889862060546875, "blocks.2.attention.w_out": 0.4126091003417969, "blocks.2.feed_forward.w1": 0.40904492139816284, "blocks.2.feed_forward.w2": 0.2498423308134079, "blocks.2.feed_forward.w3": 0.40568798780441284, "blocks.3.attention.w_q": 0.4550323486328125, "blocks.3.attention.w_k": 0.42243194580078125, "blocks.3.attention.w_v": 0.23787307739257812, "blocks.3.attention.w_out": 0.3684501647949219, "blocks.3.feed_forward.w1": 0.4074605405330658, "blocks.3.feed_forward.w2": 0.2577018737792969, "blocks.3.feed_forward.w3": 0.4036814570426941, "blocks.4.attention.w_q": 0.45922088623046875, "blocks.4.attention.w_k": 0.40639495849609375, "blocks.4.attention.w_v": 0.21609115600585938, "blocks.4.attention.w_out": 0.3006553649902344, "blocks.4.feed_forward.w1": 0.44422024488449097, "blocks.4.feed_forward.w2": 0.2781423032283783, "blocks.4.feed_forward.w3": 0.43736395239830017, "blocks.5.attention.w_q": 0.4530143737792969, "blocks.5.attention.w_k": 0.3999748229980469, "blocks.5.attention.w_v": 0.21877288818359375, "blocks.5.attention.w_out": 0.3126945495605469, "blocks.5.feed_forward.w1": 0.4464073181152344, "blocks.5.feed_forward.w2": 0.2562001645565033, "blocks.5.feed_forward.w3": 0.43728893995285034, "blocks.6.attention.w_q": 0.44966888427734375, "blocks.6.attention.w_k": 0.39307403564453125, "blocks.6.attention.w_v": 0.2424163818359375, "blocks.6.attention.w_out": 0.2993278503417969, "blocks.6.feed_forward.w1": 0.45362600684165955, "blocks.6.feed_forward.w2": 0.2501119077205658, "blocks.6.feed_forward.w3": 0.44615429639816284, "blocks.7.attention.w_q": 0.38727569580078125, "blocks.7.attention.w_k": 0.3585548400878906, "blocks.7.attention.w_v": 0.2589225769042969, "blocks.7.attention.w_out": 0.23944473266601562, "blocks.7.feed_forward.w1": 0.45036444067955017, "blocks.7.feed_forward.w2": 0.25133389234542847, "blocks.7.feed_forward.w3": 0.4364204406738281, "blocks.8.attention.w_q": 0.3977203369140625, "blocks.8.attention.w_k": 0.36658477783203125, "blocks.8.attention.w_v": 0.26712799072265625, "blocks.8.attention.w_out": 0.22702407836914062, "blocks.8.feed_forward.w1": 0.4425290524959564, "blocks.8.feed_forward.w2": 0.21996943652629852, "blocks.8.feed_forward.w3": 0.4268633723258972}, "w_mask_frac": 0.16524235216470864} +{"step": 5400, "changed_frac": 0.3400364411182893, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3721504211425781, "blocks.0.attention.w_k": 0.42378997802734375, "blocks.0.attention.w_v": 0.3932685852050781, "blocks.0.attention.w_out": 0.3146095275878906, "blocks.0.feed_forward.w1": 0.2832450866699219, "blocks.0.feed_forward.w2": 0.18328094482421875, "blocks.0.feed_forward.w3": 0.29340362548828125, "blocks.1.attention.w_q": 0.4165687561035156, "blocks.1.attention.w_k": 0.422393798828125, "blocks.1.attention.w_v": 0.3104400634765625, "blocks.1.attention.w_out": 0.4197044372558594, "blocks.1.feed_forward.w1": 0.3536008298397064, "blocks.1.feed_forward.w2": 0.2262471616268158, "blocks.1.feed_forward.w3": 0.3531430661678314, "blocks.2.attention.w_q": 0.4126777648925781, "blocks.2.attention.w_k": 0.3939094543457031, "blocks.2.attention.w_v": 0.26346588134765625, "blocks.2.attention.w_out": 0.3912200927734375, "blocks.2.feed_forward.w1": 0.3993428647518158, "blocks.2.feed_forward.w2": 0.2367350310087204, "blocks.2.feed_forward.w3": 0.3946927487850189, "blocks.3.attention.w_q": 0.43331146240234375, "blocks.3.attention.w_k": 0.4021568298339844, "blocks.3.attention.w_v": 0.21531295776367188, "blocks.3.attention.w_out": 0.34403228759765625, "blocks.3.feed_forward.w1": 0.3932940363883972, "blocks.3.feed_forward.w2": 0.2440999448299408, "blocks.3.feed_forward.w3": 0.3876190185546875, "blocks.4.attention.w_q": 0.4417381286621094, "blocks.4.attention.w_k": 0.3888587951660156, "blocks.4.attention.w_v": 0.191436767578125, "blocks.4.attention.w_out": 0.2773628234863281, "blocks.4.feed_forward.w1": 0.4307912290096283, "blocks.4.feed_forward.w2": 0.2633870542049408, "blocks.4.feed_forward.w3": 0.42523449659347534, "blocks.5.attention.w_q": 0.4234428405761719, "blocks.5.attention.w_k": 0.3734016418457031, "blocks.5.attention.w_v": 0.19800186157226562, "blocks.5.attention.w_out": 0.28662109375, "blocks.5.feed_forward.w1": 0.43454107642173767, "blocks.5.feed_forward.w2": 0.2315521240234375, "blocks.5.feed_forward.w3": 0.4280599057674408, "blocks.6.attention.w_q": 0.4296379089355469, "blocks.6.attention.w_k": 0.3733558654785156, "blocks.6.attention.w_v": 0.21324539184570312, "blocks.6.attention.w_out": 0.27362823486328125, "blocks.6.feed_forward.w1": 0.44384002685546875, "blocks.6.feed_forward.w2": 0.23300044238567352, "blocks.6.feed_forward.w3": 0.4401588439941406, "blocks.7.attention.w_q": 0.3601531982421875, "blocks.7.attention.w_k": 0.3339576721191406, "blocks.7.attention.w_v": 0.23298263549804688, "blocks.7.attention.w_out": 0.21706008911132812, "blocks.7.feed_forward.w1": 0.43709564208984375, "blocks.7.feed_forward.w2": 0.23729324340820312, "blocks.7.feed_forward.w3": 0.42556509375572205, "blocks.8.attention.w_q": 0.3575401306152344, "blocks.8.attention.w_k": 0.3383636474609375, "blocks.8.attention.w_v": 0.24217987060546875, "blocks.8.attention.w_out": 0.20629119873046875, "blocks.8.feed_forward.w1": 0.43011730909347534, "blocks.8.feed_forward.w2": 0.20805613696575165, "blocks.8.feed_forward.w3": 0.4146003723144531}, "w_mask_frac": 0.16807220615128166} +{"step": 5500, "changed_frac": 0.3289578429654113, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3581123352050781, "blocks.0.attention.w_k": 0.4134368896484375, "blocks.0.attention.w_v": 0.3847541809082031, "blocks.0.attention.w_out": 0.3028297424316406, "blocks.0.feed_forward.w1": 0.275115966796875, "blocks.0.feed_forward.w2": 0.1793263852596283, "blocks.0.feed_forward.w3": 0.28542202711105347, "blocks.1.attention.w_q": 0.396484375, "blocks.1.attention.w_k": 0.4093475341796875, "blocks.1.attention.w_v": 0.29340362548828125, "blocks.1.attention.w_out": 0.4094810485839844, "blocks.1.feed_forward.w1": 0.3469696044921875, "blocks.1.feed_forward.w2": 0.2172749936580658, "blocks.1.feed_forward.w3": 0.3460896909236908, "blocks.2.attention.w_q": 0.39887237548828125, "blocks.2.attention.w_k": 0.38726806640625, "blocks.2.attention.w_v": 0.24943923950195312, "blocks.2.attention.w_out": 0.3787956237792969, "blocks.2.feed_forward.w1": 0.3913256525993347, "blocks.2.feed_forward.w2": 0.2284647673368454, "blocks.2.feed_forward.w3": 0.3875173032283783, "blocks.3.attention.w_q": 0.4214134216308594, "blocks.3.attention.w_k": 0.3866233825683594, "blocks.3.attention.w_v": 0.20047378540039062, "blocks.3.attention.w_out": 0.3259391784667969, "blocks.3.feed_forward.w1": 0.3869819641113281, "blocks.3.feed_forward.w2": 0.23475265502929688, "blocks.3.feed_forward.w3": 0.38282012939453125, "blocks.4.attention.w_q": 0.42845916748046875, "blocks.4.attention.w_k": 0.3789405822753906, "blocks.4.attention.w_v": 0.18089675903320312, "blocks.4.attention.w_out": 0.2635040283203125, "blocks.4.feed_forward.w1": 0.4229266047477722, "blocks.4.feed_forward.w2": 0.2532399594783783, "blocks.4.feed_forward.w3": 0.41873806715011597, "blocks.5.attention.w_q": 0.4053535461425781, "blocks.5.attention.w_k": 0.3607978820800781, "blocks.5.attention.w_v": 0.1826629638671875, "blocks.5.attention.w_out": 0.27288055419921875, "blocks.5.feed_forward.w1": 0.4270591735839844, "blocks.5.feed_forward.w2": 0.22822444140911102, "blocks.5.feed_forward.w3": 0.4177500605583191, "blocks.6.attention.w_q": 0.4060478210449219, "blocks.6.attention.w_k": 0.34914398193359375, "blocks.6.attention.w_v": 0.19858169555664062, "blocks.6.attention.w_out": 0.26076507568359375, "blocks.6.feed_forward.w1": 0.4315579831600189, "blocks.6.feed_forward.w2": 0.22399012744426727, "blocks.6.feed_forward.w3": 0.4278348386287689, "blocks.7.attention.w_q": 0.3460197448730469, "blocks.7.attention.w_k": 0.3222618103027344, "blocks.7.attention.w_v": 0.215850830078125, "blocks.7.attention.w_out": 0.20641326904296875, "blocks.7.feed_forward.w1": 0.4242452085018158, "blocks.7.feed_forward.w2": 0.22358450293540955, "blocks.7.feed_forward.w3": 0.4115651547908783, "blocks.8.attention.w_q": 0.3349151611328125, "blocks.8.attention.w_k": 0.308135986328125, "blocks.8.attention.w_v": 0.216766357421875, "blocks.8.attention.w_out": 0.19728469848632812, "blocks.8.feed_forward.w1": 0.4129588007926941, "blocks.8.feed_forward.w2": 0.1940561980009079, "blocks.8.feed_forward.w3": 0.3987782895565033}, "w_mask_frac": 0.1713556177341021} +{"step": 5600, "changed_frac": 0.31619755019489515, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3388404846191406, "blocks.0.attention.w_k": 0.39836883544921875, "blocks.0.attention.w_v": 0.3661231994628906, "blocks.0.attention.w_out": 0.2853507995605469, "blocks.0.feed_forward.w1": 0.2656046748161316, "blocks.0.feed_forward.w2": 0.17122015357017517, "blocks.0.feed_forward.w3": 0.27679443359375, "blocks.1.attention.w_q": 0.37938690185546875, "blocks.1.attention.w_k": 0.3927040100097656, "blocks.1.attention.w_v": 0.2787933349609375, "blocks.1.attention.w_out": 0.3961677551269531, "blocks.1.feed_forward.w1": 0.33700817823410034, "blocks.1.feed_forward.w2": 0.21018092334270477, "blocks.1.feed_forward.w3": 0.3345845639705658, "blocks.2.attention.w_q": 0.38238525390625, "blocks.2.attention.w_k": 0.3633270263671875, "blocks.2.attention.w_v": 0.23181533813476562, "blocks.2.attention.w_out": 0.35988616943359375, "blocks.2.feed_forward.w1": 0.3804473876953125, "blocks.2.feed_forward.w2": 0.2196146696805954, "blocks.2.feed_forward.w3": 0.37611645460128784, "blocks.3.attention.w_q": 0.4039802551269531, "blocks.3.attention.w_k": 0.3652534484863281, "blocks.3.attention.w_v": 0.18629837036132812, "blocks.3.attention.w_out": 0.3046455383300781, "blocks.3.feed_forward.w1": 0.3730125427246094, "blocks.3.feed_forward.w2": 0.2236785888671875, "blocks.3.feed_forward.w3": 0.3694445490837097, "blocks.4.attention.w_q": 0.4116096496582031, "blocks.4.attention.w_k": 0.3676872253417969, "blocks.4.attention.w_v": 0.17016220092773438, "blocks.4.attention.w_out": 0.24799728393554688, "blocks.4.feed_forward.w1": 0.410797119140625, "blocks.4.feed_forward.w2": 0.244659423828125, "blocks.4.feed_forward.w3": 0.40696844458580017, "blocks.5.attention.w_q": 0.3945121765136719, "blocks.5.attention.w_k": 0.3457183837890625, "blocks.5.attention.w_v": 0.17295074462890625, "blocks.5.attention.w_out": 0.2548828125, "blocks.5.feed_forward.w1": 0.41489920020103455, "blocks.5.feed_forward.w2": 0.22130584716796875, "blocks.5.feed_forward.w3": 0.4073537290096283, "blocks.6.attention.w_q": 0.39066314697265625, "blocks.6.attention.w_k": 0.33931732177734375, "blocks.6.attention.w_v": 0.18721771240234375, "blocks.6.attention.w_out": 0.24544906616210938, "blocks.6.feed_forward.w1": 0.41646575927734375, "blocks.6.feed_forward.w2": 0.21358999609947205, "blocks.6.feed_forward.w3": 0.41053900122642517, "blocks.7.attention.w_q": 0.3394584655761719, "blocks.7.attention.w_k": 0.31156158447265625, "blocks.7.attention.w_v": 0.19625091552734375, "blocks.7.attention.w_out": 0.19503402709960938, "blocks.7.feed_forward.w1": 0.4073778986930847, "blocks.7.feed_forward.w2": 0.2115682065486908, "blocks.7.feed_forward.w3": 0.3972575068473816, "blocks.8.attention.w_q": 0.3199958801269531, "blocks.8.attention.w_k": 0.2961540222167969, "blocks.8.attention.w_v": 0.203826904296875, "blocks.8.attention.w_out": 0.18219757080078125, "blocks.8.feed_forward.w1": 0.39726004004478455, "blocks.8.feed_forward.w2": 0.18056869506835938, "blocks.8.feed_forward.w3": 0.3847287595272064}, "w_mask_frac": 0.1707861829135153} +{"step": 5700, "changed_frac": 0.30449236356295073, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3260307312011719, "blocks.0.attention.w_k": 0.3866844177246094, "blocks.0.attention.w_v": 0.3439140319824219, "blocks.0.attention.w_out": 0.2708282470703125, "blocks.0.feed_forward.w1": 0.2571207880973816, "blocks.0.feed_forward.w2": 0.16197840869426727, "blocks.0.feed_forward.w3": 0.26838430762290955, "blocks.1.attention.w_q": 0.35916900634765625, "blocks.1.attention.w_k": 0.3682746887207031, "blocks.1.attention.w_v": 0.25830078125, "blocks.1.attention.w_out": 0.3726768493652344, "blocks.1.feed_forward.w1": 0.3292350769042969, "blocks.1.feed_forward.w2": 0.19944381713867188, "blocks.1.feed_forward.w3": 0.3247884213924408, "blocks.2.attention.w_q": 0.36879730224609375, "blocks.2.attention.w_k": 0.3540763854980469, "blocks.2.attention.w_v": 0.21311569213867188, "blocks.2.attention.w_out": 0.3409538269042969, "blocks.2.feed_forward.w1": 0.3708648681640625, "blocks.2.feed_forward.w2": 0.20777639746665955, "blocks.2.feed_forward.w3": 0.36750030517578125, "blocks.3.attention.w_q": 0.3810577392578125, "blocks.3.attention.w_k": 0.3474464416503906, "blocks.3.attention.w_v": 0.1701812744140625, "blocks.3.attention.w_out": 0.28501129150390625, "blocks.3.feed_forward.w1": 0.36559805274009705, "blocks.3.feed_forward.w2": 0.21356201171875, "blocks.3.feed_forward.w3": 0.36226654052734375, "blocks.4.attention.w_q": 0.39846038818359375, "blocks.4.attention.w_k": 0.3545417785644531, "blocks.4.attention.w_v": 0.15102767944335938, "blocks.4.attention.w_out": 0.2280731201171875, "blocks.4.feed_forward.w1": 0.40595120191574097, "blocks.4.feed_forward.w2": 0.230072021484375, "blocks.4.feed_forward.w3": 0.4010416865348816, "blocks.5.attention.w_q": 0.38460540771484375, "blocks.5.attention.w_k": 0.3326416015625, "blocks.5.attention.w_v": 0.15648269653320312, "blocks.5.attention.w_out": 0.23724746704101562, "blocks.5.feed_forward.w1": 0.4076589047908783, "blocks.5.feed_forward.w2": 0.2081400603055954, "blocks.5.feed_forward.w3": 0.4002825617790222, "blocks.6.attention.w_q": 0.3828163146972656, "blocks.6.attention.w_k": 0.3302764892578125, "blocks.6.attention.w_v": 0.17411422729492188, "blocks.6.attention.w_out": 0.23028182983398438, "blocks.6.feed_forward.w1": 0.40815991163253784, "blocks.6.feed_forward.w2": 0.19904327392578125, "blocks.6.feed_forward.w3": 0.40163421630859375, "blocks.7.attention.w_q": 0.3191947937011719, "blocks.7.attention.w_k": 0.29468536376953125, "blocks.7.attention.w_v": 0.1834716796875, "blocks.7.attention.w_out": 0.181884765625, "blocks.7.feed_forward.w1": 0.39758428931236267, "blocks.7.feed_forward.w2": 0.19805781543254852, "blocks.7.feed_forward.w3": 0.3869260251522064, "blocks.8.attention.w_q": 0.30279541015625, "blocks.8.attention.w_k": 0.2801856994628906, "blocks.8.attention.w_v": 0.18667984008789062, "blocks.8.attention.w_out": 0.16769027709960938, "blocks.8.feed_forward.w1": 0.38659924268722534, "blocks.8.feed_forward.w2": 0.1688486784696579, "blocks.8.feed_forward.w3": 0.3721249997615814}, "w_mask_frac": 0.17066864535594597} +{"step": 5800, "changed_frac": 0.29282688890766895, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.31353759765625, "blocks.0.attention.w_k": 0.3770751953125, "blocks.0.attention.w_v": 0.3321495056152344, "blocks.0.attention.w_out": 0.25628662109375, "blocks.0.feed_forward.w1": 0.25143688917160034, "blocks.0.feed_forward.w2": 0.15636825561523438, "blocks.0.feed_forward.w3": 0.26267242431640625, "blocks.1.attention.w_q": 0.3436088562011719, "blocks.1.attention.w_k": 0.3566093444824219, "blocks.1.attention.w_v": 0.23827743530273438, "blocks.1.attention.w_out": 0.3623466491699219, "blocks.1.feed_forward.w1": 0.32021331787109375, "blocks.1.feed_forward.w2": 0.19161860644817352, "blocks.1.feed_forward.w3": 0.31682586669921875, "blocks.2.attention.w_q": 0.3519325256347656, "blocks.2.attention.w_k": 0.3406181335449219, "blocks.2.attention.w_v": 0.19296646118164062, "blocks.2.attention.w_out": 0.323272705078125, "blocks.2.feed_forward.w1": 0.36542510986328125, "blocks.2.feed_forward.w2": 0.20130793750286102, "blocks.2.feed_forward.w3": 0.360565185546875, "blocks.3.attention.w_q": 0.36946868896484375, "blocks.3.attention.w_k": 0.3438301086425781, "blocks.3.attention.w_v": 0.15551376342773438, "blocks.3.attention.w_out": 0.2659912109375, "blocks.3.feed_forward.w1": 0.3594970703125, "blocks.3.feed_forward.w2": 0.2042795866727829, "blocks.3.feed_forward.w3": 0.3556671142578125, "blocks.4.attention.w_q": 0.3706245422363281, "blocks.4.attention.w_k": 0.3341636657714844, "blocks.4.attention.w_v": 0.13355636596679688, "blocks.4.attention.w_out": 0.21012115478515625, "blocks.4.feed_forward.w1": 0.39477285742759705, "blocks.4.feed_forward.w2": 0.21844610571861267, "blocks.4.feed_forward.w3": 0.38835272192955017, "blocks.5.attention.w_q": 0.35089111328125, "blocks.5.attention.w_k": 0.3066062927246094, "blocks.5.attention.w_v": 0.13658523559570312, "blocks.5.attention.w_out": 0.21924591064453125, "blocks.5.feed_forward.w1": 0.3977763056755066, "blocks.5.feed_forward.w2": 0.19073233008384705, "blocks.5.feed_forward.w3": 0.38725408911705017, "blocks.6.attention.w_q": 0.3635597229003906, "blocks.6.attention.w_k": 0.3130683898925781, "blocks.6.attention.w_v": 0.1570587158203125, "blocks.6.attention.w_out": 0.21352386474609375, "blocks.6.feed_forward.w1": 0.3975919187068939, "blocks.6.feed_forward.w2": 0.18395106494426727, "blocks.6.feed_forward.w3": 0.3901329040527344, "blocks.7.attention.w_q": 0.2963752746582031, "blocks.7.attention.w_k": 0.2837181091308594, "blocks.7.attention.w_v": 0.16379928588867188, "blocks.7.attention.w_out": 0.16709518432617188, "blocks.7.feed_forward.w1": 0.39147695899009705, "blocks.7.feed_forward.w2": 0.18502554297447205, "blocks.7.feed_forward.w3": 0.37906140089035034, "blocks.8.attention.w_q": 0.2796363830566406, "blocks.8.attention.w_k": 0.2644920349121094, "blocks.8.attention.w_v": 0.172576904296875, "blocks.8.attention.w_out": 0.1533660888671875, "blocks.8.feed_forward.w1": 0.377288818359375, "blocks.8.feed_forward.w2": 0.15899023413658142, "blocks.8.feed_forward.w3": 0.362335205078125}, "w_mask_frac": 0.17023888860757536} +{"step": 5900, "changed_frac": 0.28442105676373863, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2911262512207031, "blocks.0.attention.w_k": 0.3609199523925781, "blocks.0.attention.w_v": 0.3231697082519531, "blocks.0.attention.w_out": 0.24297332763671875, "blocks.0.feed_forward.w1": 0.24326834082603455, "blocks.0.feed_forward.w2": 0.1501668393611908, "blocks.0.feed_forward.w3": 0.25760650634765625, "blocks.1.attention.w_q": 0.3292427062988281, "blocks.1.attention.w_k": 0.3428497314453125, "blocks.1.attention.w_v": 0.22402572631835938, "blocks.1.attention.w_out": 0.3529167175292969, "blocks.1.feed_forward.w1": 0.31220245361328125, "blocks.1.feed_forward.w2": 0.1848500669002533, "blocks.1.feed_forward.w3": 0.3092842102050781, "blocks.2.attention.w_q": 0.3270721435546875, "blocks.2.attention.w_k": 0.3180084228515625, "blocks.2.attention.w_v": 0.17368316650390625, "blocks.2.attention.w_out": 0.3088798522949219, "blocks.2.feed_forward.w1": 0.35958990454673767, "blocks.2.feed_forward.w2": 0.1932881772518158, "blocks.2.feed_forward.w3": 0.3527361750602722, "blocks.3.attention.w_q": 0.3517112731933594, "blocks.3.attention.w_k": 0.3281517028808594, "blocks.3.attention.w_v": 0.13345718383789062, "blocks.3.attention.w_out": 0.2497711181640625, "blocks.3.feed_forward.w1": 0.35178250074386597, "blocks.3.feed_forward.w2": 0.19867834448814392, "blocks.3.feed_forward.w3": 0.34939703345298767, "blocks.4.attention.w_q": 0.3547630310058594, "blocks.4.attention.w_k": 0.31772613525390625, "blocks.4.attention.w_v": 0.12158203125, "blocks.4.attention.w_out": 0.19646072387695312, "blocks.4.feed_forward.w1": 0.3888130187988281, "blocks.4.feed_forward.w2": 0.20998255908489227, "blocks.4.feed_forward.w3": 0.38401541113853455, "blocks.5.attention.w_q": 0.3476676940917969, "blocks.5.attention.w_k": 0.31275177001953125, "blocks.5.attention.w_v": 0.12844467163085938, "blocks.5.attention.w_out": 0.20710372924804688, "blocks.5.feed_forward.w1": 0.3922373652458191, "blocks.5.feed_forward.w2": 0.18414561450481415, "blocks.5.feed_forward.w3": 0.3822886347770691, "blocks.6.attention.w_q": 0.35588836669921875, "blocks.6.attention.w_k": 0.30558013916015625, "blocks.6.attention.w_v": 0.1419677734375, "blocks.6.attention.w_out": 0.20177841186523438, "blocks.6.feed_forward.w1": 0.39023590087890625, "blocks.6.feed_forward.w2": 0.17612330615520477, "blocks.6.feed_forward.w3": 0.38534674048423767, "blocks.7.attention.w_q": 0.30446624755859375, "blocks.7.attention.w_k": 0.2844085693359375, "blocks.7.attention.w_v": 0.15203475952148438, "blocks.7.attention.w_out": 0.15411376953125, "blocks.7.feed_forward.w1": 0.3853607177734375, "blocks.7.feed_forward.w2": 0.1757609099149704, "blocks.7.feed_forward.w3": 0.3704312741756439, "blocks.8.attention.w_q": 0.29669952392578125, "blocks.8.attention.w_k": 0.28316497802734375, "blocks.8.attention.w_v": 0.160400390625, "blocks.8.attention.w_out": 0.14015960693359375, "blocks.8.feed_forward.w1": 0.3648173213005066, "blocks.8.feed_forward.w2": 0.14767202734947205, "blocks.8.feed_forward.w3": 0.35063299536705017}, "w_mask_frac": 0.1690823516020408} +{"step": 6000, "changed_frac": 0.2766737652640057, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2793617248535156, "blocks.0.attention.w_k": 0.34642791748046875, "blocks.0.attention.w_v": 0.3101844787597656, "blocks.0.attention.w_out": 0.23114776611328125, "blocks.0.feed_forward.w1": 0.2401885986328125, "blocks.0.feed_forward.w2": 0.14333979785442352, "blocks.0.feed_forward.w3": 0.2520548701286316, "blocks.1.attention.w_q": 0.31490325927734375, "blocks.1.attention.w_k": 0.33238983154296875, "blocks.1.attention.w_v": 0.20679092407226562, "blocks.1.attention.w_out": 0.3336334228515625, "blocks.1.feed_forward.w1": 0.3072192072868347, "blocks.1.feed_forward.w2": 0.17628225684165955, "blocks.1.feed_forward.w3": 0.30291876196861267, "blocks.2.attention.w_q": 0.307830810546875, "blocks.2.attention.w_k": 0.3037261962890625, "blocks.2.attention.w_v": 0.154388427734375, "blocks.2.attention.w_out": 0.2929344177246094, "blocks.2.feed_forward.w1": 0.3530718684196472, "blocks.2.feed_forward.w2": 0.18552780151367188, "blocks.2.feed_forward.w3": 0.3459002375602722, "blocks.3.attention.w_q": 0.3352203369140625, "blocks.3.attention.w_k": 0.3169708251953125, "blocks.3.attention.w_v": 0.122283935546875, "blocks.3.attention.w_out": 0.2360687255859375, "blocks.3.feed_forward.w1": 0.35050201416015625, "blocks.3.feed_forward.w2": 0.19090017676353455, "blocks.3.feed_forward.w3": 0.3439725339412689, "blocks.4.attention.w_q": 0.3291511535644531, "blocks.4.attention.w_k": 0.3013725280761719, "blocks.4.attention.w_v": 0.10891342163085938, "blocks.4.attention.w_out": 0.18114852905273438, "blocks.4.feed_forward.w1": 0.386627197265625, "blocks.4.feed_forward.w2": 0.20413081347942352, "blocks.4.feed_forward.w3": 0.37766265869140625, "blocks.5.attention.w_q": 0.3374290466308594, "blocks.5.attention.w_k": 0.2946319580078125, "blocks.5.attention.w_v": 0.117950439453125, "blocks.5.attention.w_out": 0.1938629150390625, "blocks.5.feed_forward.w1": 0.39102429151535034, "blocks.5.feed_forward.w2": 0.18289566040039062, "blocks.5.feed_forward.w3": 0.3803914487361908, "blocks.6.attention.w_q": 0.3359375, "blocks.6.attention.w_k": 0.28896331787109375, "blocks.6.attention.w_v": 0.1318359375, "blocks.6.attention.w_out": 0.18642807006835938, "blocks.6.feed_forward.w1": 0.3888333737850189, "blocks.6.feed_forward.w2": 0.17113623023033142, "blocks.6.feed_forward.w3": 0.38350552320480347, "blocks.7.attention.w_q": 0.2870826721191406, "blocks.7.attention.w_k": 0.2762298583984375, "blocks.7.attention.w_v": 0.13970565795898438, "blocks.7.attention.w_out": 0.14260101318359375, "blocks.7.feed_forward.w1": 0.3790842890739441, "blocks.7.feed_forward.w2": 0.16631317138671875, "blocks.7.feed_forward.w3": 0.36517590284347534, "blocks.8.attention.w_q": 0.2897758483886719, "blocks.8.attention.w_k": 0.281707763671875, "blocks.8.attention.w_v": 0.14823532104492188, "blocks.8.attention.w_out": 0.13185882568359375, "blocks.8.feed_forward.w1": 0.3594093322753906, "blocks.8.feed_forward.w2": 0.13934962451457977, "blocks.8.feed_forward.w3": 0.34649786353111267}, "w_mask_frac": 0.17131825326344904} +{"step": 6100, "changed_frac": 0.2698616614708534, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2711906433105469, "blocks.0.attention.w_k": 0.3391227722167969, "blocks.0.attention.w_v": 0.30077362060546875, "blocks.0.attention.w_out": 0.22626876831054688, "blocks.0.feed_forward.w1": 0.2369791716337204, "blocks.0.feed_forward.w2": 0.14091619849205017, "blocks.0.feed_forward.w3": 0.24872589111328125, "blocks.1.attention.w_q": 0.302337646484375, "blocks.1.attention.w_k": 0.3252296447753906, "blocks.1.attention.w_v": 0.195556640625, "blocks.1.attention.w_out": 0.32866668701171875, "blocks.1.feed_forward.w1": 0.30206426978111267, "blocks.1.feed_forward.w2": 0.17333094775676727, "blocks.1.feed_forward.w3": 0.2983512878417969, "blocks.2.attention.w_q": 0.3005104064941406, "blocks.2.attention.w_k": 0.2895927429199219, "blocks.2.attention.w_v": 0.14529037475585938, "blocks.2.attention.w_out": 0.2835731506347656, "blocks.2.feed_forward.w1": 0.34520086646080017, "blocks.2.feed_forward.w2": 0.1818288266658783, "blocks.2.feed_forward.w3": 0.3401286005973816, "blocks.3.attention.w_q": 0.3293800354003906, "blocks.3.attention.w_k": 0.308929443359375, "blocks.3.attention.w_v": 0.1178741455078125, "blocks.3.attention.w_out": 0.2205810546875, "blocks.3.feed_forward.w1": 0.3423881530761719, "blocks.3.feed_forward.w2": 0.18463134765625, "blocks.3.feed_forward.w3": 0.33850860595703125, "blocks.4.attention.w_q": 0.3197288513183594, "blocks.4.attention.w_k": 0.2898406982421875, "blocks.4.attention.w_v": 0.10969161987304688, "blocks.4.attention.w_out": 0.1727752685546875, "blocks.4.feed_forward.w1": 0.3827934265136719, "blocks.4.feed_forward.w2": 0.19648361206054688, "blocks.4.feed_forward.w3": 0.3730735778808594, "blocks.5.attention.w_q": 0.32492828369140625, "blocks.5.attention.w_k": 0.28411865234375, "blocks.5.attention.w_v": 0.11831283569335938, "blocks.5.attention.w_out": 0.1801910400390625, "blocks.5.feed_forward.w1": 0.38528698682785034, "blocks.5.feed_forward.w2": 0.1789449155330658, "blocks.5.feed_forward.w3": 0.3738810420036316, "blocks.6.attention.w_q": 0.327911376953125, "blocks.6.attention.w_k": 0.2770729064941406, "blocks.6.attention.w_v": 0.12823104858398438, "blocks.6.attention.w_out": 0.17969512939453125, "blocks.6.feed_forward.w1": 0.37800854444503784, "blocks.6.feed_forward.w2": 0.162750244140625, "blocks.6.feed_forward.w3": 0.3761405944824219, "blocks.7.attention.w_q": 0.2845954895019531, "blocks.7.attention.w_k": 0.26369476318359375, "blocks.7.attention.w_v": 0.1341552734375, "blocks.7.attention.w_out": 0.13731002807617188, "blocks.7.feed_forward.w1": 0.37084197998046875, "blocks.7.feed_forward.w2": 0.16039149463176727, "blocks.7.feed_forward.w3": 0.3572425842285156, "blocks.8.attention.w_q": 0.27764892578125, "blocks.8.attention.w_k": 0.2668609619140625, "blocks.8.attention.w_v": 0.141082763671875, "blocks.8.attention.w_out": 0.12615203857421875, "blocks.8.feed_forward.w1": 0.3497607111930847, "blocks.8.feed_forward.w2": 0.1337941586971283, "blocks.8.feed_forward.w3": 0.3358650207519531}, "w_mask_frac": 0.18134839699054375} +{"step": 6200, "changed_frac": 0.2590961619320079, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2591209411621094, "blocks.0.attention.w_k": 0.32740020751953125, "blocks.0.attention.w_v": 0.29268646240234375, "blocks.0.attention.w_out": 0.22011566162109375, "blocks.0.feed_forward.w1": 0.2309773862361908, "blocks.0.feed_forward.w2": 0.13732147216796875, "blocks.0.feed_forward.w3": 0.24349212646484375, "blocks.1.attention.w_q": 0.2858543395996094, "blocks.1.attention.w_k": 0.3191261291503906, "blocks.1.attention.w_v": 0.18751907348632812, "blocks.1.attention.w_out": 0.3116188049316406, "blocks.1.feed_forward.w1": 0.292755126953125, "blocks.1.feed_forward.w2": 0.16888174414634705, "blocks.1.feed_forward.w3": 0.2885894775390625, "blocks.2.attention.w_q": 0.282928466796875, "blocks.2.attention.w_k": 0.2790565490722656, "blocks.2.attention.w_v": 0.14064788818359375, "blocks.2.attention.w_out": 0.2672119140625, "blocks.2.feed_forward.w1": 0.33724212646484375, "blocks.2.feed_forward.w2": 0.17856089770793915, "blocks.2.feed_forward.w3": 0.3304837644100189, "blocks.3.attention.w_q": 0.3178749084472656, "blocks.3.attention.w_k": 0.3076972961425781, "blocks.3.attention.w_v": 0.11264801025390625, "blocks.3.attention.w_out": 0.20547866821289062, "blocks.3.feed_forward.w1": 0.33020275831222534, "blocks.3.feed_forward.w2": 0.18038305640220642, "blocks.3.feed_forward.w3": 0.3245938718318939, "blocks.4.attention.w_q": 0.3086585998535156, "blocks.4.attention.w_k": 0.2699775695800781, "blocks.4.attention.w_v": 0.10079574584960938, "blocks.4.attention.w_out": 0.15840911865234375, "blocks.4.feed_forward.w1": 0.36866888403892517, "blocks.4.feed_forward.w2": 0.1923116147518158, "blocks.4.feed_forward.w3": 0.3605130612850189, "blocks.5.attention.w_q": 0.3083343505859375, "blocks.5.attention.w_k": 0.2677803039550781, "blocks.5.attention.w_v": 0.1072998046875, "blocks.5.attention.w_out": 0.16518020629882812, "blocks.5.feed_forward.w1": 0.3734639585018158, "blocks.5.feed_forward.w2": 0.17339834570884705, "blocks.5.feed_forward.w3": 0.3644294738769531, "blocks.6.attention.w_q": 0.31165313720703125, "blocks.6.attention.w_k": 0.2604026794433594, "blocks.6.attention.w_v": 0.11662673950195312, "blocks.6.attention.w_out": 0.15979385375976562, "blocks.6.feed_forward.w1": 0.36670559644699097, "blocks.6.feed_forward.w2": 0.15577571094036102, "blocks.6.feed_forward.w3": 0.3622118830680847, "blocks.7.attention.w_q": 0.2623710632324219, "blocks.7.attention.w_k": 0.23748397827148438, "blocks.7.attention.w_v": 0.12363815307617188, "blocks.7.attention.w_out": 0.12549591064453125, "blocks.7.feed_forward.w1": 0.35434722900390625, "blocks.7.feed_forward.w2": 0.15150325000286102, "blocks.7.feed_forward.w3": 0.3414967954158783, "blocks.8.attention.w_q": 0.26300048828125, "blocks.8.attention.w_k": 0.2455291748046875, "blocks.8.attention.w_v": 0.12683868408203125, "blocks.8.attention.w_out": 0.11228561401367188, "blocks.8.feed_forward.w1": 0.3350359797477722, "blocks.8.feed_forward.w2": 0.1222178190946579, "blocks.8.feed_forward.w3": 0.3230069577693939}, "w_mask_frac": 0.17847289705378377} +{"step": 6300, "changed_frac": 0.2490273174057659, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.24695587158203125, "blocks.0.attention.w_k": 0.3182487487792969, "blocks.0.attention.w_v": 0.277374267578125, "blocks.0.attention.w_out": 0.20712661743164062, "blocks.0.feed_forward.w1": 0.22592036426067352, "blocks.0.feed_forward.w2": 0.13311514258384705, "blocks.0.feed_forward.w3": 0.23833084106445312, "blocks.1.attention.w_q": 0.2839813232421875, "blocks.1.attention.w_k": 0.3117866516113281, "blocks.1.attention.w_v": 0.17518997192382812, "blocks.1.attention.w_out": 0.29370880126953125, "blocks.1.feed_forward.w1": 0.2867546081542969, "blocks.1.feed_forward.w2": 0.16512171924114227, "blocks.1.feed_forward.w3": 0.28224310278892517, "blocks.2.attention.w_q": 0.2761573791503906, "blocks.2.attention.w_k": 0.2816581726074219, "blocks.2.attention.w_v": 0.12960052490234375, "blocks.2.attention.w_out": 0.2522010803222656, "blocks.2.feed_forward.w1": 0.3287035822868347, "blocks.2.feed_forward.w2": 0.17336146533489227, "blocks.2.feed_forward.w3": 0.32328543066978455, "blocks.3.attention.w_q": 0.2995719909667969, "blocks.3.attention.w_k": 0.2938575744628906, "blocks.3.attention.w_v": 0.09980010986328125, "blocks.3.attention.w_out": 0.18851852416992188, "blocks.3.feed_forward.w1": 0.3243510127067566, "blocks.3.feed_forward.w2": 0.17306646704673767, "blocks.3.feed_forward.w3": 0.31917446851730347, "blocks.4.attention.w_q": 0.2929420471191406, "blocks.4.attention.w_k": 0.25487518310546875, "blocks.4.attention.w_v": 0.0899810791015625, "blocks.4.attention.w_out": 0.144866943359375, "blocks.4.feed_forward.w1": 0.35963186621665955, "blocks.4.feed_forward.w2": 0.18331655859947205, "blocks.4.feed_forward.w3": 0.3510894775390625, "blocks.5.attention.w_q": 0.29172515869140625, "blocks.5.attention.w_k": 0.24856948852539062, "blocks.5.attention.w_v": 0.0962371826171875, "blocks.5.attention.w_out": 0.15203857421875, "blocks.5.feed_forward.w1": 0.3616231381893158, "blocks.5.feed_forward.w2": 0.16081619262695312, "blocks.5.feed_forward.w3": 0.35173672437667847, "blocks.6.attention.w_q": 0.2964744567871094, "blocks.6.attention.w_k": 0.25582122802734375, "blocks.6.attention.w_v": 0.1053009033203125, "blocks.6.attention.w_out": 0.14865875244140625, "blocks.6.feed_forward.w1": 0.3521105647087097, "blocks.6.feed_forward.w2": 0.14202627539634705, "blocks.6.feed_forward.w3": 0.34700268507003784, "blocks.7.attention.w_q": 0.24827194213867188, "blocks.7.attention.w_k": 0.22596359252929688, "blocks.7.attention.w_v": 0.10672378540039062, "blocks.7.attention.w_out": 0.11280441284179688, "blocks.7.feed_forward.w1": 0.34398144483566284, "blocks.7.feed_forward.w2": 0.14215342700481415, "blocks.7.feed_forward.w3": 0.330352783203125, "blocks.8.attention.w_q": 0.2378387451171875, "blocks.8.attention.w_k": 0.22728347778320312, "blocks.8.attention.w_v": 0.1138763427734375, "blocks.8.attention.w_out": 0.10044479370117188, "blocks.8.feed_forward.w1": 0.32446417212486267, "blocks.8.feed_forward.w2": 0.11604690551757812, "blocks.8.feed_forward.w3": 0.3101399838924408}, "w_mask_frac": 0.18237549716081375} +{"step": 6400, "changed_frac": 0.23792530940129206, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.23151397705078125, "blocks.0.attention.w_k": 0.29285430908203125, "blocks.0.attention.w_v": 0.2669868469238281, "blocks.0.attention.w_out": 0.19612884521484375, "blocks.0.feed_forward.w1": 0.21677526831626892, "blocks.0.feed_forward.w2": 0.12609609961509705, "blocks.0.feed_forward.w3": 0.22924423217773438, "blocks.1.attention.w_q": 0.263153076171875, "blocks.1.attention.w_k": 0.2801513671875, "blocks.1.attention.w_v": 0.1608123779296875, "blocks.1.attention.w_out": 0.2807426452636719, "blocks.1.feed_forward.w1": 0.2748603820800781, "blocks.1.feed_forward.w2": 0.15722784399986267, "blocks.1.feed_forward.w3": 0.27105969190597534, "blocks.2.attention.w_q": 0.2546348571777344, "blocks.2.attention.w_k": 0.24435043334960938, "blocks.2.attention.w_v": 0.1156158447265625, "blocks.2.attention.w_out": 0.2381439208984375, "blocks.2.feed_forward.w1": 0.31843695044517517, "blocks.2.feed_forward.w2": 0.16464361548423767, "blocks.2.feed_forward.w3": 0.31355923414230347, "blocks.3.attention.w_q": 0.2795982360839844, "blocks.3.attention.w_k": 0.2663612365722656, "blocks.3.attention.w_v": 0.09168624877929688, "blocks.3.attention.w_out": 0.1756439208984375, "blocks.3.feed_forward.w1": 0.3118337094783783, "blocks.3.feed_forward.w2": 0.16635005176067352, "blocks.3.feed_forward.w3": 0.30733999609947205, "blocks.4.attention.w_q": 0.2755126953125, "blocks.4.attention.w_k": 0.24152755737304688, "blocks.4.attention.w_v": 0.08352279663085938, "blocks.4.attention.w_out": 0.13329315185546875, "blocks.4.feed_forward.w1": 0.34533438086509705, "blocks.4.feed_forward.w2": 0.17453384399414062, "blocks.4.feed_forward.w3": 0.3367919921875, "blocks.5.attention.w_q": 0.2768898010253906, "blocks.5.attention.w_k": 0.23665618896484375, "blocks.5.attention.w_v": 0.08821487426757812, "blocks.5.attention.w_out": 0.1443939208984375, "blocks.5.feed_forward.w1": 0.34666699171066284, "blocks.5.feed_forward.w2": 0.15359625220298767, "blocks.5.feed_forward.w3": 0.33682379126548767, "blocks.6.attention.w_q": 0.2833595275878906, "blocks.6.attention.w_k": 0.2415924072265625, "blocks.6.attention.w_v": 0.09807968139648438, "blocks.6.attention.w_out": 0.14206695556640625, "blocks.6.feed_forward.w1": 0.34101104736328125, "blocks.6.feed_forward.w2": 0.1365509033203125, "blocks.6.feed_forward.w3": 0.33505377173423767, "blocks.7.attention.w_q": 0.2379150390625, "blocks.7.attention.w_k": 0.22014236450195312, "blocks.7.attention.w_v": 0.1027069091796875, "blocks.7.attention.w_out": 0.10670089721679688, "blocks.7.feed_forward.w1": 0.33266323804855347, "blocks.7.feed_forward.w2": 0.1374053955078125, "blocks.7.feed_forward.w3": 0.3214213252067566, "blocks.8.attention.w_q": 0.22979736328125, "blocks.8.attention.w_k": 0.21595001220703125, "blocks.8.attention.w_v": 0.10973739624023438, "blocks.8.attention.w_out": 0.0938720703125, "blocks.8.feed_forward.w1": 0.3124949336051941, "blocks.8.feed_forward.w2": 0.1116943359375, "blocks.8.feed_forward.w3": 0.2995147705078125}, "w_mask_frac": 0.1853114778542111} +{"step": 6500, "changed_frac": 0.22989582811665332, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.23101425170898438, "blocks.0.attention.w_k": 0.2988319396972656, "blocks.0.attention.w_v": 0.2533302307128906, "blocks.0.attention.w_out": 0.19219207763671875, "blocks.0.feed_forward.w1": 0.2075144499540329, "blocks.0.feed_forward.w2": 0.11974462121725082, "blocks.0.feed_forward.w3": 0.21886444091796875, "blocks.1.attention.w_q": 0.2483978271484375, "blocks.1.attention.w_k": 0.27645111083984375, "blocks.1.attention.w_v": 0.14780807495117188, "blocks.1.attention.w_out": 0.270172119140625, "blocks.1.feed_forward.w1": 0.2644004821777344, "blocks.1.feed_forward.w2": 0.14920172095298767, "blocks.1.feed_forward.w3": 0.2614123225212097, "blocks.2.attention.w_q": 0.24445724487304688, "blocks.2.attention.w_k": 0.24156951904296875, "blocks.2.attention.w_v": 0.110931396484375, "blocks.2.attention.w_out": 0.23144912719726562, "blocks.2.feed_forward.w1": 0.30739468336105347, "blocks.2.feed_forward.w2": 0.1579132080078125, "blocks.2.feed_forward.w3": 0.30282720923423767, "blocks.3.attention.w_q": 0.2588844299316406, "blocks.3.attention.w_k": 0.247650146484375, "blocks.3.attention.w_v": 0.0877685546875, "blocks.3.attention.w_out": 0.16963958740234375, "blocks.3.feed_forward.w1": 0.30342358350753784, "blocks.3.feed_forward.w2": 0.16008886694908142, "blocks.3.feed_forward.w3": 0.29794439673423767, "blocks.4.attention.w_q": 0.2653007507324219, "blocks.4.attention.w_k": 0.23572158813476562, "blocks.4.attention.w_v": 0.07637786865234375, "blocks.4.attention.w_out": 0.1279449462890625, "blocks.4.feed_forward.w1": 0.3358497619628906, "blocks.4.feed_forward.w2": 0.16884613037109375, "blocks.4.feed_forward.w3": 0.3277435302734375, "blocks.5.attention.w_q": 0.26123046875, "blocks.5.attention.w_k": 0.21873855590820312, "blocks.5.attention.w_v": 0.08068466186523438, "blocks.5.attention.w_out": 0.13829421997070312, "blocks.5.feed_forward.w1": 0.3380622863769531, "blocks.5.feed_forward.w2": 0.15026602149009705, "blocks.5.feed_forward.w3": 0.3280067443847656, "blocks.6.attention.w_q": 0.26874542236328125, "blocks.6.attention.w_k": 0.22449111938476562, "blocks.6.attention.w_v": 0.09284210205078125, "blocks.6.attention.w_out": 0.13612747192382812, "blocks.6.feed_forward.w1": 0.3252143859863281, "blocks.6.feed_forward.w2": 0.13754019141197205, "blocks.6.feed_forward.w3": 0.32413357496261597, "blocks.7.attention.w_q": 0.23038101196289062, "blocks.7.attention.w_k": 0.21096420288085938, "blocks.7.attention.w_v": 0.09849166870117188, "blocks.7.attention.w_out": 0.105072021484375, "blocks.7.feed_forward.w1": 0.3234062194824219, "blocks.7.feed_forward.w2": 0.13512293994426727, "blocks.7.feed_forward.w3": 0.31174468994140625, "blocks.8.attention.w_q": 0.22173309326171875, "blocks.8.attention.w_k": 0.21490859985351562, "blocks.8.attention.w_v": 0.10446929931640625, "blocks.8.attention.w_out": 0.09535598754882812, "blocks.8.feed_forward.w1": 0.30077236890792847, "blocks.8.feed_forward.w2": 0.1111246794462204, "blocks.8.feed_forward.w3": 0.2912330627441406}, "w_mask_frac": 0.18621246344768083} +{"step": 6600, "changed_frac": 0.22639608790731838, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.21225357055664062, "blocks.0.attention.w_k": 0.2739372253417969, "blocks.0.attention.w_v": 0.24897003173828125, "blocks.0.attention.w_out": 0.1815032958984375, "blocks.0.feed_forward.w1": 0.2064565122127533, "blocks.0.feed_forward.w2": 0.11865489184856415, "blocks.0.feed_forward.w3": 0.21730677783489227, "blocks.1.attention.w_q": 0.23971176147460938, "blocks.1.attention.w_k": 0.2701072692871094, "blocks.1.attention.w_v": 0.14583206176757812, "blocks.1.attention.w_out": 0.2693214416503906, "blocks.1.feed_forward.w1": 0.26370495557785034, "blocks.1.feed_forward.w2": 0.14876429736614227, "blocks.1.feed_forward.w3": 0.2601064145565033, "blocks.2.attention.w_q": 0.24611663818359375, "blocks.2.attention.w_k": 0.24410247802734375, "blocks.2.attention.w_v": 0.10951614379882812, "blocks.2.attention.w_out": 0.22948074340820312, "blocks.2.feed_forward.w1": 0.3033790588378906, "blocks.2.feed_forward.w2": 0.15612158179283142, "blocks.2.feed_forward.w3": 0.29675930738449097, "blocks.3.attention.w_q": 0.2656402587890625, "blocks.3.attention.w_k": 0.2628746032714844, "blocks.3.attention.w_v": 0.08768081665039062, "blocks.3.attention.w_out": 0.16353607177734375, "blocks.3.feed_forward.w1": 0.30124664306640625, "blocks.3.feed_forward.w2": 0.1570383757352829, "blocks.3.feed_forward.w3": 0.29598236083984375, "blocks.4.attention.w_q": 0.2590599060058594, "blocks.4.attention.w_k": 0.23760986328125, "blocks.4.attention.w_v": 0.07735061645507812, "blocks.4.attention.w_out": 0.12556838989257812, "blocks.4.feed_forward.w1": 0.3325080871582031, "blocks.4.feed_forward.w2": 0.16611099243164062, "blocks.4.feed_forward.w3": 0.32398098707199097, "blocks.5.attention.w_q": 0.2598686218261719, "blocks.5.attention.w_k": 0.22573089599609375, "blocks.5.attention.w_v": 0.08230972290039062, "blocks.5.attention.w_out": 0.13617706298828125, "blocks.5.feed_forward.w1": 0.33371734619140625, "blocks.5.feed_forward.w2": 0.14776739478111267, "blocks.5.feed_forward.w3": 0.32337188720703125, "blocks.6.attention.w_q": 0.26178741455078125, "blocks.6.attention.w_k": 0.21645736694335938, "blocks.6.attention.w_v": 0.09272003173828125, "blocks.6.attention.w_out": 0.13358688354492188, "blocks.6.feed_forward.w1": 0.32007598876953125, "blocks.6.feed_forward.w2": 0.13631820678710938, "blocks.6.feed_forward.w3": 0.3161150813102722, "blocks.7.attention.w_q": 0.235870361328125, "blocks.7.attention.w_k": 0.21859359741210938, "blocks.7.attention.w_v": 0.093658447265625, "blocks.7.attention.w_out": 0.09870529174804688, "blocks.7.feed_forward.w1": 0.3142242431640625, "blocks.7.feed_forward.w2": 0.13168716430664062, "blocks.7.feed_forward.w3": 0.3044471740722656, "blocks.8.attention.w_q": 0.2204742431640625, "blocks.8.attention.w_k": 0.20615386962890625, "blocks.8.attention.w_v": 0.09546279907226562, "blocks.8.attention.w_out": 0.08685302734375, "blocks.8.feed_forward.w1": 0.2938677668571472, "blocks.8.feed_forward.w2": 0.10544078052043915, "blocks.8.feed_forward.w3": 0.2827657163143158}, "w_mask_frac": 0.18795587638249764} +{"step": 6700, "changed_frac": 0.21567339774889824, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.119384765625, "blocks.0.attention.w_k": 0.20597076416015625, "blocks.0.attention.w_v": 0.20856857299804688, "blocks.0.attention.w_out": 0.14011764526367188, "blocks.0.feed_forward.w1": 0.1630961149930954, "blocks.0.feed_forward.w2": 0.1139729842543602, "blocks.0.feed_forward.w3": 0.18944041430950165, "blocks.1.attention.w_q": 0.22115707397460938, "blocks.1.attention.w_k": 0.2513847351074219, "blocks.1.attention.w_v": 0.14002609252929688, "blocks.1.attention.w_out": 0.25926971435546875, "blocks.1.feed_forward.w1": 0.23819224536418915, "blocks.1.feed_forward.w2": 0.14364242553710938, "blocks.1.feed_forward.w3": 0.2359110563993454, "blocks.2.attention.w_q": 0.23154449462890625, "blocks.2.attention.w_k": 0.2257080078125, "blocks.2.attention.w_v": 0.1006011962890625, "blocks.2.attention.w_out": 0.224761962890625, "blocks.2.feed_forward.w1": 0.2863413691520691, "blocks.2.feed_forward.w2": 0.15041224658489227, "blocks.2.feed_forward.w3": 0.2816518247127533, "blocks.3.attention.w_q": 0.2528266906738281, "blocks.3.attention.w_k": 0.23810577392578125, "blocks.3.attention.w_v": 0.08080673217773438, "blocks.3.attention.w_out": 0.1602020263671875, "blocks.3.feed_forward.w1": 0.2900276184082031, "blocks.3.feed_forward.w2": 0.15243658423423767, "blocks.3.feed_forward.w3": 0.2825978696346283, "blocks.4.attention.w_q": 0.25099945068359375, "blocks.4.attention.w_k": 0.225799560546875, "blocks.4.attention.w_v": 0.07404327392578125, "blocks.4.attention.w_out": 0.1245574951171875, "blocks.4.feed_forward.w1": 0.3276557922363281, "blocks.4.feed_forward.w2": 0.16101455688476562, "blocks.4.feed_forward.w3": 0.3202311396598816, "blocks.5.attention.w_q": 0.23859024047851562, "blocks.5.attention.w_k": 0.20205307006835938, "blocks.5.attention.w_v": 0.08016204833984375, "blocks.5.attention.w_out": 0.13590240478515625, "blocks.5.feed_forward.w1": 0.3259976804256439, "blocks.5.feed_forward.w2": 0.14199194312095642, "blocks.5.feed_forward.w3": 0.3155708312988281, "blocks.6.attention.w_q": 0.25446319580078125, "blocks.6.attention.w_k": 0.21379852294921875, "blocks.6.attention.w_v": 0.09165573120117188, "blocks.6.attention.w_out": 0.13453292846679688, "blocks.6.feed_forward.w1": 0.3177528381347656, "blocks.6.feed_forward.w2": 0.1321868896484375, "blocks.6.feed_forward.w3": 0.31390127539634705, "blocks.7.attention.w_q": 0.2276153564453125, "blocks.7.attention.w_k": 0.20594024658203125, "blocks.7.attention.w_v": 0.09600448608398438, "blocks.7.attention.w_out": 0.10198974609375, "blocks.7.feed_forward.w1": 0.3115730285644531, "blocks.7.feed_forward.w2": 0.12935256958007812, "blocks.7.feed_forward.w3": 0.29924774169921875, "blocks.8.attention.w_q": 0.21117782592773438, "blocks.8.attention.w_k": 0.2045440673828125, "blocks.8.attention.w_v": 0.101837158203125, "blocks.8.attention.w_out": 0.0880584716796875, "blocks.8.feed_forward.w1": 0.2929573059082031, "blocks.8.feed_forward.w2": 0.10306549072265625, "blocks.8.feed_forward.w3": 0.28298696875572205}, "w_mask_frac": 0.18821961356279177} +{"step": 6800, "changed_frac": 0.21125669560880742, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.16009140014648438, "blocks.0.attention.w_k": 0.22245025634765625, "blocks.0.attention.w_v": 0.20178604125976562, "blocks.0.attention.w_out": 0.14321136474609375, "blocks.0.feed_forward.w1": 0.18983206152915955, "blocks.0.feed_forward.w2": 0.112518310546875, "blocks.0.feed_forward.w3": 0.20103327929973602, "blocks.1.attention.w_q": 0.21535110473632812, "blocks.1.attention.w_k": 0.24086761474609375, "blocks.1.attention.w_v": 0.1324920654296875, "blocks.1.attention.w_out": 0.24756240844726562, "blocks.1.feed_forward.w1": 0.24490229785442352, "blocks.1.feed_forward.w2": 0.14271673560142517, "blocks.1.feed_forward.w3": 0.24155554175376892, "blocks.2.attention.w_q": 0.21400070190429688, "blocks.2.attention.w_k": 0.213226318359375, "blocks.2.attention.w_v": 0.10054397583007812, "blocks.2.attention.w_out": 0.20965957641601562, "blocks.2.feed_forward.w1": 0.2861442565917969, "blocks.2.feed_forward.w2": 0.15116629004478455, "blocks.2.feed_forward.w3": 0.27995556592941284, "blocks.3.attention.w_q": 0.24496841430664062, "blocks.3.attention.w_k": 0.23318099975585938, "blocks.3.attention.w_v": 0.07887649536132812, "blocks.3.attention.w_out": 0.14934158325195312, "blocks.3.feed_forward.w1": 0.28412628173828125, "blocks.3.feed_forward.w2": 0.1514689177274704, "blocks.3.feed_forward.w3": 0.2778727412223816, "blocks.4.attention.w_q": 0.23533248901367188, "blocks.4.attention.w_k": 0.20723724365234375, "blocks.4.attention.w_v": 0.07093048095703125, "blocks.4.attention.w_out": 0.11277008056640625, "blocks.4.feed_forward.w1": 0.3148435056209564, "blocks.4.feed_forward.w2": 0.16013209521770477, "blocks.4.feed_forward.w3": 0.3099759519100189, "blocks.5.attention.w_q": 0.22412872314453125, "blocks.5.attention.w_k": 0.19898605346679688, "blocks.5.attention.w_v": 0.07471466064453125, "blocks.5.attention.w_out": 0.119873046875, "blocks.5.feed_forward.w1": 0.3148612976074219, "blocks.5.feed_forward.w2": 0.14000830054283142, "blocks.5.feed_forward.w3": 0.30404791235923767, "blocks.6.attention.w_q": 0.24051284790039062, "blocks.6.attention.w_k": 0.19244766235351562, "blocks.6.attention.w_v": 0.08449554443359375, "blocks.6.attention.w_out": 0.11661911010742188, "blocks.6.feed_forward.w1": 0.30589422583580017, "blocks.6.feed_forward.w2": 0.12619273364543915, "blocks.6.feed_forward.w3": 0.3038673400878906, "blocks.7.attention.w_q": 0.2204132080078125, "blocks.7.attention.w_k": 0.19756317138671875, "blocks.7.attention.w_v": 0.0884857177734375, "blocks.7.attention.w_out": 0.092742919921875, "blocks.7.feed_forward.w1": 0.3027687072753906, "blocks.7.feed_forward.w2": 0.1227823942899704, "blocks.7.feed_forward.w3": 0.29310864210128784, "blocks.8.attention.w_q": 0.20517730712890625, "blocks.8.attention.w_k": 0.19244003295898438, "blocks.8.attention.w_v": 0.09273147583007812, "blocks.8.attention.w_out": 0.08064651489257812, "blocks.8.feed_forward.w1": 0.284912109375, "blocks.8.feed_forward.w2": 0.09895070642232895, "blocks.8.feed_forward.w3": 0.2747535705566406}, "w_mask_frac": 0.18901734079560664} +{"step": 6900, "changed_frac": 0.21195918678218484, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.17626953125, "blocks.0.attention.w_k": 0.24477386474609375, "blocks.0.attention.w_v": 0.22584915161132812, "blocks.0.attention.w_out": 0.16151046752929688, "blocks.0.feed_forward.w1": 0.19322586059570312, "blocks.0.feed_forward.w2": 0.112457275390625, "blocks.0.feed_forward.w3": 0.20464453101158142, "blocks.1.attention.w_q": 0.22246932983398438, "blocks.1.attention.w_k": 0.251251220703125, "blocks.1.attention.w_v": 0.13219070434570312, "blocks.1.attention.w_out": 0.2476348876953125, "blocks.1.feed_forward.w1": 0.24627432227134705, "blocks.1.feed_forward.w2": 0.1429290771484375, "blocks.1.feed_forward.w3": 0.2424468994140625, "blocks.2.attention.w_q": 0.22135543823242188, "blocks.2.attention.w_k": 0.22284317016601562, "blocks.2.attention.w_v": 0.09560775756835938, "blocks.2.attention.w_out": 0.2085723876953125, "blocks.2.feed_forward.w1": 0.2886708676815033, "blocks.2.feed_forward.w2": 0.15006892383098602, "blocks.2.feed_forward.w3": 0.2828330993652344, "blocks.3.attention.w_q": 0.24756240844726562, "blocks.3.attention.w_k": 0.23575973510742188, "blocks.3.attention.w_v": 0.07379913330078125, "blocks.3.attention.w_out": 0.14273834228515625, "blocks.3.feed_forward.w1": 0.2843920588493347, "blocks.3.feed_forward.w2": 0.15003332495689392, "blocks.3.feed_forward.w3": 0.2790769040584564, "blocks.4.attention.w_q": 0.23851776123046875, "blocks.4.attention.w_k": 0.20251083374023438, "blocks.4.attention.w_v": 0.06713104248046875, "blocks.4.attention.w_out": 0.10922622680664062, "blocks.4.feed_forward.w1": 0.3153165280818939, "blocks.4.feed_forward.w2": 0.15864309668540955, "blocks.4.feed_forward.w3": 0.30906805396080017, "blocks.5.attention.w_q": 0.22506332397460938, "blocks.5.attention.w_k": 0.20137786865234375, "blocks.5.attention.w_v": 0.069793701171875, "blocks.5.attention.w_out": 0.11928558349609375, "blocks.5.feed_forward.w1": 0.31423187255859375, "blocks.5.feed_forward.w2": 0.14041265845298767, "blocks.5.feed_forward.w3": 0.30586370825767517, "blocks.6.attention.w_q": 0.23624420166015625, "blocks.6.attention.w_k": 0.200347900390625, "blocks.6.attention.w_v": 0.07837295532226562, "blocks.6.attention.w_out": 0.115478515625, "blocks.6.feed_forward.w1": 0.3047828674316406, "blocks.6.feed_forward.w2": 0.12682470679283142, "blocks.6.feed_forward.w3": 0.30229949951171875, "blocks.7.attention.w_q": 0.2119140625, "blocks.7.attention.w_k": 0.20155715942382812, "blocks.7.attention.w_v": 0.0826416015625, "blocks.7.attention.w_out": 0.08986663818359375, "blocks.7.feed_forward.w1": 0.3030599057674408, "blocks.7.feed_forward.w2": 0.12435786426067352, "blocks.7.feed_forward.w3": 0.2907447814941406, "blocks.8.attention.w_q": 0.195648193359375, "blocks.8.attention.w_k": 0.19043731689453125, "blocks.8.attention.w_v": 0.08643722534179688, "blocks.8.attention.w_out": 0.07847213745117188, "blocks.8.feed_forward.w1": 0.2844530940055847, "blocks.8.feed_forward.w2": 0.09993235766887665, "blocks.8.feed_forward.w3": 0.2725270688533783}, "w_mask_frac": 0.19244691805961806} +{"step": 7000, "changed_frac": 0.21307255671574518, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.19486618041992188, "blocks.0.attention.w_k": 0.2634849548339844, "blocks.0.attention.w_v": 0.23215103149414062, "blocks.0.attention.w_out": 0.16645050048828125, "blocks.0.feed_forward.w1": 0.19425329566001892, "blocks.0.feed_forward.w2": 0.11314519494771957, "blocks.0.feed_forward.w3": 0.20523199439048767, "blocks.1.attention.w_q": 0.22552490234375, "blocks.1.attention.w_k": 0.2610054016113281, "blocks.1.attention.w_v": 0.13646316528320312, "blocks.1.attention.w_out": 0.25121307373046875, "blocks.1.feed_forward.w1": 0.24771881103515625, "blocks.1.feed_forward.w2": 0.14233271777629852, "blocks.1.feed_forward.w3": 0.2437388151884079, "blocks.2.attention.w_q": 0.22753143310546875, "blocks.2.attention.w_k": 0.22545623779296875, "blocks.2.attention.w_v": 0.09956741333007812, "blocks.2.attention.w_out": 0.21648788452148438, "blocks.2.feed_forward.w1": 0.2902933955192566, "blocks.2.feed_forward.w2": 0.1501922607421875, "blocks.2.feed_forward.w3": 0.2848955988883972, "blocks.3.attention.w_q": 0.24645614624023438, "blocks.3.attention.w_k": 0.23104095458984375, "blocks.3.attention.w_v": 0.07714080810546875, "blocks.3.attention.w_out": 0.14879989624023438, "blocks.3.feed_forward.w1": 0.28406524658203125, "blocks.3.feed_forward.w2": 0.14874014258384705, "blocks.3.feed_forward.w3": 0.2782999873161316, "blocks.4.attention.w_q": 0.24823379516601562, "blocks.4.attention.w_k": 0.21721649169921875, "blocks.4.attention.w_v": 0.06874465942382812, "blocks.4.attention.w_out": 0.11532211303710938, "blocks.4.feed_forward.w1": 0.3163859248161316, "blocks.4.feed_forward.w2": 0.15626272559165955, "blocks.4.feed_forward.w3": 0.3097432553768158, "blocks.5.attention.w_q": 0.23134994506835938, "blocks.5.attention.w_k": 0.196990966796875, "blocks.5.attention.w_v": 0.0708160400390625, "blocks.5.attention.w_out": 0.12068939208984375, "blocks.5.feed_forward.w1": 0.3145179748535156, "blocks.5.feed_forward.w2": 0.13943354785442352, "blocks.5.feed_forward.w3": 0.30473965406417847, "blocks.6.attention.w_q": 0.2369384765625, "blocks.6.attention.w_k": 0.19488143920898438, "blocks.6.attention.w_v": 0.08392333984375, "blocks.6.attention.w_out": 0.1189117431640625, "blocks.6.feed_forward.w1": 0.30796560645103455, "blocks.6.feed_forward.w2": 0.1262664794921875, "blocks.6.feed_forward.w3": 0.3021659851074219, "blocks.7.attention.w_q": 0.21223068237304688, "blocks.7.attention.w_k": 0.19905853271484375, "blocks.7.attention.w_v": 0.08805084228515625, "blocks.7.attention.w_out": 0.09256744384765625, "blocks.7.feed_forward.w1": 0.3016255795955658, "blocks.7.feed_forward.w2": 0.12299728393554688, "blocks.7.feed_forward.w3": 0.28826904296875, "blocks.8.attention.w_q": 0.19831085205078125, "blocks.8.attention.w_k": 0.19195175170898438, "blocks.8.attention.w_v": 0.08908843994140625, "blocks.8.attention.w_out": 0.07632827758789062, "blocks.8.feed_forward.w1": 0.2821083068847656, "blocks.8.feed_forward.w2": 0.09807714074850082, "blocks.8.feed_forward.w3": 0.2712821960449219}, "w_mask_frac": 0.19563832000280038} diff --git a/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..17702e1aad9419e93d21434e29551032465532c5 --- /dev/null +++ b/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6B-muon-lr5.0e-03-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03474218025803566, + "blocks.0.attention.w_out.weight": 0.044908151030540466, + "blocks.0.attention.w_q.weight": 0.03907623142004013, + "blocks.0.attention.w_v.weight": 0.03717150166630745, + "blocks.0.feed_forward.w1.weight": 0.03356951102614403, + "blocks.0.feed_forward.w2.weight": 0.021239662542939186, + "blocks.0.feed_forward.w3.weight": 0.032106559723615646, + "blocks.1.attention.w_k.weight": 0.03286685049533844, + "blocks.1.attention.w_out.weight": 0.03342472016811371, + "blocks.1.attention.w_q.weight": 0.034714214503765106, + "blocks.1.attention.w_v.weight": 0.04287578538060188, + "blocks.1.feed_forward.w1.weight": 0.03372684866189957, + "blocks.1.feed_forward.w2.weight": 0.02066204883158207, + "blocks.1.feed_forward.w3.weight": 0.034198109060525894, + "blocks.2.attention.w_k.weight": 0.029518697410821915, + "blocks.2.attention.w_out.weight": 0.03186732158064842, + "blocks.2.attention.w_q.weight": 0.030578147619962692, + "blocks.2.attention.w_v.weight": 0.043415218591690063, + "blocks.2.feed_forward.w1.weight": 0.03319230675697327, + "blocks.2.feed_forward.w2.weight": 0.02112666517496109, + "blocks.2.feed_forward.w3.weight": 0.03387013077735901, + "blocks.3.attention.w_k.weight": 0.030551325529813766, + "blocks.3.attention.w_out.weight": 0.03590111434459686, + "blocks.3.attention.w_q.weight": 0.030868658795952797, + "blocks.3.attention.w_v.weight": 0.04522211477160454, + "blocks.3.feed_forward.w1.weight": 0.033875055611133575, + "blocks.3.feed_forward.w2.weight": 0.021725084632635117, + "blocks.3.feed_forward.w3.weight": 0.03446526825428009, + "blocks.4.attention.w_k.weight": 0.03108694963157177, + "blocks.4.attention.w_out.weight": 0.040895357728004456, + "blocks.4.attention.w_q.weight": 0.030787667259573936, + "blocks.4.attention.w_v.weight": 0.047582101076841354, + "blocks.4.feed_forward.w1.weight": 0.033528175204992294, + "blocks.4.feed_forward.w2.weight": 0.022570135071873665, + "blocks.4.feed_forward.w3.weight": 0.034178636968135834, + "blocks.5.attention.w_k.weight": 0.03290070965886116, + "blocks.5.attention.w_out.weight": 0.04242095351219177, + "blocks.5.attention.w_q.weight": 0.03248860687017441, + "blocks.5.attention.w_v.weight": 0.04927685856819153, + "blocks.5.feed_forward.w1.weight": 0.03305620700120926, + "blocks.5.feed_forward.w2.weight": 0.0215720497071743, + "blocks.5.feed_forward.w3.weight": 0.03387656807899475, + "blocks.6.attention.w_k.weight": 0.035224199295043945, + "blocks.6.attention.w_out.weight": 0.045648738741874695, + "blocks.6.attention.w_q.weight": 0.034981619566679, + "blocks.6.attention.w_v.weight": 0.04822106286883354, + "blocks.6.feed_forward.w1.weight": 0.0330008938908577, + "blocks.6.feed_forward.w2.weight": 0.022304736077785492, + "blocks.6.feed_forward.w3.weight": 0.033462945371866226, + "blocks.7.attention.w_k.weight": 0.03121311031281948, + "blocks.7.attention.w_out.weight": 0.046239499002695084, + "blocks.7.attention.w_q.weight": 0.03152060881257057, + "blocks.7.attention.w_v.weight": 0.04497750476002693, + "blocks.7.feed_forward.w1.weight": 0.03318552300333977, + "blocks.7.feed_forward.w2.weight": 0.02747611328959465, + "blocks.7.feed_forward.w3.weight": 0.03487776592373848, + "blocks.8.attention.w_k.weight": 0.02920045144855976, + "blocks.8.attention.w_out.weight": 0.0442185215651989, + "blocks.8.attention.w_q.weight": 0.02982492186129093, + "blocks.8.attention.w_v.weight": 0.04193868860602379, + "blocks.8.feed_forward.w1.weight": 0.03366883471608162, + "blocks.8.feed_forward.w2.weight": 0.02881704829633236, + "blocks.8.feed_forward.w3.weight": 0.03531993180513382 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 4.668224568905942 + }, + "base_loss": 4.668224568905942, + "loss": { + "0.005": [ + 4.667842358527936, + 4.668003006290763, + 4.66836644534818 + ], + "0.01": [ + 4.667337074326856, + 4.667997231703604, + 4.667738893262343 + ], + "0.02": [ + 4.667915284208592, + 4.668410615462698, + 4.668155798111169 + ], + "0.05": [ + 4.668838521760772, + 4.66968019522549, + 4.669524649130451 + ], + "0.1": [ + 4.673004354320406, + 4.675236216608176, + 4.6743211479841715 + ], + "0.15": [ + 4.67997435144344, + 4.683408658049793, + 4.682300954518913 + ], + "0.2": [ + 4.690193289360087, + 4.694231940158626, + 4.693384382561736 + ], + "0.3": [ + 4.720110830759339, + 4.72615964038688, + 4.725228687735866 + ], + "0.4": [ + 4.766981417660209, + 4.7731862272121335, + 4.773118551867499 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 4.667842358527936, + 4.668003006290763, + 4.66836644534818 + ], + "0.01": [ + 4.667337074326856, + 4.667997231703604, + 4.667738893262343 + ], + "0.02": [ + 4.667915284208592, + 4.668410615462698, + 4.668155798111169 + ], + "0.05": [ + 4.668838521760772, + 4.66968019522549, + 4.669524649130451 + ], + "0.1": [ + 4.673004354320406, + 4.675236216608176, + 4.6743211479841715 + ], + "0.15": [ + 4.67997435144344, + 4.683408658049793, + 4.682300954518913 + ], + "0.2": [ + 4.690193289360087, + 4.694231940158626, + 4.693384382561736 + ], + "0.3": [ + 4.720110830759339, + 4.72615964038688, + 4.725228687735866 + ], + "0.4": [ + 4.766981417660209, + 4.7731862272121335, + 4.773118551867499 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": -0.00015396551698287908, + "0.01": -0.000533502475008163, + "0.02": -6.40029784557683e-05, + "0.05": 0.0011232197996289273, + "0.1": 0.005962670731642028, + "0.15": 0.013670085764773354, + "0.2": 0.024378635120873682, + "0.3": 0.05560848405475305, + "0.4": 0.10287083000733865 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": -0.00015396551698287908, + "0.01": -0.000533502475008163, + "0.02": -6.40029784557683e-05, + "0.05": 0.0011232197996289273, + "0.1": 0.005962670731642028, + "0.15": 0.013670085764773354, + "0.2": 0.024378635120873682, + "0.3": 0.05560848405475305, + "0.4": 0.10287083000733865 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 75.4 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..7375c986d0fff85f6f8b20ed5a278a1d28c4784d --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.7610736902735544, + "heldout_cc_en_tail_4M": 4.34014044106298 + }, + "loss_bf16_master": 3.7610736902735544, + "losses_eval_model": { + "val_dolma_cc": 3.7734184156936568 + }, + "bf16_minus_quant": -0.012344725420102343, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-o9-20", + "gpu": "NVIDIA L40S", + "t_s": 17.9 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..dc004c596de3851847aa72c8445ffecaee454545 --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/finalize.json @@ -0,0 +1,100 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101", + "arm": "C", + "trust_mode": "quest", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.7734184156936568 + }, + "losses_eval_model": { + "val_dolma_cc": 3.7734184156936568 + }, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.03055771067738533, + "blocks.0.attention.w_k": 0.03054257296025753, + "blocks.0.attention.w_v": 0.030520660802721977, + "blocks.0.attention.w_out": 0.031147470697760582, + "blocks.0.feed_forward.w1": 0.030374454334378242, + "blocks.0.feed_forward.w2": 0.03329426422715187, + "blocks.0.feed_forward.w3": 0.03033650666475296, + "blocks.1.attention.w_q": 0.030535556375980377, + "blocks.1.attention.w_k": 0.030314013361930847, + "blocks.1.attention.w_v": 0.030528824776411057, + "blocks.1.attention.w_out": 0.031021375209093094, + "blocks.1.feed_forward.w1": 0.03042585961520672, + "blocks.1.feed_forward.w2": 0.033209819346666336, + "blocks.1.feed_forward.w3": 0.03055593930184841, + "blocks.2.attention.w_q": 0.030441073700785637, + "blocks.2.attention.w_k": 0.030229240655899048, + "blocks.2.attention.w_v": 0.030431099236011505, + "blocks.2.attention.w_out": 0.03118753433227539, + "blocks.2.feed_forward.w1": 0.030477704480290413, + "blocks.2.feed_forward.w2": 0.03333467245101929, + "blocks.2.feed_forward.w3": 0.030516326427459717, + "blocks.3.attention.w_q": 0.030576564371585846, + "blocks.3.attention.w_k": 0.030380144715309143, + "blocks.3.attention.w_v": 0.030463995411992073, + "blocks.3.attention.w_out": 0.030528251081705093, + "blocks.3.feed_forward.w1": 0.030655337497591972, + "blocks.3.feed_forward.w2": 0.03348798304796219, + "blocks.3.feed_forward.w3": 0.030435962602496147, + "blocks.4.attention.w_q": 0.03103587031364441, + "blocks.4.attention.w_k": 0.030571289360523224, + "blocks.4.attention.w_v": 0.030486764386296272, + "blocks.4.attention.w_out": 0.030748100951313972, + "blocks.4.feed_forward.w1": 0.030551446601748466, + "blocks.4.feed_forward.w2": 0.033396948128938675, + "blocks.4.feed_forward.w3": 0.030460765585303307, + "blocks.5.attention.w_q": 0.030523139983415604, + "blocks.5.attention.w_k": 0.030528156086802483, + "blocks.5.attention.w_v": 0.03076692670583725, + "blocks.5.attention.w_out": 0.031185312196612358, + "blocks.5.feed_forward.w1": 0.030549483373761177, + "blocks.5.feed_forward.w2": 0.03334210813045502, + "blocks.5.feed_forward.w3": 0.03047485090792179, + "blocks.6.attention.w_q": 0.030119912698864937, + "blocks.6.attention.w_k": 0.03039698675274849, + "blocks.6.attention.w_v": 0.030520282685756683, + "blocks.6.attention.w_out": 0.031199488788843155, + "blocks.6.feed_forward.w1": 0.03062213957309723, + "blocks.6.feed_forward.w2": 0.03340574726462364, + "blocks.6.feed_forward.w3": 0.030550260096788406, + "blocks.7.attention.w_q": 0.03054698370397091, + "blocks.7.attention.w_k": 0.030698178336024284, + "blocks.7.attention.w_v": 0.030340705066919327, + "blocks.7.attention.w_out": 0.03127043694257736, + "blocks.7.feed_forward.w1": 0.030562881380319595, + "blocks.7.feed_forward.w2": 0.033475637435913086, + "blocks.7.feed_forward.w3": 0.030597954988479614, + "blocks.8.attention.w_q": 0.030574167147278786, + "blocks.8.attention.w_k": 0.030287517234683037, + "blocks.8.attention.w_v": 0.030602240934967995, + "blocks.8.attention.w_out": 0.030985692515969276, + "blocks.8.feed_forward.w1": 0.03058474324643612, + "blocks.8.feed_forward.w2": 0.033415697515010834, + "blocks.8.feed_forward.w3": 0.03069550357758999 + }, + "rel_quant_noise_mean": 0.030993416491481993, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.7508323192596436, + "loss_eval_model": 3.7734184156936568, + "checks": { + "snap_reproduces_fakequant": true, + "weights_on_arm_grid": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba5868caefa4c06b4768ebaebf15addc645487ed --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,134 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.quest_train_module.QuestTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + fq_arm: C + fq_trust_mode: quest + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.02 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false + osc_probe: + _CLASS_: precq.quest_callbacks.OscillationProbeCallback + interval: 100 + bits: 6 + arm: C + trust_mode: quest +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/osc_probe.jsonl b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/osc_probe.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c7b03b511535b16c904b8c40ce9cccdb11124b4c --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/osc_probe.jsonl @@ -0,0 +1,72 @@ +{"start": true, "step": 0, "arm": "C", "bits": 6, "interval": 100} +{"step": 0} +{"step": 100, "changed_frac": 0.673385978763939, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6352691650390625, "blocks.0.attention.w_k": 0.6622772216796875, "blocks.0.attention.w_v": 0.7518844604492188, "blocks.0.attention.w_out": 0.7516517639160156, "blocks.0.feed_forward.w1": 0.6983451843261719, "blocks.0.feed_forward.w2": 0.5703277587890625, "blocks.0.feed_forward.w3": 0.7049204707145691, "blocks.1.attention.w_q": 0.6599922180175781, "blocks.1.attention.w_k": 0.6893501281738281, "blocks.1.attention.w_v": 0.7349777221679688, "blocks.1.attention.w_out": 0.7401504516601562, "blocks.1.feed_forward.w1": 0.709723174571991, "blocks.1.feed_forward.w2": 0.5876070857048035, "blocks.1.feed_forward.w3": 0.7178916931152344, "blocks.2.attention.w_q": 0.6998634338378906, "blocks.2.attention.w_k": 0.7247047424316406, "blocks.2.attention.w_v": 0.7396926879882812, "blocks.2.attention.w_out": 0.74224853515625, "blocks.2.feed_forward.w1": 0.7189903259277344, "blocks.2.feed_forward.w2": 0.5886777639389038, "blocks.2.feed_forward.w3": 0.7249984741210938, "blocks.3.attention.w_q": 0.7061920166015625, "blocks.3.attention.w_k": 0.7245826721191406, "blocks.3.attention.w_v": 0.7397689819335938, "blocks.3.attention.w_out": 0.7396469116210938, "blocks.3.feed_forward.w1": 0.7180023193359375, "blocks.3.feed_forward.w2": 0.584136962890625, "blocks.3.feed_forward.w3": 0.7250213623046875, "blocks.4.attention.w_q": 0.7152595520019531, "blocks.4.attention.w_k": 0.7297515869140625, "blocks.4.attention.w_v": 0.7319908142089844, "blocks.4.attention.w_out": 0.7321128845214844, "blocks.4.feed_forward.w1": 0.7111905813217163, "blocks.4.feed_forward.w2": 0.5770975947380066, "blocks.4.feed_forward.w3": 0.7204908132553101, "blocks.5.attention.w_q": 0.7139129638671875, "blocks.5.attention.w_k": 0.73388671875, "blocks.5.attention.w_v": 0.7311782836914062, "blocks.5.attention.w_out": 0.7275428771972656, "blocks.5.feed_forward.w1": 0.6955515742301941, "blocks.5.feed_forward.w2": 0.5648218989372253, "blocks.5.feed_forward.w3": 0.7094345092773438, "blocks.6.attention.w_q": 0.69976806640625, "blocks.6.attention.w_k": 0.7229194641113281, "blocks.6.attention.w_v": 0.7154350280761719, "blocks.6.attention.w_out": 0.7209587097167969, "blocks.6.feed_forward.w1": 0.6872279047966003, "blocks.6.feed_forward.w2": 0.5483881831169128, "blocks.6.feed_forward.w3": 0.689581573009491, "blocks.7.attention.w_q": 0.7074813842773438, "blocks.7.attention.w_k": 0.7268714904785156, "blocks.7.attention.w_v": 0.7018966674804688, "blocks.7.attention.w_out": 0.6998176574707031, "blocks.7.feed_forward.w1": 0.6718165278434753, "blocks.7.feed_forward.w2": 0.5319455862045288, "blocks.7.feed_forward.w3": 0.6853650808334351, "blocks.8.attention.w_q": 0.6942024230957031, "blocks.8.attention.w_k": 0.7147979736328125, "blocks.8.attention.w_v": 0.6819305419921875, "blocks.8.attention.w_out": 0.6830520629882812, "blocks.8.feed_forward.w1": 0.6618995666503906, "blocks.8.feed_forward.w2": 0.5147413015365601, "blocks.8.feed_forward.w3": 0.6681849360466003}} +{"step": 200, "changed_frac": 0.8803525093274239, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9156074523925781, "blocks.0.attention.w_k": 0.911163330078125, "blocks.0.attention.w_v": 0.9065895080566406, "blocks.0.attention.w_out": 0.8986282348632812, "blocks.0.feed_forward.w1": 0.8832283020019531, "blocks.0.feed_forward.w2": 0.8241831660270691, "blocks.0.feed_forward.w3": 0.8895912170410156, "blocks.1.attention.w_q": 0.8201484680175781, "blocks.1.attention.w_k": 0.8389053344726562, "blocks.1.attention.w_v": 0.8773689270019531, "blocks.1.attention.w_out": 0.8821487426757812, "blocks.1.feed_forward.w1": 0.8871650695800781, "blocks.1.feed_forward.w2": 0.8337020874023438, "blocks.1.feed_forward.w3": 0.8929163813591003, "blocks.2.attention.w_q": 0.8141899108886719, "blocks.2.attention.w_k": 0.8310585021972656, "blocks.2.attention.w_v": 0.8768463134765625, "blocks.2.attention.w_out": 0.8825340270996094, "blocks.2.feed_forward.w1": 0.8968595266342163, "blocks.2.feed_forward.w2": 0.8434880971908569, "blocks.2.feed_forward.w3": 0.8994827270507812, "blocks.3.attention.w_q": 0.8381080627441406, "blocks.3.attention.w_k": 0.8468437194824219, "blocks.3.attention.w_v": 0.8802871704101562, "blocks.3.attention.w_out": 0.88323974609375, "blocks.3.feed_forward.w1": 0.9046592712402344, "blocks.3.feed_forward.w2": 0.8463668823242188, "blocks.3.feed_forward.w3": 0.9071617126464844, "blocks.4.attention.w_q": 0.8319664001464844, "blocks.4.attention.w_k": 0.8419342041015625, "blocks.4.attention.w_v": 0.8813400268554688, "blocks.4.attention.w_out": 0.885467529296875, "blocks.4.feed_forward.w1": 0.9074249267578125, "blocks.4.feed_forward.w2": 0.8482539057731628, "blocks.4.feed_forward.w3": 0.9086341857910156, "blocks.5.attention.w_q": 0.8555679321289062, "blocks.5.attention.w_k": 0.8634147644042969, "blocks.5.attention.w_v": 0.8825187683105469, "blocks.5.attention.w_out": 0.8854446411132812, "blocks.5.feed_forward.w1": 0.9093513488769531, "blocks.5.feed_forward.w2": 0.8511034846305847, "blocks.5.feed_forward.w3": 0.9101855158805847, "blocks.6.attention.w_q": 0.8655662536621094, "blocks.6.attention.w_k": 0.8712730407714844, "blocks.6.attention.w_v": 0.883697509765625, "blocks.6.attention.w_out": 0.8859901428222656, "blocks.6.feed_forward.w1": 0.9108836054801941, "blocks.6.feed_forward.w2": 0.8521741628646851, "blocks.6.feed_forward.w3": 0.9120470881462097, "blocks.7.attention.w_q": 0.87078857421875, "blocks.7.attention.w_k": 0.8748970031738281, "blocks.7.attention.w_v": 0.8807525634765625, "blocks.7.attention.w_out": 0.8845252990722656, "blocks.7.feed_forward.w1": 0.912109375, "blocks.7.feed_forward.w2": 0.8556251525878906, "blocks.7.feed_forward.w3": 0.9128698110580444, "blocks.8.attention.w_q": 0.8646965026855469, "blocks.8.attention.w_k": 0.8666038513183594, "blocks.8.attention.w_v": 0.881591796875, "blocks.8.attention.w_out": 0.8872604370117188, "blocks.8.feed_forward.w1": 0.9129931330680847, "blocks.8.feed_forward.w2": 0.8576240539550781, "blocks.8.feed_forward.w3": 0.9140090942382812}} +{"step": 300, "changed_frac": 0.9142745254386184, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9437713623046875, "blocks.0.attention.w_k": 0.9375953674316406, "blocks.0.attention.w_v": 0.9328384399414062, "blocks.0.attention.w_out": 0.9329986572265625, "blocks.0.feed_forward.w1": 0.9235267639160156, "blocks.0.feed_forward.w2": 0.8874766230583191, "blocks.0.feed_forward.w3": 0.9251861572265625, "blocks.1.attention.w_q": 0.8864288330078125, "blocks.1.attention.w_k": 0.8903350830078125, "blocks.1.attention.w_v": 0.913818359375, "blocks.1.attention.w_out": 0.9250335693359375, "blocks.1.feed_forward.w1": 0.9267997741699219, "blocks.1.feed_forward.w2": 0.8882217407226562, "blocks.1.feed_forward.w3": 0.9301542043685913, "blocks.2.attention.w_q": 0.8413124084472656, "blocks.2.attention.w_k": 0.8346786499023438, "blocks.2.attention.w_v": 0.9106025695800781, "blocks.2.attention.w_out": 0.9249992370605469, "blocks.2.feed_forward.w1": 0.9299647212028503, "blocks.2.feed_forward.w2": 0.8962326049804688, "blocks.2.feed_forward.w3": 0.9321060180664062, "blocks.3.attention.w_q": 0.8781166076660156, "blocks.3.attention.w_k": 0.87005615234375, "blocks.3.attention.w_v": 0.9171409606933594, "blocks.3.attention.w_out": 0.9248924255371094, "blocks.3.feed_forward.w1": 0.9324226379394531, "blocks.3.feed_forward.w2": 0.8906034231185913, "blocks.3.feed_forward.w3": 0.9332898855209351, "blocks.4.attention.w_q": 0.8885383605957031, "blocks.4.attention.w_k": 0.8777122497558594, "blocks.4.attention.w_v": 0.9192962646484375, "blocks.4.attention.w_out": 0.9252166748046875, "blocks.4.feed_forward.w1": 0.9312566518783569, "blocks.4.feed_forward.w2": 0.8894666433334351, "blocks.4.feed_forward.w3": 0.930792510509491, "blocks.5.attention.w_q": 0.8893890380859375, "blocks.5.attention.w_k": 0.884521484375, "blocks.5.attention.w_v": 0.9183235168457031, "blocks.5.attention.w_out": 0.9266433715820312, "blocks.5.feed_forward.w1": 0.9304670095443726, "blocks.5.feed_forward.w2": 0.8899129629135132, "blocks.5.feed_forward.w3": 0.9299672842025757, "blocks.6.attention.w_q": 0.9099884033203125, "blocks.6.attention.w_k": 0.9116783142089844, "blocks.6.attention.w_v": 0.9208335876464844, "blocks.6.attention.w_out": 0.9246139526367188, "blocks.6.feed_forward.w1": 0.9296087026596069, "blocks.6.feed_forward.w2": 0.891204833984375, "blocks.6.feed_forward.w3": 0.9291509389877319, "blocks.7.attention.w_q": 0.9119911193847656, "blocks.7.attention.w_k": 0.9156761169433594, "blocks.7.attention.w_v": 0.9200172424316406, "blocks.7.attention.w_out": 0.92425537109375, "blocks.7.feed_forward.w1": 0.9300155639648438, "blocks.7.feed_forward.w2": 0.8941752314567566, "blocks.7.feed_forward.w3": 0.9295985102653503, "blocks.8.attention.w_q": 0.9048271179199219, "blocks.8.attention.w_k": 0.9092788696289062, "blocks.8.attention.w_v": 0.9194984436035156, "blocks.8.attention.w_out": 0.9248886108398438, "blocks.8.feed_forward.w1": 0.9307886958122253, "blocks.8.feed_forward.w2": 0.8966051936149597, "blocks.8.feed_forward.w3": 0.9304428100585938}} +{"step": 400, "changed_frac": 0.9260620051978999, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9393653869628906, "blocks.0.attention.w_k": 0.9337081909179688, "blocks.0.attention.w_v": 0.9355049133300781, "blocks.0.attention.w_out": 0.9403343200683594, "blocks.0.feed_forward.w1": 0.9373550415039062, "blocks.0.feed_forward.w2": 0.9120280146598816, "blocks.0.feed_forward.w3": 0.9363021850585938, "blocks.1.attention.w_q": 0.9255447387695312, "blocks.1.attention.w_k": 0.9208259582519531, "blocks.1.attention.w_v": 0.9252090454101562, "blocks.1.attention.w_out": 0.9348068237304688, "blocks.1.feed_forward.w1": 0.9351882934570312, "blocks.1.feed_forward.w2": 0.9084243774414062, "blocks.1.feed_forward.w3": 0.9354362487792969, "blocks.2.attention.w_q": 0.922393798828125, "blocks.2.attention.w_k": 0.9133682250976562, "blocks.2.attention.w_v": 0.9222679138183594, "blocks.2.attention.w_out": 0.9321937561035156, "blocks.2.feed_forward.w1": 0.9368604421615601, "blocks.2.feed_forward.w2": 0.9105517268180847, "blocks.2.feed_forward.w3": 0.9365641474723816, "blocks.3.attention.w_q": 0.9284782409667969, "blocks.3.attention.w_k": 0.9202880859375, "blocks.3.attention.w_v": 0.9279670715332031, "blocks.3.attention.w_out": 0.9338493347167969, "blocks.3.feed_forward.w1": 0.9347381591796875, "blocks.3.feed_forward.w2": 0.9049072265625, "blocks.3.feed_forward.w3": 0.9346262812614441, "blocks.4.attention.w_q": 0.9307098388671875, "blocks.4.attention.w_k": 0.919403076171875, "blocks.4.attention.w_v": 0.9291496276855469, "blocks.4.attention.w_out": 0.9327392578125, "blocks.4.feed_forward.w1": 0.9329948425292969, "blocks.4.feed_forward.w2": 0.9050433039665222, "blocks.4.feed_forward.w3": 0.9323158264160156, "blocks.5.attention.w_q": 0.9253120422363281, "blocks.5.attention.w_k": 0.9152145385742188, "blocks.5.attention.w_v": 0.9292221069335938, "blocks.5.attention.w_out": 0.9371719360351562, "blocks.5.feed_forward.w1": 0.9323158264160156, "blocks.5.feed_forward.w2": 0.9049924612045288, "blocks.5.feed_forward.w3": 0.9315007925033569, "blocks.6.attention.w_q": 0.9301986694335938, "blocks.6.attention.w_k": 0.9260749816894531, "blocks.6.attention.w_v": 0.9319419860839844, "blocks.6.attention.w_out": 0.9355354309082031, "blocks.6.feed_forward.w1": 0.9321670532226562, "blocks.6.feed_forward.w2": 0.9046605825424194, "blocks.6.feed_forward.w3": 0.9316266775131226, "blocks.7.attention.w_q": 0.9285697937011719, "blocks.7.attention.w_k": 0.9270362854003906, "blocks.7.attention.w_v": 0.9329681396484375, "blocks.7.attention.w_out": 0.938385009765625, "blocks.7.feed_forward.w1": 0.9322840571403503, "blocks.7.feed_forward.w2": 0.9047037959098816, "blocks.7.feed_forward.w3": 0.9312986135482788, "blocks.8.attention.w_q": 0.92919921875, "blocks.8.attention.w_k": 0.9261093139648438, "blocks.8.attention.w_v": 0.93035888671875, "blocks.8.attention.w_out": 0.9362030029296875, "blocks.8.feed_forward.w1": 0.9328753352165222, "blocks.8.feed_forward.w2": 0.903937041759491, "blocks.8.feed_forward.w3": 0.9315173029899597}} +{"step": 500, "changed_frac": 0.9287164150140225, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9375686645507812, "blocks.0.attention.w_k": 0.9346885681152344, "blocks.0.attention.w_v": 0.9354782104492188, "blocks.0.attention.w_out": 0.9388809204101562, "blocks.0.feed_forward.w1": 0.9403051137924194, "blocks.0.feed_forward.w2": 0.9237327575683594, "blocks.0.feed_forward.w3": 0.937964141368866, "blocks.1.attention.w_q": 0.9342803955078125, "blocks.1.attention.w_k": 0.9310073852539062, "blocks.1.attention.w_v": 0.9314765930175781, "blocks.1.attention.w_out": 0.9383163452148438, "blocks.1.feed_forward.w1": 0.9354934692382812, "blocks.1.feed_forward.w2": 0.9216690063476562, "blocks.1.feed_forward.w3": 0.9328638911247253, "blocks.2.attention.w_q": 0.9372406005859375, "blocks.2.attention.w_k": 0.9330940246582031, "blocks.2.attention.w_v": 0.935333251953125, "blocks.2.attention.w_out": 0.9388351440429688, "blocks.2.feed_forward.w1": 0.9348005056381226, "blocks.2.feed_forward.w2": 0.9145063161849976, "blocks.2.feed_forward.w3": 0.9335340261459351, "blocks.3.attention.w_q": 0.9343376159667969, "blocks.3.attention.w_k": 0.9319953918457031, "blocks.3.attention.w_v": 0.9325370788574219, "blocks.3.attention.w_out": 0.9367294311523438, "blocks.3.feed_forward.w1": 0.9326909780502319, "blocks.3.feed_forward.w2": 0.9123204946517944, "blocks.3.feed_forward.w3": 0.9322637319564819, "blocks.4.attention.w_q": 0.93389892578125, "blocks.4.attention.w_k": 0.9282188415527344, "blocks.4.attention.w_v": 0.9330558776855469, "blocks.4.attention.w_out": 0.9324455261230469, "blocks.4.feed_forward.w1": 0.9326680898666382, "blocks.4.feed_forward.w2": 0.9124463796615601, "blocks.4.feed_forward.w3": 0.9307162165641785, "blocks.5.attention.w_q": 0.9255752563476562, "blocks.5.attention.w_k": 0.9129714965820312, "blocks.5.attention.w_v": 0.9337310791015625, "blocks.5.attention.w_out": 0.9353561401367188, "blocks.5.feed_forward.w1": 0.9330953359603882, "blocks.5.feed_forward.w2": 0.9116719961166382, "blocks.5.feed_forward.w3": 0.9313812255859375, "blocks.6.attention.w_q": 0.9375495910644531, "blocks.6.attention.w_k": 0.93280029296875, "blocks.6.attention.w_v": 0.933258056640625, "blocks.6.attention.w_out": 0.9286422729492188, "blocks.6.feed_forward.w1": 0.9328448176383972, "blocks.6.feed_forward.w2": 0.9101728200912476, "blocks.6.feed_forward.w3": 0.9317067861557007, "blocks.7.attention.w_q": 0.9347152709960938, "blocks.7.attention.w_k": 0.9296302795410156, "blocks.7.attention.w_v": 0.9316368103027344, "blocks.7.attention.w_out": 0.9303169250488281, "blocks.7.feed_forward.w1": 0.9332873225212097, "blocks.7.feed_forward.w2": 0.9080759882926941, "blocks.7.feed_forward.w3": 0.9315414428710938, "blocks.8.attention.w_q": 0.9364700317382812, "blocks.8.attention.w_k": 0.93408203125, "blocks.8.attention.w_v": 0.9327392578125, "blocks.8.attention.w_out": 0.9326553344726562, "blocks.8.feed_forward.w1": 0.9347547292709351, "blocks.8.feed_forward.w2": 0.9040985107421875, "blocks.8.feed_forward.w3": 0.9321517944335938}} +{"step": 600, "changed_frac": 0.9279030041816907, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9397811889648438, "blocks.0.attention.w_k": 0.9377250671386719, "blocks.0.attention.w_v": 0.9351615905761719, "blocks.0.attention.w_out": 0.9353713989257812, "blocks.0.feed_forward.w1": 0.9397799372673035, "blocks.0.feed_forward.w2": 0.9263267517089844, "blocks.0.feed_forward.w3": 0.9370905756950378, "blocks.1.attention.w_q": 0.9317359924316406, "blocks.1.attention.w_k": 0.9293022155761719, "blocks.1.attention.w_v": 0.9345512390136719, "blocks.1.attention.w_out": 0.9366073608398438, "blocks.1.feed_forward.w1": 0.9331678152084351, "blocks.1.feed_forward.w2": 0.9241384267807007, "blocks.1.feed_forward.w3": 0.930578887462616, "blocks.2.attention.w_q": 0.9384956359863281, "blocks.2.attention.w_k": 0.9368476867675781, "blocks.2.attention.w_v": 0.9372787475585938, "blocks.2.attention.w_out": 0.94110107421875, "blocks.2.feed_forward.w1": 0.9320704340934753, "blocks.2.feed_forward.w2": 0.9160118103027344, "blocks.2.feed_forward.w3": 0.9304351806640625, "blocks.3.attention.w_q": 0.9377326965332031, "blocks.3.attention.w_k": 0.9338417053222656, "blocks.3.attention.w_v": 0.9329757690429688, "blocks.3.attention.w_out": 0.9379501342773438, "blocks.3.feed_forward.w1": 0.9308192133903503, "blocks.3.feed_forward.w2": 0.915924072265625, "blocks.3.feed_forward.w3": 0.9300079345703125, "blocks.4.attention.w_q": 0.9346580505371094, "blocks.4.attention.w_k": 0.9296913146972656, "blocks.4.attention.w_v": 0.9324569702148438, "blocks.4.attention.w_out": 0.9286613464355469, "blocks.4.feed_forward.w1": 0.931128203868866, "blocks.4.feed_forward.w2": 0.9148572683334351, "blocks.4.feed_forward.w3": 0.9295896291732788, "blocks.5.attention.w_q": 0.9280204772949219, "blocks.5.attention.w_k": 0.9165878295898438, "blocks.5.attention.w_v": 0.9296798706054688, "blocks.5.attention.w_out": 0.9216651916503906, "blocks.5.feed_forward.w1": 0.9313303828239441, "blocks.5.feed_forward.w2": 0.9129728078842163, "blocks.5.feed_forward.w3": 0.9294815063476562, "blocks.6.attention.w_q": 0.9376144409179688, "blocks.6.attention.w_k": 0.93560791015625, "blocks.6.attention.w_v": 0.9271049499511719, "blocks.6.attention.w_out": 0.9177284240722656, "blocks.6.feed_forward.w1": 0.9319013357162476, "blocks.6.feed_forward.w2": 0.9108861684799194, "blocks.6.feed_forward.w3": 0.9293505549430847, "blocks.7.attention.w_q": 0.9378471374511719, "blocks.7.attention.w_k": 0.9329910278320312, "blocks.7.attention.w_v": 0.9261817932128906, "blocks.7.attention.w_out": 0.9199104309082031, "blocks.7.feed_forward.w1": 0.9330024719238281, "blocks.7.feed_forward.w2": 0.9062106013298035, "blocks.7.feed_forward.w3": 0.9300016164779663, "blocks.8.attention.w_q": 0.9389305114746094, "blocks.8.attention.w_k": 0.9361572265625, "blocks.8.attention.w_v": 0.9279861450195312, "blocks.8.attention.w_out": 0.9222602844238281, "blocks.8.feed_forward.w1": 0.9341239929199219, "blocks.8.feed_forward.w2": 0.9011332392692566, "blocks.8.feed_forward.w3": 0.9298299551010132}} +{"step": 700, "changed_frac": 0.9272975106524606, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9386520385742188, "blocks.0.attention.w_k": 0.9380531311035156, "blocks.0.attention.w_v": 0.9315452575683594, "blocks.0.attention.w_out": 0.9323806762695312, "blocks.0.feed_forward.w1": 0.9381917715072632, "blocks.0.feed_forward.w2": 0.9269727468490601, "blocks.0.feed_forward.w3": 0.9361470937728882, "blocks.1.attention.w_q": 0.9281349182128906, "blocks.1.attention.w_k": 0.9273719787597656, "blocks.1.attention.w_v": 0.9330406188964844, "blocks.1.attention.w_out": 0.934295654296875, "blocks.1.feed_forward.w1": 0.9315058588981628, "blocks.1.feed_forward.w2": 0.9244639277458191, "blocks.1.feed_forward.w3": 0.92889404296875, "blocks.2.attention.w_q": 0.9400177001953125, "blocks.2.attention.w_k": 0.9386940002441406, "blocks.2.attention.w_v": 0.9366798400878906, "blocks.2.attention.w_out": 0.941192626953125, "blocks.2.feed_forward.w1": 0.9308662414550781, "blocks.2.feed_forward.w2": 0.9165992736816406, "blocks.2.feed_forward.w3": 0.9294306635856628, "blocks.3.attention.w_q": 0.9376716613769531, "blocks.3.attention.w_k": 0.9359321594238281, "blocks.3.attention.w_v": 0.9318275451660156, "blocks.3.attention.w_out": 0.9371147155761719, "blocks.3.feed_forward.w1": 0.930365264415741, "blocks.3.feed_forward.w2": 0.9177259206771851, "blocks.3.feed_forward.w3": 0.9284273982048035, "blocks.4.attention.w_q": 0.9361801147460938, "blocks.4.attention.w_k": 0.9328117370605469, "blocks.4.attention.w_v": 0.9298095703125, "blocks.4.attention.w_out": 0.9253768920898438, "blocks.4.feed_forward.w1": 0.9301910400390625, "blocks.4.feed_forward.w2": 0.9163589477539062, "blocks.4.feed_forward.w3": 0.9289703369140625, "blocks.5.attention.w_q": 0.9336090087890625, "blocks.5.attention.w_k": 0.9257354736328125, "blocks.5.attention.w_v": 0.9258689880371094, "blocks.5.attention.w_out": 0.9170608520507812, "blocks.5.feed_forward.w1": 0.9313647150993347, "blocks.5.feed_forward.w2": 0.913482666015625, "blocks.5.feed_forward.w3": 0.9273122549057007, "blocks.6.attention.w_q": 0.9406242370605469, "blocks.6.attention.w_k": 0.9375839233398438, "blocks.6.attention.w_v": 0.9217109680175781, "blocks.6.attention.w_out": 0.9120712280273438, "blocks.6.feed_forward.w1": 0.9310264587402344, "blocks.6.feed_forward.w2": 0.9114227294921875, "blocks.6.feed_forward.w3": 0.9279760122299194, "blocks.7.attention.w_q": 0.9406661987304688, "blocks.7.attention.w_k": 0.9364242553710938, "blocks.7.attention.w_v": 0.9231147766113281, "blocks.7.attention.w_out": 0.9148178100585938, "blocks.7.feed_forward.w1": 0.9320831298828125, "blocks.7.feed_forward.w2": 0.9059728384017944, "blocks.7.feed_forward.w3": 0.9289830923080444, "blocks.8.attention.w_q": 0.9410018920898438, "blocks.8.attention.w_k": 0.9376945495605469, "blocks.8.attention.w_v": 0.9238243103027344, "blocks.8.attention.w_out": 0.9167861938476562, "blocks.8.feed_forward.w1": 0.9335708618164062, "blocks.8.feed_forward.w2": 0.8991851806640625, "blocks.8.feed_forward.w3": 0.9286537170410156}} +{"step": 800, "changed_frac": 0.9241002726758647, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9380569458007812, "blocks.0.attention.w_k": 0.9373893737792969, "blocks.0.attention.w_v": 0.9273033142089844, "blocks.0.attention.w_out": 0.9273605346679688, "blocks.0.feed_forward.w1": 0.9346148371696472, "blocks.0.feed_forward.w2": 0.9236628413200378, "blocks.0.feed_forward.w3": 0.9324620962142944, "blocks.1.attention.w_q": 0.9303550720214844, "blocks.1.attention.w_k": 0.9300956726074219, "blocks.1.attention.w_v": 0.9302253723144531, "blocks.1.attention.w_out": 0.9304122924804688, "blocks.1.feed_forward.w1": 0.9280256032943726, "blocks.1.feed_forward.w2": 0.9219627380371094, "blocks.1.feed_forward.w3": 0.9253374934196472, "blocks.2.attention.w_q": 0.9393196105957031, "blocks.2.attention.w_k": 0.9387969970703125, "blocks.2.attention.w_v": 0.931884765625, "blocks.2.attention.w_out": 0.937347412109375, "blocks.2.feed_forward.w1": 0.9275996088981628, "blocks.2.feed_forward.w2": 0.9148954153060913, "blocks.2.feed_forward.w3": 0.9258410334587097, "blocks.3.attention.w_q": 0.9374771118164062, "blocks.3.attention.w_k": 0.9377899169921875, "blocks.3.attention.w_v": 0.9290046691894531, "blocks.3.attention.w_out": 0.9341850280761719, "blocks.3.feed_forward.w1": 0.9268633723258972, "blocks.3.feed_forward.w2": 0.9155782461166382, "blocks.3.feed_forward.w3": 0.9243431091308594, "blocks.4.attention.w_q": 0.93597412109375, "blocks.4.attention.w_k": 0.9343223571777344, "blocks.4.attention.w_v": 0.9250602722167969, "blocks.4.attention.w_out": 0.9175949096679688, "blocks.4.feed_forward.w1": 0.9267896413803101, "blocks.4.feed_forward.w2": 0.9130210876464844, "blocks.4.feed_forward.w3": 0.9243329763412476, "blocks.5.attention.w_q": 0.9332466125488281, "blocks.5.attention.w_k": 0.9299392700195312, "blocks.5.attention.w_v": 0.9214210510253906, "blocks.5.attention.w_out": 0.91326904296875, "blocks.5.feed_forward.w1": 0.9272906184196472, "blocks.5.feed_forward.w2": 0.9092445373535156, "blocks.5.feed_forward.w3": 0.9233297109603882, "blocks.6.attention.w_q": 0.9402999877929688, "blocks.6.attention.w_k": 0.9370079040527344, "blocks.6.attention.w_v": 0.9161300659179688, "blocks.6.attention.w_out": 0.9045448303222656, "blocks.6.feed_forward.w1": 0.9280433654785156, "blocks.6.feed_forward.w2": 0.9080988764762878, "blocks.6.feed_forward.w3": 0.9243329763412476, "blocks.7.attention.w_q": 0.9395256042480469, "blocks.7.attention.w_k": 0.9382896423339844, "blocks.7.attention.w_v": 0.9169082641601562, "blocks.7.attention.w_out": 0.9054679870605469, "blocks.7.feed_forward.w1": 0.9298655390739441, "blocks.7.feed_forward.w2": 0.9015833735466003, "blocks.7.feed_forward.w3": 0.9242998957633972, "blocks.8.attention.w_q": 0.9399681091308594, "blocks.8.attention.w_k": 0.938446044921875, "blocks.8.attention.w_v": 0.9177017211914062, "blocks.8.attention.w_out": 0.9075889587402344, "blocks.8.feed_forward.w1": 0.9308013916015625, "blocks.8.feed_forward.w2": 0.8928604125976562, "blocks.8.feed_forward.w3": 0.9249267578125}} +{"step": 900, "changed_frac": 0.9191346127762754, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9348945617675781, "blocks.0.attention.w_k": 0.9341468811035156, "blocks.0.attention.w_v": 0.9213027954101562, "blocks.0.attention.w_out": 0.919952392578125, "blocks.0.feed_forward.w1": 0.9276212453842163, "blocks.0.feed_forward.w2": 0.918524444103241, "blocks.0.feed_forward.w3": 0.9264640808105469, "blocks.1.attention.w_q": 0.9317131042480469, "blocks.1.attention.w_k": 0.9304885864257812, "blocks.1.attention.w_v": 0.9250755310058594, "blocks.1.attention.w_out": 0.9243431091308594, "blocks.1.feed_forward.w1": 0.9225108027458191, "blocks.1.feed_forward.w2": 0.9158986806869507, "blocks.1.feed_forward.w3": 0.9199727773666382, "blocks.2.attention.w_q": 0.9367599487304688, "blocks.2.attention.w_k": 0.9360885620117188, "blocks.2.attention.w_v": 0.9271926879882812, "blocks.2.attention.w_out": 0.9339332580566406, "blocks.2.feed_forward.w1": 0.9220466613769531, "blocks.2.feed_forward.w2": 0.9100379943847656, "blocks.2.feed_forward.w3": 0.9203910827636719, "blocks.3.attention.w_q": 0.9353446960449219, "blocks.3.attention.w_k": 0.9350090026855469, "blocks.3.attention.w_v": 0.9229850769042969, "blocks.3.attention.w_out": 0.9286880493164062, "blocks.3.feed_forward.w1": 0.9212939143180847, "blocks.3.feed_forward.w2": 0.9121106863021851, "blocks.3.feed_forward.w3": 0.918792724609375, "blocks.4.attention.w_q": 0.9345436096191406, "blocks.4.attention.w_k": 0.9338150024414062, "blocks.4.attention.w_v": 0.9185867309570312, "blocks.4.attention.w_out": 0.9118309020996094, "blocks.4.feed_forward.w1": 0.9217962026596069, "blocks.4.feed_forward.w2": 0.9089953303337097, "blocks.4.feed_forward.w3": 0.918432891368866, "blocks.5.attention.w_q": 0.93328857421875, "blocks.5.attention.w_k": 0.9292869567871094, "blocks.5.attention.w_v": 0.9151840209960938, "blocks.5.attention.w_out": 0.9113731384277344, "blocks.5.feed_forward.w1": 0.9223455190658569, "blocks.5.feed_forward.w2": 0.9047788381576538, "blocks.5.feed_forward.w3": 0.9179420471191406, "blocks.6.attention.w_q": 0.9379119873046875, "blocks.6.attention.w_k": 0.9360237121582031, "blocks.6.attention.w_v": 0.9101028442382812, "blocks.6.attention.w_out": 0.8962249755859375, "blocks.6.feed_forward.w1": 0.9231147766113281, "blocks.6.feed_forward.w2": 0.9010950922966003, "blocks.6.feed_forward.w3": 0.9186490774154663, "blocks.7.attention.w_q": 0.9385757446289062, "blocks.7.attention.w_k": 0.9355354309082031, "blocks.7.attention.w_v": 0.9108161926269531, "blocks.7.attention.w_out": 0.8974494934082031, "blocks.7.feed_forward.w1": 0.9253311157226562, "blocks.7.feed_forward.w2": 0.8936983942985535, "blocks.7.feed_forward.w3": 0.9195022583007812, "blocks.8.attention.w_q": 0.93902587890625, "blocks.8.attention.w_k": 0.9365310668945312, "blocks.8.attention.w_v": 0.9097709655761719, "blocks.8.attention.w_out": 0.898712158203125, "blocks.8.feed_forward.w1": 0.9275792837142944, "blocks.8.feed_forward.w2": 0.8839086294174194, "blocks.8.feed_forward.w3": 0.9192466735839844}} +{"step": 1000, "changed_frac": 0.9155295934432592, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.933441162109375, "blocks.0.attention.w_k": 0.9323806762695312, "blocks.0.attention.w_v": 0.9162025451660156, "blocks.0.attention.w_out": 0.9151763916015625, "blocks.0.feed_forward.w1": 0.9243609309196472, "blocks.0.feed_forward.w2": 0.9156469106674194, "blocks.0.feed_forward.w3": 0.9221636652946472, "blocks.1.attention.w_q": 0.9319725036621094, "blocks.1.attention.w_k": 0.93182373046875, "blocks.1.attention.w_v": 0.9214859008789062, "blocks.1.attention.w_out": 0.9213447570800781, "blocks.1.feed_forward.w1": 0.9190763235092163, "blocks.1.feed_forward.w2": 0.9120610952377319, "blocks.1.feed_forward.w3": 0.9157320857048035, "blocks.2.attention.w_q": 0.9363517761230469, "blocks.2.attention.w_k": 0.9353103637695312, "blocks.2.attention.w_v": 0.9225883483886719, "blocks.2.attention.w_out": 0.9315185546875, "blocks.2.feed_forward.w1": 0.9190038442611694, "blocks.2.feed_forward.w2": 0.9069430232048035, "blocks.2.feed_forward.w3": 0.9158872365951538, "blocks.3.attention.w_q": 0.9336357116699219, "blocks.3.attention.w_k": 0.9324150085449219, "blocks.3.attention.w_v": 0.918365478515625, "blocks.3.attention.w_out": 0.925384521484375, "blocks.3.feed_forward.w1": 0.9179916381835938, "blocks.3.feed_forward.w2": 0.9081968069076538, "blocks.3.feed_forward.w3": 0.9143270254135132, "blocks.4.attention.w_q": 0.9337539672851562, "blocks.4.attention.w_k": 0.9327583312988281, "blocks.4.attention.w_v": 0.9141921997070312, "blocks.4.attention.w_out": 0.9071846008300781, "blocks.4.feed_forward.w1": 0.9183553457260132, "blocks.4.feed_forward.w2": 0.9051971435546875, "blocks.4.feed_forward.w3": 0.9149755239486694, "blocks.5.attention.w_q": 0.9317474365234375, "blocks.5.attention.w_k": 0.9284629821777344, "blocks.5.attention.w_v": 0.9108505249023438, "blocks.5.attention.w_out": 0.9081039428710938, "blocks.5.feed_forward.w1": 0.9174537658691406, "blocks.5.feed_forward.w2": 0.9008738398551941, "blocks.5.feed_forward.w3": 0.9135932922363281, "blocks.6.attention.w_q": 0.9360618591308594, "blocks.6.attention.w_k": 0.9348678588867188, "blocks.6.attention.w_v": 0.9035606384277344, "blocks.6.attention.w_out": 0.8898887634277344, "blocks.6.feed_forward.w1": 0.9199625849723816, "blocks.6.feed_forward.w2": 0.8972193598747253, "blocks.6.feed_forward.w3": 0.9149068593978882, "blocks.7.attention.w_q": 0.9380531311035156, "blocks.7.attention.w_k": 0.9346466064453125, "blocks.7.attention.w_v": 0.9048423767089844, "blocks.7.attention.w_out": 0.8917083740234375, "blocks.7.feed_forward.w1": 0.9214210510253906, "blocks.7.feed_forward.w2": 0.888763427734375, "blocks.7.feed_forward.w3": 0.9152845144271851, "blocks.8.attention.w_q": 0.937103271484375, "blocks.8.attention.w_k": 0.9346580505371094, "blocks.8.attention.w_v": 0.9039993286132812, "blocks.8.attention.w_out": 0.8924484252929688, "blocks.8.feed_forward.w1": 0.9237162470817566, "blocks.8.feed_forward.w2": 0.8778241872787476, "blocks.8.feed_forward.w3": 0.9152870178222656}} +{"step": 1100, "changed_frac": 0.9131159334101229, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9321250915527344, "blocks.0.attention.w_k": 0.9298057556152344, "blocks.0.attention.w_v": 0.9123916625976562, "blocks.0.attention.w_out": 0.9101982116699219, "blocks.0.feed_forward.w1": 0.9214744567871094, "blocks.0.feed_forward.w2": 0.9132308959960938, "blocks.0.feed_forward.w3": 0.9204444885253906, "blocks.1.attention.w_q": 0.9315299987792969, "blocks.1.attention.w_k": 0.9321098327636719, "blocks.1.attention.w_v": 0.9180030822753906, "blocks.1.attention.w_out": 0.9193801879882812, "blocks.1.feed_forward.w1": 0.9167467951774597, "blocks.1.feed_forward.w2": 0.9106101989746094, "blocks.1.feed_forward.w3": 0.913726806640625, "blocks.2.attention.w_q": 0.9347457885742188, "blocks.2.attention.w_k": 0.9352302551269531, "blocks.2.attention.w_v": 0.9202423095703125, "blocks.2.attention.w_out": 0.9303436279296875, "blocks.2.feed_forward.w1": 0.9170316457748413, "blocks.2.feed_forward.w2": 0.9046567678451538, "blocks.2.feed_forward.w3": 0.9136060476303101, "blocks.3.attention.w_q": 0.9330940246582031, "blocks.3.attention.w_k": 0.9318962097167969, "blocks.3.attention.w_v": 0.91400146484375, "blocks.3.attention.w_out": 0.9226913452148438, "blocks.3.feed_forward.w1": 0.9155998229980469, "blocks.3.feed_forward.w2": 0.9064763784408569, "blocks.3.feed_forward.w3": 0.9112167358398438, "blocks.4.attention.w_q": 0.9342155456542969, "blocks.4.attention.w_k": 0.93157958984375, "blocks.4.attention.w_v": 0.9113693237304688, "blocks.4.attention.w_out": 0.9034690856933594, "blocks.4.feed_forward.w1": 0.9163869619369507, "blocks.4.feed_forward.w2": 0.9035797119140625, "blocks.4.feed_forward.w3": 0.9118906855583191, "blocks.5.attention.w_q": 0.9319076538085938, "blocks.5.attention.w_k": 0.9286537170410156, "blocks.5.attention.w_v": 0.9071998596191406, "blocks.5.attention.w_out": 0.9053535461425781, "blocks.5.feed_forward.w1": 0.9162305593490601, "blocks.5.feed_forward.w2": 0.8976821899414062, "blocks.5.feed_forward.w3": 0.9106191396713257, "blocks.6.attention.w_q": 0.9341392517089844, "blocks.6.attention.w_k": 0.9321746826171875, "blocks.6.attention.w_v": 0.9005012512207031, "blocks.6.attention.w_out": 0.8881263732910156, "blocks.6.feed_forward.w1": 0.9178110957145691, "blocks.6.feed_forward.w2": 0.89410400390625, "blocks.6.feed_forward.w3": 0.9114405512809753, "blocks.7.attention.w_q": 0.9369087219238281, "blocks.7.attention.w_k": 0.9342575073242188, "blocks.7.attention.w_v": 0.9007339477539062, "blocks.7.attention.w_out": 0.885894775390625, "blocks.7.feed_forward.w1": 0.9193763732910156, "blocks.7.feed_forward.w2": 0.8848826289176941, "blocks.7.feed_forward.w3": 0.9122785329818726, "blocks.8.attention.w_q": 0.9356002807617188, "blocks.8.attention.w_k": 0.9347610473632812, "blocks.8.attention.w_v": 0.9006690979003906, "blocks.8.attention.w_out": 0.8868675231933594, "blocks.8.feed_forward.w1": 0.9218699336051941, "blocks.8.feed_forward.w2": 0.8721975088119507, "blocks.8.feed_forward.w3": 0.9122937917709351}} +{"step": 1200, "changed_frac": 0.9113397190713475, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9305496215820312, "blocks.0.attention.w_k": 0.9286117553710938, "blocks.0.attention.w_v": 0.9103050231933594, "blocks.0.attention.w_out": 0.9080276489257812, "blocks.0.feed_forward.w1": 0.9195671081542969, "blocks.0.feed_forward.w2": 0.9110400080680847, "blocks.0.feed_forward.w3": 0.9186452627182007, "blocks.1.attention.w_q": 0.9318695068359375, "blocks.1.attention.w_k": 0.9337539672851562, "blocks.1.attention.w_v": 0.9183425903320312, "blocks.1.attention.w_out": 0.9191436767578125, "blocks.1.feed_forward.w1": 0.9154243469238281, "blocks.1.feed_forward.w2": 0.907660186290741, "blocks.1.feed_forward.w3": 0.9113184809684753, "blocks.2.attention.w_q": 0.9346733093261719, "blocks.2.attention.w_k": 0.9351158142089844, "blocks.2.attention.w_v": 0.9173202514648438, "blocks.2.attention.w_out": 0.9289169311523438, "blocks.2.feed_forward.w1": 0.9141845703125, "blocks.2.feed_forward.w2": 0.9043172597885132, "blocks.2.feed_forward.w3": 0.9113973379135132, "blocks.3.attention.w_q": 0.9318466186523438, "blocks.3.attention.w_k": 0.9301490783691406, "blocks.3.attention.w_v": 0.9125289916992188, "blocks.3.attention.w_out": 0.9209136962890625, "blocks.3.feed_forward.w1": 0.9135564565658569, "blocks.3.feed_forward.w2": 0.9054362177848816, "blocks.3.feed_forward.w3": 0.9090894460678101, "blocks.4.attention.w_q": 0.9333686828613281, "blocks.4.attention.w_k": 0.9314193725585938, "blocks.4.attention.w_v": 0.90924072265625, "blocks.4.attention.w_out": 0.9029006958007812, "blocks.4.feed_forward.w1": 0.9140459895133972, "blocks.4.feed_forward.w2": 0.9023348689079285, "blocks.4.feed_forward.w3": 0.909942626953125, "blocks.5.attention.w_q": 0.9311714172363281, "blocks.5.attention.w_k": 0.9281349182128906, "blocks.5.attention.w_v": 0.9052314758300781, "blocks.5.attention.w_out": 0.9041633605957031, "blocks.5.feed_forward.w1": 0.91412353515625, "blocks.5.feed_forward.w2": 0.89715576171875, "blocks.5.feed_forward.w3": 0.9082908630371094, "blocks.6.attention.w_q": 0.9348602294921875, "blocks.6.attention.w_k": 0.9317779541015625, "blocks.6.attention.w_v": 0.8973045349121094, "blocks.6.attention.w_out": 0.8851547241210938, "blocks.6.feed_forward.w1": 0.9154065847396851, "blocks.6.feed_forward.w2": 0.8917058706283569, "blocks.6.feed_forward.w3": 0.9092521667480469, "blocks.7.attention.w_q": 0.9357986450195312, "blocks.7.attention.w_k": 0.9332504272460938, "blocks.7.attention.w_v": 0.8987312316894531, "blocks.7.attention.w_out": 0.8821792602539062, "blocks.7.feed_forward.w1": 0.9171689748764038, "blocks.7.feed_forward.w2": 0.8826459646224976, "blocks.7.feed_forward.w3": 0.9112676382064819, "blocks.8.attention.w_q": 0.9352149963378906, "blocks.8.attention.w_k": 0.9333686828613281, "blocks.8.attention.w_v": 0.89715576171875, "blocks.8.attention.w_out": 0.8823013305664062, "blocks.8.feed_forward.w1": 0.9201278686523438, "blocks.8.feed_forward.w2": 0.8684349060058594, "blocks.8.feed_forward.w3": 0.9104436635971069}} +{"step": 1300, "changed_frac": 0.9100289304032285, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9289512634277344, "blocks.0.attention.w_k": 0.9274444580078125, "blocks.0.attention.w_v": 0.9073410034179688, "blocks.0.attention.w_out": 0.9048118591308594, "blocks.0.feed_forward.w1": 0.9173940420150757, "blocks.0.feed_forward.w2": 0.9099210500717163, "blocks.0.feed_forward.w3": 0.9170278310775757, "blocks.1.attention.w_q": 0.931640625, "blocks.1.attention.w_k": 0.9313011169433594, "blocks.1.attention.w_v": 0.9158935546875, "blocks.1.attention.w_out": 0.9170150756835938, "blocks.1.feed_forward.w1": 0.913909912109375, "blocks.1.feed_forward.w2": 0.9070816040039062, "blocks.1.feed_forward.w3": 0.9103279113769531, "blocks.2.attention.w_q": 0.9331741333007812, "blocks.2.attention.w_k": 0.9337272644042969, "blocks.2.attention.w_v": 0.9173011779785156, "blocks.2.attention.w_out": 0.9278373718261719, "blocks.2.feed_forward.w1": 0.9139722585678101, "blocks.2.feed_forward.w2": 0.9034335017204285, "blocks.2.feed_forward.w3": 0.9104347229003906, "blocks.3.attention.w_q": 0.9317741394042969, "blocks.3.attention.w_k": 0.9302024841308594, "blocks.3.attention.w_v": 0.9103851318359375, "blocks.3.attention.w_out": 0.9186515808105469, "blocks.3.feed_forward.w1": 0.9126104116439819, "blocks.3.feed_forward.w2": 0.9043058156967163, "blocks.3.feed_forward.w3": 0.9079208374023438, "blocks.4.attention.w_q": 0.9328804016113281, "blocks.4.attention.w_k": 0.9298019409179688, "blocks.4.attention.w_v": 0.9065589904785156, "blocks.4.attention.w_out": 0.9011154174804688, "blocks.4.feed_forward.w1": 0.9129409790039062, "blocks.4.feed_forward.w2": 0.9019101858139038, "blocks.4.feed_forward.w3": 0.9076589345932007, "blocks.5.attention.w_q": 0.9295082092285156, "blocks.5.attention.w_k": 0.9271965026855469, "blocks.5.attention.w_v": 0.9028282165527344, "blocks.5.attention.w_out": 0.9040641784667969, "blocks.5.feed_forward.w1": 0.9127769470214844, "blocks.5.feed_forward.w2": 0.8959274291992188, "blocks.5.feed_forward.w3": 0.9069722890853882, "blocks.6.attention.w_q": 0.9326744079589844, "blocks.6.attention.w_k": 0.9310111999511719, "blocks.6.attention.w_v": 0.8949928283691406, "blocks.6.attention.w_out": 0.8827323913574219, "blocks.6.feed_forward.w1": 0.9145113825798035, "blocks.6.feed_forward.w2": 0.8907915949821472, "blocks.6.feed_forward.w3": 0.9085604548454285, "blocks.7.attention.w_q": 0.9352149963378906, "blocks.7.attention.w_k": 0.9315605163574219, "blocks.7.attention.w_v": 0.8968505859375, "blocks.7.attention.w_out": 0.8791694641113281, "blocks.7.feed_forward.w1": 0.9166806936264038, "blocks.7.feed_forward.w2": 0.8812446594238281, "blocks.7.feed_forward.w3": 0.9092127680778503, "blocks.8.attention.w_q": 0.9349784851074219, "blocks.8.attention.w_k": 0.9325714111328125, "blocks.8.attention.w_v": 0.8950309753417969, "blocks.8.attention.w_out": 0.8810272216796875, "blocks.8.feed_forward.w1": 0.9197807312011719, "blocks.8.feed_forward.w2": 0.8654696345329285, "blocks.8.feed_forward.w3": 0.9086100459098816}} +{"step": 1400, "changed_frac": 0.9088740552592481, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9285202026367188, "blocks.0.attention.w_k": 0.9264640808105469, "blocks.0.attention.w_v": 0.904510498046875, "blocks.0.attention.w_out": 0.9023361206054688, "blocks.0.feed_forward.w1": 0.9164085388183594, "blocks.0.feed_forward.w2": 0.90863037109375, "blocks.0.feed_forward.w3": 0.9154345393180847, "blocks.1.attention.w_q": 0.9308013916015625, "blocks.1.attention.w_k": 0.9316444396972656, "blocks.1.attention.w_v": 0.9154472351074219, "blocks.1.attention.w_out": 0.9169120788574219, "blocks.1.feed_forward.w1": 0.912353515625, "blocks.1.feed_forward.w2": 0.9061126708984375, "blocks.1.feed_forward.w3": 0.908941924571991, "blocks.2.attention.w_q": 0.9327011108398438, "blocks.2.attention.w_k": 0.9325523376464844, "blocks.2.attention.w_v": 0.9157562255859375, "blocks.2.attention.w_out": 0.9276771545410156, "blocks.2.feed_forward.w1": 0.9121831655502319, "blocks.2.feed_forward.w2": 0.9015337824821472, "blocks.2.feed_forward.w3": 0.9093284606933594, "blocks.3.attention.w_q": 0.9297943115234375, "blocks.3.attention.w_k": 0.9294891357421875, "blocks.3.attention.w_v": 0.909149169921875, "blocks.3.attention.w_out": 0.9184074401855469, "blocks.3.feed_forward.w1": 0.9103533625602722, "blocks.3.feed_forward.w2": 0.9029401540756226, "blocks.3.feed_forward.w3": 0.9061521291732788, "blocks.4.attention.w_q": 0.9311981201171875, "blocks.4.attention.w_k": 0.92999267578125, "blocks.4.attention.w_v": 0.9047927856445312, "blocks.4.attention.w_out": 0.9003143310546875, "blocks.4.feed_forward.w1": 0.911078155040741, "blocks.4.feed_forward.w2": 0.9003868103027344, "blocks.4.feed_forward.w3": 0.9072901606559753, "blocks.5.attention.w_q": 0.9305953979492188, "blocks.5.attention.w_k": 0.9257698059082031, "blocks.5.attention.w_v": 0.9014739990234375, "blocks.5.attention.w_out": 0.9022750854492188, "blocks.5.feed_forward.w1": 0.9125328063964844, "blocks.5.feed_forward.w2": 0.894964873790741, "blocks.5.feed_forward.w3": 0.9066276550292969, "blocks.6.attention.w_q": 0.93328857421875, "blocks.6.attention.w_k": 0.9300575256347656, "blocks.6.attention.w_v": 0.8948631286621094, "blocks.6.attention.w_out": 0.8811187744140625, "blocks.6.feed_forward.w1": 0.9134470820426941, "blocks.6.feed_forward.w2": 0.8907750844955444, "blocks.6.feed_forward.w3": 0.9076678156852722, "blocks.7.attention.w_q": 0.9347610473632812, "blocks.7.attention.w_k": 0.9313812255859375, "blocks.7.attention.w_v": 0.8950729370117188, "blocks.7.attention.w_out": 0.8777999877929688, "blocks.7.feed_forward.w1": 0.9158592224121094, "blocks.7.feed_forward.w2": 0.8792966604232788, "blocks.7.feed_forward.w3": 0.9083760976791382, "blocks.8.attention.w_q": 0.9347915649414062, "blocks.8.attention.w_k": 0.933380126953125, "blocks.8.attention.w_v": 0.893463134765625, "blocks.8.attention.w_out": 0.8801307678222656, "blocks.8.feed_forward.w1": 0.917395293712616, "blocks.8.feed_forward.w2": 0.8623530268669128, "blocks.8.feed_forward.w3": 0.9081039428710938}} +{"step": 1500, "changed_frac": 0.907914283948067, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9277000427246094, "blocks.0.attention.w_k": 0.9255447387695312, "blocks.0.attention.w_v": 0.9022903442382812, "blocks.0.attention.w_out": 0.9001426696777344, "blocks.0.feed_forward.w1": 0.915435791015625, "blocks.0.feed_forward.w2": 0.9078077077865601, "blocks.0.feed_forward.w3": 0.9143702387809753, "blocks.1.attention.w_q": 0.9308700561523438, "blocks.1.attention.w_k": 0.9310302734375, "blocks.1.attention.w_v": 0.9144859313964844, "blocks.1.attention.w_out": 0.9163436889648438, "blocks.1.feed_forward.w1": 0.9119619131088257, "blocks.1.feed_forward.w2": 0.9053192138671875, "blocks.1.feed_forward.w3": 0.9081026911735535, "blocks.2.attention.w_q": 0.9311027526855469, "blocks.2.attention.w_k": 0.9320220947265625, "blocks.2.attention.w_v": 0.9155464172363281, "blocks.2.attention.w_out": 0.9277153015136719, "blocks.2.feed_forward.w1": 0.9118983149528503, "blocks.2.feed_forward.w2": 0.9013049006462097, "blocks.2.feed_forward.w3": 0.9074084162712097, "blocks.3.attention.w_q": 0.9302291870117188, "blocks.3.attention.w_k": 0.92840576171875, "blocks.3.attention.w_v": 0.9068489074707031, "blocks.3.attention.w_out": 0.9172096252441406, "blocks.3.feed_forward.w1": 0.9099159240722656, "blocks.3.feed_forward.w2": 0.9029172658920288, "blocks.3.feed_forward.w3": 0.9055086970329285, "blocks.4.attention.w_q": 0.9309310913085938, "blocks.4.attention.w_k": 0.9297866821289062, "blocks.4.attention.w_v": 0.9009971618652344, "blocks.4.attention.w_out": 0.9006462097167969, "blocks.4.feed_forward.w1": 0.9107691645622253, "blocks.4.feed_forward.w2": 0.8995780944824219, "blocks.4.feed_forward.w3": 0.9062551259994507, "blocks.5.attention.w_q": 0.9291877746582031, "blocks.5.attention.w_k": 0.9253387451171875, "blocks.5.attention.w_v": 0.8991851806640625, "blocks.5.attention.w_out": 0.9021339416503906, "blocks.5.feed_forward.w1": 0.9112104177474976, "blocks.5.feed_forward.w2": 0.8942896723747253, "blocks.5.feed_forward.w3": 0.9056002497673035, "blocks.6.attention.w_q": 0.9324722290039062, "blocks.6.attention.w_k": 0.9289817810058594, "blocks.6.attention.w_v": 0.893890380859375, "blocks.6.attention.w_out": 0.8818473815917969, "blocks.6.feed_forward.w1": 0.9126968383789062, "blocks.6.feed_forward.w2": 0.8885320425033569, "blocks.6.feed_forward.w3": 0.9059461355209351, "blocks.7.attention.w_q": 0.9328422546386719, "blocks.7.attention.w_k": 0.9306144714355469, "blocks.7.attention.w_v": 0.893035888671875, "blocks.7.attention.w_out": 0.8754539489746094, "blocks.7.feed_forward.w1": 0.9148814082145691, "blocks.7.feed_forward.w2": 0.8773676753044128, "blocks.7.feed_forward.w3": 0.9071935415267944, "blocks.8.attention.w_q": 0.9331588745117188, "blocks.8.attention.w_k": 0.93243408203125, "blocks.8.attention.w_v": 0.8922004699707031, "blocks.8.attention.w_out": 0.8795242309570312, "blocks.8.feed_forward.w1": 0.9169934988021851, "blocks.8.feed_forward.w2": 0.8613319396972656, "blocks.8.feed_forward.w3": 0.90667724609375}} +{"step": 1600, "changed_frac": 0.9069725101829594, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9272003173828125, "blocks.0.attention.w_k": 0.9235992431640625, "blocks.0.attention.w_v": 0.9011077880859375, "blocks.0.attention.w_out": 0.8994369506835938, "blocks.0.feed_forward.w1": 0.9146347045898438, "blocks.0.feed_forward.w2": 0.9073156118392944, "blocks.0.feed_forward.w3": 0.9132322072982788, "blocks.1.attention.w_q": 0.9302330017089844, "blocks.1.attention.w_k": 0.9293212890625, "blocks.1.attention.w_v": 0.9136695861816406, "blocks.1.attention.w_out": 0.9168815612792969, "blocks.1.feed_forward.w1": 0.9109281301498413, "blocks.1.feed_forward.w2": 0.9040044546127319, "blocks.1.feed_forward.w3": 0.9070943593978882, "blocks.2.attention.w_q": 0.9310646057128906, "blocks.2.attention.w_k": 0.931182861328125, "blocks.2.attention.w_v": 0.9145622253417969, "blocks.2.attention.w_out": 0.9275131225585938, "blocks.2.feed_forward.w1": 0.9103991389274597, "blocks.2.feed_forward.w2": 0.9008840322494507, "blocks.2.feed_forward.w3": 0.906707763671875, "blocks.3.attention.w_q": 0.9290542602539062, "blocks.3.attention.w_k": 0.9276962280273438, "blocks.3.attention.w_v": 0.9064559936523438, "blocks.3.attention.w_out": 0.9161911010742188, "blocks.3.feed_forward.w1": 0.9087638854980469, "blocks.3.feed_forward.w2": 0.9013570547103882, "blocks.3.feed_forward.w3": 0.9051069021224976, "blocks.4.attention.w_q": 0.9301033020019531, "blocks.4.attention.w_k": 0.9295730590820312, "blocks.4.attention.w_v": 0.90167236328125, "blocks.4.attention.w_out": 0.8997840881347656, "blocks.4.feed_forward.w1": 0.9100532531738281, "blocks.4.feed_forward.w2": 0.8991292715072632, "blocks.4.feed_forward.w3": 0.9052404165267944, "blocks.5.attention.w_q": 0.9283523559570312, "blocks.5.attention.w_k": 0.9262542724609375, "blocks.5.attention.w_v": 0.8977127075195312, "blocks.5.attention.w_out": 0.9006004333496094, "blocks.5.feed_forward.w1": 0.9107894897460938, "blocks.5.feed_forward.w2": 0.8929786682128906, "blocks.5.feed_forward.w3": 0.9038467407226562, "blocks.6.attention.w_q": 0.93182373046875, "blocks.6.attention.w_k": 0.9286384582519531, "blocks.6.attention.w_v": 0.8923416137695312, "blocks.6.attention.w_out": 0.8814277648925781, "blocks.6.feed_forward.w1": 0.912542998790741, "blocks.6.feed_forward.w2": 0.8875986933708191, "blocks.6.feed_forward.w3": 0.9048640131950378, "blocks.7.attention.w_q": 0.933074951171875, "blocks.7.attention.w_k": 0.9300346374511719, "blocks.7.attention.w_v": 0.8916893005371094, "blocks.7.attention.w_out": 0.8736457824707031, "blocks.7.feed_forward.w1": 0.913763701915741, "blocks.7.feed_forward.w2": 0.8762804865837097, "blocks.7.feed_forward.w3": 0.906524658203125, "blocks.8.attention.w_q": 0.9332733154296875, "blocks.8.attention.w_k": 0.9309501647949219, "blocks.8.attention.w_v": 0.8900260925292969, "blocks.8.attention.w_out": 0.8761062622070312, "blocks.8.feed_forward.w1": 0.9163081049919128, "blocks.8.feed_forward.w2": 0.8585192561149597, "blocks.8.feed_forward.w3": 0.9056422114372253}} +{"step": 1700, "changed_frac": 0.9060202052450588, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9247856140136719, "blocks.0.attention.w_k": 0.9240036010742188, "blocks.0.attention.w_v": 0.8990287780761719, "blocks.0.attention.w_out": 0.8971176147460938, "blocks.0.feed_forward.w1": 0.9139035940170288, "blocks.0.feed_forward.w2": 0.9063529968261719, "blocks.0.feed_forward.w3": 0.9134203791618347, "blocks.1.attention.w_q": 0.9280929565429688, "blocks.1.attention.w_k": 0.9289169311523438, "blocks.1.attention.w_v": 0.9120254516601562, "blocks.1.attention.w_out": 0.9172019958496094, "blocks.1.feed_forward.w1": 0.9099706411361694, "blocks.1.feed_forward.w2": 0.9038912653923035, "blocks.1.feed_forward.w3": 0.9065093994140625, "blocks.2.attention.w_q": 0.9308700561523438, "blocks.2.attention.w_k": 0.9299392700195312, "blocks.2.attention.w_v": 0.913543701171875, "blocks.2.attention.w_out": 0.9266548156738281, "blocks.2.feed_forward.w1": 0.9091567993164062, "blocks.2.feed_forward.w2": 0.8989944458007812, "blocks.2.feed_forward.w3": 0.905859649181366, "blocks.3.attention.w_q": 0.9276809692382812, "blocks.3.attention.w_k": 0.9274253845214844, "blocks.3.attention.w_v": 0.9053153991699219, "blocks.3.attention.w_out": 0.9150810241699219, "blocks.3.feed_forward.w1": 0.9081077575683594, "blocks.3.feed_forward.w2": 0.9003576040267944, "blocks.3.feed_forward.w3": 0.9033622741699219, "blocks.4.attention.w_q": 0.9284858703613281, "blocks.4.attention.w_k": 0.9284286499023438, "blocks.4.attention.w_v": 0.9009056091308594, "blocks.4.attention.w_out": 0.8998031616210938, "blocks.4.feed_forward.w1": 0.9092916250228882, "blocks.4.feed_forward.w2": 0.8967196345329285, "blocks.4.feed_forward.w3": 0.9039281606674194, "blocks.5.attention.w_q": 0.9281272888183594, "blocks.5.attention.w_k": 0.9254112243652344, "blocks.5.attention.w_v": 0.89715576171875, "blocks.5.attention.w_out": 0.9011268615722656, "blocks.5.feed_forward.w1": 0.9101702570915222, "blocks.5.feed_forward.w2": 0.892486572265625, "blocks.5.feed_forward.w3": 0.9025344848632812, "blocks.6.attention.w_q": 0.9313545227050781, "blocks.6.attention.w_k": 0.9278297424316406, "blocks.6.attention.w_v": 0.8903045654296875, "blocks.6.attention.w_out": 0.8801231384277344, "blocks.6.feed_forward.w1": 0.9117559194564819, "blocks.6.feed_forward.w2": 0.8874689936637878, "blocks.6.feed_forward.w3": 0.9039701223373413, "blocks.7.attention.w_q": 0.9323616027832031, "blocks.7.attention.w_k": 0.9292411804199219, "blocks.7.attention.w_v": 0.8906135559082031, "blocks.7.attention.w_out": 0.8739509582519531, "blocks.7.feed_forward.w1": 0.9130706787109375, "blocks.7.feed_forward.w2": 0.8752543330192566, "blocks.7.feed_forward.w3": 0.9046758413314819, "blocks.8.attention.w_q": 0.9324150085449219, "blocks.8.attention.w_k": 0.9301605224609375, "blocks.8.attention.w_v": 0.8896026611328125, "blocks.8.attention.w_out": 0.874420166015625, "blocks.8.feed_forward.w1": 0.9148432612419128, "blocks.8.feed_forward.w2": 0.8579063415527344, "blocks.8.feed_forward.w3": 0.9043235778808594}} +{"step": 1800, "changed_frac": 0.905346438416049, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9249954223632812, "blocks.0.attention.w_k": 0.9218559265136719, "blocks.0.attention.w_v": 0.8983306884765625, "blocks.0.attention.w_out": 0.8954124450683594, "blocks.0.feed_forward.w1": 0.9129511713981628, "blocks.0.feed_forward.w2": 0.9056129455566406, "blocks.0.feed_forward.w3": 0.9119097590446472, "blocks.1.attention.w_q": 0.9276237487792969, "blocks.1.attention.w_k": 0.9280738830566406, "blocks.1.attention.w_v": 0.9129409790039062, "blocks.1.attention.w_out": 0.9168624877929688, "blocks.1.feed_forward.w1": 0.9089902639389038, "blocks.1.feed_forward.w2": 0.9026756286621094, "blocks.1.feed_forward.w3": 0.9057032465934753, "blocks.2.attention.w_q": 0.9303741455078125, "blocks.2.attention.w_k": 0.9288749694824219, "blocks.2.attention.w_v": 0.9146537780761719, "blocks.2.attention.w_out": 0.9262199401855469, "blocks.2.feed_forward.w1": 0.9094480276107788, "blocks.2.feed_forward.w2": 0.8983573913574219, "blocks.2.feed_forward.w3": 0.9052976369857788, "blocks.3.attention.w_q": 0.9267807006835938, "blocks.3.attention.w_k": 0.9265861511230469, "blocks.3.attention.w_v": 0.9038581848144531, "blocks.3.attention.w_out": 0.9142379760742188, "blocks.3.feed_forward.w1": 0.9068374633789062, "blocks.3.feed_forward.w2": 0.9010162353515625, "blocks.3.feed_forward.w3": 0.9031473994255066, "blocks.4.attention.w_q": 0.92779541015625, "blocks.4.attention.w_k": 0.9269447326660156, "blocks.4.attention.w_v": 0.899688720703125, "blocks.4.attention.w_out": 0.9001655578613281, "blocks.4.feed_forward.w1": 0.9090919494628906, "blocks.4.feed_forward.w2": 0.8976987600326538, "blocks.4.feed_forward.w3": 0.9034830927848816, "blocks.5.attention.w_q": 0.927459716796875, "blocks.5.attention.w_k": 0.9251708984375, "blocks.5.attention.w_v": 0.8960800170898438, "blocks.5.attention.w_out": 0.9003067016601562, "blocks.5.feed_forward.w1": 0.9089622497558594, "blocks.5.feed_forward.w2": 0.8924522399902344, "blocks.5.feed_forward.w3": 0.901800811290741, "blocks.6.attention.w_q": 0.9313011169433594, "blocks.6.attention.w_k": 0.9273300170898438, "blocks.6.attention.w_v": 0.8896369934082031, "blocks.6.attention.w_out": 0.8805885314941406, "blocks.6.feed_forward.w1": 0.9111137390136719, "blocks.6.feed_forward.w2": 0.8865877985954285, "blocks.6.feed_forward.w3": 0.9034881591796875, "blocks.7.attention.w_q": 0.9323692321777344, "blocks.7.attention.w_k": 0.9289131164550781, "blocks.7.attention.w_v": 0.8900718688964844, "blocks.7.attention.w_out": 0.8718147277832031, "blocks.7.feed_forward.w1": 0.9124959707260132, "blocks.7.feed_forward.w2": 0.8735135793685913, "blocks.7.feed_forward.w3": 0.9042803645133972, "blocks.8.attention.w_q": 0.9319801330566406, "blocks.8.attention.w_k": 0.9289054870605469, "blocks.8.attention.w_v": 0.8883934020996094, "blocks.8.attention.w_out": 0.8728599548339844, "blocks.8.feed_forward.w1": 0.9137382507324219, "blocks.8.feed_forward.w2": 0.8564949035644531, "blocks.8.feed_forward.w3": 0.9028765559196472}} +{"step": 1900, "changed_frac": 0.9043384780231704, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9231300354003906, "blocks.0.attention.w_k": 0.9199256896972656, "blocks.0.attention.w_v": 0.8958549499511719, "blocks.0.attention.w_out": 0.8930206298828125, "blocks.0.feed_forward.w1": 0.9121932983398438, "blocks.0.feed_forward.w2": 0.9048246145248413, "blocks.0.feed_forward.w3": 0.9108937978744507, "blocks.1.attention.w_q": 0.9259262084960938, "blocks.1.attention.w_k": 0.9271926879882812, "blocks.1.attention.w_v": 0.909637451171875, "blocks.1.attention.w_out": 0.9163017272949219, "blocks.1.feed_forward.w1": 0.9081230163574219, "blocks.1.feed_forward.w2": 0.9024556875228882, "blocks.1.feed_forward.w3": 0.9045054316520691, "blocks.2.attention.w_q": 0.9297256469726562, "blocks.2.attention.w_k": 0.9282264709472656, "blocks.2.attention.w_v": 0.9130401611328125, "blocks.2.attention.w_out": 0.9254608154296875, "blocks.2.feed_forward.w1": 0.9072138667106628, "blocks.2.feed_forward.w2": 0.8977648615837097, "blocks.2.feed_forward.w3": 0.9040769338607788, "blocks.3.attention.w_q": 0.9258956909179688, "blocks.3.attention.w_k": 0.9254226684570312, "blocks.3.attention.w_v": 0.901702880859375, "blocks.3.attention.w_out": 0.9137763977050781, "blocks.3.feed_forward.w1": 0.90673828125, "blocks.3.feed_forward.w2": 0.9001007080078125, "blocks.3.feed_forward.w3": 0.9014549255371094, "blocks.4.attention.w_q": 0.9275665283203125, "blocks.4.attention.w_k": 0.9271354675292969, "blocks.4.attention.w_v": 0.8977546691894531, "blocks.4.attention.w_out": 0.8987350463867188, "blocks.4.feed_forward.w1": 0.908453643321991, "blocks.4.feed_forward.w2": 0.8972434997558594, "blocks.4.feed_forward.w3": 0.9012159109115601, "blocks.5.attention.w_q": 0.9269943237304688, "blocks.5.attention.w_k": 0.9249000549316406, "blocks.5.attention.w_v": 0.8947410583496094, "blocks.5.attention.w_out": 0.8984603881835938, "blocks.5.feed_forward.w1": 0.9078852534294128, "blocks.5.feed_forward.w2": 0.8915380239486694, "blocks.5.feed_forward.w3": 0.90167236328125, "blocks.6.attention.w_q": 0.9299888610839844, "blocks.6.attention.w_k": 0.9258995056152344, "blocks.6.attention.w_v": 0.8893890380859375, "blocks.6.attention.w_out": 0.88043212890625, "blocks.6.feed_forward.w1": 0.9101384878158569, "blocks.6.feed_forward.w2": 0.8858617544174194, "blocks.6.feed_forward.w3": 0.9025281667709351, "blocks.7.attention.w_q": 0.931121826171875, "blocks.7.attention.w_k": 0.9272422790527344, "blocks.7.attention.w_v": 0.8880043029785156, "blocks.7.attention.w_out": 0.8705253601074219, "blocks.7.feed_forward.w1": 0.9126485586166382, "blocks.7.feed_forward.w2": 0.8727353811264038, "blocks.7.feed_forward.w3": 0.9036840200424194, "blocks.8.attention.w_q": 0.9314002990722656, "blocks.8.attention.w_k": 0.9276046752929688, "blocks.8.attention.w_v": 0.887237548828125, "blocks.8.attention.w_out": 0.8731803894042969, "blocks.8.feed_forward.w1": 0.9124488830566406, "blocks.8.feed_forward.w2": 0.8540751338005066, "blocks.8.feed_forward.w3": 0.9025421142578125}} +{"step": 2000, "changed_frac": 0.9034646515153412, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9212913513183594, "blocks.0.attention.w_k": 0.9201889038085938, "blocks.0.attention.w_v": 0.8947525024414062, "blocks.0.attention.w_out": 0.8912925720214844, "blocks.0.feed_forward.w1": 0.9114837646484375, "blocks.0.feed_forward.w2": 0.9039230346679688, "blocks.0.feed_forward.w3": 0.9101740717887878, "blocks.1.attention.w_q": 0.9257583618164062, "blocks.1.attention.w_k": 0.9275741577148438, "blocks.1.attention.w_v": 0.9095878601074219, "blocks.1.attention.w_out": 0.9154129028320312, "blocks.1.feed_forward.w1": 0.9073423147201538, "blocks.1.feed_forward.w2": 0.9021098017692566, "blocks.1.feed_forward.w3": 0.9038073420524597, "blocks.2.attention.w_q": 0.9272232055664062, "blocks.2.attention.w_k": 0.9273414611816406, "blocks.2.attention.w_v": 0.9111442565917969, "blocks.2.attention.w_out": 0.9262542724609375, "blocks.2.feed_forward.w1": 0.9068222045898438, "blocks.2.feed_forward.w2": 0.8974736928939819, "blocks.2.feed_forward.w3": 0.9032186269760132, "blocks.3.attention.w_q": 0.9251060485839844, "blocks.3.attention.w_k": 0.9241409301757812, "blocks.3.attention.w_v": 0.900634765625, "blocks.3.attention.w_out": 0.9128646850585938, "blocks.3.feed_forward.w1": 0.9058710932731628, "blocks.3.feed_forward.w2": 0.8998743891716003, "blocks.3.feed_forward.w3": 0.9014104604721069, "blocks.4.attention.w_q": 0.9270172119140625, "blocks.4.attention.w_k": 0.9239501953125, "blocks.4.attention.w_v": 0.8954658508300781, "blocks.4.attention.w_out": 0.8976554870605469, "blocks.4.feed_forward.w1": 0.9069633483886719, "blocks.4.feed_forward.w2": 0.8961970210075378, "blocks.4.feed_forward.w3": 0.901220977306366, "blocks.5.attention.w_q": 0.9260940551757812, "blocks.5.attention.w_k": 0.92236328125, "blocks.5.attention.w_v": 0.8937263488769531, "blocks.5.attention.w_out": 0.8971939086914062, "blocks.5.feed_forward.w1": 0.9070295095443726, "blocks.5.feed_forward.w2": 0.8910955190658569, "blocks.5.feed_forward.w3": 0.90087890625, "blocks.6.attention.w_q": 0.9280128479003906, "blocks.6.attention.w_k": 0.9254302978515625, "blocks.6.attention.w_v": 0.8872032165527344, "blocks.6.attention.w_out": 0.8783912658691406, "blocks.6.feed_forward.w1": 0.9093920588493347, "blocks.6.feed_forward.w2": 0.8842176198959351, "blocks.6.feed_forward.w3": 0.9012540578842163, "blocks.7.attention.w_q": 0.929656982421875, "blocks.7.attention.w_k": 0.9279022216796875, "blocks.7.attention.w_v": 0.8878021240234375, "blocks.7.attention.w_out": 0.8710594177246094, "blocks.7.feed_forward.w1": 0.9108709096908569, "blocks.7.feed_forward.w2": 0.8708839416503906, "blocks.7.feed_forward.w3": 0.9023221731185913, "blocks.8.attention.w_q": 0.93109130859375, "blocks.8.attention.w_k": 0.9268684387207031, "blocks.8.attention.w_v": 0.8852615356445312, "blocks.8.attention.w_out": 0.8724288940429688, "blocks.8.feed_forward.w1": 0.9124667048454285, "blocks.8.feed_forward.w2": 0.8530769348144531, "blocks.8.feed_forward.w3": 0.9020271301269531}} +{"step": 2100, "changed_frac": 0.9025468092698318, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9213104248046875, "blocks.0.attention.w_k": 0.9174232482910156, "blocks.0.attention.w_v": 0.8923377990722656, "blocks.0.attention.w_out": 0.8900222778320312, "blocks.0.feed_forward.w1": 0.9108301997184753, "blocks.0.feed_forward.w2": 0.9033737182617188, "blocks.0.feed_forward.w3": 0.9086965322494507, "blocks.1.attention.w_q": 0.92498779296875, "blocks.1.attention.w_k": 0.9265594482421875, "blocks.1.attention.w_v": 0.9090347290039062, "blocks.1.attention.w_out": 0.9147720336914062, "blocks.1.feed_forward.w1": 0.9069048762321472, "blocks.1.feed_forward.w2": 0.9012998342514038, "blocks.1.feed_forward.w3": 0.9024264216423035, "blocks.2.attention.w_q": 0.9281463623046875, "blocks.2.attention.w_k": 0.9274826049804688, "blocks.2.attention.w_v": 0.9109458923339844, "blocks.2.attention.w_out": 0.925262451171875, "blocks.2.feed_forward.w1": 0.9055659174919128, "blocks.2.feed_forward.w2": 0.8969027400016785, "blocks.2.feed_forward.w3": 0.9020742177963257, "blocks.3.attention.w_q": 0.9239883422851562, "blocks.3.attention.w_k": 0.9226837158203125, "blocks.3.attention.w_v": 0.9010391235351562, "blocks.3.attention.w_out": 0.9105644226074219, "blocks.3.feed_forward.w1": 0.9052734375, "blocks.3.feed_forward.w2": 0.8983535766601562, "blocks.3.feed_forward.w3": 0.9000511169433594, "blocks.4.attention.w_q": 0.9261245727539062, "blocks.4.attention.w_k": 0.9253501892089844, "blocks.4.attention.w_v": 0.89471435546875, "blocks.4.attention.w_out": 0.8971748352050781, "blocks.4.feed_forward.w1": 0.9067205190658569, "blocks.4.feed_forward.w2": 0.8962377309799194, "blocks.4.feed_forward.w3": 0.9007619619369507, "blocks.5.attention.w_q": 0.9241447448730469, "blocks.5.attention.w_k": 0.9215812683105469, "blocks.5.attention.w_v": 0.8920555114746094, "blocks.5.attention.w_out": 0.8955535888671875, "blocks.5.feed_forward.w1": 0.9063822627067566, "blocks.5.feed_forward.w2": 0.8898748159408569, "blocks.5.feed_forward.w3": 0.8996353149414062, "blocks.6.attention.w_q": 0.9271354675292969, "blocks.6.attention.w_k": 0.9241294860839844, "blocks.6.attention.w_v": 0.8852500915527344, "blocks.6.attention.w_out": 0.8782997131347656, "blocks.6.feed_forward.w1": 0.9081293940544128, "blocks.6.feed_forward.w2": 0.8839709162712097, "blocks.6.feed_forward.w3": 0.9009780883789062, "blocks.7.attention.w_q": 0.9289970397949219, "blocks.7.attention.w_k": 0.9268569946289062, "blocks.7.attention.w_v": 0.8870849609375, "blocks.7.attention.w_out": 0.8685760498046875, "blocks.7.feed_forward.w1": 0.9093946218490601, "blocks.7.feed_forward.w2": 0.8692054748535156, "blocks.7.feed_forward.w3": 0.9020487666130066, "blocks.8.attention.w_q": 0.9289894104003906, "blocks.8.attention.w_k": 0.9274444580078125, "blocks.8.attention.w_v": 0.8849029541015625, "blocks.8.attention.w_out": 0.8699798583984375, "blocks.8.feed_forward.w1": 0.9118499755859375, "blocks.8.feed_forward.w2": 0.8512891530990601, "blocks.8.feed_forward.w3": 0.9007924795150757}} +{"step": 2200, "changed_frac": 0.9014218322232238, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9195404052734375, "blocks.0.attention.w_k": 0.916900634765625, "blocks.0.attention.w_v": 0.8898811340332031, "blocks.0.attention.w_out": 0.8876152038574219, "blocks.0.feed_forward.w1": 0.9088592529296875, "blocks.0.feed_forward.w2": 0.9019279479980469, "blocks.0.feed_forward.w3": 0.9066645503044128, "blocks.1.attention.w_q": 0.9250259399414062, "blocks.1.attention.w_k": 0.92608642578125, "blocks.1.attention.w_v": 0.9091224670410156, "blocks.1.attention.w_out": 0.9148826599121094, "blocks.1.feed_forward.w1": 0.9060770869255066, "blocks.1.feed_forward.w2": 0.8995755910873413, "blocks.1.feed_forward.w3": 0.9017728567123413, "blocks.2.attention.w_q": 0.9268798828125, "blocks.2.attention.w_k": 0.9259567260742188, "blocks.2.attention.w_v": 0.9090194702148438, "blocks.2.attention.w_out": 0.9241371154785156, "blocks.2.feed_forward.w1": 0.9048449397087097, "blocks.2.feed_forward.w2": 0.8956769704818726, "blocks.2.feed_forward.w3": 0.9005622863769531, "blocks.3.attention.w_q": 0.9235267639160156, "blocks.3.attention.w_k": 0.921783447265625, "blocks.3.attention.w_v": 0.8992462158203125, "blocks.3.attention.w_out": 0.9106063842773438, "blocks.3.feed_forward.w1": 0.9046033620834351, "blocks.3.feed_forward.w2": 0.8977826833724976, "blocks.3.feed_forward.w3": 0.8993123769760132, "blocks.4.attention.w_q": 0.925201416015625, "blocks.4.attention.w_k": 0.9240188598632812, "blocks.4.attention.w_v": 0.893798828125, "blocks.4.attention.w_out": 0.8968124389648438, "blocks.4.feed_forward.w1": 0.9051348567008972, "blocks.4.feed_forward.w2": 0.89385986328125, "blocks.4.feed_forward.w3": 0.8986663818359375, "blocks.5.attention.w_q": 0.9232177734375, "blocks.5.attention.w_k": 0.9211769104003906, "blocks.5.attention.w_v": 0.8903656005859375, "blocks.5.attention.w_out": 0.8955039978027344, "blocks.5.feed_forward.w1": 0.9059181213378906, "blocks.5.feed_forward.w2": 0.8883921504020691, "blocks.5.feed_forward.w3": 0.8990834951400757, "blocks.6.attention.w_q": 0.9269485473632812, "blocks.6.attention.w_k": 0.9233970642089844, "blocks.6.attention.w_v": 0.8847198486328125, "blocks.6.attention.w_out": 0.8777503967285156, "blocks.6.feed_forward.w1": 0.9074720144271851, "blocks.6.feed_forward.w2": 0.8827527761459351, "blocks.6.feed_forward.w3": 0.89971923828125, "blocks.7.attention.w_q": 0.9277305603027344, "blocks.7.attention.w_k": 0.9250717163085938, "blocks.7.attention.w_v": 0.885650634765625, "blocks.7.attention.w_out": 0.8671493530273438, "blocks.7.feed_forward.w1": 0.9091085195541382, "blocks.7.feed_forward.w2": 0.8683294057846069, "blocks.7.feed_forward.w3": 0.9008331298828125, "blocks.8.attention.w_q": 0.9288902282714844, "blocks.8.attention.w_k": 0.9268875122070312, "blocks.8.attention.w_v": 0.8820381164550781, "blocks.8.attention.w_out": 0.8689041137695312, "blocks.8.feed_forward.w1": 0.9110603332519531, "blocks.8.feed_forward.w2": 0.8488642573356628, "blocks.8.feed_forward.w3": 0.900115966796875}} +{"step": 2300, "changed_frac": 0.9005897065513154, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9174308776855469, "blocks.0.attention.w_k": 0.9150886535644531, "blocks.0.attention.w_v": 0.8904762268066406, "blocks.0.attention.w_out": 0.8867759704589844, "blocks.0.feed_forward.w1": 0.9077123403549194, "blocks.0.feed_forward.w2": 0.9008306264877319, "blocks.0.feed_forward.w3": 0.9053586721420288, "blocks.1.attention.w_q": 0.9232215881347656, "blocks.1.attention.w_k": 0.9237213134765625, "blocks.1.attention.w_v": 0.9071464538574219, "blocks.1.attention.w_out": 0.9141807556152344, "blocks.1.feed_forward.w1": 0.9039294123649597, "blocks.1.feed_forward.w2": 0.8979314565658569, "blocks.1.feed_forward.w3": 0.9009959101676941, "blocks.2.attention.w_q": 0.9262809753417969, "blocks.2.attention.w_k": 0.9256095886230469, "blocks.2.attention.w_v": 0.9090690612792969, "blocks.2.attention.w_out": 0.924835205078125, "blocks.2.feed_forward.w1": 0.904571533203125, "blocks.2.feed_forward.w2": 0.8947118520736694, "blocks.2.feed_forward.w3": 0.9001731872558594, "blocks.3.attention.w_q": 0.9226570129394531, "blocks.3.attention.w_k": 0.9214210510253906, "blocks.3.attention.w_v": 0.8993873596191406, "blocks.3.attention.w_out": 0.9114265441894531, "blocks.3.feed_forward.w1": 0.9038810729980469, "blocks.3.feed_forward.w2": 0.896887481212616, "blocks.3.feed_forward.w3": 0.8984998464584351, "blocks.4.attention.w_q": 0.9241447448730469, "blocks.4.attention.w_k": 0.9233551025390625, "blocks.4.attention.w_v": 0.8933982849121094, "blocks.4.attention.w_out": 0.8970184326171875, "blocks.4.feed_forward.w1": 0.9052709341049194, "blocks.4.feed_forward.w2": 0.8933436274528503, "blocks.4.feed_forward.w3": 0.8982213735580444, "blocks.5.attention.w_q": 0.9229469299316406, "blocks.5.attention.w_k": 0.9204483032226562, "blocks.5.attention.w_v": 0.8886222839355469, "blocks.5.attention.w_out": 0.894683837890625, "blocks.5.feed_forward.w1": 0.9050814509391785, "blocks.5.feed_forward.w2": 0.8883680105209351, "blocks.5.feed_forward.w3": 0.8969535827636719, "blocks.6.attention.w_q": 0.9260635375976562, "blocks.6.attention.w_k": 0.9226837158203125, "blocks.6.attention.w_v": 0.882659912109375, "blocks.6.attention.w_out": 0.8754768371582031, "blocks.6.feed_forward.w1": 0.9069849848747253, "blocks.6.feed_forward.w2": 0.8819808959960938, "blocks.6.feed_forward.w3": 0.8987541198730469, "blocks.7.attention.w_q": 0.9268913269042969, "blocks.7.attention.w_k": 0.9238700866699219, "blocks.7.attention.w_v": 0.8850784301757812, "blocks.7.attention.w_out": 0.86651611328125, "blocks.7.feed_forward.w1": 0.9086647033691406, "blocks.7.feed_forward.w2": 0.8672498464584351, "blocks.7.feed_forward.w3": 0.8990669250488281, "blocks.8.attention.w_q": 0.9287109375, "blocks.8.attention.w_k": 0.9255790710449219, "blocks.8.attention.w_v": 0.8828277587890625, "blocks.8.attention.w_out": 0.8699798583984375, "blocks.8.feed_forward.w1": 0.9103279113769531, "blocks.8.feed_forward.w2": 0.8480606079101562, "blocks.8.feed_forward.w3": 0.8992919921875}} +{"step": 2400, "changed_frac": 0.8993476802467281, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.91729736328125, "blocks.0.attention.w_k": 0.9125938415527344, "blocks.0.attention.w_v": 0.8880043029785156, "blocks.0.attention.w_out": 0.8856658935546875, "blocks.0.feed_forward.w1": 0.9066047668457031, "blocks.0.feed_forward.w2": 0.8997879028320312, "blocks.0.feed_forward.w3": 0.905157744884491, "blocks.1.attention.w_q": 0.9225349426269531, "blocks.1.attention.w_k": 0.9230308532714844, "blocks.1.attention.w_v": 0.9060592651367188, "blocks.1.attention.w_out": 0.9150123596191406, "blocks.1.feed_forward.w1": 0.904211699962616, "blocks.1.feed_forward.w2": 0.8969854116439819, "blocks.1.feed_forward.w3": 0.8991254568099976, "blocks.2.attention.w_q": 0.9238510131835938, "blocks.2.attention.w_k": 0.9243049621582031, "blocks.2.attention.w_v": 0.9071159362792969, "blocks.2.attention.w_out": 0.9245834350585938, "blocks.2.feed_forward.w1": 0.9026324152946472, "blocks.2.feed_forward.w2": 0.8935572504997253, "blocks.2.feed_forward.w3": 0.8987668752670288, "blocks.3.attention.w_q": 0.9205322265625, "blocks.3.attention.w_k": 0.9195823669433594, "blocks.3.attention.w_v": 0.8973159790039062, "blocks.3.attention.w_out": 0.9080238342285156, "blocks.3.feed_forward.w1": 0.9029783010482788, "blocks.3.feed_forward.w2": 0.8946317434310913, "blocks.3.feed_forward.w3": 0.8975232839584351, "blocks.4.attention.w_q": 0.9226417541503906, "blocks.4.attention.w_k": 0.921783447265625, "blocks.4.attention.w_v": 0.8912773132324219, "blocks.4.attention.w_out": 0.8956832885742188, "blocks.4.feed_forward.w1": 0.9041354060173035, "blocks.4.feed_forward.w2": 0.8923263549804688, "blocks.4.feed_forward.w3": 0.896307647228241, "blocks.5.attention.w_q": 0.9209518432617188, "blocks.5.attention.w_k": 0.9199943542480469, "blocks.5.attention.w_v": 0.8879928588867188, "blocks.5.attention.w_out": 0.8930549621582031, "blocks.5.feed_forward.w1": 0.9035886526107788, "blocks.5.feed_forward.w2": 0.8862330317497253, "blocks.5.feed_forward.w3": 0.8969167470932007, "blocks.6.attention.w_q": 0.9258270263671875, "blocks.6.attention.w_k": 0.9210433959960938, "blocks.6.attention.w_v": 0.8817939758300781, "blocks.6.attention.w_out": 0.8748283386230469, "blocks.6.feed_forward.w1": 0.9053726196289062, "blocks.6.feed_forward.w2": 0.8808441162109375, "blocks.6.feed_forward.w3": 0.8975741267204285, "blocks.7.attention.w_q": 0.9255409240722656, "blocks.7.attention.w_k": 0.924102783203125, "blocks.7.attention.w_v": 0.8849639892578125, "blocks.7.attention.w_out": 0.8648185729980469, "blocks.7.feed_forward.w1": 0.9069862365722656, "blocks.7.feed_forward.w2": 0.8669179677963257, "blocks.7.feed_forward.w3": 0.8982671499252319, "blocks.8.attention.w_q": 0.9266777038574219, "blocks.8.attention.w_k": 0.9251976013183594, "blocks.8.attention.w_v": 0.88055419921875, "blocks.8.attention.w_out": 0.8667106628417969, "blocks.8.feed_forward.w1": 0.9090309143066406, "blocks.8.feed_forward.w2": 0.8467509150505066, "blocks.8.feed_forward.w3": 0.8976961970329285}} +{"step": 2500, "changed_frac": 0.8982515253572383, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9151573181152344, "blocks.0.attention.w_k": 0.9139747619628906, "blocks.0.attention.w_v": 0.8850326538085938, "blocks.0.attention.w_out": 0.8819847106933594, "blocks.0.feed_forward.w1": 0.9047966003417969, "blocks.0.feed_forward.w2": 0.8990325927734375, "blocks.0.feed_forward.w3": 0.9028193354606628, "blocks.1.attention.w_q": 0.9217185974121094, "blocks.1.attention.w_k": 0.9216384887695312, "blocks.1.attention.w_v": 0.9050750732421875, "blocks.1.attention.w_out": 0.91265869140625, "blocks.1.feed_forward.w1": 0.9025319814682007, "blocks.1.feed_forward.w2": 0.8965060114860535, "blocks.1.feed_forward.w3": 0.8977140188217163, "blocks.2.attention.w_q": 0.9225234985351562, "blocks.2.attention.w_k": 0.9232635498046875, "blocks.2.attention.w_v": 0.9062347412109375, "blocks.2.attention.w_out": 0.9231452941894531, "blocks.2.feed_forward.w1": 0.9022852778434753, "blocks.2.feed_forward.w2": 0.892248809337616, "blocks.2.feed_forward.w3": 0.8970222473144531, "blocks.3.attention.w_q": 0.9189682006835938, "blocks.3.attention.w_k": 0.9184532165527344, "blocks.3.attention.w_v": 0.8959312438964844, "blocks.3.attention.w_out": 0.9076461791992188, "blocks.3.feed_forward.w1": 0.9013748168945312, "blocks.3.feed_forward.w2": 0.8943532705307007, "blocks.3.feed_forward.w3": 0.8957036733627319, "blocks.4.attention.w_q": 0.9217071533203125, "blocks.4.attention.w_k": 0.9205665588378906, "blocks.4.attention.w_v": 0.8901557922363281, "blocks.4.attention.w_out": 0.8949317932128906, "blocks.4.feed_forward.w1": 0.9028905630111694, "blocks.4.feed_forward.w2": 0.891302764415741, "blocks.4.feed_forward.w3": 0.8951873779296875, "blocks.5.attention.w_q": 0.9203910827636719, "blocks.5.attention.w_k": 0.9202079772949219, "blocks.5.attention.w_v": 0.8855514526367188, "blocks.5.attention.w_out": 0.8929481506347656, "blocks.5.feed_forward.w1": 0.9030927419662476, "blocks.5.feed_forward.w2": 0.8852819204330444, "blocks.5.feed_forward.w3": 0.8951250910758972, "blocks.6.attention.w_q": 0.9244041442871094, "blocks.6.attention.w_k": 0.9219093322753906, "blocks.6.attention.w_v": 0.8798866271972656, "blocks.6.attention.w_out": 0.8752403259277344, "blocks.6.feed_forward.w1": 0.90533447265625, "blocks.6.feed_forward.w2": 0.8796348571777344, "blocks.6.feed_forward.w3": 0.8970286250114441, "blocks.7.attention.w_q": 0.9242439270019531, "blocks.7.attention.w_k": 0.9225273132324219, "blocks.7.attention.w_v": 0.883026123046875, "blocks.7.attention.w_out": 0.8640213012695312, "blocks.7.feed_forward.w1": 0.9068412780761719, "blocks.7.feed_forward.w2": 0.8651809692382812, "blocks.7.feed_forward.w3": 0.8977369070053101, "blocks.8.attention.w_q": 0.9259910583496094, "blocks.8.attention.w_k": 0.9233818054199219, "blocks.8.attention.w_v": 0.878204345703125, "blocks.8.attention.w_out": 0.8657150268554688, "blocks.8.feed_forward.w1": 0.9083506464958191, "blocks.8.feed_forward.w2": 0.8462817072868347, "blocks.8.feed_forward.w3": 0.8966789245605469}} +{"step": 2600, "changed_frac": 0.8969170333992722, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9139976501464844, "blocks.0.attention.w_k": 0.9115371704101562, "blocks.0.attention.w_v": 0.8848152160644531, "blocks.0.attention.w_out": 0.8811492919921875, "blocks.0.feed_forward.w1": 0.9042651057243347, "blocks.0.feed_forward.w2": 0.8982760310173035, "blocks.0.feed_forward.w3": 0.9014371633529663, "blocks.1.attention.w_q": 0.9209976196289062, "blocks.1.attention.w_k": 0.9215049743652344, "blocks.1.attention.w_v": 0.902801513671875, "blocks.1.attention.w_out": 0.9116744995117188, "blocks.1.feed_forward.w1": 0.9013087153434753, "blocks.1.feed_forward.w2": 0.8955816030502319, "blocks.1.feed_forward.w3": 0.897253692150116, "blocks.2.attention.w_q": 0.9230766296386719, "blocks.2.attention.w_k": 0.9222640991210938, "blocks.2.attention.w_v": 0.9053611755371094, "blocks.2.attention.w_out": 0.9217338562011719, "blocks.2.feed_forward.w1": 0.9003435969352722, "blocks.2.feed_forward.w2": 0.8907076716423035, "blocks.2.feed_forward.w3": 0.8963623046875, "blocks.3.attention.w_q": 0.9191436767578125, "blocks.3.attention.w_k": 0.9183235168457031, "blocks.3.attention.w_v": 0.8942222595214844, "blocks.3.attention.w_out": 0.9057960510253906, "blocks.3.feed_forward.w1": 0.900115966796875, "blocks.3.feed_forward.w2": 0.8927447199821472, "blocks.3.feed_forward.w3": 0.8944638967514038, "blocks.4.attention.w_q": 0.9208755493164062, "blocks.4.attention.w_k": 0.9199981689453125, "blocks.4.attention.w_v": 0.8892135620117188, "blocks.4.attention.w_out": 0.8940353393554688, "blocks.4.feed_forward.w1": 0.9010722041130066, "blocks.4.feed_forward.w2": 0.8904393911361694, "blocks.4.feed_forward.w3": 0.8940709829330444, "blocks.5.attention.w_q": 0.9203414916992188, "blocks.5.attention.w_k": 0.9191703796386719, "blocks.5.attention.w_v": 0.8848762512207031, "blocks.5.attention.w_out": 0.8912925720214844, "blocks.5.feed_forward.w1": 0.90167236328125, "blocks.5.feed_forward.w2": 0.8830312490463257, "blocks.5.feed_forward.w3": 0.8934962153434753, "blocks.6.attention.w_q": 0.9223556518554688, "blocks.6.attention.w_k": 0.9207077026367188, "blocks.6.attention.w_v": 0.8774948120117188, "blocks.6.attention.w_out": 0.8730583190917969, "blocks.6.feed_forward.w1": 0.9037196040153503, "blocks.6.feed_forward.w2": 0.8770663142204285, "blocks.6.feed_forward.w3": 0.8954086303710938, "blocks.7.attention.w_q": 0.9237251281738281, "blocks.7.attention.w_k": 0.9217796325683594, "blocks.7.attention.w_v": 0.882080078125, "blocks.7.attention.w_out": 0.8642692565917969, "blocks.7.feed_forward.w1": 0.9051437377929688, "blocks.7.feed_forward.w2": 0.8629061579704285, "blocks.7.feed_forward.w3": 0.8953107595443726, "blocks.8.attention.w_q": 0.9248504638671875, "blocks.8.attention.w_k": 0.9234962463378906, "blocks.8.attention.w_v": 0.8774185180664062, "blocks.8.attention.w_out": 0.8636093139648438, "blocks.8.feed_forward.w1": 0.9072647094726562, "blocks.8.feed_forward.w2": 0.8437131643295288, "blocks.8.feed_forward.w3": 0.8949064016342163}} +{"step": 2700, "changed_frac": 0.8957798297588642, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9117012023925781, "blocks.0.attention.w_k": 0.9096717834472656, "blocks.0.attention.w_v": 0.8805389404296875, "blocks.0.attention.w_out": 0.8793182373046875, "blocks.0.feed_forward.w1": 0.9034818410873413, "blocks.0.feed_forward.w2": 0.8969065546989441, "blocks.0.feed_forward.w3": 0.8993034362792969, "blocks.1.attention.w_q": 0.9193191528320312, "blocks.1.attention.w_k": 0.9193840026855469, "blocks.1.attention.w_v": 0.903533935546875, "blocks.1.attention.w_out": 0.9108657836914062, "blocks.1.feed_forward.w1": 0.9001414179801941, "blocks.1.feed_forward.w2": 0.8938026428222656, "blocks.1.feed_forward.w3": 0.8949699401855469, "blocks.2.attention.w_q": 0.9210357666015625, "blocks.2.attention.w_k": 0.9202499389648438, "blocks.2.attention.w_v": 0.9059333801269531, "blocks.2.attention.w_out": 0.9203262329101562, "blocks.2.feed_forward.w1": 0.8993810415267944, "blocks.2.feed_forward.w2": 0.8907546997070312, "blocks.2.feed_forward.w3": 0.8948606252670288, "blocks.3.attention.w_q": 0.918701171875, "blocks.3.attention.w_k": 0.917388916015625, "blocks.3.attention.w_v": 0.8915214538574219, "blocks.3.attention.w_out": 0.9056396484375, "blocks.3.feed_forward.w1": 0.8997230529785156, "blocks.3.feed_forward.w2": 0.8923658132553101, "blocks.3.feed_forward.w3": 0.89300537109375, "blocks.4.attention.w_q": 0.9197196960449219, "blocks.4.attention.w_k": 0.9191970825195312, "blocks.4.attention.w_v": 0.8881454467773438, "blocks.4.attention.w_out": 0.8932838439941406, "blocks.4.feed_forward.w1": 0.9005152583122253, "blocks.4.feed_forward.w2": 0.8895047903060913, "blocks.4.feed_forward.w3": 0.8934186697006226, "blocks.5.attention.w_q": 0.91845703125, "blocks.5.attention.w_k": 0.9181327819824219, "blocks.5.attention.w_v": 0.8828506469726562, "blocks.5.attention.w_out": 0.8892745971679688, "blocks.5.feed_forward.w1": 0.9009590148925781, "blocks.5.feed_forward.w2": 0.8821983337402344, "blocks.5.feed_forward.w3": 0.8924484252929688, "blocks.6.attention.w_q": 0.9228897094726562, "blocks.6.attention.w_k": 0.9188880920410156, "blocks.6.attention.w_v": 0.8760185241699219, "blocks.6.attention.w_out": 0.8721389770507812, "blocks.6.feed_forward.w1": 0.9027201533317566, "blocks.6.feed_forward.w2": 0.8766822814941406, "blocks.6.feed_forward.w3": 0.8948695063591003, "blocks.7.attention.w_q": 0.9224777221679688, "blocks.7.attention.w_k": 0.9199676513671875, "blocks.7.attention.w_v": 0.8794631958007812, "blocks.7.attention.w_out": 0.8619499206542969, "blocks.7.feed_forward.w1": 0.9045041799545288, "blocks.7.feed_forward.w2": 0.8617503046989441, "blocks.7.feed_forward.w3": 0.8943036794662476, "blocks.8.attention.w_q": 0.9249305725097656, "blocks.8.attention.w_k": 0.92071533203125, "blocks.8.attention.w_v": 0.8766059875488281, "blocks.8.attention.w_out": 0.8635902404785156, "blocks.8.feed_forward.w1": 0.9060007929801941, "blocks.8.feed_forward.w2": 0.8416036367416382, "blocks.8.feed_forward.w3": 0.8939628601074219}} +{"step": 2800, "changed_frac": 0.8940334157047108, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9099388122558594, "blocks.0.attention.w_k": 0.9066543579101562, "blocks.0.attention.w_v": 0.8796310424804688, "blocks.0.attention.w_out": 0.8776512145996094, "blocks.0.feed_forward.w1": 0.9012769460678101, "blocks.0.feed_forward.w2": 0.8949229121208191, "blocks.0.feed_forward.w3": 0.8973745107650757, "blocks.1.attention.w_q": 0.9185791015625, "blocks.1.attention.w_k": 0.917266845703125, "blocks.1.attention.w_v": 0.9021186828613281, "blocks.1.attention.w_out": 0.9086341857910156, "blocks.1.feed_forward.w1": 0.8982633352279663, "blocks.1.feed_forward.w2": 0.8924471735954285, "blocks.1.feed_forward.w3": 0.8931922912597656, "blocks.2.attention.w_q": 0.9210777282714844, "blocks.2.attention.w_k": 0.9201850891113281, "blocks.2.attention.w_v": 0.9027290344238281, "blocks.2.attention.w_out": 0.9212150573730469, "blocks.2.feed_forward.w1": 0.8984896540641785, "blocks.2.feed_forward.w2": 0.8892809748649597, "blocks.2.feed_forward.w3": 0.893042266368866, "blocks.3.attention.w_q": 0.9150924682617188, "blocks.3.attention.w_k": 0.914093017578125, "blocks.3.attention.w_v": 0.8900489807128906, "blocks.3.attention.w_out": 0.904449462890625, "blocks.3.feed_forward.w1": 0.8973274230957031, "blocks.3.feed_forward.w2": 0.8897578120231628, "blocks.3.feed_forward.w3": 0.8922449946403503, "blocks.4.attention.w_q": 0.9175224304199219, "blocks.4.attention.w_k": 0.9161338806152344, "blocks.4.attention.w_v": 0.8867034912109375, "blocks.4.attention.w_out": 0.8906936645507812, "blocks.4.feed_forward.w1": 0.8991050720214844, "blocks.4.feed_forward.w2": 0.8875020742416382, "blocks.4.feed_forward.w3": 0.8916130065917969, "blocks.5.attention.w_q": 0.9163627624511719, "blocks.5.attention.w_k": 0.915313720703125, "blocks.5.attention.w_v": 0.8807258605957031, "blocks.5.attention.w_out": 0.88775634765625, "blocks.5.feed_forward.w1": 0.899444580078125, "blocks.5.feed_forward.w2": 0.8804461359977722, "blocks.5.feed_forward.w3": 0.8909925222396851, "blocks.6.attention.w_q": 0.9208526611328125, "blocks.6.attention.w_k": 0.9176445007324219, "blocks.6.attention.w_v": 0.8753509521484375, "blocks.6.attention.w_out": 0.8698387145996094, "blocks.6.feed_forward.w1": 0.9012298583984375, "blocks.6.feed_forward.w2": 0.8744761347770691, "blocks.6.feed_forward.w3": 0.8936780691146851, "blocks.7.attention.w_q": 0.9211616516113281, "blocks.7.attention.w_k": 0.9187889099121094, "blocks.7.attention.w_v": 0.87896728515625, "blocks.7.attention.w_out": 0.8594779968261719, "blocks.7.feed_forward.w1": 0.9033343195915222, "blocks.7.feed_forward.w2": 0.8591117858886719, "blocks.7.feed_forward.w3": 0.89324951171875, "blocks.8.attention.w_q": 0.9220466613769531, "blocks.8.attention.w_k": 0.9198036193847656, "blocks.8.attention.w_v": 0.8758659362792969, "blocks.8.attention.w_out": 0.861541748046875, "blocks.8.feed_forward.w1": 0.9045626521110535, "blocks.8.feed_forward.w2": 0.8386065363883972, "blocks.8.feed_forward.w3": 0.8916918635368347}} +{"step": 2900, "changed_frac": 0.8928236676077558, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9093208312988281, "blocks.0.attention.w_k": 0.9067001342773438, "blocks.0.attention.w_v": 0.8780975341796875, "blocks.0.attention.w_out": 0.8756179809570312, "blocks.0.feed_forward.w1": 0.9001376032829285, "blocks.0.feed_forward.w2": 0.8935546875, "blocks.0.feed_forward.w3": 0.8957405090332031, "blocks.1.attention.w_q": 0.9164810180664062, "blocks.1.attention.w_k": 0.9167060852050781, "blocks.1.attention.w_v": 0.89996337890625, "blocks.1.attention.w_out": 0.9088401794433594, "blocks.1.feed_forward.w1": 0.8975169062614441, "blocks.1.feed_forward.w2": 0.8915876150131226, "blocks.1.feed_forward.w3": 0.8919525146484375, "blocks.2.attention.w_q": 0.9184150695800781, "blocks.2.attention.w_k": 0.9185142517089844, "blocks.2.attention.w_v": 0.9020423889160156, "blocks.2.attention.w_out": 0.9201278686523438, "blocks.2.feed_forward.w1": 0.8973363637924194, "blocks.2.feed_forward.w2": 0.8881200551986694, "blocks.2.feed_forward.w3": 0.8916740417480469, "blocks.3.attention.w_q": 0.9145622253417969, "blocks.3.attention.w_k": 0.9127464294433594, "blocks.3.attention.w_v": 0.8886985778808594, "blocks.3.attention.w_out": 0.9034805297851562, "blocks.3.feed_forward.w1": 0.8970476984977722, "blocks.3.feed_forward.w2": 0.8885841369628906, "blocks.3.feed_forward.w3": 0.8906351923942566, "blocks.4.attention.w_q": 0.9173316955566406, "blocks.4.attention.w_k": 0.9152030944824219, "blocks.4.attention.w_v": 0.8838958740234375, "blocks.4.attention.w_out": 0.8892440795898438, "blocks.4.feed_forward.w1": 0.8981272578239441, "blocks.4.feed_forward.w2": 0.8860791921615601, "blocks.4.feed_forward.w3": 0.8902969360351562, "blocks.5.attention.w_q": 0.9152374267578125, "blocks.5.attention.w_k": 0.9136009216308594, "blocks.5.attention.w_v": 0.8805694580078125, "blocks.5.attention.w_out": 0.8873519897460938, "blocks.5.feed_forward.w1": 0.8979721069335938, "blocks.5.feed_forward.w2": 0.8792610168457031, "blocks.5.feed_forward.w3": 0.8895772695541382, "blocks.6.attention.w_q": 0.9206809997558594, "blocks.6.attention.w_k": 0.915924072265625, "blocks.6.attention.w_v": 0.8732681274414062, "blocks.6.attention.w_out": 0.8692245483398438, "blocks.6.feed_forward.w1": 0.9006093740463257, "blocks.6.feed_forward.w2": 0.8741544485092163, "blocks.6.feed_forward.w3": 0.8920224905014038, "blocks.7.attention.w_q": 0.9185943603515625, "blocks.7.attention.w_k": 0.9177513122558594, "blocks.7.attention.w_v": 0.8766708374023438, "blocks.7.attention.w_out": 0.8580551147460938, "blocks.7.feed_forward.w1": 0.9024289846420288, "blocks.7.feed_forward.w2": 0.8575363159179688, "blocks.7.feed_forward.w3": 0.891326904296875, "blocks.8.attention.w_q": 0.9215736389160156, "blocks.8.attention.w_k": 0.9198493957519531, "blocks.8.attention.w_v": 0.8727264404296875, "blocks.8.attention.w_out": 0.85845947265625, "blocks.8.feed_forward.w1": 0.9037551879882812, "blocks.8.feed_forward.w2": 0.836889922618866, "blocks.8.feed_forward.w3": 0.8910230398178101}} +{"step": 3000, "changed_frac": 0.8912147130721655, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9073143005371094, "blocks.0.attention.w_k": 0.903594970703125, "blocks.0.attention.w_v": 0.8752403259277344, "blocks.0.attention.w_out": 0.8746490478515625, "blocks.0.feed_forward.w1": 0.8983739614486694, "blocks.0.feed_forward.w2": 0.8926519155502319, "blocks.0.feed_forward.w3": 0.8948020935058594, "blocks.1.attention.w_q": 0.9145278930664062, "blocks.1.attention.w_k": 0.916229248046875, "blocks.1.attention.w_v": 0.8983650207519531, "blocks.1.attention.w_out": 0.9075088500976562, "blocks.1.feed_forward.w1": 0.8948389887809753, "blocks.1.feed_forward.w2": 0.8898557424545288, "blocks.1.feed_forward.w3": 0.8895480036735535, "blocks.2.attention.w_q": 0.9183578491210938, "blocks.2.attention.w_k": 0.9170684814453125, "blocks.2.attention.w_v": 0.9002494812011719, "blocks.2.attention.w_out": 0.9187469482421875, "blocks.2.feed_forward.w1": 0.8952153921127319, "blocks.2.feed_forward.w2": 0.8865814208984375, "blocks.2.feed_forward.w3": 0.8896357417106628, "blocks.3.attention.w_q": 0.9134483337402344, "blocks.3.attention.w_k": 0.9131546020507812, "blocks.3.attention.w_v": 0.8864936828613281, "blocks.3.attention.w_out": 0.9011268615722656, "blocks.3.feed_forward.w1": 0.89471435546875, "blocks.3.feed_forward.w2": 0.888336181640625, "blocks.3.feed_forward.w3": 0.8892911672592163, "blocks.4.attention.w_q": 0.9163818359375, "blocks.4.attention.w_k": 0.9155311584472656, "blocks.4.attention.w_v": 0.8821449279785156, "blocks.4.attention.w_out": 0.8894996643066406, "blocks.4.feed_forward.w1": 0.8965632319450378, "blocks.4.feed_forward.w2": 0.8845939636230469, "blocks.4.feed_forward.w3": 0.88836669921875, "blocks.5.attention.w_q": 0.9139175415039062, "blocks.5.attention.w_k": 0.9124603271484375, "blocks.5.attention.w_v": 0.8787002563476562, "blocks.5.attention.w_out": 0.8842315673828125, "blocks.5.feed_forward.w1": 0.8965530395507812, "blocks.5.feed_forward.w2": 0.8776321411132812, "blocks.5.feed_forward.w3": 0.8883743286132812, "blocks.6.attention.w_q": 0.918121337890625, "blocks.6.attention.w_k": 0.9142227172851562, "blocks.6.attention.w_v": 0.8709716796875, "blocks.6.attention.w_out": 0.8681373596191406, "blocks.6.feed_forward.w1": 0.8984565734863281, "blocks.6.feed_forward.w2": 0.8724963068962097, "blocks.6.feed_forward.w3": 0.8901494741439819, "blocks.7.attention.w_q": 0.9178276062011719, "blocks.7.attention.w_k": 0.9160842895507812, "blocks.7.attention.w_v": 0.8747291564941406, "blocks.7.attention.w_out": 0.8560867309570312, "blocks.7.feed_forward.w1": 0.9010633230209351, "blocks.7.feed_forward.w2": 0.8555437922477722, "blocks.7.feed_forward.w3": 0.8904368281364441, "blocks.8.attention.w_q": 0.9201202392578125, "blocks.8.attention.w_k": 0.9179649353027344, "blocks.8.attention.w_v": 0.8704643249511719, "blocks.8.attention.w_out": 0.856048583984375, "blocks.8.feed_forward.w1": 0.9019635915756226, "blocks.8.feed_forward.w2": 0.8350664973258972, "blocks.8.feed_forward.w3": 0.8896955251693726}} +{"step": 3100, "changed_frac": 0.8894707443367722, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9045562744140625, "blocks.0.attention.w_k": 0.9019927978515625, "blocks.0.attention.w_v": 0.87347412109375, "blocks.0.attention.w_out": 0.8717727661132812, "blocks.0.feed_forward.w1": 0.8956858515739441, "blocks.0.feed_forward.w2": 0.8908170461654663, "blocks.0.feed_forward.w3": 0.8919525146484375, "blocks.1.attention.w_q": 0.9118309020996094, "blocks.1.attention.w_k": 0.9138145446777344, "blocks.1.attention.w_v": 0.8980178833007812, "blocks.1.attention.w_out": 0.9058494567871094, "blocks.1.feed_forward.w1": 0.893927276134491, "blocks.1.feed_forward.w2": 0.888220489025116, "blocks.1.feed_forward.w3": 0.8880335688591003, "blocks.2.attention.w_q": 0.9168548583984375, "blocks.2.attention.w_k": 0.9162483215332031, "blocks.2.attention.w_v": 0.8982429504394531, "blocks.2.attention.w_out": 0.9183197021484375, "blocks.2.feed_forward.w1": 0.8931821584701538, "blocks.2.feed_forward.w2": 0.8849004507064819, "blocks.2.feed_forward.w3": 0.8879331350326538, "blocks.3.attention.w_q": 0.9129409790039062, "blocks.3.attention.w_k": 0.910430908203125, "blocks.3.attention.w_v": 0.8846626281738281, "blocks.3.attention.w_out": 0.8992729187011719, "blocks.3.feed_forward.w1": 0.8934809565544128, "blocks.3.feed_forward.w2": 0.8866246938705444, "blocks.3.feed_forward.w3": 0.8876661062240601, "blocks.4.attention.w_q": 0.9135894775390625, "blocks.4.attention.w_k": 0.9134597778320312, "blocks.4.attention.w_v": 0.879638671875, "blocks.4.attention.w_out": 0.8863716125488281, "blocks.4.feed_forward.w1": 0.8944791555404663, "blocks.4.feed_forward.w2": 0.8835551142692566, "blocks.4.feed_forward.w3": 0.8873113393783569, "blocks.5.attention.w_q": 0.9127731323242188, "blocks.5.attention.w_k": 0.9103546142578125, "blocks.5.attention.w_v": 0.8762855529785156, "blocks.5.attention.w_out": 0.8841476440429688, "blocks.5.feed_forward.w1": 0.8949750661849976, "blocks.5.feed_forward.w2": 0.8763300776481628, "blocks.5.feed_forward.w3": 0.8866755366325378, "blocks.6.attention.w_q": 0.91748046875, "blocks.6.attention.w_k": 0.9128189086914062, "blocks.6.attention.w_v": 0.8692665100097656, "blocks.6.attention.w_out": 0.8661041259765625, "blocks.6.feed_forward.w1": 0.8966941833496094, "blocks.6.feed_forward.w2": 0.8700205683708191, "blocks.6.feed_forward.w3": 0.8894373774528503, "blocks.7.attention.w_q": 0.9167594909667969, "blocks.7.attention.w_k": 0.9139480590820312, "blocks.7.attention.w_v": 0.8733749389648438, "blocks.7.attention.w_out": 0.8545455932617188, "blocks.7.feed_forward.w1": 0.8989741206169128, "blocks.7.feed_forward.w2": 0.8525225520133972, "blocks.7.feed_forward.w3": 0.8889071345329285, "blocks.8.attention.w_q": 0.9188690185546875, "blocks.8.attention.w_k": 0.9163475036621094, "blocks.8.attention.w_v": 0.8692855834960938, "blocks.8.attention.w_out": 0.8555068969726562, "blocks.8.feed_forward.w1": 0.9014753103256226, "blocks.8.feed_forward.w2": 0.8311042785644531, "blocks.8.feed_forward.w3": 0.8880704641342163}} +{"step": 3200, "changed_frac": 0.8873739650106838, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.9021530151367188, "blocks.0.attention.w_k": 0.9000587463378906, "blocks.0.attention.w_v": 0.86993408203125, "blocks.0.attention.w_out": 0.8689422607421875, "blocks.0.feed_forward.w1": 0.8935890197753906, "blocks.0.feed_forward.w2": 0.8884366750717163, "blocks.0.feed_forward.w3": 0.8895136713981628, "blocks.1.attention.w_q": 0.9109344482421875, "blocks.1.attention.w_k": 0.91302490234375, "blocks.1.attention.w_v": 0.895294189453125, "blocks.1.attention.w_out": 0.90460205078125, "blocks.1.feed_forward.w1": 0.8908869624137878, "blocks.1.feed_forward.w2": 0.8858197927474976, "blocks.1.feed_forward.w3": 0.8861643671989441, "blocks.2.attention.w_q": 0.9144668579101562, "blocks.2.attention.w_k": 0.9143829345703125, "blocks.2.attention.w_v": 0.8957901000976562, "blocks.2.attention.w_out": 0.9170341491699219, "blocks.2.feed_forward.w1": 0.8925921320915222, "blocks.2.feed_forward.w2": 0.8822339773178101, "blocks.2.feed_forward.w3": 0.8858286738395691, "blocks.3.attention.w_q": 0.9107208251953125, "blocks.3.attention.w_k": 0.9088668823242188, "blocks.3.attention.w_v": 0.8838462829589844, "blocks.3.attention.w_out": 0.8989982604980469, "blocks.3.feed_forward.w1": 0.8917719721794128, "blocks.3.feed_forward.w2": 0.8841069936752319, "blocks.3.feed_forward.w3": 0.8852221369743347, "blocks.4.attention.w_q": 0.9126548767089844, "blocks.4.attention.w_k": 0.9116783142089844, "blocks.4.attention.w_v": 0.8776512145996094, "blocks.4.attention.w_out": 0.8851890563964844, "blocks.4.feed_forward.w1": 0.8932711482048035, "blocks.4.feed_forward.w2": 0.8804423213005066, "blocks.4.feed_forward.w3": 0.8843765258789062, "blocks.5.attention.w_q": 0.9119873046875, "blocks.5.attention.w_k": 0.9104881286621094, "blocks.5.attention.w_v": 0.8735809326171875, "blocks.5.attention.w_out": 0.8815345764160156, "blocks.5.feed_forward.w1": 0.8939082026481628, "blocks.5.feed_forward.w2": 0.8739471435546875, "blocks.5.feed_forward.w3": 0.8839823603630066, "blocks.6.attention.w_q": 0.9152565002441406, "blocks.6.attention.w_k": 0.9113044738769531, "blocks.6.attention.w_v": 0.8667945861816406, "blocks.6.attention.w_out": 0.8643341064453125, "blocks.6.feed_forward.w1": 0.8954963684082031, "blocks.6.feed_forward.w2": 0.8674240112304688, "blocks.6.feed_forward.w3": 0.8862686157226562, "blocks.7.attention.w_q": 0.9153900146484375, "blocks.7.attention.w_k": 0.9119834899902344, "blocks.7.attention.w_v": 0.8703041076660156, "blocks.7.attention.w_out": 0.8518753051757812, "blocks.7.feed_forward.w1": 0.8974851369857788, "blocks.7.feed_forward.w2": 0.8505338430404663, "blocks.7.feed_forward.w3": 0.8864771723747253, "blocks.8.attention.w_q": 0.9176292419433594, "blocks.8.attention.w_k": 0.9148902893066406, "blocks.8.attention.w_v": 0.8681564331054688, "blocks.8.attention.w_out": 0.8524398803710938, "blocks.8.feed_forward.w1": 0.8992564082145691, "blocks.8.feed_forward.w2": 0.8279902338981628, "blocks.8.feed_forward.w3": 0.8858350515365601}} +{"step": 3300, "changed_frac": 0.885517837654831, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8996086120605469, "blocks.0.attention.w_k": 0.897430419921875, "blocks.0.attention.w_v": 0.8680801391601562, "blocks.0.attention.w_out": 0.8672332763671875, "blocks.0.feed_forward.w1": 0.8920428156852722, "blocks.0.feed_forward.w2": 0.8871740102767944, "blocks.0.feed_forward.w3": 0.8873023986816406, "blocks.1.attention.w_q": 0.9096755981445312, "blocks.1.attention.w_k": 0.9120101928710938, "blocks.1.attention.w_v": 0.8949851989746094, "blocks.1.attention.w_out": 0.9027938842773438, "blocks.1.feed_forward.w1": 0.8894056081771851, "blocks.1.feed_forward.w2": 0.8839098811149597, "blocks.1.feed_forward.w3": 0.8836530447006226, "blocks.2.attention.w_q": 0.9125442504882812, "blocks.2.attention.w_k": 0.9127159118652344, "blocks.2.attention.w_v": 0.8941993713378906, "blocks.2.attention.w_out": 0.915557861328125, "blocks.2.feed_forward.w1": 0.8908056020736694, "blocks.2.feed_forward.w2": 0.880591094493866, "blocks.2.feed_forward.w3": 0.8841819763183594, "blocks.3.attention.w_q": 0.9082565307617188, "blocks.3.attention.w_k": 0.907379150390625, "blocks.3.attention.w_v": 0.8809013366699219, "blocks.3.attention.w_out": 0.8956527709960938, "blocks.3.feed_forward.w1": 0.8896128535270691, "blocks.3.feed_forward.w2": 0.8823992609977722, "blocks.3.feed_forward.w3": 0.88360595703125, "blocks.4.attention.w_q": 0.9112472534179688, "blocks.4.attention.w_k": 0.9086570739746094, "blocks.4.attention.w_v": 0.8747062683105469, "blocks.4.attention.w_out": 0.8839263916015625, "blocks.4.feed_forward.w1": 0.8912226557731628, "blocks.4.feed_forward.w2": 0.8796806335449219, "blocks.4.feed_forward.w3": 0.8827921748161316, "blocks.5.attention.w_q": 0.9100723266601562, "blocks.5.attention.w_k": 0.9079360961914062, "blocks.5.attention.w_v": 0.8708953857421875, "blocks.5.attention.w_out": 0.8801612854003906, "blocks.5.feed_forward.w1": 0.8918063044548035, "blocks.5.feed_forward.w2": 0.8726171255111694, "blocks.5.feed_forward.w3": 0.8817037343978882, "blocks.6.attention.w_q": 0.9137725830078125, "blocks.6.attention.w_k": 0.909088134765625, "blocks.6.attention.w_v": 0.8646202087402344, "blocks.6.attention.w_out": 0.8611679077148438, "blocks.6.feed_forward.w1": 0.8927116394042969, "blocks.6.feed_forward.w2": 0.8663368225097656, "blocks.6.feed_forward.w3": 0.8849691152572632, "blocks.7.attention.w_q": 0.9139595031738281, "blocks.7.attention.w_k": 0.9111785888671875, "blocks.7.attention.w_v": 0.8700714111328125, "blocks.7.attention.w_out": 0.8487625122070312, "blocks.7.feed_forward.w1": 0.8957952260971069, "blocks.7.feed_forward.w2": 0.8477045893669128, "blocks.7.feed_forward.w3": 0.8851356506347656, "blocks.8.attention.w_q": 0.916595458984375, "blocks.8.attention.w_k": 0.9143295288085938, "blocks.8.attention.w_v": 0.8638496398925781, "blocks.8.attention.w_out": 0.8499717712402344, "blocks.8.feed_forward.w1": 0.897101104259491, "blocks.8.feed_forward.w2": 0.8251661062240601, "blocks.8.feed_forward.w3": 0.8844375610351562}} +{"step": 3400, "changed_frac": 0.8833223652635884, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8975791931152344, "blocks.0.attention.w_k": 0.8942031860351562, "blocks.0.attention.w_v": 0.8646812438964844, "blocks.0.attention.w_out": 0.8645782470703125, "blocks.0.feed_forward.w1": 0.8890088796615601, "blocks.0.feed_forward.w2": 0.885992705821991, "blocks.0.feed_forward.w3": 0.8848254084587097, "blocks.1.attention.w_q": 0.9082794189453125, "blocks.1.attention.w_k": 0.9087333679199219, "blocks.1.attention.w_v": 0.8914985656738281, "blocks.1.attention.w_out": 0.9006690979003906, "blocks.1.feed_forward.w1": 0.8869056701660156, "blocks.1.feed_forward.w2": 0.8819580078125, "blocks.1.feed_forward.w3": 0.8820775747299194, "blocks.2.attention.w_q": 0.9112167358398438, "blocks.2.attention.w_k": 0.910614013671875, "blocks.2.attention.w_v": 0.8920822143554688, "blocks.2.attention.w_out": 0.913330078125, "blocks.2.feed_forward.w1": 0.887908935546875, "blocks.2.feed_forward.w2": 0.8789037466049194, "blocks.2.feed_forward.w3": 0.8824476003646851, "blocks.3.attention.w_q": 0.905029296875, "blocks.3.attention.w_k": 0.904815673828125, "blocks.3.attention.w_v": 0.8783111572265625, "blocks.3.attention.w_out": 0.8942756652832031, "blocks.3.feed_forward.w1": 0.8880577087402344, "blocks.3.feed_forward.w2": 0.88031005859375, "blocks.3.feed_forward.w3": 0.8811849355697632, "blocks.4.attention.w_q": 0.9090919494628906, "blocks.4.attention.w_k": 0.9071846008300781, "blocks.4.attention.w_v": 0.873199462890625, "blocks.4.attention.w_out": 0.8813323974609375, "blocks.4.feed_forward.w1": 0.8896675109863281, "blocks.4.feed_forward.w2": 0.8776487112045288, "blocks.4.feed_forward.w3": 0.8810157775878906, "blocks.5.attention.w_q": 0.9075088500976562, "blocks.5.attention.w_k": 0.9063949584960938, "blocks.5.attention.w_v": 0.8690834045410156, "blocks.5.attention.w_out": 0.8779258728027344, "blocks.5.feed_forward.w1": 0.8897552490234375, "blocks.5.feed_forward.w2": 0.8702990412712097, "blocks.5.feed_forward.w3": 0.880316436290741, "blocks.6.attention.w_q": 0.9110832214355469, "blocks.6.attention.w_k": 0.9072647094726562, "blocks.6.attention.w_v": 0.8629035949707031, "blocks.6.attention.w_out": 0.8586387634277344, "blocks.6.feed_forward.w1": 0.8906021118164062, "blocks.6.feed_forward.w2": 0.8635902404785156, "blocks.6.feed_forward.w3": 0.8821678161621094, "blocks.7.attention.w_q": 0.9114036560058594, "blocks.7.attention.w_k": 0.9091911315917969, "blocks.7.attention.w_v": 0.8679237365722656, "blocks.7.attention.w_out": 0.8470878601074219, "blocks.7.feed_forward.w1": 0.8931986689567566, "blocks.7.feed_forward.w2": 0.8454526662826538, "blocks.7.feed_forward.w3": 0.8826307058334351, "blocks.8.attention.w_q": 0.9138336181640625, "blocks.8.attention.w_k": 0.9095115661621094, "blocks.8.attention.w_v": 0.8633499145507812, "blocks.8.attention.w_out": 0.8461456298828125, "blocks.8.feed_forward.w1": 0.8950106501579285, "blocks.8.feed_forward.w2": 0.8224958181381226, "blocks.8.feed_forward.w3": 0.8828214406967163}} +{"step": 3500, "changed_frac": 0.881139412904397, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.895751953125, "blocks.0.attention.w_k": 0.8913383483886719, "blocks.0.attention.w_v": 0.8622169494628906, "blocks.0.attention.w_out": 0.8622779846191406, "blocks.0.feed_forward.w1": 0.8857218623161316, "blocks.0.feed_forward.w2": 0.8833643794059753, "blocks.0.feed_forward.w3": 0.8828824758529663, "blocks.1.attention.w_q": 0.905670166015625, "blocks.1.attention.w_k": 0.9063072204589844, "blocks.1.attention.w_v": 0.8888397216796875, "blocks.1.attention.w_out": 0.8983726501464844, "blocks.1.feed_forward.w1": 0.8845685720443726, "blocks.1.feed_forward.w2": 0.8795801997184753, "blocks.1.feed_forward.w3": 0.8786519765853882, "blocks.2.attention.w_q": 0.9079627990722656, "blocks.2.attention.w_k": 0.9075965881347656, "blocks.2.attention.w_v": 0.8900527954101562, "blocks.2.attention.w_out": 0.9130287170410156, "blocks.2.feed_forward.w1": 0.8860855102539062, "blocks.2.feed_forward.w2": 0.8771337270736694, "blocks.2.feed_forward.w3": 0.8801536560058594, "blocks.3.attention.w_q": 0.904296875, "blocks.3.attention.w_k": 0.90179443359375, "blocks.3.attention.w_v": 0.8775672912597656, "blocks.3.attention.w_out": 0.8929634094238281, "blocks.3.feed_forward.w1": 0.8867658376693726, "blocks.3.feed_forward.w2": 0.8781293630599976, "blocks.3.feed_forward.w3": 0.8797544240951538, "blocks.4.attention.w_q": 0.9076919555664062, "blocks.4.attention.w_k": 0.9050865173339844, "blocks.4.attention.w_v": 0.8701896667480469, "blocks.4.attention.w_out": 0.8783950805664062, "blocks.4.feed_forward.w1": 0.887786865234375, "blocks.4.feed_forward.w2": 0.8749427795410156, "blocks.4.feed_forward.w3": 0.8782743215560913, "blocks.5.attention.w_q": 0.9054412841796875, "blocks.5.attention.w_k": 0.9042434692382812, "blocks.5.attention.w_v": 0.8651199340820312, "blocks.5.attention.w_out": 0.8760795593261719, "blocks.5.feed_forward.w1": 0.8874486684799194, "blocks.5.feed_forward.w2": 0.8682721853256226, "blocks.5.feed_forward.w3": 0.8775215148925781, "blocks.6.attention.w_q": 0.9097213745117188, "blocks.6.attention.w_k": 0.9042167663574219, "blocks.6.attention.w_v": 0.8603172302246094, "blocks.6.attention.w_out": 0.8572158813476562, "blocks.6.feed_forward.w1": 0.8891448974609375, "blocks.6.feed_forward.w2": 0.8611412048339844, "blocks.6.feed_forward.w3": 0.8801664113998413, "blocks.7.attention.w_q": 0.910064697265625, "blocks.7.attention.w_k": 0.9086189270019531, "blocks.7.attention.w_v": 0.8623580932617188, "blocks.7.attention.w_out": 0.8439712524414062, "blocks.7.feed_forward.w1": 0.8917274475097656, "blocks.7.feed_forward.w2": 0.8429107666015625, "blocks.7.feed_forward.w3": 0.880743682384491, "blocks.8.attention.w_q": 0.9113807678222656, "blocks.8.attention.w_k": 0.9077796936035156, "blocks.8.attention.w_v": 0.8590965270996094, "blocks.8.attention.w_out": 0.845428466796875, "blocks.8.feed_forward.w1": 0.8941434621810913, "blocks.8.feed_forward.w2": 0.8204154968261719, "blocks.8.feed_forward.w3": 0.8808543086051941}} +{"step": 3600, "changed_frac": 0.8785843808426816, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8910484313964844, "blocks.0.attention.w_k": 0.8886337280273438, "blocks.0.attention.w_v": 0.8593482971191406, "blocks.0.attention.w_out": 0.8574371337890625, "blocks.0.feed_forward.w1": 0.8841896057128906, "blocks.0.feed_forward.w2": 0.8814888000488281, "blocks.0.feed_forward.w3": 0.8792673945426941, "blocks.1.attention.w_q": 0.9010581970214844, "blocks.1.attention.w_k": 0.9061927795410156, "blocks.1.attention.w_v": 0.8857383728027344, "blocks.1.attention.w_out": 0.8975257873535156, "blocks.1.feed_forward.w1": 0.8822174072265625, "blocks.1.feed_forward.w2": 0.8767560720443726, "blocks.1.feed_forward.w3": 0.8766657710075378, "blocks.2.attention.w_q": 0.9063644409179688, "blocks.2.attention.w_k": 0.9059638977050781, "blocks.2.attention.w_v": 0.8879165649414062, "blocks.2.attention.w_out": 0.9107704162597656, "blocks.2.feed_forward.w1": 0.8838895559310913, "blocks.2.feed_forward.w2": 0.8742713928222656, "blocks.2.feed_forward.w3": 0.8771299123764038, "blocks.3.attention.w_q": 0.9014396667480469, "blocks.3.attention.w_k": 0.8995361328125, "blocks.3.attention.w_v": 0.8728675842285156, "blocks.3.attention.w_out": 0.892303466796875, "blocks.3.feed_forward.w1": 0.8837661743164062, "blocks.3.feed_forward.w2": 0.8758570551872253, "blocks.3.feed_forward.w3": 0.8772939443588257, "blocks.4.attention.w_q": 0.9054489135742188, "blocks.4.attention.w_k": 0.9024467468261719, "blocks.4.attention.w_v": 0.8671150207519531, "blocks.4.attention.w_out": 0.8774566650390625, "blocks.4.feed_forward.w1": 0.8860830068588257, "blocks.4.feed_forward.w2": 0.8724288940429688, "blocks.4.feed_forward.w3": 0.8759320974349976, "blocks.5.attention.w_q": 0.9022712707519531, "blocks.5.attention.w_k": 0.9007644653320312, "blocks.5.attention.w_v": 0.8626708984375, "blocks.5.attention.w_out": 0.8734893798828125, "blocks.5.feed_forward.w1": 0.8851585388183594, "blocks.5.feed_forward.w2": 0.8656654357910156, "blocks.5.feed_forward.w3": 0.8752390742301941, "blocks.6.attention.w_q": 0.9068222045898438, "blocks.6.attention.w_k": 0.9029960632324219, "blocks.6.attention.w_v": 0.8574676513671875, "blocks.6.attention.w_out": 0.8553047180175781, "blocks.6.feed_forward.w1": 0.8870710134506226, "blocks.6.feed_forward.w2": 0.8582484126091003, "blocks.6.feed_forward.w3": 0.8777847290039062, "blocks.7.attention.w_q": 0.9079971313476562, "blocks.7.attention.w_k": 0.9054222106933594, "blocks.7.attention.w_v": 0.8611564636230469, "blocks.7.attention.w_out": 0.839813232421875, "blocks.7.feed_forward.w1": 0.889227569103241, "blocks.7.feed_forward.w2": 0.8394126892089844, "blocks.7.feed_forward.w3": 0.8780288696289062, "blocks.8.attention.w_q": 0.9098625183105469, "blocks.8.attention.w_k": 0.907135009765625, "blocks.8.attention.w_v": 0.8561744689941406, "blocks.8.attention.w_out": 0.8407745361328125, "blocks.8.feed_forward.w1": 0.8920809626579285, "blocks.8.feed_forward.w2": 0.8152580261230469, "blocks.8.feed_forward.w3": 0.8788007497787476}} +{"step": 3700, "changed_frac": 0.8761659931932759, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.887451171875, "blocks.0.attention.w_k": 0.8848304748535156, "blocks.0.attention.w_v": 0.8551712036132812, "blocks.0.attention.w_out": 0.8563804626464844, "blocks.0.feed_forward.w1": 0.8813056945800781, "blocks.0.feed_forward.w2": 0.8789558410644531, "blocks.0.feed_forward.w3": 0.877716064453125, "blocks.1.attention.w_q": 0.8991966247558594, "blocks.1.attention.w_k": 0.9025382995605469, "blocks.1.attention.w_v": 0.8848457336425781, "blocks.1.attention.w_out": 0.8957939147949219, "blocks.1.feed_forward.w1": 0.8797454833984375, "blocks.1.feed_forward.w2": 0.8742345571517944, "blocks.1.feed_forward.w3": 0.8744761347770691, "blocks.2.attention.w_q": 0.9031562805175781, "blocks.2.attention.w_k": 0.9051628112792969, "blocks.2.attention.w_v": 0.8865280151367188, "blocks.2.attention.w_out": 0.9088172912597656, "blocks.2.feed_forward.w1": 0.8812357783317566, "blocks.2.feed_forward.w2": 0.8734054565429688, "blocks.2.feed_forward.w3": 0.874823272228241, "blocks.3.attention.w_q": 0.8985633850097656, "blocks.3.attention.w_k": 0.8970222473144531, "blocks.3.attention.w_v": 0.8710441589355469, "blocks.3.attention.w_out": 0.8887710571289062, "blocks.3.feed_forward.w1": 0.8821322321891785, "blocks.3.feed_forward.w2": 0.8741595149040222, "blocks.3.feed_forward.w3": 0.8739497065544128, "blocks.4.attention.w_q": 0.9024505615234375, "blocks.4.attention.w_k": 0.9003639221191406, "blocks.4.attention.w_v": 0.8652305603027344, "blocks.4.attention.w_out": 0.8751220703125, "blocks.4.feed_forward.w1": 0.8835271596908569, "blocks.4.feed_forward.w2": 0.8701909780502319, "blocks.4.feed_forward.w3": 0.8736000061035156, "blocks.5.attention.w_q": 0.9009590148925781, "blocks.5.attention.w_k": 0.8996620178222656, "blocks.5.attention.w_v": 0.8598365783691406, "blocks.5.attention.w_out": 0.8699684143066406, "blocks.5.feed_forward.w1": 0.8821767568588257, "blocks.5.feed_forward.w2": 0.8626722097396851, "blocks.5.feed_forward.w3": 0.8730685114860535, "blocks.6.attention.w_q": 0.9051284790039062, "blocks.6.attention.w_k": 0.9003067016601562, "blocks.6.attention.w_v": 0.8543968200683594, "blocks.6.attention.w_out": 0.8532180786132812, "blocks.6.feed_forward.w1": 0.8846651911735535, "blocks.6.feed_forward.w2": 0.8550008535385132, "blocks.6.feed_forward.w3": 0.8755671381950378, "blocks.7.attention.w_q": 0.9056549072265625, "blocks.7.attention.w_k": 0.9039649963378906, "blocks.7.attention.w_v": 0.85833740234375, "blocks.7.attention.w_out": 0.8388290405273438, "blocks.7.feed_forward.w1": 0.8864479064941406, "blocks.7.feed_forward.w2": 0.8359197378158569, "blocks.7.feed_forward.w3": 0.8769671320915222, "blocks.8.attention.w_q": 0.9078330993652344, "blocks.8.attention.w_k": 0.9038505554199219, "blocks.8.attention.w_v": 0.8529319763183594, "blocks.8.attention.w_out": 0.8376846313476562, "blocks.8.feed_forward.w1": 0.8893038630485535, "blocks.8.feed_forward.w2": 0.8124250173568726, "blocks.8.feed_forward.w3": 0.8758010864257812}} +{"step": 3800, "changed_frac": 0.873184562748314, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8856887817382812, "blocks.0.attention.w_k": 0.88262939453125, "blocks.0.attention.w_v": 0.8512420654296875, "blocks.0.attention.w_out": 0.8532600402832031, "blocks.0.feed_forward.w1": 0.8788058161735535, "blocks.0.feed_forward.w2": 0.8769709467887878, "blocks.0.feed_forward.w3": 0.8733953237533569, "blocks.1.attention.w_q": 0.8985939025878906, "blocks.1.attention.w_k": 0.9004135131835938, "blocks.1.attention.w_v": 0.8820304870605469, "blocks.1.attention.w_out": 0.8949813842773438, "blocks.1.feed_forward.w1": 0.8764266967773438, "blocks.1.feed_forward.w2": 0.8725230097770691, "blocks.1.feed_forward.w3": 0.870611846446991, "blocks.2.attention.w_q": 0.9013214111328125, "blocks.2.attention.w_k": 0.9032440185546875, "blocks.2.attention.w_v": 0.8832206726074219, "blocks.2.attention.w_out": 0.906463623046875, "blocks.2.feed_forward.w1": 0.8776575922966003, "blocks.2.feed_forward.w2": 0.869049072265625, "blocks.2.feed_forward.w3": 0.8707593679428101, "blocks.3.attention.w_q": 0.8968734741210938, "blocks.3.attention.w_k": 0.8931808471679688, "blocks.3.attention.w_v": 0.867950439453125, "blocks.3.attention.w_out": 0.8875808715820312, "blocks.3.feed_forward.w1": 0.8791605830192566, "blocks.3.feed_forward.w2": 0.8717092275619507, "blocks.3.feed_forward.w3": 0.8712705373764038, "blocks.4.attention.w_q": 0.9005165100097656, "blocks.4.attention.w_k": 0.897247314453125, "blocks.4.attention.w_v": 0.8624038696289062, "blocks.4.attention.w_out": 0.8717803955078125, "blocks.4.feed_forward.w1": 0.8804296255111694, "blocks.4.feed_forward.w2": 0.8666941523551941, "blocks.4.feed_forward.w3": 0.8703193664550781, "blocks.5.attention.w_q": 0.8974266052246094, "blocks.5.attention.w_k": 0.8954849243164062, "blocks.5.attention.w_v": 0.8568649291992188, "blocks.5.attention.w_out": 0.8677482604980469, "blocks.5.feed_forward.w1": 0.879614531993866, "blocks.5.feed_forward.w2": 0.8600019216537476, "blocks.5.feed_forward.w3": 0.8700956106185913, "blocks.6.attention.w_q": 0.9029655456542969, "blocks.6.attention.w_k": 0.89776611328125, "blocks.6.attention.w_v": 0.8510513305664062, "blocks.6.attention.w_out": 0.8497657775878906, "blocks.6.feed_forward.w1": 0.8824247121810913, "blocks.6.feed_forward.w2": 0.8519389033317566, "blocks.6.feed_forward.w3": 0.872961699962616, "blocks.7.attention.w_q": 0.9024467468261719, "blocks.7.attention.w_k": 0.9006309509277344, "blocks.7.attention.w_v": 0.856201171875, "blocks.7.attention.w_out": 0.8349037170410156, "blocks.7.feed_forward.w1": 0.8839645385742188, "blocks.7.feed_forward.w2": 0.832275390625, "blocks.7.feed_forward.w3": 0.8738288879394531, "blocks.8.attention.w_q": 0.9051628112792969, "blocks.8.attention.w_k": 0.9010047912597656, "blocks.8.attention.w_v": 0.8502006530761719, "blocks.8.attention.w_out": 0.83343505859375, "blocks.8.feed_forward.w1": 0.8863881826400757, "blocks.8.feed_forward.w2": 0.8072497248649597, "blocks.8.feed_forward.w3": 0.8731104731559753}} +{"step": 3900, "changed_frac": 0.8704983637883112, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8820381164550781, "blocks.0.attention.w_k": 0.88043212890625, "blocks.0.attention.w_v": 0.8497962951660156, "blocks.0.attention.w_out": 0.8493995666503906, "blocks.0.feed_forward.w1": 0.8755277395248413, "blocks.0.feed_forward.w2": 0.8742116689682007, "blocks.0.feed_forward.w3": 0.8703638911247253, "blocks.1.attention.w_q": 0.8945884704589844, "blocks.1.attention.w_k": 0.8980903625488281, "blocks.1.attention.w_v": 0.8793373107910156, "blocks.1.attention.w_out": 0.8919296264648438, "blocks.1.feed_forward.w1": 0.8743426203727722, "blocks.1.feed_forward.w2": 0.8692843317985535, "blocks.1.feed_forward.w3": 0.8677991628646851, "blocks.2.attention.w_q": 0.8985099792480469, "blocks.2.attention.w_k": 0.8992805480957031, "blocks.2.attention.w_v": 0.8804550170898438, "blocks.2.attention.w_out": 0.9052276611328125, "blocks.2.feed_forward.w1": 0.8745778799057007, "blocks.2.feed_forward.w2": 0.867468535900116, "blocks.2.feed_forward.w3": 0.868841826915741, "blocks.3.attention.w_q": 0.893463134765625, "blocks.3.attention.w_k": 0.8908157348632812, "blocks.3.attention.w_v": 0.8655281066894531, "blocks.3.attention.w_out": 0.8826408386230469, "blocks.3.feed_forward.w1": 0.8761914968490601, "blocks.3.feed_forward.w2": 0.8686968684196472, "blocks.3.feed_forward.w3": 0.8691635131835938, "blocks.4.attention.w_q": 0.8969497680664062, "blocks.4.attention.w_k": 0.8951301574707031, "blocks.4.attention.w_v": 0.8576736450195312, "blocks.4.attention.w_out": 0.8703575134277344, "blocks.4.feed_forward.w1": 0.8780072927474976, "blocks.4.feed_forward.w2": 0.8643519282341003, "blocks.4.feed_forward.w3": 0.8674036860466003, "blocks.5.attention.w_q": 0.8938713073730469, "blocks.5.attention.w_k": 0.8931465148925781, "blocks.5.attention.w_v": 0.8534736633300781, "blocks.5.attention.w_out": 0.8639602661132812, "blocks.5.feed_forward.w1": 0.8779678344726562, "blocks.5.feed_forward.w2": 0.8580373525619507, "blocks.5.feed_forward.w3": 0.866736114025116, "blocks.6.attention.w_q": 0.8994140625, "blocks.6.attention.w_k": 0.8943557739257812, "blocks.6.attention.w_v": 0.8473320007324219, "blocks.6.attention.w_out": 0.8481483459472656, "blocks.6.feed_forward.w1": 0.8797925710678101, "blocks.6.feed_forward.w2": 0.8487447500228882, "blocks.6.feed_forward.w3": 0.8702201843261719, "blocks.7.attention.w_q": 0.9005889892578125, "blocks.7.attention.w_k": 0.898529052734375, "blocks.7.attention.w_v": 0.8517913818359375, "blocks.7.attention.w_out": 0.8321495056152344, "blocks.7.feed_forward.w1": 0.8813909292221069, "blocks.7.feed_forward.w2": 0.8292109370231628, "blocks.7.feed_forward.w3": 0.8713048696517944, "blocks.8.attention.w_q": 0.9029273986816406, "blocks.8.attention.w_k": 0.8989219665527344, "blocks.8.attention.w_v": 0.8467979431152344, "blocks.8.attention.w_out": 0.8307304382324219, "blocks.8.feed_forward.w1": 0.8845609426498413, "blocks.8.feed_forward.w2": 0.8043798208236694, "blocks.8.feed_forward.w3": 0.8715972900390625}} +{"step": 4000, "changed_frac": 0.8670687879252638, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8779792785644531, "blocks.0.attention.w_k": 0.8757705688476562, "blocks.0.attention.w_v": 0.845001220703125, "blocks.0.attention.w_out": 0.8449897766113281, "blocks.0.feed_forward.w1": 0.8715426325798035, "blocks.0.feed_forward.w2": 0.8706550598144531, "blocks.0.feed_forward.w3": 0.8669027090072632, "blocks.1.attention.w_q": 0.892730712890625, "blocks.1.attention.w_k": 0.8950653076171875, "blocks.1.attention.w_v": 0.8770065307617188, "blocks.1.attention.w_out": 0.8903579711914062, "blocks.1.feed_forward.w1": 0.8709640502929688, "blocks.1.feed_forward.w2": 0.864935576915741, "blocks.1.feed_forward.w3": 0.8637580871582031, "blocks.2.attention.w_q": 0.8961219787597656, "blocks.2.attention.w_k": 0.8961677551269531, "blocks.2.attention.w_v": 0.8773422241210938, "blocks.2.attention.w_out": 0.9026412963867188, "blocks.2.feed_forward.w1": 0.8724924921989441, "blocks.2.feed_forward.w2": 0.8632138967514038, "blocks.2.feed_forward.w3": 0.8649889826774597, "blocks.3.attention.w_q": 0.8901405334472656, "blocks.3.attention.w_k": 0.8882293701171875, "blocks.3.attention.w_v": 0.8610610961914062, "blocks.3.attention.w_out": 0.8801460266113281, "blocks.3.feed_forward.w1": 0.8733291625976562, "blocks.3.feed_forward.w2": 0.8659591674804688, "blocks.3.feed_forward.w3": 0.8663215637207031, "blocks.4.attention.w_q": 0.8936386108398438, "blocks.4.attention.w_k": 0.8912086486816406, "blocks.4.attention.w_v": 0.8553619384765625, "blocks.4.attention.w_out": 0.8670463562011719, "blocks.4.feed_forward.w1": 0.8751856684684753, "blocks.4.feed_forward.w2": 0.8612226247787476, "blocks.4.feed_forward.w3": 0.8642145991325378, "blocks.5.attention.w_q": 0.8913154602050781, "blocks.5.attention.w_k": 0.8904953002929688, "blocks.5.attention.w_v": 0.8493461608886719, "blocks.5.attention.w_out": 0.8623085021972656, "blocks.5.feed_forward.w1": 0.8749377131462097, "blocks.5.feed_forward.w2": 0.8546231985092163, "blocks.5.feed_forward.w3": 0.8642107844352722, "blocks.6.attention.w_q": 0.8970069885253906, "blocks.6.attention.w_k": 0.8907852172851562, "blocks.6.attention.w_v": 0.8428230285644531, "blocks.6.attention.w_out": 0.8437309265136719, "blocks.6.feed_forward.w1": 0.8769073486328125, "blocks.6.feed_forward.w2": 0.8454246520996094, "blocks.6.feed_forward.w3": 0.8658892512321472, "blocks.7.attention.w_q": 0.8973274230957031, "blocks.7.attention.w_k": 0.8954086303710938, "blocks.7.attention.w_v": 0.847930908203125, "blocks.7.attention.w_out": 0.8268051147460938, "blocks.7.feed_forward.w1": 0.8790881037712097, "blocks.7.feed_forward.w2": 0.8250097036361694, "blocks.7.feed_forward.w3": 0.8673223257064819, "blocks.8.attention.w_q": 0.9004859924316406, "blocks.8.attention.w_k": 0.8974990844726562, "blocks.8.attention.w_v": 0.8416099548339844, "blocks.8.attention.w_out": 0.8249969482421875, "blocks.8.feed_forward.w1": 0.8818563222885132, "blocks.8.feed_forward.w2": 0.7976812124252319, "blocks.8.feed_forward.w3": 0.8677520751953125}} +{"step": 4100, "changed_frac": 0.863782475137303, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8726959228515625, "blocks.0.attention.w_k": 0.8719444274902344, "blocks.0.attention.w_v": 0.8423004150390625, "blocks.0.attention.w_out": 0.8426666259765625, "blocks.0.feed_forward.w1": 0.8681272268295288, "blocks.0.feed_forward.w2": 0.8678767085075378, "blocks.0.feed_forward.w3": 0.8637301325798035, "blocks.1.attention.w_q": 0.8897476196289062, "blocks.1.attention.w_k": 0.89019775390625, "blocks.1.attention.w_v": 0.8730697631835938, "blocks.1.attention.w_out": 0.8874435424804688, "blocks.1.feed_forward.w1": 0.8663864135742188, "blocks.1.feed_forward.w2": 0.8615519404411316, "blocks.1.feed_forward.w3": 0.8613446950912476, "blocks.2.attention.w_q": 0.8934516906738281, "blocks.2.attention.w_k": 0.8933563232421875, "blocks.2.attention.w_v": 0.8749313354492188, "blocks.2.attention.w_out": 0.9001884460449219, "blocks.2.feed_forward.w1": 0.8690910339355469, "blocks.2.feed_forward.w2": 0.862311065196991, "blocks.2.feed_forward.w3": 0.8617566823959351, "blocks.3.attention.w_q": 0.8866424560546875, "blocks.3.attention.w_k": 0.8845901489257812, "blocks.3.attention.w_v": 0.8572654724121094, "blocks.3.attention.w_out": 0.877716064453125, "blocks.3.feed_forward.w1": 0.8701553344726562, "blocks.3.feed_forward.w2": 0.8642158508300781, "blocks.3.feed_forward.w3": 0.8623288869857788, "blocks.4.attention.w_q": 0.8901901245117188, "blocks.4.attention.w_k": 0.8888702392578125, "blocks.4.attention.w_v": 0.8512001037597656, "blocks.4.attention.w_out": 0.8655433654785156, "blocks.4.feed_forward.w1": 0.8723742365837097, "blocks.4.feed_forward.w2": 0.8584442138671875, "blocks.4.feed_forward.w3": 0.8609632253646851, "blocks.5.attention.w_q": 0.8886528015136719, "blocks.5.attention.w_k": 0.885650634765625, "blocks.5.attention.w_v": 0.8462753295898438, "blocks.5.attention.w_out": 0.8592987060546875, "blocks.5.feed_forward.w1": 0.8721110224723816, "blocks.5.feed_forward.w2": 0.8497556447982788, "blocks.5.feed_forward.w3": 0.8609263300895691, "blocks.6.attention.w_q": 0.8941688537597656, "blocks.6.attention.w_k": 0.8889083862304688, "blocks.6.attention.w_v": 0.839508056640625, "blocks.6.attention.w_out": 0.8400993347167969, "blocks.6.feed_forward.w1": 0.8735949397087097, "blocks.6.feed_forward.w2": 0.8406359553337097, "blocks.6.feed_forward.w3": 0.8630282282829285, "blocks.7.attention.w_q": 0.8935813903808594, "blocks.7.attention.w_k": 0.8911514282226562, "blocks.7.attention.w_v": 0.8453102111816406, "blocks.7.attention.w_out": 0.8239974975585938, "blocks.7.feed_forward.w1": 0.8763224482536316, "blocks.7.feed_forward.w2": 0.8199259638786316, "blocks.7.feed_forward.w3": 0.8641624450683594, "blocks.8.attention.w_q": 0.8973617553710938, "blocks.8.attention.w_k": 0.8925285339355469, "blocks.8.attention.w_v": 0.8383064270019531, "blocks.8.attention.w_out": 0.82177734375, "blocks.8.feed_forward.w1": 0.8787244558334351, "blocks.8.feed_forward.w2": 0.7927119135856628, "blocks.8.feed_forward.w3": 0.8647639155387878}} +{"step": 4200, "changed_frac": 0.8599661150549212, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.86810302734375, "blocks.0.attention.w_k": 0.8659858703613281, "blocks.0.attention.w_v": 0.837127685546875, "blocks.0.attention.w_out": 0.8387794494628906, "blocks.0.feed_forward.w1": 0.8640302419662476, "blocks.0.feed_forward.w2": 0.8635101318359375, "blocks.0.feed_forward.w3": 0.8601875305175781, "blocks.1.attention.w_q": 0.8850173950195312, "blocks.1.attention.w_k": 0.8875198364257812, "blocks.1.attention.w_v": 0.868621826171875, "blocks.1.attention.w_out": 0.8844070434570312, "blocks.1.feed_forward.w1": 0.8626226186752319, "blocks.1.feed_forward.w2": 0.8585790395736694, "blocks.1.feed_forward.w3": 0.8572909235954285, "blocks.2.attention.w_q": 0.8902740478515625, "blocks.2.attention.w_k": 0.8895683288574219, "blocks.2.attention.w_v": 0.8701934814453125, "blocks.2.attention.w_out": 0.8974533081054688, "blocks.2.feed_forward.w1": 0.8649393916130066, "blocks.2.feed_forward.w2": 0.8589388728141785, "blocks.2.feed_forward.w3": 0.8570684194564819, "blocks.3.attention.w_q": 0.8831291198730469, "blocks.3.attention.w_k": 0.8801193237304688, "blocks.3.attention.w_v": 0.8523750305175781, "blocks.3.attention.w_out": 0.8751373291015625, "blocks.3.feed_forward.w1": 0.8668658137321472, "blocks.3.feed_forward.w2": 0.8595708608627319, "blocks.3.feed_forward.w3": 0.8582484126091003, "blocks.4.attention.w_q": 0.8870124816894531, "blocks.4.attention.w_k": 0.8852005004882812, "blocks.4.attention.w_v": 0.8468246459960938, "blocks.4.attention.w_out": 0.8619041442871094, "blocks.4.feed_forward.w1": 0.8701032400131226, "blocks.4.feed_forward.w2": 0.8538996577262878, "blocks.4.feed_forward.w3": 0.8579025268554688, "blocks.5.attention.w_q": 0.8841476440429688, "blocks.5.attention.w_k": 0.8833160400390625, "blocks.5.attention.w_v": 0.8424072265625, "blocks.5.attention.w_out": 0.8548698425292969, "blocks.5.feed_forward.w1": 0.8677101135253906, "blocks.5.feed_forward.w2": 0.8472201228141785, "blocks.5.feed_forward.w3": 0.8563232421875, "blocks.6.attention.w_q": 0.8905296325683594, "blocks.6.attention.w_k": 0.8847427368164062, "blocks.6.attention.w_v": 0.8348388671875, "blocks.6.attention.w_out": 0.8369407653808594, "blocks.6.feed_forward.w1": 0.8710174560546875, "blocks.6.feed_forward.w2": 0.8359527587890625, "blocks.6.feed_forward.w3": 0.8597730398178101, "blocks.7.attention.w_q": 0.8903656005859375, "blocks.7.attention.w_k": 0.8896331787109375, "blocks.7.attention.w_v": 0.8396034240722656, "blocks.7.attention.w_out": 0.8178138732910156, "blocks.7.feed_forward.w1": 0.8730964660644531, "blocks.7.feed_forward.w2": 0.8151639699935913, "blocks.7.feed_forward.w3": 0.8610954284667969, "blocks.8.attention.w_q": 0.8945426940917969, "blocks.8.attention.w_k": 0.8890876770019531, "blocks.8.attention.w_v": 0.8347282409667969, "blocks.8.attention.w_out": 0.8178291320800781, "blocks.8.feed_forward.w1": 0.8752886652946472, "blocks.8.feed_forward.w2": 0.7881304621696472, "blocks.8.feed_forward.w3": 0.8607661128044128}} +{"step": 4300, "changed_frac": 0.8559163085415832, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8642616271972656, "blocks.0.attention.w_k": 0.8627738952636719, "blocks.0.attention.w_v": 0.8322029113769531, "blocks.0.attention.w_out": 0.8343238830566406, "blocks.0.feed_forward.w1": 0.8603948354721069, "blocks.0.feed_forward.w2": 0.8590635061264038, "blocks.0.feed_forward.w3": 0.8546371459960938, "blocks.1.attention.w_q": 0.8814506530761719, "blocks.1.attention.w_k": 0.8831024169921875, "blocks.1.attention.w_v": 0.8653488159179688, "blocks.1.attention.w_out": 0.8829345703125, "blocks.1.feed_forward.w1": 0.8584200739860535, "blocks.1.feed_forward.w2": 0.8547528982162476, "blocks.1.feed_forward.w3": 0.8522211909294128, "blocks.2.attention.w_q": 0.8869361877441406, "blocks.2.attention.w_k": 0.8857460021972656, "blocks.2.attention.w_v": 0.8670120239257812, "blocks.2.attention.w_out": 0.8947410583496094, "blocks.2.feed_forward.w1": 0.8609021902084351, "blocks.2.feed_forward.w2": 0.8540865778923035, "blocks.2.feed_forward.w3": 0.8542086482048035, "blocks.3.attention.w_q": 0.8773918151855469, "blocks.3.attention.w_k": 0.8763313293457031, "blocks.3.attention.w_v": 0.8503913879394531, "blocks.3.attention.w_out": 0.8710975646972656, "blocks.3.feed_forward.w1": 0.8632189631462097, "blocks.3.feed_forward.w2": 0.8554534912109375, "blocks.3.feed_forward.w3": 0.8548558950424194, "blocks.4.attention.w_q": 0.8810844421386719, "blocks.4.attention.w_k": 0.8820075988769531, "blocks.4.attention.w_v": 0.84259033203125, "blocks.4.attention.w_out": 0.8577079772949219, "blocks.4.feed_forward.w1": 0.8657633662223816, "blocks.4.feed_forward.w2": 0.8506202697753906, "blocks.4.feed_forward.w3": 0.8539378046989441, "blocks.5.attention.w_q": 0.8803863525390625, "blocks.5.attention.w_k": 0.8780670166015625, "blocks.5.attention.w_v": 0.8382225036621094, "blocks.5.attention.w_out": 0.8517227172851562, "blocks.5.feed_forward.w1": 0.8645871877670288, "blocks.5.feed_forward.w2": 0.8431816101074219, "blocks.5.feed_forward.w3": 0.8522580862045288, "blocks.6.attention.w_q": 0.8883132934570312, "blocks.6.attention.w_k": 0.8808212280273438, "blocks.6.attention.w_v": 0.8305854797363281, "blocks.6.attention.w_out": 0.8329887390136719, "blocks.6.feed_forward.w1": 0.8664487600326538, "blocks.6.feed_forward.w2": 0.8313166499137878, "blocks.6.feed_forward.w3": 0.855688750743866, "blocks.7.attention.w_q": 0.8858871459960938, "blocks.7.attention.w_k": 0.8858680725097656, "blocks.7.attention.w_v": 0.8358535766601562, "blocks.7.attention.w_out": 0.8142852783203125, "blocks.7.feed_forward.w1": 0.8689460754394531, "blocks.7.feed_forward.w2": 0.8103218078613281, "blocks.7.feed_forward.w3": 0.8576139211654663, "blocks.8.attention.w_q": 0.8904228210449219, "blocks.8.attention.w_k": 0.8853569030761719, "blocks.8.attention.w_v": 0.8314552307128906, "blocks.8.attention.w_out": 0.8142852783203125, "blocks.8.feed_forward.w1": 0.8718312978744507, "blocks.8.feed_forward.w2": 0.7807146906852722, "blocks.8.feed_forward.w3": 0.8573049306869507}} +{"step": 4400, "changed_frac": 0.8516640296349158, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.860198974609375, "blocks.0.attention.w_k": 0.85638427734375, "blocks.0.attention.w_v": 0.8275337219238281, "blocks.0.attention.w_out": 0.8326187133789062, "blocks.0.feed_forward.w1": 0.8562825918197632, "blocks.0.feed_forward.w2": 0.8549436330795288, "blocks.0.feed_forward.w3": 0.8506711721420288, "blocks.1.attention.w_q": 0.8772773742675781, "blocks.1.attention.w_k": 0.8791007995605469, "blocks.1.attention.w_v": 0.8616523742675781, "blocks.1.attention.w_out": 0.878021240234375, "blocks.1.feed_forward.w1": 0.8538526296615601, "blocks.1.feed_forward.w2": 0.851111114025116, "blocks.1.feed_forward.w3": 0.8479232788085938, "blocks.2.attention.w_q": 0.8835945129394531, "blocks.2.attention.w_k": 0.8824729919433594, "blocks.2.attention.w_v": 0.8616867065429688, "blocks.2.attention.w_out": 0.890960693359375, "blocks.2.feed_forward.w1": 0.8573252558708191, "blocks.2.feed_forward.w2": 0.8503303527832031, "blocks.2.feed_forward.w3": 0.8500036001205444, "blocks.3.attention.w_q": 0.8747444152832031, "blocks.3.attention.w_k": 0.8720474243164062, "blocks.3.attention.w_v": 0.8441314697265625, "blocks.3.attention.w_out": 0.8678474426269531, "blocks.3.feed_forward.w1": 0.8599802851676941, "blocks.3.feed_forward.w2": 0.8521347045898438, "blocks.3.feed_forward.w3": 0.8498243093490601, "blocks.4.attention.w_q": 0.8765640258789062, "blocks.4.attention.w_k": 0.8758010864257812, "blocks.4.attention.w_v": 0.8378982543945312, "blocks.4.attention.w_out": 0.8533058166503906, "blocks.4.feed_forward.w1": 0.8610738515853882, "blocks.4.feed_forward.w2": 0.8461253046989441, "blocks.4.feed_forward.w3": 0.8494453430175781, "blocks.5.attention.w_q": 0.8763351440429688, "blocks.5.attention.w_k": 0.8757858276367188, "blocks.5.attention.w_v": 0.8316650390625, "blocks.5.attention.w_out": 0.8470191955566406, "blocks.5.feed_forward.w1": 0.861792266368866, "blocks.5.feed_forward.w2": 0.8385302424430847, "blocks.5.feed_forward.w3": 0.8486074209213257, "blocks.6.attention.w_q": 0.8824996948242188, "blocks.6.attention.w_k": 0.874908447265625, "blocks.6.attention.w_v": 0.8247184753417969, "blocks.6.attention.w_out": 0.8277130126953125, "blocks.6.feed_forward.w1": 0.8623594045639038, "blocks.6.feed_forward.w2": 0.8270683288574219, "blocks.6.feed_forward.w3": 0.8511708974838257, "blocks.7.attention.w_q": 0.8836822509765625, "blocks.7.attention.w_k": 0.8804550170898438, "blocks.7.attention.w_v": 0.8313751220703125, "blocks.7.attention.w_out": 0.8094482421875, "blocks.7.feed_forward.w1": 0.8651059865951538, "blocks.7.feed_forward.w2": 0.804779052734375, "blocks.7.feed_forward.w3": 0.8525556325912476, "blocks.8.attention.w_q": 0.886505126953125, "blocks.8.attention.w_k": 0.8817710876464844, "blocks.8.attention.w_v": 0.8265342712402344, "blocks.8.attention.w_out": 0.8087921142578125, "blocks.8.feed_forward.w1": 0.8681640625, "blocks.8.feed_forward.w2": 0.7753614187240601, "blocks.8.feed_forward.w3": 0.8540255427360535}} +{"step": 4500, "changed_frac": 0.8471123785035223, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8541374206542969, "blocks.0.attention.w_k": 0.851470947265625, "blocks.0.attention.w_v": 0.8214378356933594, "blocks.0.attention.w_out": 0.8272972106933594, "blocks.0.feed_forward.w1": 0.8517138361930847, "blocks.0.feed_forward.w2": 0.8510729670524597, "blocks.0.feed_forward.w3": 0.8458951711654663, "blocks.1.attention.w_q": 0.8720741271972656, "blocks.1.attention.w_k": 0.8742408752441406, "blocks.1.attention.w_v": 0.8586235046386719, "blocks.1.attention.w_out": 0.8738822937011719, "blocks.1.feed_forward.w1": 0.8496818542480469, "blocks.1.feed_forward.w2": 0.8458163142204285, "blocks.1.feed_forward.w3": 0.8435516357421875, "blocks.2.attention.w_q": 0.8776130676269531, "blocks.2.attention.w_k": 0.8772811889648438, "blocks.2.attention.w_v": 0.85784912109375, "blocks.2.attention.w_out": 0.887420654296875, "blocks.2.feed_forward.w1": 0.8530260920524597, "blocks.2.feed_forward.w2": 0.8463847041130066, "blocks.2.feed_forward.w3": 0.8452262878417969, "blocks.3.attention.w_q": 0.8701972961425781, "blocks.3.attention.w_k": 0.8681526184082031, "blocks.3.attention.w_v": 0.838836669921875, "blocks.3.attention.w_out": 0.8636627197265625, "blocks.3.feed_forward.w1": 0.8553314208984375, "blocks.3.feed_forward.w2": 0.8477821350097656, "blocks.3.feed_forward.w3": 0.8460744619369507, "blocks.4.attention.w_q": 0.8722496032714844, "blocks.4.attention.w_k": 0.8718185424804688, "blocks.4.attention.w_v": 0.8335227966308594, "blocks.4.attention.w_out": 0.8503837585449219, "blocks.4.feed_forward.w1": 0.856475830078125, "blocks.4.feed_forward.w2": 0.8416849970817566, "blocks.4.feed_forward.w3": 0.8442739248275757, "blocks.5.attention.w_q": 0.8727455139160156, "blocks.5.attention.w_k": 0.8716049194335938, "blocks.5.attention.w_v": 0.8267478942871094, "blocks.5.attention.w_out": 0.8426780700683594, "blocks.5.feed_forward.w1": 0.8572120666503906, "blocks.5.feed_forward.w2": 0.8337821960449219, "blocks.5.feed_forward.w3": 0.8438491821289062, "blocks.6.attention.w_q": 0.8785285949707031, "blocks.6.attention.w_k": 0.8727912902832031, "blocks.6.attention.w_v": 0.8216514587402344, "blocks.6.attention.w_out": 0.8227806091308594, "blocks.6.feed_forward.w1": 0.8588129878044128, "blocks.6.feed_forward.w2": 0.8226484060287476, "blocks.6.feed_forward.w3": 0.84674072265625, "blocks.7.attention.w_q": 0.8791961669921875, "blocks.7.attention.w_k": 0.8767776489257812, "blocks.7.attention.w_v": 0.826202392578125, "blocks.7.attention.w_out": 0.8030815124511719, "blocks.7.feed_forward.w1": 0.8608385920524597, "blocks.7.feed_forward.w2": 0.7993202209472656, "blocks.7.feed_forward.w3": 0.8484815359115601, "blocks.8.attention.w_q": 0.8832893371582031, "blocks.8.attention.w_k": 0.8771133422851562, "blocks.8.attention.w_v": 0.8207740783691406, "blocks.8.attention.w_out": 0.8014793395996094, "blocks.8.feed_forward.w1": 0.8644968867301941, "blocks.8.feed_forward.w2": 0.7676722407341003, "blocks.8.feed_forward.w3": 0.8496717214584351}} +{"step": 4600, "changed_frac": 0.8417545220790765, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8462944030761719, "blocks.0.attention.w_k": 0.8453636169433594, "blocks.0.attention.w_v": 0.8143577575683594, "blocks.0.attention.w_out": 0.8210487365722656, "blocks.0.feed_forward.w1": 0.8453483581542969, "blocks.0.feed_forward.w2": 0.8456535339355469, "blocks.0.feed_forward.w3": 0.839422881603241, "blocks.1.attention.w_q": 0.8675994873046875, "blocks.1.attention.w_k": 0.8697242736816406, "blocks.1.attention.w_v": 0.8537788391113281, "blocks.1.attention.w_out": 0.8693428039550781, "blocks.1.feed_forward.w1": 0.8446807861328125, "blocks.1.feed_forward.w2": 0.8387807607650757, "blocks.1.feed_forward.w3": 0.8380661010742188, "blocks.2.attention.w_q": 0.8723335266113281, "blocks.2.attention.w_k": 0.87286376953125, "blocks.2.attention.w_v": 0.8534011840820312, "blocks.2.attention.w_out": 0.8835983276367188, "blocks.2.feed_forward.w1": 0.84783935546875, "blocks.2.feed_forward.w2": 0.8405253291130066, "blocks.2.feed_forward.w3": 0.8395945429801941, "blocks.3.attention.w_q": 0.8658218383789062, "blocks.3.attention.w_k": 0.8629074096679688, "blocks.3.attention.w_v": 0.8353958129882812, "blocks.3.attention.w_out": 0.8605537414550781, "blocks.3.feed_forward.w1": 0.850860595703125, "blocks.3.feed_forward.w2": 0.844268798828125, "blocks.3.feed_forward.w3": 0.8400408625602722, "blocks.4.attention.w_q": 0.8687820434570312, "blocks.4.attention.w_k": 0.8664398193359375, "blocks.4.attention.w_v": 0.8273048400878906, "blocks.4.attention.w_out": 0.8435287475585938, "blocks.4.feed_forward.w1": 0.8523368835449219, "blocks.4.feed_forward.w2": 0.8366228938102722, "blocks.4.feed_forward.w3": 0.839385986328125, "blocks.5.attention.w_q": 0.8670463562011719, "blocks.5.attention.w_k": 0.8655738830566406, "blocks.5.attention.w_v": 0.8194580078125, "blocks.5.attention.w_out": 0.8374481201171875, "blocks.5.feed_forward.w1": 0.8520686030387878, "blocks.5.feed_forward.w2": 0.8289451599121094, "blocks.5.feed_forward.w3": 0.8383395075798035, "blocks.6.attention.w_q": 0.8744316101074219, "blocks.6.attention.w_k": 0.867523193359375, "blocks.6.attention.w_v": 0.8149909973144531, "blocks.6.attention.w_out": 0.8191375732421875, "blocks.6.feed_forward.w1": 0.853644073009491, "blocks.6.feed_forward.w2": 0.8178736567497253, "blocks.6.feed_forward.w3": 0.8423792719841003, "blocks.7.attention.w_q": 0.8756256103515625, "blocks.7.attention.w_k": 0.8709907531738281, "blocks.7.attention.w_v": 0.8191795349121094, "blocks.7.attention.w_out": 0.7937355041503906, "blocks.7.feed_forward.w1": 0.8562088012695312, "blocks.7.feed_forward.w2": 0.7924601435661316, "blocks.7.feed_forward.w3": 0.8439611196517944, "blocks.8.attention.w_q": 0.8788223266601562, "blocks.8.attention.w_k": 0.8727493286132812, "blocks.8.attention.w_v": 0.8150634765625, "blocks.8.attention.w_out": 0.7948951721191406, "blocks.8.feed_forward.w1": 0.8594347834587097, "blocks.8.feed_forward.w2": 0.7591832876205444, "blocks.8.feed_forward.w3": 0.8447965383529663}} +{"step": 4700, "changed_frac": 0.8362237979204227, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8402366638183594, "blocks.0.attention.w_k": 0.8377647399902344, "blocks.0.attention.w_v": 0.8070907592773438, "blocks.0.attention.w_out": 0.8163719177246094, "blocks.0.feed_forward.w1": 0.8401908874511719, "blocks.0.feed_forward.w2": 0.8408076167106628, "blocks.0.feed_forward.w3": 0.8340314626693726, "blocks.1.attention.w_q": 0.8617095947265625, "blocks.1.attention.w_k": 0.8643760681152344, "blocks.1.attention.w_v": 0.8477783203125, "blocks.1.attention.w_out": 0.8641014099121094, "blocks.1.feed_forward.w1": 0.837774932384491, "blocks.1.feed_forward.w2": 0.8342565298080444, "blocks.1.feed_forward.w3": 0.8317362666130066, "blocks.2.attention.w_q": 0.868377685546875, "blocks.2.attention.w_k": 0.8676109313964844, "blocks.2.attention.w_v": 0.8474960327148438, "blocks.2.attention.w_out": 0.8811912536621094, "blocks.2.feed_forward.w1": 0.8423576354980469, "blocks.2.feed_forward.w2": 0.8364270925521851, "blocks.2.feed_forward.w3": 0.8340162038803101, "blocks.3.attention.w_q": 0.8596534729003906, "blocks.3.attention.w_k": 0.8565177917480469, "blocks.3.attention.w_v": 0.8295822143554688, "blocks.3.attention.w_out": 0.8556289672851562, "blocks.3.feed_forward.w1": 0.8452441096305847, "blocks.3.feed_forward.w2": 0.8382314443588257, "blocks.3.feed_forward.w3": 0.8353564143180847, "blocks.4.attention.w_q": 0.8646697998046875, "blocks.4.attention.w_k": 0.8622093200683594, "blocks.4.attention.w_v": 0.8215408325195312, "blocks.4.attention.w_out": 0.8387832641601562, "blocks.4.feed_forward.w1": 0.846594512462616, "blocks.4.feed_forward.w2": 0.8304990530014038, "blocks.4.feed_forward.w3": 0.8350397944450378, "blocks.5.attention.w_q": 0.8635635375976562, "blocks.5.attention.w_k": 0.8613128662109375, "blocks.5.attention.w_v": 0.8131446838378906, "blocks.5.attention.w_out": 0.8327064514160156, "blocks.5.feed_forward.w1": 0.8474388122558594, "blocks.5.feed_forward.w2": 0.8227819204330444, "blocks.5.feed_forward.w3": 0.8323554992675781, "blocks.6.attention.w_q": 0.867767333984375, "blocks.6.attention.w_k": 0.8613662719726562, "blocks.6.attention.w_v": 0.8080253601074219, "blocks.6.attention.w_out": 0.8131599426269531, "blocks.6.feed_forward.w1": 0.8494822382926941, "blocks.6.feed_forward.w2": 0.8107961416244507, "blocks.6.feed_forward.w3": 0.8366165161132812, "blocks.7.attention.w_q": 0.8692474365234375, "blocks.7.attention.w_k": 0.8663597106933594, "blocks.7.attention.w_v": 0.8141441345214844, "blocks.7.attention.w_out": 0.7895278930664062, "blocks.7.feed_forward.w1": 0.8517951965332031, "blocks.7.feed_forward.w2": 0.7843437194824219, "blocks.7.feed_forward.w3": 0.8385238647460938, "blocks.8.attention.w_q": 0.8743667602539062, "blocks.8.attention.w_k": 0.8678016662597656, "blocks.8.attention.w_v": 0.8083686828613281, "blocks.8.attention.w_out": 0.7877578735351562, "blocks.8.feed_forward.w1": 0.8548355102539062, "blocks.8.feed_forward.w2": 0.7514928579330444, "blocks.8.feed_forward.w3": 0.8392652273178101}} +{"step": 4800, "changed_frac": 0.8303122235159589, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8322792053222656, "blocks.0.attention.w_k": 0.8291435241699219, "blocks.0.attention.w_v": 0.8007431030273438, "blocks.0.attention.w_out": 0.81036376953125, "blocks.0.feed_forward.w1": 0.8334249258041382, "blocks.0.feed_forward.w2": 0.8355700373649597, "blocks.0.feed_forward.w3": 0.8270772695541382, "blocks.1.attention.w_q": 0.8567123413085938, "blocks.1.attention.w_k": 0.8573188781738281, "blocks.1.attention.w_v": 0.8425407409667969, "blocks.1.attention.w_out": 0.8602714538574219, "blocks.1.feed_forward.w1": 0.8320121765136719, "blocks.1.feed_forward.w2": 0.8289566040039062, "blocks.1.feed_forward.w3": 0.8248723745346069, "blocks.2.attention.w_q": 0.8633804321289062, "blocks.2.attention.w_k": 0.8621444702148438, "blocks.2.attention.w_v": 0.840301513671875, "blocks.2.attention.w_out": 0.8765449523925781, "blocks.2.feed_forward.w1": 0.8372611999511719, "blocks.2.feed_forward.w2": 0.8314056396484375, "blocks.2.feed_forward.w3": 0.8283373713493347, "blocks.3.attention.w_q": 0.8528327941894531, "blocks.3.attention.w_k": 0.8500213623046875, "blocks.3.attention.w_v": 0.8229026794433594, "blocks.3.attention.w_out": 0.8506011962890625, "blocks.3.feed_forward.w1": 0.8382695913314819, "blocks.3.feed_forward.w2": 0.8334426879882812, "blocks.3.feed_forward.w3": 0.8284391164779663, "blocks.4.attention.w_q": 0.857635498046875, "blocks.4.attention.w_k": 0.8552093505859375, "blocks.4.attention.w_v": 0.8157997131347656, "blocks.4.attention.w_out": 0.8349990844726562, "blocks.4.feed_forward.w1": 0.8419939875602722, "blocks.4.feed_forward.w2": 0.8253555297851562, "blocks.4.feed_forward.w3": 0.8294919729232788, "blocks.5.attention.w_q": 0.8560295104980469, "blocks.5.attention.w_k": 0.8547210693359375, "blocks.5.attention.w_v": 0.8071556091308594, "blocks.5.attention.w_out": 0.8261947631835938, "blocks.5.feed_forward.w1": 0.8410428762435913, "blocks.5.feed_forward.w2": 0.8161227107048035, "blocks.5.feed_forward.w3": 0.8274091482162476, "blocks.6.attention.w_q": 0.8635597229003906, "blocks.6.attention.w_k": 0.8566818237304688, "blocks.6.attention.w_v": 0.8028144836425781, "blocks.6.attention.w_out": 0.8071136474609375, "blocks.6.feed_forward.w1": 0.8438619375228882, "blocks.6.feed_forward.w2": 0.8040771484375, "blocks.6.feed_forward.w3": 0.8314336538314819, "blocks.7.attention.w_q": 0.8642463684082031, "blocks.7.attention.w_k": 0.8608055114746094, "blocks.7.attention.w_v": 0.8072242736816406, "blocks.7.attention.w_out": 0.7828712463378906, "blocks.7.feed_forward.w1": 0.8472315669059753, "blocks.7.feed_forward.w2": 0.7771021723747253, "blocks.7.feed_forward.w3": 0.8329582214355469, "blocks.8.attention.w_q": 0.8679275512695312, "blocks.8.attention.w_k": 0.8621597290039062, "blocks.8.attention.w_v": 0.8014450073242188, "blocks.8.attention.w_out": 0.7814178466796875, "blocks.8.feed_forward.w1": 0.8496920466423035, "blocks.8.feed_forward.w2": 0.7428525686264038, "blocks.8.feed_forward.w3": 0.8344447016716003}} +{"step": 4900, "changed_frac": 0.8242310091980503, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8248329162597656, "blocks.0.attention.w_k": 0.8215522766113281, "blocks.0.attention.w_v": 0.7964515686035156, "blocks.0.attention.w_out": 0.8058586120605469, "blocks.0.feed_forward.w1": 0.8268038630485535, "blocks.0.feed_forward.w2": 0.8278440237045288, "blocks.0.feed_forward.w3": 0.8213310241699219, "blocks.1.attention.w_q": 0.8488731384277344, "blocks.1.attention.w_k": 0.853179931640625, "blocks.1.attention.w_v": 0.83734130859375, "blocks.1.attention.w_out": 0.8561553955078125, "blocks.1.feed_forward.w1": 0.8264757990837097, "blocks.1.feed_forward.w2": 0.8230375051498413, "blocks.1.feed_forward.w3": 0.8184636831283569, "blocks.2.attention.w_q": 0.8564071655273438, "blocks.2.attention.w_k": 0.8562736511230469, "blocks.2.attention.w_v": 0.834716796875, "blocks.2.attention.w_out": 0.8741455078125, "blocks.2.feed_forward.w1": 0.830291748046875, "blocks.2.feed_forward.w2": 0.8265762329101562, "blocks.2.feed_forward.w3": 0.8223966360092163, "blocks.3.attention.w_q": 0.8468971252441406, "blocks.3.attention.w_k": 0.8425941467285156, "blocks.3.attention.w_v": 0.8167381286621094, "blocks.3.attention.w_out": 0.8465156555175781, "blocks.3.feed_forward.w1": 0.8335584402084351, "blocks.3.feed_forward.w2": 0.8274091482162476, "blocks.3.feed_forward.w3": 0.8245658874511719, "blocks.4.attention.w_q": 0.8525466918945312, "blocks.4.attention.w_k": 0.8502616882324219, "blocks.4.attention.w_v": 0.8074302673339844, "blocks.4.attention.w_out": 0.8313484191894531, "blocks.4.feed_forward.w1": 0.8366915583610535, "blocks.4.feed_forward.w2": 0.8189951777458191, "blocks.4.feed_forward.w3": 0.8228811025619507, "blocks.5.attention.w_q": 0.8509979248046875, "blocks.5.attention.w_k": 0.8487510681152344, "blocks.5.attention.w_v": 0.8005561828613281, "blocks.5.attention.w_out": 0.8201713562011719, "blocks.5.feed_forward.w1": 0.8353182673454285, "blocks.5.feed_forward.w2": 0.8101781606674194, "blocks.5.feed_forward.w3": 0.8216692805290222, "blocks.6.attention.w_q": 0.8577499389648438, "blocks.6.attention.w_k": 0.8479995727539062, "blocks.6.attention.w_v": 0.7939109802246094, "blocks.6.attention.w_out": 0.8011054992675781, "blocks.6.feed_forward.w1": 0.8381462097167969, "blocks.6.feed_forward.w2": 0.7968864440917969, "blocks.6.feed_forward.w3": 0.8249092102050781, "blocks.7.attention.w_q": 0.8596229553222656, "blocks.7.attention.w_k": 0.8558082580566406, "blocks.7.attention.w_v": 0.7996559143066406, "blocks.7.attention.w_out": 0.7766571044921875, "blocks.7.feed_forward.w1": 0.8407478332519531, "blocks.7.feed_forward.w2": 0.7684326171875, "blocks.7.feed_forward.w3": 0.8273506164550781, "blocks.8.attention.w_q": 0.8618927001953125, "blocks.8.attention.w_k": 0.8566780090332031, "blocks.8.attention.w_v": 0.7942771911621094, "blocks.8.attention.w_out": 0.7722663879394531, "blocks.8.feed_forward.w1": 0.8449300527572632, "blocks.8.feed_forward.w2": 0.73358154296875, "blocks.8.feed_forward.w3": 0.8294639587402344}} +{"step": 5000, "changed_frac": 0.8170777671357505, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8150520324707031, "blocks.0.attention.w_k": 0.8120269775390625, "blocks.0.attention.w_v": 0.7857589721679688, "blocks.0.attention.w_out": 0.799285888671875, "blocks.0.feed_forward.w1": 0.8198903799057007, "blocks.0.feed_forward.w2": 0.8218358755111694, "blocks.0.feed_forward.w3": 0.8146553039550781, "blocks.1.attention.w_q": 0.8429489135742188, "blocks.1.attention.w_k": 0.8429718017578125, "blocks.1.attention.w_v": 0.8296661376953125, "blocks.1.attention.w_out": 0.8493499755859375, "blocks.1.feed_forward.w1": 0.819098174571991, "blocks.1.feed_forward.w2": 0.8165690302848816, "blocks.1.feed_forward.w3": 0.8118693232536316, "blocks.2.attention.w_q": 0.8503227233886719, "blocks.2.attention.w_k": 0.8508415222167969, "blocks.2.attention.w_v": 0.8287086486816406, "blocks.2.attention.w_out": 0.866180419921875, "blocks.2.feed_forward.w1": 0.8230603933334351, "blocks.2.feed_forward.w2": 0.8196119070053101, "blocks.2.feed_forward.w3": 0.814825713634491, "blocks.3.attention.w_q": 0.8411865234375, "blocks.3.attention.w_k": 0.835693359375, "blocks.3.attention.w_v": 0.8086814880371094, "blocks.3.attention.w_out": 0.8400039672851562, "blocks.3.feed_forward.w1": 0.82781982421875, "blocks.3.feed_forward.w2": 0.8214124441146851, "blocks.3.feed_forward.w3": 0.8172137141227722, "blocks.4.attention.w_q": 0.8452415466308594, "blocks.4.attention.w_k": 0.8409042358398438, "blocks.4.attention.w_v": 0.8008499145507812, "blocks.4.attention.w_out": 0.8232688903808594, "blocks.4.feed_forward.w1": 0.8302599787712097, "blocks.4.feed_forward.w2": 0.8117777705192566, "blocks.4.feed_forward.w3": 0.8165842890739441, "blocks.5.attention.w_q": 0.8428611755371094, "blocks.5.attention.w_k": 0.8409767150878906, "blocks.5.attention.w_v": 0.7916908264160156, "blocks.5.attention.w_out": 0.812408447265625, "blocks.5.feed_forward.w1": 0.8290685415267944, "blocks.5.feed_forward.w2": 0.8025220632553101, "blocks.5.feed_forward.w3": 0.8135948181152344, "blocks.6.attention.w_q": 0.8503456115722656, "blocks.6.attention.w_k": 0.8438301086425781, "blocks.6.attention.w_v": 0.7884063720703125, "blocks.6.attention.w_out": 0.7949180603027344, "blocks.6.feed_forward.w1": 0.8315646052360535, "blocks.6.feed_forward.w2": 0.7890625, "blocks.6.feed_forward.w3": 0.8183149099349976, "blocks.7.attention.w_q": 0.8523330688476562, "blocks.7.attention.w_k": 0.8493118286132812, "blocks.7.attention.w_v": 0.79437255859375, "blocks.7.attention.w_out": 0.7682304382324219, "blocks.7.feed_forward.w1": 0.8346939086914062, "blocks.7.feed_forward.w2": 0.7604230642318726, "blocks.7.feed_forward.w3": 0.8185081481933594, "blocks.8.attention.w_q": 0.8541412353515625, "blocks.8.attention.w_k": 0.8495941162109375, "blocks.8.attention.w_v": 0.7870216369628906, "blocks.8.attention.w_out": 0.76513671875, "blocks.8.feed_forward.w1": 0.8385111689567566, "blocks.8.feed_forward.w2": 0.7234268188476562, "blocks.8.feed_forward.w3": 0.8216845393180847}} +{"step": 5100, "changed_frac": 0.8097166729788495, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8077468872070312, "blocks.0.attention.w_k": 0.8046722412109375, "blocks.0.attention.w_v": 0.7788124084472656, "blocks.0.attention.w_out": 0.7919540405273438, "blocks.0.feed_forward.w1": 0.8124529719352722, "blocks.0.feed_forward.w2": 0.8132985830307007, "blocks.0.feed_forward.w3": 0.8069165945053101, "blocks.1.attention.w_q": 0.8341941833496094, "blocks.1.attention.w_k": 0.8376045227050781, "blocks.1.attention.w_v": 0.8231544494628906, "blocks.1.attention.w_out": 0.8437423706054688, "blocks.1.feed_forward.w1": 0.8117231130599976, "blocks.1.feed_forward.w2": 0.8094953298568726, "blocks.1.feed_forward.w3": 0.8033790588378906, "blocks.2.attention.w_q": 0.8444900512695312, "blocks.2.attention.w_k": 0.8408470153808594, "blocks.2.attention.w_v": 0.8223457336425781, "blocks.2.attention.w_out": 0.8628616333007812, "blocks.2.feed_forward.w1": 0.8164507746696472, "blocks.2.feed_forward.w2": 0.8126907348632812, "blocks.2.feed_forward.w3": 0.8069559931755066, "blocks.3.attention.w_q": 0.8323249816894531, "blocks.3.attention.w_k": 0.8264617919921875, "blocks.3.attention.w_v": 0.8010711669921875, "blocks.3.attention.w_out": 0.8353080749511719, "blocks.3.feed_forward.w1": 0.8202896118164062, "blocks.3.feed_forward.w2": 0.8153953552246094, "blocks.3.feed_forward.w3": 0.8084895014762878, "blocks.4.attention.w_q": 0.8373756408691406, "blocks.4.attention.w_k": 0.8357582092285156, "blocks.4.attention.w_v": 0.7924232482910156, "blocks.4.attention.w_out": 0.8169937133789062, "blocks.4.feed_forward.w1": 0.8236579895019531, "blocks.4.feed_forward.w2": 0.8061981201171875, "blocks.4.feed_forward.w3": 0.807952880859375, "blocks.5.attention.w_q": 0.8358879089355469, "blocks.5.attention.w_k": 0.8350296020507812, "blocks.5.attention.w_v": 0.7823486328125, "blocks.5.attention.w_out": 0.8045921325683594, "blocks.5.feed_forward.w1": 0.8228302001953125, "blocks.5.feed_forward.w2": 0.7950388789176941, "blocks.5.feed_forward.w3": 0.8077634572982788, "blocks.6.attention.w_q": 0.8436851501464844, "blocks.6.attention.w_k": 0.8344573974609375, "blocks.6.attention.w_v": 0.7802276611328125, "blocks.6.attention.w_out": 0.7856903076171875, "blocks.6.feed_forward.w1": 0.8249448537826538, "blocks.6.feed_forward.w2": 0.7804591059684753, "blocks.6.feed_forward.w3": 0.8118883967399597, "blocks.7.attention.w_q": 0.844696044921875, "blocks.7.attention.w_k": 0.8416175842285156, "blocks.7.attention.w_v": 0.7864952087402344, "blocks.7.attention.w_out": 0.7609329223632812, "blocks.7.feed_forward.w1": 0.8283004760742188, "blocks.7.feed_forward.w2": 0.7505950927734375, "blocks.7.feed_forward.w3": 0.812408447265625, "blocks.8.attention.w_q": 0.8476066589355469, "blocks.8.attention.w_k": 0.8422660827636719, "blocks.8.attention.w_v": 0.7805213928222656, "blocks.8.attention.w_out": 0.7536201477050781, "blocks.8.feed_forward.w1": 0.8318099975585938, "blocks.8.feed_forward.w2": 0.7130483388900757, "blocks.8.feed_forward.w3": 0.8145777583122253}} +{"step": 5200, "changed_frac": 0.8017884604951255, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7987556457519531, "blocks.0.attention.w_k": 0.793853759765625, "blocks.0.attention.w_v": 0.7675857543945312, "blocks.0.attention.w_out": 0.7818183898925781, "blocks.0.feed_forward.w1": 0.8040975332260132, "blocks.0.feed_forward.w2": 0.806280791759491, "blocks.0.feed_forward.w3": 0.7981898188591003, "blocks.1.attention.w_q": 0.8284721374511719, "blocks.1.attention.w_k": 0.8290748596191406, "blocks.1.attention.w_v": 0.8147163391113281, "blocks.1.attention.w_out": 0.8384933471679688, "blocks.1.feed_forward.w1": 0.8038927912712097, "blocks.1.feed_forward.w2": 0.8007444143295288, "blocks.1.feed_forward.w3": 0.7961985468864441, "blocks.2.attention.w_q": 0.8353614807128906, "blocks.2.attention.w_k": 0.8340415954589844, "blocks.2.attention.w_v": 0.8127861022949219, "blocks.2.attention.w_out": 0.85675048828125, "blocks.2.feed_forward.w1": 0.8082771301269531, "blocks.2.feed_forward.w2": 0.8049876093864441, "blocks.2.feed_forward.w3": 0.7991765737533569, "blocks.3.attention.w_q": 0.8251724243164062, "blocks.3.attention.w_k": 0.8194923400878906, "blocks.3.attention.w_v": 0.7925071716308594, "blocks.3.attention.w_out": 0.8270187377929688, "blocks.3.feed_forward.w1": 0.8132438659667969, "blocks.3.feed_forward.w2": 0.8072382807731628, "blocks.3.feed_forward.w3": 0.8012822866439819, "blocks.4.attention.w_q": 0.8292961120605469, "blocks.4.attention.w_k": 0.8274574279785156, "blocks.4.attention.w_v": 0.7845840454101562, "blocks.4.attention.w_out": 0.8093986511230469, "blocks.4.feed_forward.w1": 0.8162816762924194, "blocks.4.feed_forward.w2": 0.7986869812011719, "blocks.4.feed_forward.w3": 0.8009859919548035, "blocks.5.attention.w_q": 0.8290176391601562, "blocks.5.attention.w_k": 0.8266220092773438, "blocks.5.attention.w_v": 0.7733650207519531, "blocks.5.attention.w_out": 0.7976646423339844, "blocks.5.feed_forward.w1": 0.8155912160873413, "blocks.5.feed_forward.w2": 0.7879880666732788, "blocks.5.feed_forward.w3": 0.7999242544174194, "blocks.6.attention.w_q": 0.8369102478027344, "blocks.6.attention.w_k": 0.8283653259277344, "blocks.6.attention.w_v": 0.7704505920410156, "blocks.6.attention.w_out": 0.7761688232421875, "blocks.6.feed_forward.w1": 0.8172760009765625, "blocks.6.feed_forward.w2": 0.7725741267204285, "blocks.6.feed_forward.w3": 0.8041572570800781, "blocks.7.attention.w_q": 0.836822509765625, "blocks.7.attention.w_k": 0.8340034484863281, "blocks.7.attention.w_v": 0.7762413024902344, "blocks.7.attention.w_out": 0.7511367797851562, "blocks.7.feed_forward.w1": 0.8207855224609375, "blocks.7.feed_forward.w2": 0.7406158447265625, "blocks.7.feed_forward.w3": 0.8054250478744507, "blocks.8.attention.w_q": 0.8402366638183594, "blocks.8.attention.w_k": 0.8338088989257812, "blocks.8.attention.w_v": 0.7715644836425781, "blocks.8.attention.w_out": 0.7427215576171875, "blocks.8.feed_forward.w1": 0.8255348205566406, "blocks.8.feed_forward.w2": 0.70208740234375, "blocks.8.feed_forward.w3": 0.807647705078125}} +{"step": 5300, "changed_frac": 0.7931445847209703, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7871780395507812, "blocks.0.attention.w_k": 0.7832489013671875, "blocks.0.attention.w_v": 0.7599906921386719, "blocks.0.attention.w_out": 0.7758064270019531, "blocks.0.feed_forward.w1": 0.7968737483024597, "blocks.0.feed_forward.w2": 0.7974433898925781, "blocks.0.feed_forward.w3": 0.7884559631347656, "blocks.1.attention.w_q": 0.816436767578125, "blocks.1.attention.w_k": 0.8200187683105469, "blocks.1.attention.w_v": 0.8081626892089844, "blocks.1.attention.w_out": 0.8295745849609375, "blocks.1.feed_forward.w1": 0.7947438955307007, "blocks.1.feed_forward.w2": 0.7938944697380066, "blocks.1.feed_forward.w3": 0.7868182063102722, "blocks.2.attention.w_q": 0.8295402526855469, "blocks.2.attention.w_k": 0.8244094848632812, "blocks.2.attention.w_v": 0.8054542541503906, "blocks.2.attention.w_out": 0.8500633239746094, "blocks.2.feed_forward.w1": 0.8011690974235535, "blocks.2.feed_forward.w2": 0.796362578868866, "blocks.2.feed_forward.w3": 0.7907575368881226, "blocks.3.attention.w_q": 0.817291259765625, "blocks.3.attention.w_k": 0.8085365295410156, "blocks.3.attention.w_v": 0.7822494506835938, "blocks.3.attention.w_out": 0.8197059631347656, "blocks.3.feed_forward.w1": 0.8046442866325378, "blocks.3.feed_forward.w2": 0.7997233271598816, "blocks.3.feed_forward.w3": 0.7924105525016785, "blocks.4.attention.w_q": 0.8212432861328125, "blocks.4.attention.w_k": 0.8193016052246094, "blocks.4.attention.w_v": 0.7725868225097656, "blocks.4.attention.w_out": 0.8020057678222656, "blocks.4.feed_forward.w1": 0.8081601858139038, "blocks.4.feed_forward.w2": 0.7902514338493347, "blocks.4.feed_forward.w3": 0.7927144765853882, "blocks.5.attention.w_q": 0.820404052734375, "blocks.5.attention.w_k": 0.8191146850585938, "blocks.5.attention.w_v": 0.7657203674316406, "blocks.5.attention.w_out": 0.7912254333496094, "blocks.5.feed_forward.w1": 0.8080381155014038, "blocks.5.feed_forward.w2": 0.7785835266113281, "blocks.5.feed_forward.w3": 0.7905197143554688, "blocks.6.attention.w_q": 0.8285026550292969, "blocks.6.attention.w_k": 0.8172225952148438, "blocks.6.attention.w_v": 0.7605094909667969, "blocks.6.attention.w_out": 0.7679328918457031, "blocks.6.feed_forward.w1": 0.8101603388786316, "blocks.6.feed_forward.w2": 0.7630577087402344, "blocks.6.feed_forward.w3": 0.79534912109375, "blocks.7.attention.w_q": 0.8295021057128906, "blocks.7.attention.w_k": 0.8253898620605469, "blocks.7.attention.w_v": 0.7660980224609375, "blocks.7.attention.w_out": 0.7409248352050781, "blocks.7.feed_forward.w1": 0.8131688833236694, "blocks.7.feed_forward.w2": 0.729925811290741, "blocks.7.feed_forward.w3": 0.7972323298454285, "blocks.8.attention.w_q": 0.8317184448242188, "blocks.8.attention.w_k": 0.8260993957519531, "blocks.8.attention.w_v": 0.7612495422363281, "blocks.8.attention.w_out": 0.7338676452636719, "blocks.8.feed_forward.w1": 0.8172264099121094, "blocks.8.feed_forward.w2": 0.6889533996582031, "blocks.8.feed_forward.w3": 0.7999051809310913}} +{"step": 5400, "changed_frac": 0.783744648990468, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7755966186523438, "blocks.0.attention.w_k": 0.7723197937011719, "blocks.0.attention.w_v": 0.7480430603027344, "blocks.0.attention.w_out": 0.7682075500488281, "blocks.0.feed_forward.w1": 0.7873471975326538, "blocks.0.feed_forward.w2": 0.7888628840446472, "blocks.0.feed_forward.w3": 0.7810313105583191, "blocks.1.attention.w_q": 0.8085365295410156, "blocks.1.attention.w_k": 0.8090896606445312, "blocks.1.attention.w_v": 0.7998161315917969, "blocks.1.attention.w_out": 0.8230171203613281, "blocks.1.feed_forward.w1": 0.7843856811523438, "blocks.1.feed_forward.w2": 0.785430908203125, "blocks.1.feed_forward.w3": 0.7778066396713257, "blocks.2.attention.w_q": 0.8193321228027344, "blocks.2.attention.w_k": 0.815582275390625, "blocks.2.attention.w_v": 0.794708251953125, "blocks.2.attention.w_out": 0.8425674438476562, "blocks.2.feed_forward.w1": 0.7924296259880066, "blocks.2.feed_forward.w2": 0.7880033254623413, "blocks.2.feed_forward.w3": 0.7808558344841003, "blocks.3.attention.w_q": 0.8072090148925781, "blocks.3.attention.w_k": 0.7994308471679688, "blocks.3.attention.w_v": 0.7721672058105469, "blocks.3.attention.w_out": 0.8136062622070312, "blocks.3.feed_forward.w1": 0.7955958247184753, "blocks.3.feed_forward.w2": 0.7917582392692566, "blocks.3.feed_forward.w3": 0.7847124934196472, "blocks.4.attention.w_q": 0.8130340576171875, "blocks.4.attention.w_k": 0.8085136413574219, "blocks.4.attention.w_v": 0.7636947631835938, "blocks.4.attention.w_out": 0.7930908203125, "blocks.4.feed_forward.w1": 0.7999216914176941, "blocks.4.feed_forward.w2": 0.7810796499252319, "blocks.4.feed_forward.w3": 0.7829526662826538, "blocks.5.attention.w_q": 0.8099098205566406, "blocks.5.attention.w_k": 0.8093643188476562, "blocks.5.attention.w_v": 0.7548942565917969, "blocks.5.attention.w_out": 0.7820930480957031, "blocks.5.feed_forward.w1": 0.7977854609489441, "blocks.5.feed_forward.w2": 0.7696101069450378, "blocks.5.feed_forward.w3": 0.7823982238769531, "blocks.6.attention.w_q": 0.8204727172851562, "blocks.6.attention.w_k": 0.8095970153808594, "blocks.6.attention.w_v": 0.7498245239257812, "blocks.6.attention.w_out": 0.7581939697265625, "blocks.6.feed_forward.w1": 0.8009669184684753, "blocks.6.feed_forward.w2": 0.7510694265365601, "blocks.6.feed_forward.w3": 0.7861226797103882, "blocks.7.attention.w_q": 0.8197212219238281, "blocks.7.attention.w_k": 0.8153457641601562, "blocks.7.attention.w_v": 0.7570915222167969, "blocks.7.attention.w_out": 0.7295341491699219, "blocks.7.feed_forward.w1": 0.8038749694824219, "blocks.7.feed_forward.w2": 0.7172927856445312, "blocks.7.feed_forward.w3": 0.7884737849235535, "blocks.8.attention.w_q": 0.8236236572265625, "blocks.8.attention.w_k": 0.8175277709960938, "blocks.8.attention.w_v": 0.7511253356933594, "blocks.8.attention.w_out": 0.7218208312988281, "blocks.8.feed_forward.w1": 0.8086166381835938, "blocks.8.feed_forward.w2": 0.6734873652458191, "blocks.8.feed_forward.w3": 0.791601836681366}} +{"step": 5500, "changed_frac": 0.7739279005262587, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7649955749511719, "blocks.0.attention.w_k": 0.7606925964355469, "blocks.0.attention.w_v": 0.7369804382324219, "blocks.0.attention.w_out": 0.7585220336914062, "blocks.0.feed_forward.w1": 0.7774556875228882, "blocks.0.feed_forward.w2": 0.7784907221794128, "blocks.0.feed_forward.w3": 0.7708231806755066, "blocks.1.attention.w_q": 0.7989082336425781, "blocks.1.attention.w_k": 0.8001937866210938, "blocks.1.attention.w_v": 0.7905387878417969, "blocks.1.attention.w_out": 0.8161239624023438, "blocks.1.feed_forward.w1": 0.7743682861328125, "blocks.1.feed_forward.w2": 0.7769254446029663, "blocks.1.feed_forward.w3": 0.7670428156852722, "blocks.2.attention.w_q": 0.8088302612304688, "blocks.2.attention.w_k": 0.8064079284667969, "blocks.2.attention.w_v": 0.7868766784667969, "blocks.2.attention.w_out": 0.8363418579101562, "blocks.2.feed_forward.w1": 0.7821706533432007, "blocks.2.feed_forward.w2": 0.78094482421875, "blocks.2.feed_forward.w3": 0.7712669372558594, "blocks.3.attention.w_q": 0.7972564697265625, "blocks.3.attention.w_k": 0.7885551452636719, "blocks.3.attention.w_v": 0.7609062194824219, "blocks.3.attention.w_out": 0.8034591674804688, "blocks.3.feed_forward.w1": 0.7866033315658569, "blocks.3.feed_forward.w2": 0.7832998037338257, "blocks.3.feed_forward.w3": 0.7736778259277344, "blocks.4.attention.w_q": 0.8031234741210938, "blocks.4.attention.w_k": 0.8003959655761719, "blocks.4.attention.w_v": 0.7538681030273438, "blocks.4.attention.w_out": 0.7827033996582031, "blocks.4.feed_forward.w1": 0.7898877859115601, "blocks.4.feed_forward.w2": 0.772070586681366, "blocks.4.feed_forward.w3": 0.7735010981559753, "blocks.5.attention.w_q": 0.8017463684082031, "blocks.5.attention.w_k": 0.7987442016601562, "blocks.5.attention.w_v": 0.7440910339355469, "blocks.5.attention.w_out": 0.772186279296875, "blocks.5.feed_forward.w1": 0.7893155813217163, "blocks.5.feed_forward.w2": 0.7601407766342163, "blocks.5.feed_forward.w3": 0.7725334167480469, "blocks.6.attention.w_q": 0.8087501525878906, "blocks.6.attention.w_k": 0.7981910705566406, "blocks.6.attention.w_v": 0.7387619018554688, "blocks.6.attention.w_out": 0.7499351501464844, "blocks.6.feed_forward.w1": 0.7908515930175781, "blocks.6.feed_forward.w2": 0.7402229309082031, "blocks.6.feed_forward.w3": 0.7754135131835938, "blocks.7.attention.w_q": 0.8104286193847656, "blocks.7.attention.w_k": 0.8052024841308594, "blocks.7.attention.w_v": 0.7459945678710938, "blocks.7.attention.w_out": 0.7186965942382812, "blocks.7.feed_forward.w1": 0.7967147827148438, "blocks.7.feed_forward.w2": 0.7046229243278503, "blocks.7.feed_forward.w3": 0.7792078852653503, "blocks.8.attention.w_q": 0.8160171508789062, "blocks.8.attention.w_k": 0.8083992004394531, "blocks.8.attention.w_v": 0.7397613525390625, "blocks.8.attention.w_out": 0.7094306945800781, "blocks.8.feed_forward.w1": 0.8012046813964844, "blocks.8.feed_forward.w2": 0.6585540771484375, "blocks.8.feed_forward.w3": 0.7818717956542969}} +{"step": 5600, "changed_frac": 0.7630530137282151, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7510604858398438, "blocks.0.attention.w_k": 0.7446250915527344, "blocks.0.attention.w_v": 0.7252578735351562, "blocks.0.attention.w_out": 0.7472305297851562, "blocks.0.feed_forward.w1": 0.7666906118392944, "blocks.0.feed_forward.w2": 0.7669474482536316, "blocks.0.feed_forward.w3": 0.7587776184082031, "blocks.1.attention.w_q": 0.7890548706054688, "blocks.1.attention.w_k": 0.7891464233398438, "blocks.1.attention.w_v": 0.77984619140625, "blocks.1.attention.w_out": 0.8064498901367188, "blocks.1.feed_forward.w1": 0.7632688283920288, "blocks.1.feed_forward.w2": 0.7671445608139038, "blocks.1.feed_forward.w3": 0.7563718557357788, "blocks.2.attention.w_q": 0.7989768981933594, "blocks.2.attention.w_k": 0.79364013671875, "blocks.2.attention.w_v": 0.7751083374023438, "blocks.2.attention.w_out": 0.8293380737304688, "blocks.2.feed_forward.w1": 0.7719777822494507, "blocks.2.feed_forward.w2": 0.7701517939567566, "blocks.2.feed_forward.w3": 0.7601420283317566, "blocks.3.attention.w_q": 0.7851409912109375, "blocks.3.attention.w_k": 0.7784271240234375, "blocks.3.attention.w_v": 0.7485618591308594, "blocks.3.attention.w_out": 0.7963638305664062, "blocks.3.feed_forward.w1": 0.7762934565544128, "blocks.3.feed_forward.w2": 0.7743123769760132, "blocks.3.feed_forward.w3": 0.7626661062240601, "blocks.4.attention.w_q": 0.792877197265625, "blocks.4.attention.w_k": 0.7879409790039062, "blocks.4.attention.w_v": 0.7420654296875, "blocks.4.attention.w_out": 0.77606201171875, "blocks.4.feed_forward.w1": 0.7803637385368347, "blocks.4.feed_forward.w2": 0.7629508972167969, "blocks.4.feed_forward.w3": 0.7618586421012878, "blocks.5.attention.w_q": 0.7895736694335938, "blocks.5.attention.w_k": 0.7872543334960938, "blocks.5.attention.w_v": 0.733917236328125, "blocks.5.attention.w_out": 0.7613449096679688, "blocks.5.feed_forward.w1": 0.7790324091911316, "blocks.5.feed_forward.w2": 0.7478739619255066, "blocks.5.feed_forward.w3": 0.7618128657341003, "blocks.6.attention.w_q": 0.7982902526855469, "blocks.6.attention.w_k": 0.7860488891601562, "blocks.6.attention.w_v": 0.7279586791992188, "blocks.6.attention.w_out": 0.7381134033203125, "blocks.6.feed_forward.w1": 0.7813631892204285, "blocks.6.feed_forward.w2": 0.7290815114974976, "blocks.6.feed_forward.w3": 0.7649752497673035, "blocks.7.attention.w_q": 0.8001174926757812, "blocks.7.attention.w_k": 0.7960166931152344, "blocks.7.attention.w_v": 0.73492431640625, "blocks.7.attention.w_out": 0.7079505920410156, "blocks.7.feed_forward.w1": 0.7862752676010132, "blocks.7.feed_forward.w2": 0.6920293569564819, "blocks.7.feed_forward.w3": 0.7679952383041382, "blocks.8.attention.w_q": 0.8041419982910156, "blocks.8.attention.w_k": 0.7997665405273438, "blocks.8.attention.w_v": 0.7286872863769531, "blocks.8.attention.w_out": 0.6943016052246094, "blocks.8.feed_forward.w1": 0.7912585139274597, "blocks.8.feed_forward.w2": 0.6441167593002319, "blocks.8.feed_forward.w3": 0.7714754939079285}} +{"step": 5700, "changed_frac": 0.7516274737496661, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.738800048828125, "blocks.0.attention.w_k": 0.7356491088867188, "blocks.0.attention.w_v": 0.71051025390625, "blocks.0.attention.w_out": 0.7388458251953125, "blocks.0.feed_forward.w1": 0.756109893321991, "blocks.0.feed_forward.w2": 0.7567037343978882, "blocks.0.feed_forward.w3": 0.7474149465560913, "blocks.1.attention.w_q": 0.7781143188476562, "blocks.1.attention.w_k": 0.7768020629882812, "blocks.1.attention.w_v": 0.7696952819824219, "blocks.1.attention.w_out": 0.7979621887207031, "blocks.1.feed_forward.w1": 0.7527593374252319, "blocks.1.feed_forward.w2": 0.7581812739372253, "blocks.1.feed_forward.w3": 0.7462489008903503, "blocks.2.attention.w_q": 0.7877769470214844, "blocks.2.attention.w_k": 0.7841453552246094, "blocks.2.attention.w_v": 0.7632293701171875, "blocks.2.attention.w_out": 0.8210830688476562, "blocks.2.feed_forward.w1": 0.7590128779411316, "blocks.2.feed_forward.w2": 0.7603315114974976, "blocks.2.feed_forward.w3": 0.7497266530990601, "blocks.3.attention.w_q": 0.7722244262695312, "blocks.3.attention.w_k": 0.7659416198730469, "blocks.3.attention.w_v": 0.7388725280761719, "blocks.3.attention.w_out": 0.7858123779296875, "blocks.3.feed_forward.w1": 0.7657203674316406, "blocks.3.feed_forward.w2": 0.7621955871582031, "blocks.3.feed_forward.w3": 0.751678466796875, "blocks.4.attention.w_q": 0.7813835144042969, "blocks.4.attention.w_k": 0.7753562927246094, "blocks.4.attention.w_v": 0.7281036376953125, "blocks.4.attention.w_out": 0.7642974853515625, "blocks.4.feed_forward.w1": 0.7701658010482788, "blocks.4.feed_forward.w2": 0.7504934072494507, "blocks.4.feed_forward.w3": 0.7515830993652344, "blocks.5.attention.w_q": 0.7782783508300781, "blocks.5.attention.w_k": 0.7742538452148438, "blocks.5.attention.w_v": 0.7185935974121094, "blocks.5.attention.w_out": 0.749725341796875, "blocks.5.feed_forward.w1": 0.768829345703125, "blocks.5.feed_forward.w2": 0.7355384826660156, "blocks.5.feed_forward.w3": 0.7505569458007812, "blocks.6.attention.w_q": 0.7857093811035156, "blocks.6.attention.w_k": 0.7742843627929688, "blocks.6.attention.w_v": 0.7157516479492188, "blocks.6.attention.w_out": 0.7273750305175781, "blocks.6.feed_forward.w1": 0.7706832885742188, "blocks.6.feed_forward.w2": 0.7163276672363281, "blocks.6.feed_forward.w3": 0.7541122436523438, "blocks.7.attention.w_q": 0.7897071838378906, "blocks.7.attention.w_k": 0.7835731506347656, "blocks.7.attention.w_v": 0.7220535278320312, "blocks.7.attention.w_out": 0.6924247741699219, "blocks.7.feed_forward.w1": 0.7744001150131226, "blocks.7.feed_forward.w2": 0.6766497492790222, "blocks.7.feed_forward.w3": 0.7571945190429688, "blocks.8.attention.w_q": 0.7932395935058594, "blocks.8.attention.w_k": 0.7875480651855469, "blocks.8.attention.w_v": 0.7166862487792969, "blocks.8.attention.w_out": 0.6810150146484375, "blocks.8.feed_forward.w1": 0.78009033203125, "blocks.8.feed_forward.w2": 0.6290906667709351, "blocks.8.feed_forward.w3": 0.7600644826889038}} +{"step": 5800, "changed_frac": 0.7398463844234108, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7235374450683594, "blocks.0.attention.w_k": 0.7178306579589844, "blocks.0.attention.w_v": 0.7003707885742188, "blocks.0.attention.w_out": 0.7256011962890625, "blocks.0.feed_forward.w1": 0.7440605163574219, "blocks.0.feed_forward.w2": 0.7431144714355469, "blocks.0.feed_forward.w3": 0.7362810969352722, "blocks.1.attention.w_q": 0.7647628784179688, "blocks.1.attention.w_k": 0.7652969360351562, "blocks.1.attention.w_v": 0.7578620910644531, "blocks.1.attention.w_out": 0.7874794006347656, "blocks.1.feed_forward.w1": 0.7401339411735535, "blocks.1.feed_forward.w2": 0.7473831176757812, "blocks.1.feed_forward.w3": 0.7335001826286316, "blocks.2.attention.w_q": 0.7761039733886719, "blocks.2.attention.w_k": 0.7713661193847656, "blocks.2.attention.w_v": 0.7524948120117188, "blocks.2.attention.w_out": 0.8116798400878906, "blocks.2.feed_forward.w1": 0.7497749328613281, "blocks.2.feed_forward.w2": 0.7509193420410156, "blocks.2.feed_forward.w3": 0.7374624013900757, "blocks.3.attention.w_q": 0.7599868774414062, "blocks.3.attention.w_k": 0.7503738403320312, "blocks.3.attention.w_v": 0.7258033752441406, "blocks.3.attention.w_out": 0.7780876159667969, "blocks.3.feed_forward.w1": 0.7540982961654663, "blocks.3.feed_forward.w2": 0.7505111694335938, "blocks.3.feed_forward.w3": 0.7402178645133972, "blocks.4.attention.w_q": 0.7700843811035156, "blocks.4.attention.w_k": 0.7626457214355469, "blocks.4.attention.w_v": 0.7135658264160156, "blocks.4.attention.w_out": 0.7529144287109375, "blocks.4.feed_forward.w1": 0.7590345144271851, "blocks.4.feed_forward.w2": 0.7403780817985535, "blocks.4.feed_forward.w3": 0.7397702932357788, "blocks.5.attention.w_q": 0.7663307189941406, "blocks.5.attention.w_k": 0.7649497985839844, "blocks.5.attention.w_v": 0.7040596008300781, "blocks.5.attention.w_out": 0.7399139404296875, "blocks.5.feed_forward.w1": 0.7573776245117188, "blocks.5.feed_forward.w2": 0.7250417470932007, "blocks.5.feed_forward.w3": 0.7404531240463257, "blocks.6.attention.w_q": 0.7747154235839844, "blocks.6.attention.w_k": 0.76202392578125, "blocks.6.attention.w_v": 0.7027854919433594, "blocks.6.attention.w_out": 0.7155265808105469, "blocks.6.feed_forward.w1": 0.7593892812728882, "blocks.6.feed_forward.w2": 0.7030016779899597, "blocks.6.feed_forward.w3": 0.7425765991210938, "blocks.7.attention.w_q": 0.7766876220703125, "blocks.7.attention.w_k": 0.771087646484375, "blocks.7.attention.w_v": 0.708740234375, "blocks.7.attention.w_out": 0.6780662536621094, "blocks.7.feed_forward.w1": 0.7629992365837097, "blocks.7.feed_forward.w2": 0.6626676321029663, "blocks.7.feed_forward.w3": 0.746863067150116, "blocks.8.attention.w_q": 0.7825088500976562, "blocks.8.attention.w_k": 0.7744102478027344, "blocks.8.attention.w_v": 0.7020492553710938, "blocks.8.attention.w_out": 0.6687736511230469, "blocks.8.feed_forward.w1": 0.7711805105209351, "blocks.8.feed_forward.w2": 0.6129786372184753, "blocks.8.feed_forward.w3": 0.7493476867675781}} +{"step": 5900, "changed_frac": 0.7267828721266526, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.70831298828125, "blocks.0.attention.w_k": 0.7019920349121094, "blocks.0.attention.w_v": 0.6860694885253906, "blocks.0.attention.w_out": 0.7133827209472656, "blocks.0.feed_forward.w1": 0.7300466299057007, "blocks.0.feed_forward.w2": 0.7313117980957031, "blocks.0.feed_forward.w3": 0.7230581045150757, "blocks.1.attention.w_q": 0.75323486328125, "blocks.1.attention.w_k": 0.7535324096679688, "blocks.1.attention.w_v": 0.7442741394042969, "blocks.1.attention.w_out": 0.7789955139160156, "blocks.1.feed_forward.w1": 0.7266718745231628, "blocks.1.feed_forward.w2": 0.7344856262207031, "blocks.1.feed_forward.w3": 0.7195600271224976, "blocks.2.attention.w_q": 0.7639503479003906, "blocks.2.attention.w_k": 0.7577476501464844, "blocks.2.attention.w_v": 0.7387962341308594, "blocks.2.attention.w_out": 0.8034858703613281, "blocks.2.feed_forward.w1": 0.7359733581542969, "blocks.2.feed_forward.w2": 0.7373365163803101, "blocks.2.feed_forward.w3": 0.7240995168685913, "blocks.3.attention.w_q": 0.7484359741210938, "blocks.3.attention.w_k": 0.736846923828125, "blocks.3.attention.w_v": 0.7144546508789062, "blocks.3.attention.w_out": 0.7655715942382812, "blocks.3.feed_forward.w1": 0.7424176931381226, "blocks.3.feed_forward.w2": 0.7396901845932007, "blocks.3.feed_forward.w3": 0.7272936701774597, "blocks.4.attention.w_q": 0.7572479248046875, "blocks.4.attention.w_k": 0.7508201599121094, "blocks.4.attention.w_v": 0.70159912109375, "blocks.4.attention.w_out": 0.7409477233886719, "blocks.4.feed_forward.w1": 0.7477137446403503, "blocks.4.feed_forward.w2": 0.7270253896713257, "blocks.4.feed_forward.w3": 0.7266896963119507, "blocks.5.attention.w_q": 0.7537269592285156, "blocks.5.attention.w_k": 0.7476615905761719, "blocks.5.attention.w_v": 0.6883621215820312, "blocks.5.attention.w_out": 0.7270851135253906, "blocks.5.feed_forward.w1": 0.7451273798942566, "blocks.5.feed_forward.w2": 0.7106196284294128, "blocks.5.feed_forward.w3": 0.7267049551010132, "blocks.6.attention.w_q": 0.7632522583007812, "blocks.6.attention.w_k": 0.7490959167480469, "blocks.6.attention.w_v": 0.6877822875976562, "blocks.6.attention.w_out": 0.7013397216796875, "blocks.6.feed_forward.w1": 0.7474543452262878, "blocks.6.feed_forward.w2": 0.6894111633300781, "blocks.6.feed_forward.w3": 0.7306035757064819, "blocks.7.attention.w_q": 0.7647323608398438, "blocks.7.attention.w_k": 0.7576141357421875, "blocks.7.attention.w_v": 0.69677734375, "blocks.7.attention.w_out": 0.663665771484375, "blocks.7.feed_forward.w1": 0.7520154714584351, "blocks.7.feed_forward.w2": 0.6476173400878906, "blocks.7.feed_forward.w3": 0.7339770197868347, "blocks.8.attention.w_q": 0.7681999206542969, "blocks.8.attention.w_k": 0.7622489929199219, "blocks.8.attention.w_v": 0.6890220642089844, "blocks.8.attention.w_out": 0.6530075073242188, "blocks.8.feed_forward.w1": 0.7580465078353882, "blocks.8.feed_forward.w2": 0.5953840017318726, "blocks.8.feed_forward.w3": 0.7364400625228882}} +{"step": 6000, "changed_frac": 0.7134052504841079, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6915054321289062, "blocks.0.attention.w_k": 0.6835365295410156, "blocks.0.attention.w_v": 0.6728134155273438, "blocks.0.attention.w_out": 0.7042465209960938, "blocks.0.feed_forward.w1": 0.7178446650505066, "blocks.0.feed_forward.w2": 0.7180086970329285, "blocks.0.feed_forward.w3": 0.7101415395736694, "blocks.1.attention.w_q": 0.7382469177246094, "blocks.1.attention.w_k": 0.7381744384765625, "blocks.1.attention.w_v": 0.7312545776367188, "blocks.1.attention.w_out": 0.7677459716796875, "blocks.1.feed_forward.w1": 0.7130279541015625, "blocks.1.feed_forward.w2": 0.7223294973373413, "blocks.1.feed_forward.w3": 0.7065760493278503, "blocks.2.attention.w_q": 0.7505836486816406, "blocks.2.attention.w_k": 0.7458419799804688, "blocks.2.attention.w_v": 0.7254295349121094, "blocks.2.attention.w_out": 0.7932586669921875, "blocks.2.feed_forward.w1": 0.7249768972396851, "blocks.2.feed_forward.w2": 0.7251523733139038, "blocks.2.feed_forward.w3": 0.7106005549430847, "blocks.3.attention.w_q": 0.7310218811035156, "blocks.3.attention.w_k": 0.7217903137207031, "blocks.3.attention.w_v": 0.6974143981933594, "blocks.3.attention.w_out": 0.7527389526367188, "blocks.3.feed_forward.w1": 0.7305883169174194, "blocks.3.feed_forward.w2": 0.7265256643295288, "blocks.3.feed_forward.w3": 0.7139244079589844, "blocks.4.attention.w_q": 0.74237060546875, "blocks.4.attention.w_k": 0.7364768981933594, "blocks.4.attention.w_v": 0.6838417053222656, "blocks.4.attention.w_out": 0.729461669921875, "blocks.4.feed_forward.w1": 0.7332687377929688, "blocks.4.feed_forward.w2": 0.7147929072380066, "blocks.4.feed_forward.w3": 0.7141355276107788, "blocks.5.attention.w_q": 0.74029541015625, "blocks.5.attention.w_k": 0.7338027954101562, "blocks.5.attention.w_v": 0.6730499267578125, "blocks.5.attention.w_out": 0.7144317626953125, "blocks.5.feed_forward.w1": 0.7344589233398438, "blocks.5.feed_forward.w2": 0.6983884572982788, "blocks.5.feed_forward.w3": 0.7120794057846069, "blocks.6.attention.w_q": 0.7472000122070312, "blocks.6.attention.w_k": 0.73529052734375, "blocks.6.attention.w_v": 0.6720390319824219, "blocks.6.attention.w_out": 0.6895484924316406, "blocks.6.feed_forward.w1": 0.7353528738021851, "blocks.6.feed_forward.w2": 0.6742693781852722, "blocks.6.feed_forward.w3": 0.717413604259491, "blocks.7.attention.w_q": 0.7488555908203125, "blocks.7.attention.w_k": 0.7449493408203125, "blocks.7.attention.w_v": 0.6821174621582031, "blocks.7.attention.w_out": 0.6502037048339844, "blocks.7.feed_forward.w1": 0.7404620051383972, "blocks.7.feed_forward.w2": 0.631317138671875, "blocks.7.feed_forward.w3": 0.7218615412712097, "blocks.8.attention.w_q": 0.7550125122070312, "blocks.8.attention.w_k": 0.7489967346191406, "blocks.8.attention.w_v": 0.6732063293457031, "blocks.8.attention.w_out": 0.63494873046875, "blocks.8.feed_forward.w1": 0.7451171875, "blocks.8.feed_forward.w2": 0.5778160095214844, "blocks.8.feed_forward.w3": 0.7251409292221069}} +{"step": 6100, "changed_frac": 0.7000772851145166, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6752243041992188, "blocks.0.attention.w_k": 0.6674766540527344, "blocks.0.attention.w_v": 0.6555252075195312, "blocks.0.attention.w_out": 0.6914176940917969, "blocks.0.feed_forward.w1": 0.7045161128044128, "blocks.0.feed_forward.w2": 0.7029838562011719, "blocks.0.feed_forward.w3": 0.6959394216537476, "blocks.1.attention.w_q": 0.7231216430664062, "blocks.1.attention.w_k": 0.7256278991699219, "blocks.1.attention.w_v": 0.7194480895996094, "blocks.1.attention.w_out": 0.7562942504882812, "blocks.1.feed_forward.w1": 0.699896514415741, "blocks.1.feed_forward.w2": 0.71038818359375, "blocks.1.feed_forward.w3": 0.6934814453125, "blocks.2.attention.w_q": 0.7378425598144531, "blocks.2.attention.w_k": 0.7291717529296875, "blocks.2.attention.w_v": 0.7124824523925781, "blocks.2.attention.w_out": 0.786346435546875, "blocks.2.feed_forward.w1": 0.7104835510253906, "blocks.2.feed_forward.w2": 0.7144584655761719, "blocks.2.feed_forward.w3": 0.6975237727165222, "blocks.3.attention.w_q": 0.7155952453613281, "blocks.3.attention.w_k": 0.7052536010742188, "blocks.3.attention.w_v": 0.682525634765625, "blocks.3.attention.w_out": 0.7407379150390625, "blocks.3.feed_forward.w1": 0.7165247797966003, "blocks.3.feed_forward.w2": 0.7145195007324219, "blocks.3.feed_forward.w3": 0.7007039785385132, "blocks.4.attention.w_q": 0.7261734008789062, "blocks.4.attention.w_k": 0.7210922241210938, "blocks.4.attention.w_v": 0.6708602905273438, "blocks.4.attention.w_out": 0.7155914306640625, "blocks.4.feed_forward.w1": 0.7211698293685913, "blocks.4.feed_forward.w2": 0.7034441828727722, "blocks.4.feed_forward.w3": 0.7014122009277344, "blocks.5.attention.w_q": 0.7269668579101562, "blocks.5.attention.w_k": 0.7197608947753906, "blocks.5.attention.w_v": 0.6599082946777344, "blocks.5.attention.w_out": 0.7013778686523438, "blocks.5.feed_forward.w1": 0.7211889028549194, "blocks.5.feed_forward.w2": 0.6835607290267944, "blocks.5.feed_forward.w3": 0.699225127696991, "blocks.6.attention.w_q": 0.7333641052246094, "blocks.6.attention.w_k": 0.7220802307128906, "blocks.6.attention.w_v": 0.6605148315429688, "blocks.6.attention.w_out": 0.6753082275390625, "blocks.6.feed_forward.w1": 0.7232831716537476, "blocks.6.feed_forward.w2": 0.6593233942985535, "blocks.6.feed_forward.w3": 0.7050310969352722, "blocks.7.attention.w_q": 0.7371139526367188, "blocks.7.attention.w_k": 0.7337913513183594, "blocks.7.attention.w_v": 0.6679840087890625, "blocks.7.attention.w_out": 0.6335029602050781, "blocks.7.feed_forward.w1": 0.7271283864974976, "blocks.7.feed_forward.w2": 0.6154937744140625, "blocks.7.feed_forward.w3": 0.7097702026367188, "blocks.8.attention.w_q": 0.7420005798339844, "blocks.8.attention.w_k": 0.7361640930175781, "blocks.8.attention.w_v": 0.657989501953125, "blocks.8.attention.w_out": 0.6220169067382812, "blocks.8.feed_forward.w1": 0.7355130910873413, "blocks.8.feed_forward.w2": 0.5608482360839844, "blocks.8.feed_forward.w3": 0.7126516103744507}} +{"step": 6200, "changed_frac": 0.6862413781320947, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6580772399902344, "blocks.0.attention.w_k": 0.6530418395996094, "blocks.0.attention.w_v": 0.6417465209960938, "blocks.0.attention.w_out": 0.6798439025878906, "blocks.0.feed_forward.w1": 0.690399169921875, "blocks.0.feed_forward.w2": 0.68963623046875, "blocks.0.feed_forward.w3": 0.6826490163803101, "blocks.1.attention.w_q": 0.7065620422363281, "blocks.1.attention.w_k": 0.7103080749511719, "blocks.1.attention.w_v": 0.7069587707519531, "blocks.1.attention.w_out": 0.7464408874511719, "blocks.1.feed_forward.w1": 0.6855367422103882, "blocks.1.feed_forward.w2": 0.6987292170524597, "blocks.1.feed_forward.w3": 0.6809806823730469, "blocks.2.attention.w_q": 0.7223777770996094, "blocks.2.attention.w_k": 0.7147903442382812, "blocks.2.attention.w_v": 0.7008285522460938, "blocks.2.attention.w_out": 0.7743644714355469, "blocks.2.feed_forward.w1": 0.6964938044548035, "blocks.2.feed_forward.w2": 0.7000236511230469, "blocks.2.feed_forward.w3": 0.6836522817611694, "blocks.3.attention.w_q": 0.7028388977050781, "blocks.3.attention.w_k": 0.6901512145996094, "blocks.3.attention.w_v": 0.6685600280761719, "blocks.3.attention.w_out": 0.7309379577636719, "blocks.3.feed_forward.w1": 0.7030957937240601, "blocks.3.feed_forward.w2": 0.7011985778808594, "blocks.3.feed_forward.w3": 0.6875826716423035, "blocks.4.attention.w_q": 0.7118034362792969, "blocks.4.attention.w_k": 0.708526611328125, "blocks.4.attention.w_v": 0.6572914123535156, "blocks.4.attention.w_out": 0.7033653259277344, "blocks.4.feed_forward.w1": 0.7091687917709351, "blocks.4.feed_forward.w2": 0.6885833740234375, "blocks.4.feed_forward.w3": 0.6873703002929688, "blocks.5.attention.w_q": 0.7111473083496094, "blocks.5.attention.w_k": 0.7054023742675781, "blocks.5.attention.w_v": 0.6418609619140625, "blocks.5.attention.w_out": 0.6870994567871094, "blocks.5.feed_forward.w1": 0.7090174555778503, "blocks.5.feed_forward.w2": 0.6699689626693726, "blocks.5.feed_forward.w3": 0.6874351501464844, "blocks.6.attention.w_q": 0.7193794250488281, "blocks.6.attention.w_k": 0.7069778442382812, "blocks.6.attention.w_v": 0.6465492248535156, "blocks.6.attention.w_out": 0.6627044677734375, "blocks.6.feed_forward.w1": 0.7093887329101562, "blocks.6.feed_forward.w2": 0.6444091796875, "blocks.6.feed_forward.w3": 0.6908887624740601, "blocks.7.attention.w_q": 0.7253074645996094, "blocks.7.attention.w_k": 0.7172698974609375, "blocks.7.attention.w_v": 0.6533279418945312, "blocks.7.attention.w_out": 0.6183929443359375, "blocks.7.feed_forward.w1": 0.7136217951774597, "blocks.7.feed_forward.w2": 0.5996882319450378, "blocks.7.feed_forward.w3": 0.6955286860466003, "blocks.8.attention.w_q": 0.7286148071289062, "blocks.8.attention.w_k": 0.7189865112304688, "blocks.8.attention.w_v": 0.6447067260742188, "blocks.8.attention.w_out": 0.6050453186035156, "blocks.8.feed_forward.w1": 0.7224897146224976, "blocks.8.feed_forward.w2": 0.5422210693359375, "blocks.8.feed_forward.w3": 0.6997935175895691}} +{"step": 6300, "changed_frac": 0.6725203033186432, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6400642395019531, "blocks.0.attention.w_k": 0.633880615234375, "blocks.0.attention.w_v": 0.6289100646972656, "blocks.0.attention.w_out": 0.6665458679199219, "blocks.0.feed_forward.w1": 0.6767718195915222, "blocks.0.feed_forward.w2": 0.6759898066520691, "blocks.0.feed_forward.w3": 0.6696815490722656, "blocks.1.attention.w_q": 0.6920433044433594, "blocks.1.attention.w_k": 0.6961746215820312, "blocks.1.attention.w_v": 0.692169189453125, "blocks.1.attention.w_out": 0.7361831665039062, "blocks.1.feed_forward.w1": 0.6708692312240601, "blocks.1.feed_forward.w2": 0.686621367931366, "blocks.1.feed_forward.w3": 0.6656990051269531, "blocks.2.attention.w_q": 0.7094535827636719, "blocks.2.attention.w_k": 0.702972412109375, "blocks.2.attention.w_v": 0.68505859375, "blocks.2.attention.w_out": 0.7662467956542969, "blocks.2.feed_forward.w1": 0.684179961681366, "blocks.2.feed_forward.w2": 0.6886736750602722, "blocks.2.feed_forward.w3": 0.6707344055175781, "blocks.3.attention.w_q": 0.6874008178710938, "blocks.3.attention.w_k": 0.6776237487792969, "blocks.3.attention.w_v": 0.6538276672363281, "blocks.3.attention.w_out": 0.7187957763671875, "blocks.3.feed_forward.w1": 0.6907552480697632, "blocks.3.feed_forward.w2": 0.6896514892578125, "blocks.3.feed_forward.w3": 0.6743940114974976, "blocks.4.attention.w_q": 0.6972198486328125, "blocks.4.attention.w_k": 0.6909980773925781, "blocks.4.attention.w_v": 0.6415672302246094, "blocks.4.attention.w_out": 0.6920051574707031, "blocks.4.feed_forward.w1": 0.696234405040741, "blocks.4.feed_forward.w2": 0.6753400564193726, "blocks.4.feed_forward.w3": 0.6757774353027344, "blocks.5.attention.w_q": 0.6978874206542969, "blocks.5.attention.w_k": 0.6879806518554688, "blocks.5.attention.w_v": 0.6295509338378906, "blocks.5.attention.w_out": 0.6737899780273438, "blocks.5.feed_forward.w1": 0.695709228515625, "blocks.5.feed_forward.w2": 0.6565157771110535, "blocks.5.feed_forward.w3": 0.6724840998649597, "blocks.6.attention.w_q": 0.7047386169433594, "blocks.6.attention.w_k": 0.6904029846191406, "blocks.6.attention.w_v": 0.6292686462402344, "blocks.6.attention.w_out": 0.6471519470214844, "blocks.6.feed_forward.w1": 0.6967138051986694, "blocks.6.feed_forward.w2": 0.6313667297363281, "blocks.6.feed_forward.w3": 0.6786092519760132, "blocks.7.attention.w_q": 0.7101669311523438, "blocks.7.attention.w_k": 0.7009735107421875, "blocks.7.attention.w_v": 0.6355743408203125, "blocks.7.attention.w_out": 0.6022262573242188, "blocks.7.feed_forward.w1": 0.7015851736068726, "blocks.7.feed_forward.w2": 0.5814260244369507, "blocks.7.feed_forward.w3": 0.6821988821029663, "blocks.8.attention.w_q": 0.7140998840332031, "blocks.8.attention.w_k": 0.7039604187011719, "blocks.8.attention.w_v": 0.6286773681640625, "blocks.8.attention.w_out": 0.5861778259277344, "blocks.8.feed_forward.w1": 0.7097829580307007, "blocks.8.feed_forward.w2": 0.5261001586914062, "blocks.8.feed_forward.w3": 0.687170684337616}} +{"step": 6400, "changed_frac": 0.6591744382157285, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6227149963378906, "blocks.0.attention.w_k": 0.6153030395507812, "blocks.0.attention.w_v": 0.6141242980957031, "blocks.0.attention.w_out": 0.6568183898925781, "blocks.0.feed_forward.w1": 0.665136992931366, "blocks.0.feed_forward.w2": 0.6630897521972656, "blocks.0.feed_forward.w3": 0.6563466787338257, "blocks.1.attention.w_q": 0.678314208984375, "blocks.1.attention.w_k": 0.6793899536132812, "blocks.1.attention.w_v": 0.6801338195800781, "blocks.1.attention.w_out": 0.7251358032226562, "blocks.1.feed_forward.w1": 0.6586965322494507, "blocks.1.feed_forward.w2": 0.6721751093864441, "blocks.1.feed_forward.w3": 0.6519826650619507, "blocks.2.attention.w_q": 0.6959075927734375, "blocks.2.attention.w_k": 0.6896400451660156, "blocks.2.attention.w_v": 0.6713142395019531, "blocks.2.attention.w_out": 0.7550048828125, "blocks.2.feed_forward.w1": 0.6694208979606628, "blocks.2.feed_forward.w2": 0.6747856140136719, "blocks.2.feed_forward.w3": 0.6586927175521851, "blocks.3.attention.w_q": 0.6743965148925781, "blocks.3.attention.w_k": 0.6618423461914062, "blocks.3.attention.w_v": 0.639801025390625, "blocks.3.attention.w_out": 0.706634521484375, "blocks.3.feed_forward.w1": 0.6773999929428101, "blocks.3.feed_forward.w2": 0.6780878901481628, "blocks.3.feed_forward.w3": 0.6617622375488281, "blocks.4.attention.w_q": 0.6835784912109375, "blocks.4.attention.w_k": 0.6791839599609375, "blocks.4.attention.w_v": 0.6262664794921875, "blocks.4.attention.w_out": 0.6802291870117188, "blocks.4.feed_forward.w1": 0.6832427978515625, "blocks.4.feed_forward.w2": 0.6624705195426941, "blocks.4.feed_forward.w3": 0.6620966792106628, "blocks.5.attention.w_q": 0.6838531494140625, "blocks.5.attention.w_k": 0.6779861450195312, "blocks.5.attention.w_v": 0.6138801574707031, "blocks.5.attention.w_out": 0.6592597961425781, "blocks.5.feed_forward.w1": 0.6836103200912476, "blocks.5.feed_forward.w2": 0.6435903310775757, "blocks.5.feed_forward.w3": 0.6607310175895691, "blocks.6.attention.w_q": 0.6908149719238281, "blocks.6.attention.w_k": 0.6765213012695312, "blocks.6.attention.w_v": 0.617156982421875, "blocks.6.attention.w_out": 0.6324348449707031, "blocks.6.feed_forward.w1": 0.6838951110839844, "blocks.6.feed_forward.w2": 0.6149101257324219, "blocks.6.feed_forward.w3": 0.665167510509491, "blocks.7.attention.w_q": 0.6976470947265625, "blocks.7.attention.w_k": 0.6881828308105469, "blocks.7.attention.w_v": 0.6194801330566406, "blocks.7.attention.w_out": 0.5866622924804688, "blocks.7.feed_forward.w1": 0.6896616816520691, "blocks.7.feed_forward.w2": 0.5654983520507812, "blocks.7.feed_forward.w3": 0.6705716848373413, "blocks.8.attention.w_q": 0.7014198303222656, "blocks.8.attention.w_k": 0.6902313232421875, "blocks.8.attention.w_v": 0.6159210205078125, "blocks.8.attention.w_out": 0.5717391967773438, "blocks.8.feed_forward.w1": 0.6970354914665222, "blocks.8.feed_forward.w2": 0.5098037719726562, "blocks.8.feed_forward.w3": 0.6749662160873413}} +{"step": 6500, "changed_frac": 0.6472507052951388, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6084518432617188, "blocks.0.attention.w_k": 0.6021652221679688, "blocks.0.attention.w_v": 0.6009902954101562, "blocks.0.attention.w_out": 0.6439056396484375, "blocks.0.feed_forward.w1": 0.6540679931640625, "blocks.0.feed_forward.w2": 0.6517435908317566, "blocks.0.feed_forward.w3": 0.645965576171875, "blocks.1.attention.w_q": 0.6672554016113281, "blocks.1.attention.w_k": 0.6664390563964844, "blocks.1.attention.w_v": 0.6661949157714844, "blocks.1.attention.w_out": 0.7163619995117188, "blocks.1.feed_forward.w1": 0.6462033987045288, "blocks.1.feed_forward.w2": 0.6647529602050781, "blocks.1.feed_forward.w3": 0.6410509943962097, "blocks.2.attention.w_q": 0.6811294555664062, "blocks.2.attention.w_k": 0.673431396484375, "blocks.2.attention.w_v": 0.661468505859375, "blocks.2.attention.w_out": 0.7447967529296875, "blocks.2.feed_forward.w1": 0.6595408320426941, "blocks.2.feed_forward.w2": 0.6648585200309753, "blocks.2.feed_forward.w3": 0.646979033946991, "blocks.3.attention.w_q": 0.6607475280761719, "blocks.3.attention.w_k": 0.647369384765625, "blocks.3.attention.w_v": 0.6262245178222656, "blocks.3.attention.w_out": 0.6974525451660156, "blocks.3.feed_forward.w1": 0.6671854853630066, "blocks.3.feed_forward.w2": 0.6670812368392944, "blocks.3.feed_forward.w3": 0.6485239863395691, "blocks.4.attention.w_q": 0.6710281372070312, "blocks.4.attention.w_k": 0.6640663146972656, "blocks.4.attention.w_v": 0.6150703430175781, "blocks.4.attention.w_out": 0.6694259643554688, "blocks.4.feed_forward.w1": 0.6722831726074219, "blocks.4.feed_forward.w2": 0.6511484980583191, "blocks.4.feed_forward.w3": 0.6504351496696472, "blocks.5.attention.w_q": 0.6711616516113281, "blocks.5.attention.w_k": 0.6646995544433594, "blocks.5.attention.w_v": 0.59783935546875, "blocks.5.attention.w_out": 0.6502952575683594, "blocks.5.feed_forward.w1": 0.6710523366928101, "blocks.5.feed_forward.w2": 0.6301612854003906, "blocks.5.feed_forward.w3": 0.6487770080566406, "blocks.6.attention.w_q": 0.6766357421875, "blocks.6.attention.w_k": 0.6610794067382812, "blocks.6.attention.w_v": 0.6035003662109375, "blocks.6.attention.w_out": 0.6210784912109375, "blocks.6.feed_forward.w1": 0.6721509695053101, "blocks.6.feed_forward.w2": 0.6031684875488281, "blocks.6.feed_forward.w3": 0.653235137462616, "blocks.7.attention.w_q": 0.6827583312988281, "blocks.7.attention.w_k": 0.6762657165527344, "blocks.7.attention.w_v": 0.6085052490234375, "blocks.7.attention.w_out": 0.5753669738769531, "blocks.7.feed_forward.w1": 0.6779022216796875, "blocks.7.feed_forward.w2": 0.5520948171615601, "blocks.7.feed_forward.w3": 0.6584523916244507, "blocks.8.attention.w_q": 0.6884498596191406, "blocks.8.attention.w_k": 0.6775093078613281, "blocks.8.attention.w_v": 0.6021690368652344, "blocks.8.attention.w_out": 0.5584449768066406, "blocks.8.feed_forward.w1": 0.6860173940658569, "blocks.8.feed_forward.w2": 0.4954974055290222, "blocks.8.feed_forward.w3": 0.66253662109375}} +{"step": 6600, "changed_frac": 0.636053721110026, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5942916870117188, "blocks.0.attention.w_k": 0.5871734619140625, "blocks.0.attention.w_v": 0.5905609130859375, "blocks.0.attention.w_out": 0.6354293823242188, "blocks.0.feed_forward.w1": 0.6418317556381226, "blocks.0.feed_forward.w2": 0.6413688659667969, "blocks.0.feed_forward.w3": 0.6342519521713257, "blocks.1.attention.w_q": 0.6530609130859375, "blocks.1.attention.w_k": 0.6559524536132812, "blocks.1.attention.w_v": 0.6564750671386719, "blocks.1.attention.w_out": 0.7051048278808594, "blocks.1.feed_forward.w1": 0.6354891657829285, "blocks.1.feed_forward.w2": 0.6533305048942566, "blocks.1.feed_forward.w3": 0.6293309926986694, "blocks.2.attention.w_q": 0.6691017150878906, "blocks.2.attention.w_k": 0.6606292724609375, "blocks.2.attention.w_v": 0.6476325988769531, "blocks.2.attention.w_out": 0.734588623046875, "blocks.2.feed_forward.w1": 0.6469981074333191, "blocks.2.feed_forward.w2": 0.6558278799057007, "blocks.2.feed_forward.w3": 0.6361821889877319, "blocks.3.attention.w_q": 0.6485977172851562, "blocks.3.attention.w_k": 0.6341934204101562, "blocks.3.attention.w_v": 0.6142807006835938, "blocks.3.attention.w_out": 0.6861419677734375, "blocks.3.feed_forward.w1": 0.6545118093490601, "blocks.3.feed_forward.w2": 0.6564878225326538, "blocks.3.feed_forward.w3": 0.6392568349838257, "blocks.4.attention.w_q": 0.6587257385253906, "blocks.4.attention.w_k": 0.6530532836914062, "blocks.4.attention.w_v": 0.6037139892578125, "blocks.4.attention.w_out": 0.6589088439941406, "blocks.4.feed_forward.w1": 0.6621882319450378, "blocks.4.feed_forward.w2": 0.6398544311523438, "blocks.4.feed_forward.w3": 0.6383794546127319, "blocks.5.attention.w_q": 0.6579208374023438, "blocks.5.attention.w_k": 0.6511154174804688, "blocks.5.attention.w_v": 0.5864753723144531, "blocks.5.attention.w_out": 0.6380233764648438, "blocks.5.feed_forward.w1": 0.6602134704589844, "blocks.5.feed_forward.w2": 0.6208407282829285, "blocks.5.feed_forward.w3": 0.6382879018783569, "blocks.6.attention.w_q": 0.6654090881347656, "blocks.6.attention.w_k": 0.6507797241210938, "blocks.6.attention.w_v": 0.5913314819335938, "blocks.6.attention.w_out": 0.6099739074707031, "blocks.6.feed_forward.w1": 0.662054717540741, "blocks.6.feed_forward.w2": 0.5906219482421875, "blocks.6.feed_forward.w3": 0.6426277160644531, "blocks.7.attention.w_q": 0.6688270568847656, "blocks.7.attention.w_k": 0.6636428833007812, "blocks.7.attention.w_v": 0.595733642578125, "blocks.7.attention.w_out": 0.5633316040039062, "blocks.7.feed_forward.w1": 0.6682714223861694, "blocks.7.feed_forward.w2": 0.5391731262207031, "blocks.7.feed_forward.w3": 0.6490211486816406, "blocks.8.attention.w_q": 0.677825927734375, "blocks.8.attention.w_k": 0.6664695739746094, "blocks.8.attention.w_v": 0.5932121276855469, "blocks.8.attention.w_out": 0.5477066040039062, "blocks.8.feed_forward.w1": 0.676483154296875, "blocks.8.feed_forward.w2": 0.4823646545410156, "blocks.8.feed_forward.w3": 0.652380645275116}} +{"step": 6700, "changed_frac": 0.6268856668064737, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.58367919921875, "blocks.0.attention.w_k": 0.5736122131347656, "blocks.0.attention.w_v": 0.578704833984375, "blocks.0.attention.w_out": 0.6278419494628906, "blocks.0.feed_forward.w1": 0.6329600214958191, "blocks.0.feed_forward.w2": 0.6332029104232788, "blocks.0.feed_forward.w3": 0.6248690485954285, "blocks.1.attention.w_q": 0.64263916015625, "blocks.1.attention.w_k": 0.643310546875, "blocks.1.attention.w_v": 0.6489181518554688, "blocks.1.attention.w_out": 0.6988525390625, "blocks.1.feed_forward.w1": 0.6253179311752319, "blocks.1.feed_forward.w2": 0.6450729370117188, "blocks.1.feed_forward.w3": 0.6222178339958191, "blocks.2.attention.w_q": 0.661865234375, "blocks.2.attention.w_k": 0.6529808044433594, "blocks.2.attention.w_v": 0.6400527954101562, "blocks.2.attention.w_out": 0.7285270690917969, "blocks.2.feed_forward.w1": 0.6383463740348816, "blocks.2.feed_forward.w2": 0.6453692317008972, "blocks.2.feed_forward.w3": 0.6259638667106628, "blocks.3.attention.w_q": 0.6363258361816406, "blocks.3.attention.w_k": 0.625579833984375, "blocks.3.attention.w_v": 0.6066856384277344, "blocks.3.attention.w_out": 0.6804428100585938, "blocks.3.feed_forward.w1": 0.6465136408805847, "blocks.3.feed_forward.w2": 0.6472816467285156, "blocks.3.feed_forward.w3": 0.6301765441894531, "blocks.4.attention.w_q": 0.6482963562011719, "blocks.4.attention.w_k": 0.6409378051757812, "blocks.4.attention.w_v": 0.5925102233886719, "blocks.4.attention.w_out": 0.6498298645019531, "blocks.4.feed_forward.w1": 0.6526820063591003, "blocks.4.feed_forward.w2": 0.6308873891830444, "blocks.4.feed_forward.w3": 0.6310781240463257, "blocks.5.attention.w_q": 0.6479721069335938, "blocks.5.attention.w_k": 0.6415901184082031, "blocks.5.attention.w_v": 0.5775871276855469, "blocks.5.attention.w_out": 0.6295166015625, "blocks.5.feed_forward.w1": 0.6522483825683594, "blocks.5.feed_forward.w2": 0.6127523183822632, "blocks.5.feed_forward.w3": 0.6299833059310913, "blocks.6.attention.w_q": 0.6567916870117188, "blocks.6.attention.w_k": 0.6409187316894531, "blocks.6.attention.w_v": 0.5816688537597656, "blocks.6.attention.w_out": 0.59954833984375, "blocks.6.feed_forward.w1": 0.6507810354232788, "blocks.6.feed_forward.w2": 0.5818799734115601, "blocks.6.feed_forward.w3": 0.6331431269645691, "blocks.7.attention.w_q": 0.66046142578125, "blocks.7.attention.w_k": 0.6526145935058594, "blocks.7.attention.w_v": 0.5891914367675781, "blocks.7.attention.w_out": 0.5518913269042969, "blocks.7.feed_forward.w1": 0.6592229604721069, "blocks.7.feed_forward.w2": 0.5284957885742188, "blocks.7.feed_forward.w3": 0.6399650573730469, "blocks.8.attention.w_q": 0.6663589477539062, "blocks.8.attention.w_k": 0.6548233032226562, "blocks.8.attention.w_v": 0.5840415954589844, "blocks.8.attention.w_out": 0.5346412658691406, "blocks.8.feed_forward.w1": 0.6678136587142944, "blocks.8.feed_forward.w2": 0.4720090329647064, "blocks.8.feed_forward.w3": 0.6445707082748413}} +{"step": 6800, "changed_frac": 0.6201322832678118, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5744895935058594, "blocks.0.attention.w_k": 0.5654106140136719, "blocks.0.attention.w_v": 0.5730514526367188, "blocks.0.attention.w_out": 0.6220512390136719, "blocks.0.feed_forward.w1": 0.6266860961914062, "blocks.0.feed_forward.w2": 0.62652587890625, "blocks.0.feed_forward.w3": 0.6203562617301941, "blocks.1.attention.w_q": 0.6356925964355469, "blocks.1.attention.w_k": 0.6343154907226562, "blocks.1.attention.w_v": 0.6397323608398438, "blocks.1.attention.w_out": 0.694183349609375, "blocks.1.feed_forward.w1": 0.6185557246208191, "blocks.1.feed_forward.w2": 0.641180694103241, "blocks.1.feed_forward.w3": 0.615362823009491, "blocks.2.attention.w_q": 0.6549301147460938, "blocks.2.attention.w_k": 0.6431083679199219, "blocks.2.attention.w_v": 0.6309928894042969, "blocks.2.attention.w_out": 0.7243080139160156, "blocks.2.feed_forward.w1": 0.631323516368866, "blocks.2.feed_forward.w2": 0.6398481130599976, "blocks.2.feed_forward.w3": 0.6202380061149597, "blocks.3.attention.w_q": 0.6309356689453125, "blocks.3.attention.w_k": 0.6137351989746094, "blocks.3.attention.w_v": 0.5982475280761719, "blocks.3.attention.w_out": 0.6761932373046875, "blocks.3.feed_forward.w1": 0.6395899653434753, "blocks.3.feed_forward.w2": 0.6417669057846069, "blocks.3.feed_forward.w3": 0.6238365173339844, "blocks.4.attention.w_q": 0.6407546997070312, "blocks.4.attention.w_k": 0.6336135864257812, "blocks.4.attention.w_v": 0.5847053527832031, "blocks.4.attention.w_out": 0.6456642150878906, "blocks.4.feed_forward.w1": 0.6459058523178101, "blocks.4.feed_forward.w2": 0.6251564025878906, "blocks.4.feed_forward.w3": 0.6241595149040222, "blocks.5.attention.w_q": 0.6397018432617188, "blocks.5.attention.w_k": 0.6321945190429688, "blocks.5.attention.w_v": 0.5700111389160156, "blocks.5.attention.w_out": 0.6224479675292969, "blocks.5.feed_forward.w1": 0.6439527273178101, "blocks.5.feed_forward.w2": 0.6057383418083191, "blocks.5.feed_forward.w3": 0.6229743957519531, "blocks.6.attention.w_q": 0.6481781005859375, "blocks.6.attention.w_k": 0.6316871643066406, "blocks.6.attention.w_v": 0.5752716064453125, "blocks.6.attention.w_out": 0.5918769836425781, "blocks.6.feed_forward.w1": 0.6457812190055847, "blocks.6.feed_forward.w2": 0.5725225210189819, "blocks.6.feed_forward.w3": 0.6264025568962097, "blocks.7.attention.w_q": 0.6513099670410156, "blocks.7.attention.w_k": 0.6441459655761719, "blocks.7.attention.w_v": 0.5783157348632812, "blocks.7.attention.w_out": 0.5475196838378906, "blocks.7.feed_forward.w1": 0.6533076167106628, "blocks.7.feed_forward.w2": 0.5227165222167969, "blocks.7.feed_forward.w3": 0.6336822509765625, "blocks.8.attention.w_q": 0.659149169921875, "blocks.8.attention.w_k": 0.646697998046875, "blocks.8.attention.w_v": 0.57733154296875, "blocks.8.attention.w_out": 0.5261192321777344, "blocks.8.feed_forward.w1": 0.6621335744857788, "blocks.8.feed_forward.w2": 0.46421942114830017, "blocks.8.feed_forward.w3": 0.6385447382926941}} +{"step": 6900, "changed_frac": 0.615669185279781, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5630149841308594, "blocks.0.attention.w_k": 0.5593948364257812, "blocks.0.attention.w_v": 0.5660667419433594, "blocks.0.attention.w_out": 0.6175956726074219, "blocks.0.feed_forward.w1": 0.6218795776367188, "blocks.0.feed_forward.w2": 0.6241073608398438, "blocks.0.feed_forward.w3": 0.6167488098144531, "blocks.1.attention.w_q": 0.6285743713378906, "blocks.1.attention.w_k": 0.6314277648925781, "blocks.1.attention.w_v": 0.6355705261230469, "blocks.1.attention.w_out": 0.6912956237792969, "blocks.1.feed_forward.w1": 0.6145490407943726, "blocks.1.feed_forward.w2": 0.6360041499137878, "blocks.1.feed_forward.w3": 0.6106974482536316, "blocks.2.attention.w_q": 0.6472854614257812, "blocks.2.attention.w_k": 0.6412582397460938, "blocks.2.attention.w_v": 0.626068115234375, "blocks.2.attention.w_out": 0.7204475402832031, "blocks.2.feed_forward.w1": 0.626898467540741, "blocks.2.feed_forward.w2": 0.6348495483398438, "blocks.2.feed_forward.w3": 0.6146151423454285, "blocks.3.attention.w_q": 0.6232223510742188, "blocks.3.attention.w_k": 0.6088066101074219, "blocks.3.attention.w_v": 0.5920524597167969, "blocks.3.attention.w_out": 0.672393798828125, "blocks.3.feed_forward.w1": 0.6362165212631226, "blocks.3.feed_forward.w2": 0.6376978754997253, "blocks.3.feed_forward.w3": 0.6202341914176941, "blocks.4.attention.w_q": 0.6354255676269531, "blocks.4.attention.w_k": 0.6274795532226562, "blocks.4.attention.w_v": 0.5813255310058594, "blocks.4.attention.w_out": 0.6412925720214844, "blocks.4.feed_forward.w1": 0.6422742605209351, "blocks.4.feed_forward.w2": 0.6212781667709351, "blocks.4.feed_forward.w3": 0.6198107600212097, "blocks.5.attention.w_q": 0.6339836120605469, "blocks.5.attention.w_k": 0.6269493103027344, "blocks.5.attention.w_v": 0.5614280700683594, "blocks.5.attention.w_out": 0.6206779479980469, "blocks.5.feed_forward.w1": 0.6408576965332031, "blocks.5.feed_forward.w2": 0.6004041433334351, "blocks.5.feed_forward.w3": 0.6188952326774597, "blocks.6.attention.w_q": 0.6412620544433594, "blocks.6.attention.w_k": 0.6245994567871094, "blocks.6.attention.w_v": 0.5707015991210938, "blocks.6.attention.w_out": 0.5889854431152344, "blocks.6.feed_forward.w1": 0.6419080495834351, "blocks.6.feed_forward.w2": 0.5685908198356628, "blocks.6.feed_forward.w3": 0.6247953176498413, "blocks.7.attention.w_q": 0.6472206115722656, "blocks.7.attention.w_k": 0.6394157409667969, "blocks.7.attention.w_v": 0.5758285522460938, "blocks.7.attention.w_out": 0.5415458679199219, "blocks.7.feed_forward.w1": 0.6486104726791382, "blocks.7.feed_forward.w2": 0.5186576843261719, "blocks.7.feed_forward.w3": 0.6298116445541382, "blocks.8.attention.w_q": 0.6513671875, "blocks.8.attention.w_k": 0.6407356262207031, "blocks.8.attention.w_v": 0.5744247436523438, "blocks.8.attention.w_out": 0.5247459411621094, "blocks.8.feed_forward.w1": 0.6575266718864441, "blocks.8.feed_forward.w2": 0.4579353332519531, "blocks.8.feed_forward.w3": 0.6339544057846069}} +{"step": 7000, "changed_frac": 0.6147573096120459, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5643997192382812, "blocks.0.attention.w_k": 0.5538063049316406, "blocks.0.attention.w_v": 0.5666770935058594, "blocks.0.attention.w_out": 0.6176643371582031, "blocks.0.feed_forward.w1": 0.6216367483139038, "blocks.0.feed_forward.w2": 0.6257108449935913, "blocks.0.feed_forward.w3": 0.6149406433105469, "blocks.1.attention.w_q": 0.6253204345703125, "blocks.1.attention.w_k": 0.6261329650878906, "blocks.1.attention.w_v": 0.6357765197753906, "blocks.1.attention.w_out": 0.6912689208984375, "blocks.1.feed_forward.w1": 0.6153450012207031, "blocks.1.feed_forward.w2": 0.6351191401481628, "blocks.1.feed_forward.w3": 0.6096140742301941, "blocks.2.attention.w_q": 0.6460647583007812, "blocks.2.attention.w_k": 0.6382179260253906, "blocks.2.attention.w_v": 0.6255950927734375, "blocks.2.attention.w_out": 0.71954345703125, "blocks.2.feed_forward.w1": 0.6267077326774597, "blocks.2.feed_forward.w2": 0.6346995234489441, "blocks.2.feed_forward.w3": 0.6135292053222656, "blocks.3.attention.w_q": 0.6225242614746094, "blocks.3.attention.w_k": 0.6052932739257812, "blocks.3.attention.w_v": 0.5934715270996094, "blocks.3.attention.w_out": 0.6699295043945312, "blocks.3.feed_forward.w1": 0.6347185969352722, "blocks.3.feed_forward.w2": 0.6390419006347656, "blocks.3.feed_forward.w3": 0.618536651134491, "blocks.4.attention.w_q": 0.6325225830078125, "blocks.4.attention.w_k": 0.6235733032226562, "blocks.4.attention.w_v": 0.5798187255859375, "blocks.4.attention.w_out": 0.6397018432617188, "blocks.4.feed_forward.w1": 0.6422335505485535, "blocks.4.feed_forward.w2": 0.6205279231071472, "blocks.4.feed_forward.w3": 0.6202163696289062, "blocks.5.attention.w_q": 0.6301345825195312, "blocks.5.attention.w_k": 0.6225204467773438, "blocks.5.attention.w_v": 0.5648460388183594, "blocks.5.attention.w_out": 0.6166458129882812, "blocks.5.feed_forward.w1": 0.6401329040527344, "blocks.5.feed_forward.w2": 0.6022517085075378, "blocks.5.feed_forward.w3": 0.6177698969841003, "blocks.6.attention.w_q": 0.6414222717285156, "blocks.6.attention.w_k": 0.6231498718261719, "blocks.6.attention.w_v": 0.5698928833007812, "blocks.6.attention.w_out": 0.5900650024414062, "blocks.6.feed_forward.w1": 0.6405080556869507, "blocks.6.feed_forward.w2": 0.5685641169548035, "blocks.6.feed_forward.w3": 0.6236228942871094, "blocks.7.attention.w_q": 0.6435737609863281, "blocks.7.attention.w_k": 0.6361007690429688, "blocks.7.attention.w_v": 0.5733757019042969, "blocks.7.attention.w_out": 0.5382308959960938, "blocks.7.feed_forward.w1": 0.6475957632064819, "blocks.7.feed_forward.w2": 0.5163409113883972, "blocks.7.feed_forward.w3": 0.6290397644042969, "blocks.8.attention.w_q": 0.6517715454101562, "blocks.8.attention.w_k": 0.639556884765625, "blocks.8.attention.w_v": 0.5727157592773438, "blocks.8.attention.w_out": 0.5194587707519531, "blocks.8.feed_forward.w1": 0.6571159362792969, "blocks.8.feed_forward.w2": 0.4573008418083191, "blocks.8.feed_forward.w3": 0.6324602961540222}} diff --git a/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..d622e7d41f175dc329527e8e46ff56ed9302cb4c --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr2.0e-02-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.038499865680933, + "blocks.0.attention.w_out.weight": 0.027206694707274437, + "blocks.0.attention.w_q.weight": 0.037828486412763596, + "blocks.0.attention.w_v.weight": 0.03433013707399368, + "blocks.0.feed_forward.w1.weight": 0.03357727825641632, + "blocks.0.feed_forward.w2.weight": 0.016322316601872444, + "blocks.0.feed_forward.w3.weight": 0.033964551985263824, + "blocks.1.attention.w_k.weight": 0.032908305525779724, + "blocks.1.attention.w_out.weight": 0.022957412526011467, + "blocks.1.attention.w_q.weight": 0.032475970685482025, + "blocks.1.attention.w_v.weight": 0.028816066682338715, + "blocks.1.feed_forward.w1.weight": 0.03485385701060295, + "blocks.1.feed_forward.w2.weight": 0.01603315770626068, + "blocks.1.feed_forward.w3.weight": 0.03442353382706642, + "blocks.2.attention.w_k.weight": 0.032416973263025284, + "blocks.2.attention.w_out.weight": 0.020846648141741753, + "blocks.2.attention.w_q.weight": 0.03133060038089752, + "blocks.2.attention.w_v.weight": 0.03030882216989994, + "blocks.2.feed_forward.w1.weight": 0.034399911761283875, + "blocks.2.feed_forward.w2.weight": 0.01665172353386879, + "blocks.2.feed_forward.w3.weight": 0.03443540260195732, + "blocks.3.attention.w_k.weight": 0.0349750854074955, + "blocks.3.attention.w_out.weight": 0.025004277005791664, + "blocks.3.attention.w_q.weight": 0.03328290209174156, + "blocks.3.attention.w_v.weight": 0.03251136839389801, + "blocks.3.feed_forward.w1.weight": 0.03327028080821037, + "blocks.3.feed_forward.w2.weight": 0.01668485999107361, + "blocks.3.feed_forward.w3.weight": 0.034158460795879364, + "blocks.4.attention.w_k.weight": 0.03413629159331322, + "blocks.4.attention.w_out.weight": 0.028049327433109283, + "blocks.4.attention.w_q.weight": 0.033892326056957245, + "blocks.4.attention.w_v.weight": 0.034088898450136185, + "blocks.4.feed_forward.w1.weight": 0.032840244472026825, + "blocks.4.feed_forward.w2.weight": 0.017762355506420135, + "blocks.4.feed_forward.w3.weight": 0.03416663780808449, + "blocks.5.attention.w_k.weight": 0.0345296636223793, + "blocks.5.attention.w_out.weight": 0.03008638508617878, + "blocks.5.attention.w_q.weight": 0.03413122519850731, + "blocks.5.attention.w_v.weight": 0.03572705015540123, + "blocks.5.feed_forward.w1.weight": 0.033185530453920364, + "blocks.5.feed_forward.w2.weight": 0.019158657640218735, + "blocks.5.feed_forward.w3.weight": 0.034439411014318466, + "blocks.6.attention.w_k.weight": 0.03546879068017006, + "blocks.6.attention.w_out.weight": 0.03304358571767807, + "blocks.6.attention.w_q.weight": 0.03404225781559944, + "blocks.6.attention.w_v.weight": 0.034208666533231735, + "blocks.6.feed_forward.w1.weight": 0.03351958841085434, + "blocks.6.feed_forward.w2.weight": 0.021552393212914467, + "blocks.6.feed_forward.w3.weight": 0.03429644927382469, + "blocks.7.attention.w_k.weight": 0.03330676257610321, + "blocks.7.attention.w_out.weight": 0.03942330554127693, + "blocks.7.attention.w_q.weight": 0.03317943587899208, + "blocks.7.attention.w_v.weight": 0.03500346839427948, + "blocks.7.feed_forward.w1.weight": 0.033117130398750305, + "blocks.7.feed_forward.w2.weight": 0.025365209206938744, + "blocks.7.feed_forward.w3.weight": 0.03389021009206772, + "blocks.8.attention.w_k.weight": 0.03294213488698006, + "blocks.8.attention.w_out.weight": 0.042533401399850845, + "blocks.8.attention.w_q.weight": 0.031727541238069534, + "blocks.8.attention.w_v.weight": 0.03506490960717201, + "blocks.8.feed_forward.w1.weight": 0.03176509588956833, + "blocks.8.feed_forward.w2.weight": 0.030179372057318687, + "blocks.8.feed_forward.w3.weight": 0.032679636031389236 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.7734184156936568 + }, + "base_loss": 3.7734184156936568, + "loss": { + "0.005": [ + 3.773475394750152, + 3.7733615148835993, + 3.7734141277888282 + ], + "0.01": [ + 3.7737136473306574, + 3.773248585711274, + 3.7736084958543157 + ], + "0.02": [ + 3.7747984246550645, + 3.774996904570894, + 3.773975827672047 + ], + "0.05": [ + 3.78265694138553, + 3.7831493657844475, + 3.7816878944559456 + ], + "0.1": [ + 3.813233067408117, + 3.8129424092754456, + 3.8118619550667634 + ], + "0.15": [ + 3.86940770748076, + 3.8677722473716707, + 3.869796709649626 + ], + "0.2": [ + 3.9592853968734474, + 3.9574374976839053, + 3.965224504994251 + ], + "0.3": [ + 4.302851032615119, + 4.308630502352151, + 4.345787384677269 + ], + "0.4": [ + 5.016516281706709, + 5.043407376060145, + 5.138693345822717 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.773475394750152, + 3.7733615148835993, + 3.7734141277888282 + ], + "0.01": [ + 3.7737136473306574, + 3.773248585711274, + 3.7736084958543157 + ], + "0.02": [ + 3.7747984246550645, + 3.774996904570894, + 3.773975827672047 + ], + "0.05": [ + 3.78265694138553, + 3.7831493657844475, + 3.7816878944559456 + ], + "0.1": [ + 3.813233067408117, + 3.8129424092754456, + 3.8118619550667634 + ], + "0.15": [ + 3.86940770748076, + 3.8677722473716707, + 3.869796709649626 + ], + "0.2": [ + 3.9592853968734474, + 3.9574374976839053, + 3.965224504994251 + ], + "0.3": [ + 4.302851032615119, + 4.308630502352151, + 4.345787384677269 + ], + "0.4": [ + 5.016516281706709, + 5.043407376060145, + 5.138693345822717 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": -1.4032194637358941e-06, + "0.01": 0.00010516060509191405, + "0.02": 0.0011719699390115856, + "0.05": 0.009079651514984466, + "0.1": 0.039260728223118324, + "0.15": 0.09557380580702857, + "0.2": 0.1872307174902108, + "0.3": 0.5456712241878563, + "0.4": 1.2927872521695338 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": -1.4032194637358941e-06, + "0.01": 0.00010516060509191405, + "0.02": 0.0011719699390115856, + "0.05": 0.009079651514984466, + "0.1": 0.039260728223118324, + "0.15": 0.09557380580702857, + "0.2": 0.1872307174902108, + "0.3": 0.5456712241878563, + "0.4": 1.2927872521695338 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 94.4 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba23dafb662ae0310655854e4c0434594ee9e23 --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.762961123476135, + "heldout_cc_en_tail_4M": 4.347372184167976 + }, + "loss_bf16_master": 3.762961123476135, + "losses_eval_model": { + "val_dolma_cc": 3.77718177596004 + }, + "bf16_minus_quant": -0.014220652483905027, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-q9-20", + "gpu": "NVIDIA L40S", + "t_s": 19.9 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..ff2cebe3d1ddfc4876bcf3a9c76c32bb9997577e --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/finalize.json @@ -0,0 +1,100 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101", + "arm": "C", + "trust_mode": "quest", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.77718177596004 + }, + "losses_eval_model": { + "val_dolma_cc": 3.77718177596004 + }, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.03083246946334839, + "blocks.0.attention.w_k": 0.030721670016646385, + "blocks.0.attention.w_v": 0.03151591494679451, + "blocks.0.attention.w_out": 0.032137431204319, + "blocks.0.feed_forward.w1": 0.030710268765687943, + "blocks.0.feed_forward.w2": 0.033606283366680145, + "blocks.0.feed_forward.w3": 0.030482860282063484, + "blocks.1.attention.w_q": 0.03076275810599327, + "blocks.1.attention.w_k": 0.03064279817044735, + "blocks.1.attention.w_v": 0.030609792098402977, + "blocks.1.attention.w_out": 0.03286907821893692, + "blocks.1.feed_forward.w1": 0.030649911612272263, + "blocks.1.feed_forward.w2": 0.03368881717324257, + "blocks.1.feed_forward.w3": 0.03062477707862854, + "blocks.2.attention.w_q": 0.030424609780311584, + "blocks.2.attention.w_k": 0.030494600534439087, + "blocks.2.attention.w_v": 0.030865278095006943, + "blocks.2.attention.w_out": 0.03127046301960945, + "blocks.2.feed_forward.w1": 0.030852487310767174, + "blocks.2.feed_forward.w2": 0.03348724916577339, + "blocks.2.feed_forward.w3": 0.030702536925673485, + "blocks.3.attention.w_q": 0.030627639964222908, + "blocks.3.attention.w_k": 0.030585728585720062, + "blocks.3.attention.w_v": 0.030913786962628365, + "blocks.3.attention.w_out": 0.031005097553133965, + "blocks.3.feed_forward.w1": 0.030687401071190834, + "blocks.3.feed_forward.w2": 0.03368067368865013, + "blocks.3.feed_forward.w3": 0.030721545219421387, + "blocks.4.attention.w_q": 0.030744265764951706, + "blocks.4.attention.w_k": 0.030621489509940147, + "blocks.4.attention.w_v": 0.03083016164600849, + "blocks.4.attention.w_out": 0.031112372875213623, + "blocks.4.feed_forward.w1": 0.030928540974855423, + "blocks.4.feed_forward.w2": 0.03337927907705307, + "blocks.4.feed_forward.w3": 0.030801206827163696, + "blocks.5.attention.w_q": 0.03097427263855934, + "blocks.5.attention.w_k": 0.030929019674658775, + "blocks.5.attention.w_v": 0.0307600237429142, + "blocks.5.attention.w_out": 0.031408339738845825, + "blocks.5.feed_forward.w1": 0.030655372887849808, + "blocks.5.feed_forward.w2": 0.03401470556855202, + "blocks.5.feed_forward.w3": 0.03081686794757843, + "blocks.6.attention.w_q": 0.030848776921629906, + "blocks.6.attention.w_k": 0.030810333788394928, + "blocks.6.attention.w_v": 0.031020324677228928, + "blocks.6.attention.w_out": 0.03343423455953598, + "blocks.6.feed_forward.w1": 0.030885523185133934, + "blocks.6.feed_forward.w2": 0.033757444471120834, + "blocks.6.feed_forward.w3": 0.030721010640263557, + "blocks.7.attention.w_q": 0.03082958422601223, + "blocks.7.attention.w_k": 0.031044142320752144, + "blocks.7.attention.w_v": 0.03102583810687065, + "blocks.7.attention.w_out": 0.03260786831378937, + "blocks.7.feed_forward.w1": 0.030825665220618248, + "blocks.7.feed_forward.w2": 0.033770423382520676, + "blocks.7.feed_forward.w3": 0.03068596124649048, + "blocks.8.attention.w_q": 0.03075864166021347, + "blocks.8.attention.w_k": 0.03075345605611801, + "blocks.8.attention.w_v": 0.03106779232621193, + "blocks.8.attention.w_out": 0.03136279061436653, + "blocks.8.feed_forward.w1": 0.03073519468307495, + "blocks.8.feed_forward.w2": 0.034182872623205185, + "blocks.8.feed_forward.w3": 0.030833741649985313 + }, + "rel_quant_noise_mean": 0.03136684869726499, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 0.7899808287620544, + "loss_eval_model": 3.77718177596004, + "checks": { + "snap_reproduces_fakequant": true, + "weights_on_arm_grid": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4facef0aa94f104b9dcdc35f7940096073cbdf7 --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/launch.yaml @@ -0,0 +1,134 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.quest_train_module.QuestTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + fq_arm: C + fq_trust_mode: quest + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: precq.muon26.MuonConfig + lr: 0.005 + weight_decay: 0.1 + momentum: 0.95 + nesterov: true + ns_steps: 5 + eps: 1.0e-07 + adjust_lr_fn: null + adamw_lr: 0.0024 + adamw_betas: + - 0.9 + - 0.95 + adamw_eps: 1.0e-15 + adamw_weight_decay: 0.1 + adamw_fused: true + muon_on_lm_head: false + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false + osc_probe: + _CLASS_: precq.quest_callbacks.OscillationProbeCallback + interval: 100 + bits: 6 + arm: C + trust_mode: quest +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/osc_probe.jsonl b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/osc_probe.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..26459ae498f6b7ff878e9263b4c7d40f64c959b2 --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/osc_probe.jsonl @@ -0,0 +1,72 @@ +{"start": true, "step": 0, "arm": "C", "bits": 6, "interval": 100} +{"step": 0} +{"step": 100, "changed_frac": 0.2549811469184028, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.27996826171875, "blocks.0.attention.w_k": 0.3110466003417969, "blocks.0.attention.w_v": 0.31456756591796875, "blocks.0.attention.w_out": 0.3260498046875, "blocks.0.feed_forward.w1": 0.29759472608566284, "blocks.0.feed_forward.w2": 0.15982310473918915, "blocks.0.feed_forward.w3": 0.2916921079158783, "blocks.1.attention.w_q": 0.27339935302734375, "blocks.1.attention.w_k": 0.3009529113769531, "blocks.1.attention.w_v": 0.28687286376953125, "blocks.1.attention.w_out": 0.2956581115722656, "blocks.1.feed_forward.w1": 0.2979087829589844, "blocks.1.feed_forward.w2": 0.1600697934627533, "blocks.1.feed_forward.w3": 0.2988739013671875, "blocks.2.attention.w_q": 0.3116950988769531, "blocks.2.attention.w_k": 0.33412933349609375, "blocks.2.attention.w_v": 0.282989501953125, "blocks.2.attention.w_out": 0.28571319580078125, "blocks.2.feed_forward.w1": 0.29857510328292847, "blocks.2.feed_forward.w2": 0.1570027768611908, "blocks.2.feed_forward.w3": 0.29994964599609375, "blocks.3.attention.w_q": 0.3122825622558594, "blocks.3.attention.w_k": 0.3331108093261719, "blocks.3.attention.w_v": 0.2776947021484375, "blocks.3.attention.w_out": 0.28041839599609375, "blocks.3.feed_forward.w1": 0.2895596921443939, "blocks.3.feed_forward.w2": 0.15480296313762665, "blocks.3.feed_forward.w3": 0.29360073804855347, "blocks.4.attention.w_q": 0.309661865234375, "blocks.4.attention.w_k": 0.337799072265625, "blocks.4.attention.w_v": 0.2680816650390625, "blocks.4.attention.w_out": 0.267486572265625, "blocks.4.feed_forward.w1": 0.28429922461509705, "blocks.4.feed_forward.w2": 0.1524810791015625, "blocks.4.feed_forward.w3": 0.2903709411621094, "blocks.5.attention.w_q": 0.3192710876464844, "blocks.5.attention.w_k": 0.34786224365234375, "blocks.5.attention.w_v": 0.26519775390625, "blocks.5.attention.w_out": 0.26136016845703125, "blocks.5.feed_forward.w1": 0.27285513281822205, "blocks.5.feed_forward.w2": 0.14642207324504852, "blocks.5.feed_forward.w3": 0.27760952711105347, "blocks.6.attention.w_q": 0.3246345520019531, "blocks.6.attention.w_k": 0.3532600402832031, "blocks.6.attention.w_v": 0.2495880126953125, "blocks.6.attention.w_out": 0.2574882507324219, "blocks.6.feed_forward.w1": 0.27282842993736267, "blocks.6.feed_forward.w2": 0.1422831267118454, "blocks.6.feed_forward.w3": 0.2695973813533783, "blocks.7.attention.w_q": 0.32422637939453125, "blocks.7.attention.w_k": 0.3497428894042969, "blocks.7.attention.w_v": 0.24201583862304688, "blocks.7.attention.w_out": 0.2407073974609375, "blocks.7.feed_forward.w1": 0.2584317624568939, "blocks.7.feed_forward.w2": 0.13858795166015625, "blocks.7.feed_forward.w3": 0.2601127624511719, "blocks.8.attention.w_q": 0.3038825988769531, "blocks.8.attention.w_k": 0.33075714111328125, "blocks.8.attention.w_v": 0.22893524169921875, "blocks.8.attention.w_out": 0.23040771484375, "blocks.8.feed_forward.w1": 0.2541491389274597, "blocks.8.feed_forward.w2": 0.13497671484947205, "blocks.8.feed_forward.w3": 0.2501678466796875}} +{"step": 200, "changed_frac": 0.5974661020132211, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6404342651367188, "blocks.0.attention.w_k": 0.653900146484375, "blocks.0.attention.w_v": 0.6912918090820312, "blocks.0.attention.w_out": 0.6628608703613281, "blocks.0.feed_forward.w1": 0.6159769892692566, "blocks.0.feed_forward.w2": 0.4821408689022064, "blocks.0.feed_forward.w3": 0.6269607543945312, "blocks.1.attention.w_q": 0.4767494201660156, "blocks.1.attention.w_k": 0.5086212158203125, "blocks.1.attention.w_v": 0.6205863952636719, "blocks.1.attention.w_out": 0.623779296875, "blocks.1.feed_forward.w1": 0.6237525939941406, "blocks.1.feed_forward.w2": 0.48932647705078125, "blocks.1.feed_forward.w3": 0.6372935175895691, "blocks.2.attention.w_q": 0.5072059631347656, "blocks.2.attention.w_k": 0.5382843017578125, "blocks.2.attention.w_v": 0.6240806579589844, "blocks.2.attention.w_out": 0.6244697570800781, "blocks.2.feed_forward.w1": 0.6290105581283569, "blocks.2.feed_forward.w2": 0.49625906348228455, "blocks.2.feed_forward.w3": 0.6407687067985535, "blocks.3.attention.w_q": 0.5096817016601562, "blocks.3.attention.w_k": 0.5298385620117188, "blocks.3.attention.w_v": 0.6221122741699219, "blocks.3.attention.w_out": 0.6267662048339844, "blocks.3.feed_forward.w1": 0.6388015747070312, "blocks.3.feed_forward.w2": 0.5043742060661316, "blocks.3.feed_forward.w3": 0.6543617248535156, "blocks.4.attention.w_q": 0.5236434936523438, "blocks.4.attention.w_k": 0.5598335266113281, "blocks.4.attention.w_v": 0.6222877502441406, "blocks.4.attention.w_out": 0.6218528747558594, "blocks.4.feed_forward.w1": 0.6465365290641785, "blocks.4.feed_forward.w2": 0.5090306997299194, "blocks.4.feed_forward.w3": 0.6617393493652344, "blocks.5.attention.w_q": 0.5306625366210938, "blocks.5.attention.w_k": 0.5567970275878906, "blocks.5.attention.w_v": 0.6185646057128906, "blocks.5.attention.w_out": 0.6239662170410156, "blocks.5.feed_forward.w1": 0.6521784663200378, "blocks.5.feed_forward.w2": 0.5146535634994507, "blocks.5.feed_forward.w3": 0.6647211909294128, "blocks.6.attention.w_q": 0.5384178161621094, "blocks.6.attention.w_k": 0.5630416870117188, "blocks.6.attention.w_v": 0.6129226684570312, "blocks.6.attention.w_out": 0.6236572265625, "blocks.6.feed_forward.w1": 0.6635894775390625, "blocks.6.feed_forward.w2": 0.5141843557357788, "blocks.6.feed_forward.w3": 0.6665369868278503, "blocks.7.attention.w_q": 0.5455741882324219, "blocks.7.attention.w_k": 0.5584983825683594, "blocks.7.attention.w_v": 0.6150093078613281, "blocks.7.attention.w_out": 0.6176528930664062, "blocks.7.feed_forward.w1": 0.6669210195541382, "blocks.7.feed_forward.w2": 0.5184275507926941, "blocks.7.feed_forward.w3": 0.6758588552474976, "blocks.8.attention.w_q": 0.5454902648925781, "blocks.8.attention.w_k": 0.5591049194335938, "blocks.8.attention.w_v": 0.6099815368652344, "blocks.8.attention.w_out": 0.6187667846679688, "blocks.8.feed_forward.w1": 0.672149658203125, "blocks.8.feed_forward.w2": 0.5164909362792969, "blocks.8.feed_forward.w3": 0.6770032644271851}} +{"step": 300, "changed_frac": 0.7473787813105135, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8089179992675781, "blocks.0.attention.w_k": 0.8134574890136719, "blocks.0.attention.w_v": 0.7958831787109375, "blocks.0.attention.w_out": 0.7775306701660156, "blocks.0.feed_forward.w1": 0.7568613886833191, "blocks.0.feed_forward.w2": 0.6441777944564819, "blocks.0.feed_forward.w3": 0.7647565603256226, "blocks.1.attention.w_q": 0.5922584533691406, "blocks.1.attention.w_k": 0.6137351989746094, "blocks.1.attention.w_v": 0.7413902282714844, "blocks.1.attention.w_out": 0.7525367736816406, "blocks.1.feed_forward.w1": 0.7738520503044128, "blocks.1.feed_forward.w2": 0.6524226069450378, "blocks.1.feed_forward.w3": 0.7837613821029663, "blocks.2.attention.w_q": 0.6160888671875, "blocks.2.attention.w_k": 0.649993896484375, "blocks.2.attention.w_v": 0.7406959533691406, "blocks.2.attention.w_out": 0.7505569458007812, "blocks.2.feed_forward.w1": 0.7815157771110535, "blocks.2.feed_forward.w2": 0.6612955927848816, "blocks.2.feed_forward.w3": 0.7895342707633972, "blocks.3.attention.w_q": 0.6613044738769531, "blocks.3.attention.w_k": 0.6754035949707031, "blocks.3.attention.w_v": 0.7426300048828125, "blocks.3.attention.w_out": 0.753265380859375, "blocks.3.feed_forward.w1": 0.7954877614974976, "blocks.3.feed_forward.w2": 0.6627375483512878, "blocks.3.feed_forward.w3": 0.8028768301010132, "blocks.4.attention.w_q": 0.6820564270019531, "blocks.4.attention.w_k": 0.7122077941894531, "blocks.4.attention.w_v": 0.7431144714355469, "blocks.4.attention.w_out": 0.7517204284667969, "blocks.4.feed_forward.w1": 0.8017387390136719, "blocks.4.feed_forward.w2": 0.6644122004508972, "blocks.4.feed_forward.w3": 0.8057441711425781, "blocks.5.attention.w_q": 0.7111740112304688, "blocks.5.attention.w_k": 0.7309608459472656, "blocks.5.attention.w_v": 0.7455062866210938, "blocks.5.attention.w_out": 0.7512321472167969, "blocks.5.feed_forward.w1": 0.806158721446991, "blocks.5.feed_forward.w2": 0.6670214533805847, "blocks.5.feed_forward.w3": 0.8109156489372253, "blocks.6.attention.w_q": 0.7330131530761719, "blocks.6.attention.w_k": 0.7515945434570312, "blocks.6.attention.w_v": 0.7462120056152344, "blocks.6.attention.w_out": 0.7568397521972656, "blocks.6.feed_forward.w1": 0.8123041987419128, "blocks.6.feed_forward.w2": 0.6705220937728882, "blocks.6.feed_forward.w3": 0.8130073547363281, "blocks.7.attention.w_q": 0.749755859375, "blocks.7.attention.w_k": 0.7586708068847656, "blocks.7.attention.w_v": 0.74945068359375, "blocks.7.attention.w_out": 0.7599258422851562, "blocks.7.feed_forward.w1": 0.8154258728027344, "blocks.7.feed_forward.w2": 0.6791979670524597, "blocks.7.feed_forward.w3": 0.8163121938705444, "blocks.8.attention.w_q": 0.7589454650878906, "blocks.8.attention.w_k": 0.7670173645019531, "blocks.8.attention.w_v": 0.7556686401367188, "blocks.8.attention.w_out": 0.7622795104980469, "blocks.8.feed_forward.w1": 0.8188006281852722, "blocks.8.feed_forward.w2": 0.6896032094955444, "blocks.8.feed_forward.w3": 0.8196640014648438}} +{"step": 400, "changed_frac": 0.7989679646288228, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8689994812011719, "blocks.0.attention.w_k": 0.8670158386230469, "blocks.0.attention.w_v": 0.8475227355957031, "blocks.0.attention.w_out": 0.8394508361816406, "blocks.0.feed_forward.w1": 0.8201281428337097, "blocks.0.feed_forward.w2": 0.7250506281852722, "blocks.0.feed_forward.w3": 0.8263105154037476, "blocks.1.attention.w_q": 0.49663543701171875, "blocks.1.attention.w_k": 0.5246124267578125, "blocks.1.attention.w_v": 0.80010986328125, "blocks.1.attention.w_out": 0.8219451904296875, "blocks.1.feed_forward.w1": 0.8342310786247253, "blocks.1.feed_forward.w2": 0.7334556579589844, "blocks.1.feed_forward.w3": 0.8405138850212097, "blocks.2.attention.w_q": 0.6695899963378906, "blocks.2.attention.w_k": 0.6997566223144531, "blocks.2.attention.w_v": 0.8048095703125, "blocks.2.attention.w_out": 0.8233795166015625, "blocks.2.feed_forward.w1": 0.840545654296875, "blocks.2.feed_forward.w2": 0.73858642578125, "blocks.2.feed_forward.w3": 0.8457069396972656, "blocks.3.attention.w_q": 0.7612686157226562, "blocks.3.attention.w_k": 0.7785606384277344, "blocks.3.attention.w_v": 0.8075180053710938, "blocks.3.attention.w_out": 0.8233833312988281, "blocks.3.feed_forward.w1": 0.8480199575424194, "blocks.3.feed_forward.w2": 0.7300071716308594, "blocks.3.feed_forward.w3": 0.8518626093864441, "blocks.4.attention.w_q": 0.5996170043945312, "blocks.4.attention.w_k": 0.6418647766113281, "blocks.4.attention.w_v": 0.8130455017089844, "blocks.4.attention.w_out": 0.8223190307617188, "blocks.4.feed_forward.w1": 0.8481254577636719, "blocks.4.feed_forward.w2": 0.7213656306266785, "blocks.4.feed_forward.w3": 0.8500785827636719, "blocks.5.attention.w_q": 0.7928199768066406, "blocks.5.attention.w_k": 0.8075523376464844, "blocks.5.attention.w_v": 0.8145217895507812, "blocks.5.attention.w_out": 0.823150634765625, "blocks.5.feed_forward.w1": 0.8456599116325378, "blocks.5.feed_forward.w2": 0.7209663391113281, "blocks.5.feed_forward.w3": 0.8472722768783569, "blocks.6.attention.w_q": 0.802154541015625, "blocks.6.attention.w_k": 0.8165779113769531, "blocks.6.attention.w_v": 0.8159713745117188, "blocks.6.attention.w_out": 0.8267250061035156, "blocks.6.feed_forward.w1": 0.8460859060287476, "blocks.6.feed_forward.w2": 0.7269210815429688, "blocks.6.feed_forward.w3": 0.8454119563102722, "blocks.7.attention.w_q": 0.8170814514160156, "blocks.7.attention.w_k": 0.8247718811035156, "blocks.7.attention.w_v": 0.8187332153320312, "blocks.7.attention.w_out": 0.8259735107421875, "blocks.7.feed_forward.w1": 0.8475621938705444, "blocks.7.feed_forward.w2": 0.737250030040741, "blocks.7.feed_forward.w3": 0.8470649719238281, "blocks.8.attention.w_q": 0.8181114196777344, "blocks.8.attention.w_k": 0.8264999389648438, "blocks.8.attention.w_v": 0.8216171264648438, "blocks.8.attention.w_out": 0.8278579711914062, "blocks.8.feed_forward.w1": 0.8496310114860535, "blocks.8.feed_forward.w2": 0.7457542419433594, "blocks.8.feed_forward.w3": 0.8490079641342163}} +{"step": 500, "changed_frac": 0.836403056087657, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8893241882324219, "blocks.0.attention.w_k": 0.8855056762695312, "blocks.0.attention.w_v": 0.8622932434082031, "blocks.0.attention.w_out": 0.8704795837402344, "blocks.0.feed_forward.w1": 0.8566996455192566, "blocks.0.feed_forward.w2": 0.7836431264877319, "blocks.0.feed_forward.w3": 0.8612022399902344, "blocks.1.attention.w_q": 0.6936607360839844, "blocks.1.attention.w_k": 0.6987457275390625, "blocks.1.attention.w_v": 0.8181686401367188, "blocks.1.attention.w_out": 0.8523483276367188, "blocks.1.feed_forward.w1": 0.8654212951660156, "blocks.1.feed_forward.w2": 0.7844161987304688, "blocks.1.feed_forward.w3": 0.8700587153434753, "blocks.2.attention.w_q": 0.748291015625, "blocks.2.attention.w_k": 0.7643814086914062, "blocks.2.attention.w_v": 0.8220062255859375, "blocks.2.attention.w_out": 0.8506851196289062, "blocks.2.feed_forward.w1": 0.8727874755859375, "blocks.2.feed_forward.w2": 0.7884877920150757, "blocks.2.feed_forward.w3": 0.8757667541503906, "blocks.3.attention.w_q": 0.8215103149414062, "blocks.3.attention.w_k": 0.8276138305664062, "blocks.3.attention.w_v": 0.8328170776367188, "blocks.3.attention.w_out": 0.8515396118164062, "blocks.3.feed_forward.w1": 0.8766352534294128, "blocks.3.feed_forward.w2": 0.7790057063102722, "blocks.3.feed_forward.w3": 0.8793589472770691, "blocks.4.attention.w_q": 0.7500686645507812, "blocks.4.attention.w_k": 0.7217788696289062, "blocks.4.attention.w_v": 0.82928466796875, "blocks.4.attention.w_out": 0.8538398742675781, "blocks.4.feed_forward.w1": 0.8744494318962097, "blocks.4.feed_forward.w2": 0.7701377868652344, "blocks.4.feed_forward.w3": 0.8742561340332031, "blocks.5.attention.w_q": 0.8195915222167969, "blocks.5.attention.w_k": 0.8328514099121094, "blocks.5.attention.w_v": 0.8447265625, "blocks.5.attention.w_out": 0.8568611145019531, "blocks.5.feed_forward.w1": 0.8709615468978882, "blocks.5.feed_forward.w2": 0.7761052846908569, "blocks.5.feed_forward.w3": 0.87139892578125, "blocks.6.attention.w_q": 0.8206405639648438, "blocks.6.attention.w_k": 0.8287620544433594, "blocks.6.attention.w_v": 0.8462715148925781, "blocks.6.attention.w_out": 0.8598594665527344, "blocks.6.feed_forward.w1": 0.8718592524528503, "blocks.6.feed_forward.w2": 0.7784335017204285, "blocks.6.feed_forward.w3": 0.8697840571403503, "blocks.7.attention.w_q": 0.8318824768066406, "blocks.7.attention.w_k": 0.8427543640136719, "blocks.7.attention.w_v": 0.8526039123535156, "blocks.7.attention.w_out": 0.8616294860839844, "blocks.7.feed_forward.w1": 0.8726412653923035, "blocks.7.feed_forward.w2": 0.7824478149414062, "blocks.7.feed_forward.w3": 0.8710479736328125, "blocks.8.attention.w_q": 0.8395271301269531, "blocks.8.attention.w_k": 0.8427886962890625, "blocks.8.attention.w_v": 0.8518409729003906, "blocks.8.attention.w_out": 0.8658828735351562, "blocks.8.feed_forward.w1": 0.8730430603027344, "blocks.8.feed_forward.w2": 0.7828267812728882, "blocks.8.feed_forward.w3": 0.8725713491439819}} +{"step": 600, "changed_frac": 0.8574406224438268, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8904266357421875, "blocks.0.attention.w_k": 0.8847579956054688, "blocks.0.attention.w_v": 0.8648185729980469, "blocks.0.attention.w_out": 0.8812026977539062, "blocks.0.feed_forward.w1": 0.8758112788200378, "blocks.0.feed_forward.w2": 0.8156954646110535, "blocks.0.feed_forward.w3": 0.8788350820541382, "blocks.1.attention.w_q": 0.7842903137207031, "blocks.1.attention.w_k": 0.78515625, "blocks.1.attention.w_v": 0.837005615234375, "blocks.1.attention.w_out": 0.8683662414550781, "blocks.1.feed_forward.w1": 0.8861579895019531, "blocks.1.feed_forward.w2": 0.8158429861068726, "blocks.1.feed_forward.w3": 0.8862711787223816, "blocks.2.attention.w_q": 0.83050537109375, "blocks.2.attention.w_k": 0.8275489807128906, "blocks.2.attention.w_v": 0.8389663696289062, "blocks.2.attention.w_out": 0.8698692321777344, "blocks.2.feed_forward.w1": 0.8887189626693726, "blocks.2.feed_forward.w2": 0.8136800527572632, "blocks.2.feed_forward.w3": 0.8892707824707031, "blocks.3.attention.w_q": 0.8520469665527344, "blocks.3.attention.w_k": 0.8532867431640625, "blocks.3.attention.w_v": 0.8514747619628906, "blocks.3.attention.w_out": 0.8725814819335938, "blocks.3.feed_forward.w1": 0.8862254023551941, "blocks.3.feed_forward.w2": 0.7997245788574219, "blocks.3.feed_forward.w3": 0.8866018056869507, "blocks.4.attention.w_q": 0.8432388305664062, "blocks.4.attention.w_k": 0.8105239868164062, "blocks.4.attention.w_v": 0.8496208190917969, "blocks.4.attention.w_out": 0.8734512329101562, "blocks.4.feed_forward.w1": 0.8825594782829285, "blocks.4.feed_forward.w2": 0.7953338623046875, "blocks.4.feed_forward.w3": 0.8817011713981628, "blocks.5.attention.w_q": 0.861724853515625, "blocks.5.attention.w_k": 0.8583450317382812, "blocks.5.attention.w_v": 0.8650970458984375, "blocks.5.attention.w_out": 0.8823585510253906, "blocks.5.feed_forward.w1": 0.8799349665641785, "blocks.5.feed_forward.w2": 0.8018926382064819, "blocks.5.feed_forward.w3": 0.8800023794174194, "blocks.6.attention.w_q": 0.862548828125, "blocks.6.attention.w_k": 0.8589286804199219, "blocks.6.attention.w_v": 0.8659820556640625, "blocks.6.attention.w_out": 0.8829917907714844, "blocks.6.feed_forward.w1": 0.8814646601676941, "blocks.6.feed_forward.w2": 0.8025398254394531, "blocks.6.feed_forward.w3": 0.8806623220443726, "blocks.7.attention.w_q": 0.8614768981933594, "blocks.7.attention.w_k": 0.8648681640625, "blocks.7.attention.w_v": 0.8733673095703125, "blocks.7.attention.w_out": 0.8861541748046875, "blocks.7.feed_forward.w1": 0.8834737539291382, "blocks.7.feed_forward.w2": 0.8041496276855469, "blocks.7.feed_forward.w3": 0.8808886408805847, "blocks.8.attention.w_q": 0.8666954040527344, "blocks.8.attention.w_k": 0.8675994873046875, "blocks.8.attention.w_v": 0.8721275329589844, "blocks.8.attention.w_out": 0.8873863220214844, "blocks.8.feed_forward.w1": 0.8842328786849976, "blocks.8.feed_forward.w2": 0.8009821772575378, "blocks.8.feed_forward.w3": 0.881933867931366}} +{"step": 700, "changed_frac": 0.8694946745522002, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8863525390625, "blocks.0.attention.w_k": 0.87982177734375, "blocks.0.attention.w_v": 0.8704795837402344, "blocks.0.attention.w_out": 0.8878059387207031, "blocks.0.feed_forward.w1": 0.8907585144042969, "blocks.0.feed_forward.w2": 0.8410758972167969, "blocks.0.feed_forward.w3": 0.8909810781478882, "blocks.1.attention.w_q": 0.820648193359375, "blocks.1.attention.w_k": 0.8199958801269531, "blocks.1.attention.w_v": 0.8584709167480469, "blocks.1.attention.w_out": 0.8808860778808594, "blocks.1.feed_forward.w1": 0.8960851430892944, "blocks.1.feed_forward.w2": 0.8385569453239441, "blocks.1.feed_forward.w3": 0.8951670527458191, "blocks.2.attention.w_q": 0.8579330444335938, "blocks.2.attention.w_k": 0.8491897583007812, "blocks.2.attention.w_v": 0.8568649291992188, "blocks.2.attention.w_out": 0.8820724487304688, "blocks.2.feed_forward.w1": 0.8932571411132812, "blocks.2.feed_forward.w2": 0.8298530578613281, "blocks.2.feed_forward.w3": 0.8925590515136719, "blocks.3.attention.w_q": 0.8765068054199219, "blocks.3.attention.w_k": 0.8707923889160156, "blocks.3.attention.w_v": 0.870697021484375, "blocks.3.attention.w_out": 0.8882331848144531, "blocks.3.feed_forward.w1": 0.8895645141601562, "blocks.3.feed_forward.w2": 0.8165995478630066, "blocks.3.feed_forward.w3": 0.8891856074333191, "blocks.4.attention.w_q": 0.8646888732910156, "blocks.4.attention.w_k": 0.8388404846191406, "blocks.4.attention.w_v": 0.8671722412109375, "blocks.4.attention.w_out": 0.8867416381835938, "blocks.4.feed_forward.w1": 0.8871790766716003, "blocks.4.feed_forward.w2": 0.81646728515625, "blocks.4.feed_forward.w3": 0.8857714533805847, "blocks.5.attention.w_q": 0.8720016479492188, "blocks.5.attention.w_k": 0.8644866943359375, "blocks.5.attention.w_v": 0.8781852722167969, "blocks.5.attention.w_out": 0.8957328796386719, "blocks.5.feed_forward.w1": 0.8871498107910156, "blocks.5.feed_forward.w2": 0.8214607238769531, "blocks.5.feed_forward.w3": 0.8866780996322632, "blocks.6.attention.w_q": 0.8726387023925781, "blocks.6.attention.w_k": 0.8657798767089844, "blocks.6.attention.w_v": 0.8815727233886719, "blocks.6.attention.w_out": 0.8927803039550781, "blocks.6.feed_forward.w1": 0.8889198303222656, "blocks.6.feed_forward.w2": 0.8195050954818726, "blocks.6.feed_forward.w3": 0.8870290517807007, "blocks.7.attention.w_q": 0.8775291442871094, "blocks.7.attention.w_k": 0.8720588684082031, "blocks.7.attention.w_v": 0.8867263793945312, "blocks.7.attention.w_out": 0.8957481384277344, "blocks.7.feed_forward.w1": 0.8914159536361694, "blocks.7.feed_forward.w2": 0.8199781179428101, "blocks.7.feed_forward.w3": 0.888647735118866, "blocks.8.attention.w_q": 0.8803634643554688, "blocks.8.attention.w_k": 0.8764305114746094, "blocks.8.attention.w_v": 0.8876304626464844, "blocks.8.attention.w_out": 0.8954734802246094, "blocks.8.feed_forward.w1": 0.8914235830307007, "blocks.8.feed_forward.w2": 0.8150405883789062, "blocks.8.feed_forward.w3": 0.8902053833007812}} +{"step": 800, "changed_frac": 0.8706873054178352, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8795890808105469, "blocks.0.attention.w_k": 0.8757591247558594, "blocks.0.attention.w_v": 0.8702011108398438, "blocks.0.attention.w_out": 0.8850975036621094, "blocks.0.feed_forward.w1": 0.893798828125, "blocks.0.feed_forward.w2": 0.8490142822265625, "blocks.0.feed_forward.w3": 0.8918966054916382, "blocks.1.attention.w_q": 0.8434715270996094, "blocks.1.attention.w_k": 0.8439903259277344, "blocks.1.attention.w_v": 0.8678131103515625, "blocks.1.attention.w_out": 0.8879203796386719, "blocks.1.feed_forward.w1": 0.8926824331283569, "blocks.1.feed_forward.w2": 0.8460973501205444, "blocks.1.feed_forward.w3": 0.8913663625717163, "blocks.2.attention.w_q": 0.8667488098144531, "blocks.2.attention.w_k": 0.8587875366210938, "blocks.2.attention.w_v": 0.8676376342773438, "blocks.2.attention.w_out": 0.885162353515625, "blocks.2.feed_forward.w1": 0.8878504633903503, "blocks.2.feed_forward.w2": 0.8328183889389038, "blocks.2.feed_forward.w3": 0.8866615295410156, "blocks.3.attention.w_q": 0.8777618408203125, "blocks.3.attention.w_k": 0.8732452392578125, "blocks.3.attention.w_v": 0.87567138671875, "blocks.3.attention.w_out": 0.8924484252929688, "blocks.3.feed_forward.w1": 0.8850517272949219, "blocks.3.feed_forward.w2": 0.8224322199821472, "blocks.3.feed_forward.w3": 0.8844515681266785, "blocks.4.attention.w_q": 0.8762702941894531, "blocks.4.attention.w_k": 0.86236572265625, "blocks.4.attention.w_v": 0.8771514892578125, "blocks.4.attention.w_out": 0.8886833190917969, "blocks.4.feed_forward.w1": 0.8858782649040222, "blocks.4.feed_forward.w2": 0.823150634765625, "blocks.4.feed_forward.w3": 0.8830999135971069, "blocks.5.attention.w_q": 0.8676109313964844, "blocks.5.attention.w_k": 0.8564872741699219, "blocks.5.attention.w_v": 0.8821220397949219, "blocks.5.attention.w_out": 0.892852783203125, "blocks.5.feed_forward.w1": 0.8873329162597656, "blocks.5.feed_forward.w2": 0.8279584646224976, "blocks.5.feed_forward.w3": 0.8858999013900757, "blocks.6.attention.w_q": 0.87335205078125, "blocks.6.attention.w_k": 0.8613967895507812, "blocks.6.attention.w_v": 0.8844108581542969, "blocks.6.attention.w_out": 0.8784255981445312, "blocks.6.feed_forward.w1": 0.8884722590446472, "blocks.6.feed_forward.w2": 0.8261197805404663, "blocks.6.feed_forward.w3": 0.8872718811035156, "blocks.7.attention.w_q": 0.8751335144042969, "blocks.7.attention.w_k": 0.8665275573730469, "blocks.7.attention.w_v": 0.8832855224609375, "blocks.7.attention.w_out": 0.8878021240234375, "blocks.7.feed_forward.w1": 0.8908576965332031, "blocks.7.feed_forward.w2": 0.8230565786361694, "blocks.7.feed_forward.w3": 0.8884315490722656, "blocks.8.attention.w_q": 0.8785820007324219, "blocks.8.attention.w_k": 0.8724441528320312, "blocks.8.attention.w_v": 0.8842811584472656, "blocks.8.attention.w_out": 0.8894081115722656, "blocks.8.feed_forward.w1": 0.8920783996582031, "blocks.8.feed_forward.w2": 0.8170509338378906, "blocks.8.feed_forward.w3": 0.8893916010856628}} +{"step": 900, "changed_frac": 0.8647148262741219, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8697128295898438, "blocks.0.attention.w_k": 0.8657913208007812, "blocks.0.attention.w_v": 0.8674850463867188, "blocks.0.attention.w_out": 0.8774070739746094, "blocks.0.feed_forward.w1": 0.8901596069335938, "blocks.0.feed_forward.w2": 0.8471196889877319, "blocks.0.feed_forward.w3": 0.8863360285758972, "blocks.1.attention.w_q": 0.8545951843261719, "blocks.1.attention.w_k": 0.8533744812011719, "blocks.1.attention.w_v": 0.8700027465820312, "blocks.1.attention.w_out": 0.8871994018554688, "blocks.1.feed_forward.w1": 0.883544921875, "blocks.1.feed_forward.w2": 0.8420804738998413, "blocks.1.feed_forward.w3": 0.881506621837616, "blocks.2.attention.w_q": 0.8648109436035156, "blocks.2.attention.w_k": 0.8586463928222656, "blocks.2.attention.w_v": 0.8672447204589844, "blocks.2.attention.w_out": 0.88226318359375, "blocks.2.feed_forward.w1": 0.8781420588493347, "blocks.2.feed_forward.w2": 0.8277384638786316, "blocks.2.feed_forward.w3": 0.8765208125114441, "blocks.3.attention.w_q": 0.8694496154785156, "blocks.3.attention.w_k": 0.8643379211425781, "blocks.3.attention.w_v": 0.8731651306152344, "blocks.3.attention.w_out": 0.8889312744140625, "blocks.3.feed_forward.w1": 0.8785591125488281, "blocks.3.feed_forward.w2": 0.8201014399528503, "blocks.3.feed_forward.w3": 0.8749160766601562, "blocks.4.attention.w_q": 0.8734931945800781, "blocks.4.attention.w_k": 0.8632354736328125, "blocks.4.attention.w_v": 0.8771743774414062, "blocks.4.attention.w_out": 0.8837013244628906, "blocks.4.feed_forward.w1": 0.8798421621322632, "blocks.4.feed_forward.w2": 0.8216158747673035, "blocks.4.feed_forward.w3": 0.8754247426986694, "blocks.5.attention.w_q": 0.8614768981933594, "blocks.5.attention.w_k": 0.8474693298339844, "blocks.5.attention.w_v": 0.8744850158691406, "blocks.5.attention.w_out": 0.8748207092285156, "blocks.5.feed_forward.w1": 0.8810603022575378, "blocks.5.feed_forward.w2": 0.8249435424804688, "blocks.5.feed_forward.w3": 0.8794454336166382, "blocks.6.attention.w_q": 0.8690681457519531, "blocks.6.attention.w_k": 0.8565826416015625, "blocks.6.attention.w_v": 0.8760719299316406, "blocks.6.attention.w_out": 0.8515434265136719, "blocks.6.feed_forward.w1": 0.8832740783691406, "blocks.6.feed_forward.w2": 0.8198903799057007, "blocks.6.feed_forward.w3": 0.8817278742790222, "blocks.7.attention.w_q": 0.8687553405761719, "blocks.7.attention.w_k": 0.8590965270996094, "blocks.7.attention.w_v": 0.8718719482421875, "blocks.7.attention.w_out": 0.8729934692382812, "blocks.7.feed_forward.w1": 0.8846715688705444, "blocks.7.feed_forward.w2": 0.8154220581054688, "blocks.7.feed_forward.w3": 0.8818473815917969, "blocks.8.attention.w_q": 0.87237548828125, "blocks.8.attention.w_k": 0.8664474487304688, "blocks.8.attention.w_v": 0.87396240234375, "blocks.8.attention.w_out": 0.875244140625, "blocks.8.feed_forward.w1": 0.8868637084960938, "blocks.8.feed_forward.w2": 0.808233916759491, "blocks.8.feed_forward.w3": 0.8847948908805847}} +{"step": 1000, "changed_frac": 0.8580362694895166, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8591346740722656, "blocks.0.attention.w_k": 0.8520965576171875, "blocks.0.attention.w_v": 0.8609352111816406, "blocks.0.attention.w_out": 0.8704605102539062, "blocks.0.feed_forward.w1": 0.8845138549804688, "blocks.0.feed_forward.w2": 0.8437487483024597, "blocks.0.feed_forward.w3": 0.8803634643554688, "blocks.1.attention.w_q": 0.8591957092285156, "blocks.1.attention.w_k": 0.85711669921875, "blocks.1.attention.w_v": 0.8698234558105469, "blocks.1.attention.w_out": 0.8856582641601562, "blocks.1.feed_forward.w1": 0.8744939565658569, "blocks.1.feed_forward.w2": 0.8372230529785156, "blocks.1.feed_forward.w3": 0.8717066645622253, "blocks.2.attention.w_q": 0.859527587890625, "blocks.2.attention.w_k": 0.8557662963867188, "blocks.2.attention.w_v": 0.8636932373046875, "blocks.2.attention.w_out": 0.8797187805175781, "blocks.2.feed_forward.w1": 0.8703104853630066, "blocks.2.feed_forward.w2": 0.8211250305175781, "blocks.2.feed_forward.w3": 0.8681716918945312, "blocks.3.attention.w_q": 0.8614349365234375, "blocks.3.attention.w_k": 0.8556938171386719, "blocks.3.attention.w_v": 0.8666191101074219, "blocks.3.attention.w_out": 0.8845024108886719, "blocks.3.feed_forward.w1": 0.8710746765136719, "blocks.3.feed_forward.w2": 0.8170331716537476, "blocks.3.feed_forward.w3": 0.8675346374511719, "blocks.4.attention.w_q": 0.868133544921875, "blocks.4.attention.w_k": 0.8629188537597656, "blocks.4.attention.w_v": 0.8720474243164062, "blocks.4.attention.w_out": 0.8800315856933594, "blocks.4.feed_forward.w1": 0.8729400634765625, "blocks.4.feed_forward.w2": 0.8189367055892944, "blocks.4.feed_forward.w3": 0.8685099482536316, "blocks.5.attention.w_q": 0.8563804626464844, "blocks.5.attention.w_k": 0.8391838073730469, "blocks.5.attention.w_v": 0.8648414611816406, "blocks.5.attention.w_out": 0.8611488342285156, "blocks.5.feed_forward.w1": 0.8756434321403503, "blocks.5.feed_forward.w2": 0.8203989863395691, "blocks.5.feed_forward.w3": 0.8721746206283569, "blocks.6.attention.w_q": 0.8619575500488281, "blocks.6.attention.w_k": 0.8477554321289062, "blocks.6.attention.w_v": 0.8648109436035156, "blocks.6.attention.w_out": 0.8343620300292969, "blocks.6.feed_forward.w1": 0.8768157958984375, "blocks.6.feed_forward.w2": 0.8136266469955444, "blocks.6.feed_forward.w3": 0.8741289973258972, "blocks.7.attention.w_q": 0.8647003173828125, "blocks.7.attention.w_k": 0.8525886535644531, "blocks.7.attention.w_v": 0.8616256713867188, "blocks.7.attention.w_out": 0.8558845520019531, "blocks.7.feed_forward.w1": 0.8795446157455444, "blocks.7.feed_forward.w2": 0.8074747920036316, "blocks.7.feed_forward.w3": 0.8747024536132812, "blocks.8.attention.w_q": 0.8655586242675781, "blocks.8.attention.w_k": 0.8577461242675781, "blocks.8.attention.w_v": 0.8647499084472656, "blocks.8.attention.w_out": 0.8630638122558594, "blocks.8.feed_forward.w1": 0.8808237910270691, "blocks.8.feed_forward.w2": 0.7964045405387878, "blocks.8.feed_forward.w3": 0.8770345449447632}} +{"step": 1100, "changed_frac": 0.8527983967055622, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8483505249023438, "blocks.0.attention.w_k": 0.840911865234375, "blocks.0.attention.w_v": 0.8535499572753906, "blocks.0.attention.w_out": 0.8645286560058594, "blocks.0.feed_forward.w1": 0.8802172541618347, "blocks.0.feed_forward.w2": 0.8408088684082031, "blocks.0.feed_forward.w3": 0.87396240234375, "blocks.1.attention.w_q": 0.8590469360351562, "blocks.1.attention.w_k": 0.8567390441894531, "blocks.1.attention.w_v": 0.8671150207519531, "blocks.1.attention.w_out": 0.8823738098144531, "blocks.1.feed_forward.w1": 0.8684603571891785, "blocks.1.feed_forward.w2": 0.8355178833007812, "blocks.1.feed_forward.w3": 0.8642743825912476, "blocks.2.attention.w_q": 0.858001708984375, "blocks.2.attention.w_k": 0.8537712097167969, "blocks.2.attention.w_v": 0.8617515563964844, "blocks.2.attention.w_out": 0.8763542175292969, "blocks.2.feed_forward.w1": 0.8639068603515625, "blocks.2.feed_forward.w2": 0.8179118037223816, "blocks.2.feed_forward.w3": 0.8615061640739441, "blocks.3.attention.w_q": 0.8541259765625, "blocks.3.attention.w_k": 0.8486557006835938, "blocks.3.attention.w_v": 0.8636245727539062, "blocks.3.attention.w_out": 0.8804435729980469, "blocks.3.feed_forward.w1": 0.8653272390365601, "blocks.3.feed_forward.w2": 0.816076934337616, "blocks.3.feed_forward.w3": 0.8617503046989441, "blocks.4.attention.w_q": 0.86383056640625, "blocks.4.attention.w_k": 0.8614387512207031, "blocks.4.attention.w_v": 0.8677978515625, "blocks.4.attention.w_out": 0.874755859375, "blocks.4.feed_forward.w1": 0.8666394948959351, "blocks.4.feed_forward.w2": 0.816802978515625, "blocks.4.feed_forward.w3": 0.8639628291130066, "blocks.5.attention.w_q": 0.8528518676757812, "blocks.5.attention.w_k": 0.8350372314453125, "blocks.5.attention.w_v": 0.8571815490722656, "blocks.5.attention.w_out": 0.8538398742675781, "blocks.5.feed_forward.w1": 0.8703283071517944, "blocks.5.feed_forward.w2": 0.8170573115348816, "blocks.5.feed_forward.w3": 0.8671290278434753, "blocks.6.attention.w_q": 0.8558578491210938, "blocks.6.attention.w_k": 0.8425331115722656, "blocks.6.attention.w_v": 0.8558464050292969, "blocks.6.attention.w_out": 0.8201026916503906, "blocks.6.feed_forward.w1": 0.8720130920410156, "blocks.6.feed_forward.w2": 0.8086967468261719, "blocks.6.feed_forward.w3": 0.8684031367301941, "blocks.7.attention.w_q": 0.8603553771972656, "blocks.7.attention.w_k": 0.84796142578125, "blocks.7.attention.w_v": 0.8535270690917969, "blocks.7.attention.w_out": 0.8422088623046875, "blocks.7.feed_forward.w1": 0.8748830556869507, "blocks.7.feed_forward.w2": 0.7998670339584351, "blocks.7.feed_forward.w3": 0.8689588308334351, "blocks.8.attention.w_q": 0.8623542785644531, "blocks.8.attention.w_k": 0.8528976440429688, "blocks.8.attention.w_v": 0.8567085266113281, "blocks.8.attention.w_out": 0.8530693054199219, "blocks.8.feed_forward.w1": 0.8754285573959351, "blocks.8.feed_forward.w2": 0.7875391840934753, "blocks.8.feed_forward.w3": 0.8718745112419128}} +{"step": 1200, "changed_frac": 0.8480732909634582, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8393096923828125, "blocks.0.attention.w_k": 0.8360481262207031, "blocks.0.attention.w_v": 0.8473663330078125, "blocks.0.attention.w_out": 0.860382080078125, "blocks.0.feed_forward.w1": 0.8753013610839844, "blocks.0.feed_forward.w2": 0.8377202749252319, "blocks.0.feed_forward.w3": 0.8707923889160156, "blocks.1.attention.w_q": 0.8568191528320312, "blocks.1.attention.w_k": 0.8539924621582031, "blocks.1.attention.w_v": 0.8656539916992188, "blocks.1.attention.w_out": 0.8815650939941406, "blocks.1.feed_forward.w1": 0.8621711730957031, "blocks.1.feed_forward.w2": 0.832672119140625, "blocks.1.feed_forward.w3": 0.8575553894042969, "blocks.2.attention.w_q": 0.8557777404785156, "blocks.2.attention.w_k": 0.849822998046875, "blocks.2.attention.w_v": 0.8560218811035156, "blocks.2.attention.w_out": 0.87359619140625, "blocks.2.feed_forward.w1": 0.8584480285644531, "blocks.2.feed_forward.w2": 0.8154017329216003, "blocks.2.feed_forward.w3": 0.8552322387695312, "blocks.3.attention.w_q": 0.8494873046875, "blocks.3.attention.w_k": 0.8452110290527344, "blocks.3.attention.w_v": 0.85870361328125, "blocks.3.attention.w_out": 0.8755683898925781, "blocks.3.feed_forward.w1": 0.8604418635368347, "blocks.3.feed_forward.w2": 0.8152517080307007, "blocks.3.feed_forward.w3": 0.8565508723258972, "blocks.4.attention.w_q": 0.85906982421875, "blocks.4.attention.w_k": 0.8585586547851562, "blocks.4.attention.w_v": 0.8631591796875, "blocks.4.attention.w_out": 0.8704452514648438, "blocks.4.feed_forward.w1": 0.862555205821991, "blocks.4.feed_forward.w2": 0.8164914846420288, "blocks.4.feed_forward.w3": 0.8594144582748413, "blocks.5.attention.w_q": 0.8476142883300781, "blocks.5.attention.w_k": 0.830047607421875, "blocks.5.attention.w_v": 0.8497848510742188, "blocks.5.attention.w_out": 0.8446922302246094, "blocks.5.feed_forward.w1": 0.8654429316520691, "blocks.5.feed_forward.w2": 0.8133748769760132, "blocks.5.feed_forward.w3": 0.8612327575683594, "blocks.6.attention.w_q": 0.8528289794921875, "blocks.6.attention.w_k": 0.837799072265625, "blocks.6.attention.w_v": 0.8481788635253906, "blocks.6.attention.w_out": 0.8085403442382812, "blocks.6.feed_forward.w1": 0.8676058650016785, "blocks.6.feed_forward.w2": 0.8043073415756226, "blocks.6.feed_forward.w3": 0.8629557490348816, "blocks.7.attention.w_q": 0.85626220703125, "blocks.7.attention.w_k": 0.845001220703125, "blocks.7.attention.w_v": 0.8448715209960938, "blocks.7.attention.w_out": 0.8286666870117188, "blocks.7.feed_forward.w1": 0.8698171377182007, "blocks.7.feed_forward.w2": 0.7957369685173035, "blocks.7.feed_forward.w3": 0.8647779226303101, "blocks.8.attention.w_q": 0.8583946228027344, "blocks.8.attention.w_k": 0.8485107421875, "blocks.8.attention.w_v": 0.8489608764648438, "blocks.8.attention.w_out": 0.8430137634277344, "blocks.8.feed_forward.w1": 0.8720588684082031, "blocks.8.feed_forward.w2": 0.7789205312728882, "blocks.8.feed_forward.w3": 0.8660520315170288}} +{"step": 1300, "changed_frac": 0.8439422020545373, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8326873779296875, "blocks.0.attention.w_k": 0.8303680419921875, "blocks.0.attention.w_v": 0.8409996032714844, "blocks.0.attention.w_out": 0.8549118041992188, "blocks.0.feed_forward.w1": 0.8719393610954285, "blocks.0.feed_forward.w2": 0.8367919921875, "blocks.0.feed_forward.w3": 0.8668493032455444, "blocks.1.attention.w_q": 0.8513336181640625, "blocks.1.attention.w_k": 0.8479690551757812, "blocks.1.attention.w_v": 0.8666305541992188, "blocks.1.attention.w_out": 0.8791694641113281, "blocks.1.feed_forward.w1": 0.8577817678451538, "blocks.1.feed_forward.w2": 0.8296610713005066, "blocks.1.feed_forward.w3": 0.8529269099235535, "blocks.2.attention.w_q": 0.853424072265625, "blocks.2.attention.w_k": 0.8479881286621094, "blocks.2.attention.w_v": 0.8535232543945312, "blocks.2.attention.w_out": 0.8700294494628906, "blocks.2.feed_forward.w1": 0.8537915945053101, "blocks.2.feed_forward.w2": 0.813812255859375, "blocks.2.feed_forward.w3": 0.8511034846305847, "blocks.3.attention.w_q": 0.8473396301269531, "blocks.3.attention.w_k": 0.8400421142578125, "blocks.3.attention.w_v": 0.8522529602050781, "blocks.3.attention.w_out": 0.871917724609375, "blocks.3.feed_forward.w1": 0.8561834096908569, "blocks.3.feed_forward.w2": 0.8143247365951538, "blocks.3.feed_forward.w3": 0.850860595703125, "blocks.4.attention.w_q": 0.854583740234375, "blocks.4.attention.w_k": 0.8545608520507812, "blocks.4.attention.w_v": 0.8584175109863281, "blocks.4.attention.w_out": 0.8687973022460938, "blocks.4.feed_forward.w1": 0.8584365844726562, "blocks.4.feed_forward.w2": 0.8128802180290222, "blocks.4.feed_forward.w3": 0.8533465266227722, "blocks.5.attention.w_q": 0.8445663452148438, "blocks.5.attention.w_k": 0.8289108276367188, "blocks.5.attention.w_v": 0.842529296875, "blocks.5.attention.w_out": 0.8379745483398438, "blocks.5.feed_forward.w1": 0.8619168996810913, "blocks.5.feed_forward.w2": 0.8111648559570312, "blocks.5.feed_forward.w3": 0.8553657531738281, "blocks.6.attention.w_q": 0.8480339050292969, "blocks.6.attention.w_k": 0.8356170654296875, "blocks.6.attention.w_v": 0.8392333984375, "blocks.6.attention.w_out": 0.8026924133300781, "blocks.6.feed_forward.w1": 0.8639640808105469, "blocks.6.feed_forward.w2": 0.8016154170036316, "blocks.6.feed_forward.w3": 0.8576062917709351, "blocks.7.attention.w_q": 0.8537483215332031, "blocks.7.attention.w_k": 0.8422317504882812, "blocks.7.attention.w_v": 0.83746337890625, "blocks.7.attention.w_out": 0.820587158203125, "blocks.7.feed_forward.w1": 0.866095244884491, "blocks.7.feed_forward.w2": 0.7910296320915222, "blocks.7.feed_forward.w3": 0.8599777221679688, "blocks.8.attention.w_q": 0.8546867370605469, "blocks.8.attention.w_k": 0.8425140380859375, "blocks.8.attention.w_v": 0.8423271179199219, "blocks.8.attention.w_out": 0.8339195251464844, "blocks.8.feed_forward.w1": 0.869574248790741, "blocks.8.feed_forward.w2": 0.7721532583236694, "blocks.8.feed_forward.w3": 0.8612658381462097}} +{"step": 1400, "changed_frac": 0.8403317378117487, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8259658813476562, "blocks.0.attention.w_k": 0.8242416381835938, "blocks.0.attention.w_v": 0.8364524841308594, "blocks.0.attention.w_out": 0.8501548767089844, "blocks.0.feed_forward.w1": 0.8683204650878906, "blocks.0.feed_forward.w2": 0.8326441645622253, "blocks.0.feed_forward.w3": 0.8623682856559753, "blocks.1.attention.w_q": 0.8466644287109375, "blocks.1.attention.w_k": 0.8431472778320312, "blocks.1.attention.w_v": 0.8625450134277344, "blocks.1.attention.w_out": 0.8773078918457031, "blocks.1.feed_forward.w1": 0.8519935607910156, "blocks.1.feed_forward.w2": 0.8262113332748413, "blocks.1.feed_forward.w3": 0.8480631709098816, "blocks.2.attention.w_q": 0.8510475158691406, "blocks.2.attention.w_k": 0.847412109375, "blocks.2.attention.w_v": 0.8497848510742188, "blocks.2.attention.w_out": 0.8661079406738281, "blocks.2.feed_forward.w1": 0.85101318359375, "blocks.2.feed_forward.w2": 0.8119685053825378, "blocks.2.feed_forward.w3": 0.8462486267089844, "blocks.3.attention.w_q": 0.8441352844238281, "blocks.3.attention.w_k": 0.8398590087890625, "blocks.3.attention.w_v": 0.85003662109375, "blocks.3.attention.w_out": 0.8696937561035156, "blocks.3.feed_forward.w1": 0.8526535034179688, "blocks.3.feed_forward.w2": 0.8140093684196472, "blocks.3.feed_forward.w3": 0.8467509150505066, "blocks.4.attention.w_q": 0.852020263671875, "blocks.4.attention.w_k": 0.8513031005859375, "blocks.4.attention.w_v": 0.8529090881347656, "blocks.4.attention.w_out": 0.8641090393066406, "blocks.4.feed_forward.w1": 0.8564847707748413, "blocks.4.feed_forward.w2": 0.8144493103027344, "blocks.4.feed_forward.w3": 0.8497365713119507, "blocks.5.attention.w_q": 0.839630126953125, "blocks.5.attention.w_k": 0.8272705078125, "blocks.5.attention.w_v": 0.8368644714355469, "blocks.5.attention.w_out": 0.831512451171875, "blocks.5.feed_forward.w1": 0.8578224182128906, "blocks.5.feed_forward.w2": 0.8086268305778503, "blocks.5.feed_forward.w3": 0.8525378108024597, "blocks.6.attention.w_q": 0.8447151184082031, "blocks.6.attention.w_k": 0.8335685729980469, "blocks.6.attention.w_v": 0.8351249694824219, "blocks.6.attention.w_out": 0.7968788146972656, "blocks.6.feed_forward.w1": 0.8610865473747253, "blocks.6.feed_forward.w2": 0.79791259765625, "blocks.6.feed_forward.w3": 0.8533796072006226, "blocks.7.attention.w_q": 0.850799560546875, "blocks.7.attention.w_k": 0.8390960693359375, "blocks.7.attention.w_v": 0.8309402465820312, "blocks.7.attention.w_out": 0.8117523193359375, "blocks.7.feed_forward.w1": 0.8622220754623413, "blocks.7.feed_forward.w2": 0.7864062190055847, "blocks.7.feed_forward.w3": 0.8564669489860535, "blocks.8.attention.w_q": 0.8512382507324219, "blocks.8.attention.w_k": 0.8414497375488281, "blocks.8.attention.w_v": 0.8371925354003906, "blocks.8.attention.w_out": 0.82489013671875, "blocks.8.feed_forward.w1": 0.8664843440055847, "blocks.8.feed_forward.w2": 0.7668660879135132, "blocks.8.feed_forward.w3": 0.8576037287712097}} +{"step": 1500, "changed_frac": 0.8369660010704627, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8236656188964844, "blocks.0.attention.w_k": 0.82000732421875, "blocks.0.attention.w_v": 0.8328132629394531, "blocks.0.attention.w_out": 0.8472099304199219, "blocks.0.feed_forward.w1": 0.8659045100212097, "blocks.0.feed_forward.w2": 0.8317159414291382, "blocks.0.feed_forward.w3": 0.8598658442497253, "blocks.1.attention.w_q": 0.8435440063476562, "blocks.1.attention.w_k": 0.8399734497070312, "blocks.1.attention.w_v": 0.8598518371582031, "blocks.1.attention.w_out": 0.8761138916015625, "blocks.1.feed_forward.w1": 0.8493868708610535, "blocks.1.feed_forward.w2": 0.8250007629394531, "blocks.1.feed_forward.w3": 0.8439509272575378, "blocks.2.attention.w_q": 0.848846435546875, "blocks.2.attention.w_k": 0.8445587158203125, "blocks.2.attention.w_v": 0.8478851318359375, "blocks.2.attention.w_out": 0.8638877868652344, "blocks.2.feed_forward.w1": 0.846221923828125, "blocks.2.feed_forward.w2": 0.811370849609375, "blocks.2.feed_forward.w3": 0.8434244990348816, "blocks.3.attention.w_q": 0.8406753540039062, "blocks.3.attention.w_k": 0.8358230590820312, "blocks.3.attention.w_v": 0.8446731567382812, "blocks.3.attention.w_out": 0.8656272888183594, "blocks.3.feed_forward.w1": 0.8488757014274597, "blocks.3.feed_forward.w2": 0.8129043579101562, "blocks.3.feed_forward.w3": 0.8434867858886719, "blocks.4.attention.w_q": 0.84893798828125, "blocks.4.attention.w_k": 0.8470230102539062, "blocks.4.attention.w_v": 0.8479690551757812, "blocks.4.attention.w_out": 0.8609046936035156, "blocks.4.feed_forward.w1": 0.8516731262207031, "blocks.4.feed_forward.w2": 0.8130442500114441, "blocks.4.feed_forward.w3": 0.8463796377182007, "blocks.5.attention.w_q": 0.8377151489257812, "blocks.5.attention.w_k": 0.8237228393554688, "blocks.5.attention.w_v": 0.8304710388183594, "blocks.5.attention.w_out": 0.8284187316894531, "blocks.5.feed_forward.w1": 0.8542976379394531, "blocks.5.feed_forward.w2": 0.8081309199333191, "blocks.5.feed_forward.w3": 0.8481801748275757, "blocks.6.attention.w_q": 0.8415794372558594, "blocks.6.attention.w_k": 0.8292121887207031, "blocks.6.attention.w_v": 0.8292312622070312, "blocks.6.attention.w_out": 0.7921905517578125, "blocks.6.feed_forward.w1": 0.8573951721191406, "blocks.6.feed_forward.w2": 0.7934913635253906, "blocks.6.feed_forward.w3": 0.8502413630485535, "blocks.7.attention.w_q": 0.8461875915527344, "blocks.7.attention.w_k": 0.8349075317382812, "blocks.7.attention.w_v": 0.8273124694824219, "blocks.7.attention.w_out": 0.8030357360839844, "blocks.7.feed_forward.w1": 0.8594767451286316, "blocks.7.feed_forward.w2": 0.7805506587028503, "blocks.7.feed_forward.w3": 0.8522847890853882, "blocks.8.attention.w_q": 0.8486137390136719, "blocks.8.attention.w_k": 0.8390388488769531, "blocks.8.attention.w_v": 0.83251953125, "blocks.8.attention.w_out": 0.8178558349609375, "blocks.8.feed_forward.w1": 0.8623988032341003, "blocks.8.feed_forward.w2": 0.76123046875, "blocks.8.feed_forward.w3": 0.8534559011459351}} +{"step": 1600, "changed_frac": 0.8332964579264323, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8179244995117188, "blocks.0.attention.w_k": 0.8169631958007812, "blocks.0.attention.w_v": 0.8253402709960938, "blocks.0.attention.w_out": 0.8414077758789062, "blocks.0.feed_forward.w1": 0.8608970642089844, "blocks.0.feed_forward.w2": 0.8295237421989441, "blocks.0.feed_forward.w3": 0.8560943603515625, "blocks.1.attention.w_q": 0.8399467468261719, "blocks.1.attention.w_k": 0.8357048034667969, "blocks.1.attention.w_v": 0.8601951599121094, "blocks.1.attention.w_out": 0.8731422424316406, "blocks.1.feed_forward.w1": 0.8455416560173035, "blocks.1.feed_forward.w2": 0.821319580078125, "blocks.1.feed_forward.w3": 0.8404172658920288, "blocks.2.attention.w_q": 0.8458824157714844, "blocks.2.attention.w_k": 0.8416213989257812, "blocks.2.attention.w_v": 0.8428001403808594, "blocks.2.attention.w_out": 0.8639106750488281, "blocks.2.feed_forward.w1": 0.8427963256835938, "blocks.2.feed_forward.w2": 0.8102086782455444, "blocks.2.feed_forward.w3": 0.8392257690429688, "blocks.3.attention.w_q": 0.8368873596191406, "blocks.3.attention.w_k": 0.8298149108886719, "blocks.3.attention.w_v": 0.8404350280761719, "blocks.3.attention.w_out": 0.8611793518066406, "blocks.3.feed_forward.w1": 0.8457006216049194, "blocks.3.feed_forward.w2": 0.8131612539291382, "blocks.3.feed_forward.w3": 0.8400892019271851, "blocks.4.attention.w_q": 0.8452110290527344, "blocks.4.attention.w_k": 0.8424301147460938, "blocks.4.attention.w_v": 0.8456153869628906, "blocks.4.attention.w_out": 0.8576812744140625, "blocks.4.feed_forward.w1": 0.8490486145019531, "blocks.4.feed_forward.w2": 0.8113390803337097, "blocks.4.feed_forward.w3": 0.8411725759506226, "blocks.5.attention.w_q": 0.8355941772460938, "blocks.5.attention.w_k": 0.8217391967773438, "blocks.5.attention.w_v": 0.8253974914550781, "blocks.5.attention.w_out": 0.8230476379394531, "blocks.5.feed_forward.w1": 0.8510780334472656, "blocks.5.feed_forward.w2": 0.8034681081771851, "blocks.5.feed_forward.w3": 0.8445142507553101, "blocks.6.attention.w_q": 0.8390769958496094, "blocks.6.attention.w_k": 0.8271369934082031, "blocks.6.attention.w_v": 0.8196678161621094, "blocks.6.attention.w_out": 0.7863388061523438, "blocks.6.feed_forward.w1": 0.8530973196029663, "blocks.6.feed_forward.w2": 0.7910079956054688, "blocks.6.feed_forward.w3": 0.845709502696991, "blocks.7.attention.w_q": 0.8429985046386719, "blocks.7.attention.w_k": 0.8344802856445312, "blocks.7.attention.w_v": 0.8199348449707031, "blocks.7.attention.w_out": 0.7954750061035156, "blocks.7.feed_forward.w1": 0.8561655879020691, "blocks.7.feed_forward.w2": 0.7771415710449219, "blocks.7.feed_forward.w3": 0.848211944103241, "blocks.8.attention.w_q": 0.8466262817382812, "blocks.8.attention.w_k": 0.8356132507324219, "blocks.8.attention.w_v": 0.8268318176269531, "blocks.8.attention.w_out": 0.8115081787109375, "blocks.8.feed_forward.w1": 0.8601722717285156, "blocks.8.feed_forward.w2": 0.7529920339584351, "blocks.8.feed_forward.w3": 0.8499476313591003}} +{"step": 1700, "changed_frac": 0.8303791274372329, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8111915588378906, "blocks.0.attention.w_k": 0.81109619140625, "blocks.0.attention.w_v": 0.8212471008300781, "blocks.0.attention.w_out": 0.838134765625, "blocks.0.feed_forward.w1": 0.8591436147689819, "blocks.0.feed_forward.w2": 0.8281949758529663, "blocks.0.feed_forward.w3": 0.852508544921875, "blocks.1.attention.w_q": 0.83837890625, "blocks.1.attention.w_k": 0.8355712890625, "blocks.1.attention.w_v": 0.8580398559570312, "blocks.1.attention.w_out": 0.8727378845214844, "blocks.1.feed_forward.w1": 0.8416671752929688, "blocks.1.feed_forward.w2": 0.819891631603241, "blocks.1.feed_forward.w3": 0.8366788625717163, "blocks.2.attention.w_q": 0.8461265563964844, "blocks.2.attention.w_k": 0.8391990661621094, "blocks.2.attention.w_v": 0.8411216735839844, "blocks.2.attention.w_out": 0.8613128662109375, "blocks.2.feed_forward.w1": 0.8403257131576538, "blocks.2.feed_forward.w2": 0.8079719543457031, "blocks.2.feed_forward.w3": 0.836798369884491, "blocks.3.attention.w_q": 0.83447265625, "blocks.3.attention.w_k": 0.8273849487304688, "blocks.3.attention.w_v": 0.8366928100585938, "blocks.3.attention.w_out": 0.8593177795410156, "blocks.3.feed_forward.w1": 0.8424352407455444, "blocks.3.feed_forward.w2": 0.8119291067123413, "blocks.3.feed_forward.w3": 0.8356641530990601, "blocks.4.attention.w_q": 0.8451309204101562, "blocks.4.attention.w_k": 0.8398284912109375, "blocks.4.attention.w_v": 0.8394546508789062, "blocks.4.attention.w_out": 0.853363037109375, "blocks.4.feed_forward.w1": 0.8454310297966003, "blocks.4.feed_forward.w2": 0.810736358165741, "blocks.4.feed_forward.w3": 0.838446319103241, "blocks.5.attention.w_q": 0.8338623046875, "blocks.5.attention.w_k": 0.8202285766601562, "blocks.5.attention.w_v": 0.8224639892578125, "blocks.5.attention.w_out": 0.8201980590820312, "blocks.5.feed_forward.w1": 0.8476829528808594, "blocks.5.feed_forward.w2": 0.8029416799545288, "blocks.5.feed_forward.w3": 0.8408915400505066, "blocks.6.attention.w_q": 0.8353157043457031, "blocks.6.attention.w_k": 0.8256568908691406, "blocks.6.attention.w_v": 0.8148002624511719, "blocks.6.attention.w_out": 0.782073974609375, "blocks.6.feed_forward.w1": 0.8498942255973816, "blocks.6.feed_forward.w2": 0.7889531850814819, "blocks.6.feed_forward.w3": 0.841705322265625, "blocks.7.attention.w_q": 0.8404312133789062, "blocks.7.attention.w_k": 0.8303680419921875, "blocks.7.attention.w_v": 0.8135757446289062, "blocks.7.attention.w_out": 0.7896690368652344, "blocks.7.feed_forward.w1": 0.8529103994369507, "blocks.7.feed_forward.w2": 0.7726580500602722, "blocks.7.feed_forward.w3": 0.8452581167221069, "blocks.8.attention.w_q": 0.8445587158203125, "blocks.8.attention.w_k": 0.8348236083984375, "blocks.8.attention.w_v": 0.8194313049316406, "blocks.8.attention.w_out": 0.8046684265136719, "blocks.8.feed_forward.w1": 0.8573278188705444, "blocks.8.feed_forward.w2": 0.7489331960678101, "blocks.8.feed_forward.w3": 0.8471641540527344}} +{"step": 1800, "changed_frac": 0.8274173899593517, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8084945678710938, "blocks.0.attention.w_k": 0.8079757690429688, "blocks.0.attention.w_v": 0.8168601989746094, "blocks.0.attention.w_out": 0.8345909118652344, "blocks.0.feed_forward.w1": 0.8563741445541382, "blocks.0.feed_forward.w2": 0.826019287109375, "blocks.0.feed_forward.w3": 0.8506126403808594, "blocks.1.attention.w_q": 0.8353805541992188, "blocks.1.attention.w_k": 0.8317680358886719, "blocks.1.attention.w_v": 0.8550529479980469, "blocks.1.attention.w_out": 0.8698387145996094, "blocks.1.feed_forward.w1": 0.8379974365234375, "blocks.1.feed_forward.w2": 0.8179461359977722, "blocks.1.feed_forward.w3": 0.8336080312728882, "blocks.2.attention.w_q": 0.8411445617675781, "blocks.2.attention.w_k": 0.8366165161132812, "blocks.2.attention.w_v": 0.8383026123046875, "blocks.2.attention.w_out": 0.8590888977050781, "blocks.2.feed_forward.w1": 0.8375155329704285, "blocks.2.feed_forward.w2": 0.8067550659179688, "blocks.2.feed_forward.w3": 0.8330078125, "blocks.3.attention.w_q": 0.83050537109375, "blocks.3.attention.w_k": 0.8242301940917969, "blocks.3.attention.w_v": 0.8315811157226562, "blocks.3.attention.w_out": 0.8566970825195312, "blocks.3.feed_forward.w1": 0.8398641347885132, "blocks.3.feed_forward.w2": 0.8105659484863281, "blocks.3.feed_forward.w3": 0.8324064016342163, "blocks.4.attention.w_q": 0.8410987854003906, "blocks.4.attention.w_k": 0.8365402221679688, "blocks.4.attention.w_v": 0.83612060546875, "blocks.4.attention.w_out": 0.8517265319824219, "blocks.4.feed_forward.w1": 0.841949462890625, "blocks.4.feed_forward.w2": 0.8089739680290222, "blocks.4.feed_forward.w3": 0.8350499868392944, "blocks.5.attention.w_q": 0.8312492370605469, "blocks.5.attention.w_k": 0.819305419921875, "blocks.5.attention.w_v": 0.8169288635253906, "blocks.5.attention.w_out": 0.8159141540527344, "blocks.5.feed_forward.w1": 0.8442128896713257, "blocks.5.feed_forward.w2": 0.8017616271972656, "blocks.5.feed_forward.w3": 0.8383700251579285, "blocks.6.attention.w_q": 0.8343544006347656, "blocks.6.attention.w_k": 0.8215866088867188, "blocks.6.attention.w_v": 0.8111534118652344, "blocks.6.attention.w_out": 0.7786178588867188, "blocks.6.feed_forward.w1": 0.8471082448959351, "blocks.6.feed_forward.w2": 0.7857666015625, "blocks.6.feed_forward.w3": 0.8377279043197632, "blocks.7.attention.w_q": 0.8389320373535156, "blocks.7.attention.w_k": 0.8291511535644531, "blocks.7.attention.w_v": 0.810821533203125, "blocks.7.attention.w_out": 0.7843894958496094, "blocks.7.feed_forward.w1": 0.8506698608398438, "blocks.7.feed_forward.w2": 0.7687925100326538, "blocks.7.feed_forward.w3": 0.8407415151596069, "blocks.8.attention.w_q": 0.8429946899414062, "blocks.8.attention.w_k": 0.8321456909179688, "blocks.8.attention.w_v": 0.8179397583007812, "blocks.8.attention.w_out": 0.8009147644042969, "blocks.8.feed_forward.w1": 0.854254424571991, "blocks.8.feed_forward.w2": 0.7441622614860535, "blocks.8.feed_forward.w3": 0.8437271118164062}} +{"step": 1900, "changed_frac": 0.8239782284467648, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8050727844238281, "blocks.0.attention.w_k": 0.8045845031738281, "blocks.0.attention.w_v": 0.8123626708984375, "blocks.0.attention.w_out": 0.8310585021972656, "blocks.0.feed_forward.w1": 0.8529307246208191, "blocks.0.feed_forward.w2": 0.8233274221420288, "blocks.0.feed_forward.w3": 0.8454272150993347, "blocks.1.attention.w_q": 0.8317527770996094, "blocks.1.attention.w_k": 0.8255081176757812, "blocks.1.attention.w_v": 0.8527107238769531, "blocks.1.attention.w_out": 0.868408203125, "blocks.1.feed_forward.w1": 0.8345273733139038, "blocks.1.feed_forward.w2": 0.8142382502555847, "blocks.1.feed_forward.w3": 0.8306071162223816, "blocks.2.attention.w_q": 0.8390960693359375, "blocks.2.attention.w_k": 0.8345870971679688, "blocks.2.attention.w_v": 0.8350486755371094, "blocks.2.attention.w_out": 0.8574905395507812, "blocks.2.feed_forward.w1": 0.8348159790039062, "blocks.2.feed_forward.w2": 0.8043658137321472, "blocks.2.feed_forward.w3": 0.8286959528923035, "blocks.3.attention.w_q": 0.8287010192871094, "blocks.3.attention.w_k": 0.8216705322265625, "blocks.3.attention.w_v": 0.8264274597167969, "blocks.3.attention.w_out": 0.8533058166503906, "blocks.3.feed_forward.w1": 0.8371989130973816, "blocks.3.feed_forward.w2": 0.8094329833984375, "blocks.3.feed_forward.w3": 0.8298937678337097, "blocks.4.attention.w_q": 0.8390960693359375, "blocks.4.attention.w_k": 0.8327102661132812, "blocks.4.attention.w_v": 0.8310356140136719, "blocks.4.attention.w_out": 0.8473472595214844, "blocks.4.feed_forward.w1": 0.8398272395133972, "blocks.4.feed_forward.w2": 0.8072052001953125, "blocks.4.feed_forward.w3": 0.831976592540741, "blocks.5.attention.w_q": 0.8264579772949219, "blocks.5.attention.w_k": 0.8150978088378906, "blocks.5.attention.w_v": 0.8130111694335938, "blocks.5.attention.w_out": 0.81402587890625, "blocks.5.feed_forward.w1": 0.8417231440544128, "blocks.5.feed_forward.w2": 0.7972132563591003, "blocks.5.feed_forward.w3": 0.8330497741699219, "blocks.6.attention.w_q": 0.8300209045410156, "blocks.6.attention.w_k": 0.8187942504882812, "blocks.6.attention.w_v": 0.8042984008789062, "blocks.6.attention.w_out": 0.7749061584472656, "blocks.6.feed_forward.w1": 0.8441734313964844, "blocks.6.feed_forward.w2": 0.7837423086166382, "blocks.6.feed_forward.w3": 0.835540771484375, "blocks.7.attention.w_q": 0.8375244140625, "blocks.7.attention.w_k": 0.8243942260742188, "blocks.7.attention.w_v": 0.8060264587402344, "blocks.7.attention.w_out": 0.7768630981445312, "blocks.7.feed_forward.w1": 0.8472747802734375, "blocks.7.feed_forward.w2": 0.7634977102279663, "blocks.7.feed_forward.w3": 0.8377279043197632, "blocks.8.attention.w_q": 0.8384819030761719, "blocks.8.attention.w_k": 0.8293991088867188, "blocks.8.attention.w_v": 0.811279296875, "blocks.8.attention.w_out": 0.7920494079589844, "blocks.8.feed_forward.w1": 0.852508544921875, "blocks.8.feed_forward.w2": 0.7377332448959351, "blocks.8.feed_forward.w3": 0.8396276235580444}} +{"step": 2000, "changed_frac": 0.8209723122099526, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8001480102539062, "blocks.0.attention.w_k": 0.7985687255859375, "blocks.0.attention.w_v": 0.8064689636230469, "blocks.0.attention.w_out": 0.8259696960449219, "blocks.0.feed_forward.w1": 0.8504282832145691, "blocks.0.feed_forward.w2": 0.8213793635368347, "blocks.0.feed_forward.w3": 0.8434168696403503, "blocks.1.attention.w_q": 0.8315963745117188, "blocks.1.attention.w_k": 0.8245201110839844, "blocks.1.attention.w_v": 0.8499794006347656, "blocks.1.attention.w_out": 0.8670997619628906, "blocks.1.feed_forward.w1": 0.8315519094467163, "blocks.1.feed_forward.w2": 0.8111928701400757, "blocks.1.feed_forward.w3": 0.8264821767807007, "blocks.2.attention.w_q": 0.8369216918945312, "blocks.2.attention.w_k": 0.8329277038574219, "blocks.2.attention.w_v": 0.831787109375, "blocks.2.attention.w_out": 0.8539810180664062, "blocks.2.feed_forward.w1": 0.8313993215560913, "blocks.2.feed_forward.w2": 0.8031298518180847, "blocks.2.feed_forward.w3": 0.8267390131950378, "blocks.3.attention.w_q": 0.8251609802246094, "blocks.3.attention.w_k": 0.8198928833007812, "blocks.3.attention.w_v": 0.8248634338378906, "blocks.3.attention.w_out": 0.8496742248535156, "blocks.3.feed_forward.w1": 0.833594024181366, "blocks.3.feed_forward.w2": 0.8083839416503906, "blocks.3.feed_forward.w3": 0.8260549306869507, "blocks.4.attention.w_q": 0.8348236083984375, "blocks.4.attention.w_k": 0.8295402526855469, "blocks.4.attention.w_v": 0.8276443481445312, "blocks.4.attention.w_out": 0.8440208435058594, "blocks.4.feed_forward.w1": 0.8367869257926941, "blocks.4.feed_forward.w2": 0.8054746389389038, "blocks.4.feed_forward.w3": 0.828131377696991, "blocks.5.attention.w_q": 0.8246536254882812, "blocks.5.attention.w_k": 0.8139762878417969, "blocks.5.attention.w_v": 0.8089675903320312, "blocks.5.attention.w_out": 0.809051513671875, "blocks.5.feed_forward.w1": 0.8389803767204285, "blocks.5.feed_forward.w2": 0.7958335876464844, "blocks.5.feed_forward.w3": 0.8311628103256226, "blocks.6.attention.w_q": 0.8287086486816406, "blocks.6.attention.w_k": 0.8181419372558594, "blocks.6.attention.w_v": 0.7998733520507812, "blocks.6.attention.w_out": 0.7706565856933594, "blocks.6.feed_forward.w1": 0.8412336111068726, "blocks.6.feed_forward.w2": 0.7793134450912476, "blocks.6.feed_forward.w3": 0.8316981196403503, "blocks.7.attention.w_q": 0.8331413269042969, "blocks.7.attention.w_k": 0.8222618103027344, "blocks.7.attention.w_v": 0.8002204895019531, "blocks.7.attention.w_out": 0.7705650329589844, "blocks.7.feed_forward.w1": 0.8451805114746094, "blocks.7.feed_forward.w2": 0.7598088979721069, "blocks.7.feed_forward.w3": 0.8341547846794128, "blocks.8.attention.w_q": 0.8365631103515625, "blocks.8.attention.w_k": 0.8265914916992188, "blocks.8.attention.w_v": 0.8073081970214844, "blocks.8.attention.w_out": 0.7863655090332031, "blocks.8.feed_forward.w1": 0.8494352102279663, "blocks.8.feed_forward.w2": 0.7333450317382812, "blocks.8.feed_forward.w3": 0.8360837697982788}} +{"step": 2100, "changed_frac": 0.8178094065087473, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7959403991699219, "blocks.0.attention.w_k": 0.7942390441894531, "blocks.0.attention.w_v": 0.8018684387207031, "blocks.0.attention.w_out": 0.8232498168945312, "blocks.0.feed_forward.w1": 0.8454450368881226, "blocks.0.feed_forward.w2": 0.8189405202865601, "blocks.0.feed_forward.w3": 0.840277373790741, "blocks.1.attention.w_q": 0.8263168334960938, "blocks.1.attention.w_k": 0.8213577270507812, "blocks.1.attention.w_v": 0.8466720581054688, "blocks.1.attention.w_out": 0.8639564514160156, "blocks.1.feed_forward.w1": 0.8294347524642944, "blocks.1.feed_forward.w2": 0.8090667724609375, "blocks.1.feed_forward.w3": 0.8243128657341003, "blocks.2.attention.w_q": 0.8329048156738281, "blocks.2.attention.w_k": 0.8292083740234375, "blocks.2.attention.w_v": 0.8275032043457031, "blocks.2.attention.w_out": 0.8521156311035156, "blocks.2.feed_forward.w1": 0.828589141368866, "blocks.2.feed_forward.w2": 0.80181884765625, "blocks.2.feed_forward.w3": 0.823699951171875, "blocks.3.attention.w_q": 0.8232574462890625, "blocks.3.attention.w_k": 0.8148841857910156, "blocks.3.attention.w_v": 0.8190994262695312, "blocks.3.attention.w_out": 0.8468780517578125, "blocks.3.feed_forward.w1": 0.8302955627441406, "blocks.3.feed_forward.w2": 0.8066915273666382, "blocks.3.feed_forward.w3": 0.8227030634880066, "blocks.4.attention.w_q": 0.8336372375488281, "blocks.4.attention.w_k": 0.8256797790527344, "blocks.4.attention.w_v": 0.8238487243652344, "blocks.4.attention.w_out": 0.8416328430175781, "blocks.4.feed_forward.w1": 0.8340123891830444, "blocks.4.feed_forward.w2": 0.8039411306381226, "blocks.4.feed_forward.w3": 0.8254305720329285, "blocks.5.attention.w_q": 0.8203659057617188, "blocks.5.attention.w_k": 0.8101425170898438, "blocks.5.attention.w_v": 0.8049278259277344, "blocks.5.attention.w_out": 0.8063392639160156, "blocks.5.feed_forward.w1": 0.8357696533203125, "blocks.5.feed_forward.w2": 0.7930641174316406, "blocks.5.feed_forward.w3": 0.8276456594467163, "blocks.6.attention.w_q": 0.8269004821777344, "blocks.6.attention.w_k": 0.8153877258300781, "blocks.6.attention.w_v": 0.7944450378417969, "blocks.6.attention.w_out": 0.7629508972167969, "blocks.6.feed_forward.w1": 0.8390274047851562, "blocks.6.feed_forward.w2": 0.7771224975585938, "blocks.6.feed_forward.w3": 0.8287557363510132, "blocks.7.attention.w_q": 0.8309173583984375, "blocks.7.attention.w_k": 0.8171844482421875, "blocks.7.attention.w_v": 0.7953758239746094, "blocks.7.attention.w_out": 0.7660140991210938, "blocks.7.feed_forward.w1": 0.8426641225814819, "blocks.7.feed_forward.w2": 0.7577362060546875, "blocks.7.feed_forward.w3": 0.8302536010742188, "blocks.8.attention.w_q": 0.8343162536621094, "blocks.8.attention.w_k": 0.8243751525878906, "blocks.8.attention.w_v": 0.8023490905761719, "blocks.8.attention.w_out": 0.7793006896972656, "blocks.8.feed_forward.w1": 0.8465474843978882, "blocks.8.feed_forward.w2": 0.7272224426269531, "blocks.8.feed_forward.w3": 0.832251250743866}} +{"step": 2200, "changed_frac": 0.814566424769214, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7906532287597656, "blocks.0.attention.w_k": 0.7909965515136719, "blocks.0.attention.w_v": 0.7975196838378906, "blocks.0.attention.w_out": 0.8220062255859375, "blocks.0.feed_forward.w1": 0.8429985046386719, "blocks.0.feed_forward.w2": 0.8160337209701538, "blocks.0.feed_forward.w3": 0.8369089961051941, "blocks.1.attention.w_q": 0.8258895874023438, "blocks.1.attention.w_k": 0.8194427490234375, "blocks.1.attention.w_v": 0.8460769653320312, "blocks.1.attention.w_out": 0.8611907958984375, "blocks.1.feed_forward.w1": 0.8255615234375, "blocks.1.feed_forward.w2": 0.8077608942985535, "blocks.1.feed_forward.w3": 0.8209813833236694, "blocks.2.attention.w_q": 0.8319091796875, "blocks.2.attention.w_k": 0.8260459899902344, "blocks.2.attention.w_v": 0.8237571716308594, "blocks.2.attention.w_out": 0.8500595092773438, "blocks.2.feed_forward.w1": 0.825506865978241, "blocks.2.feed_forward.w2": 0.7997614741325378, "blocks.2.feed_forward.w3": 0.8194745779037476, "blocks.3.attention.w_q": 0.8182640075683594, "blocks.3.attention.w_k": 0.811614990234375, "blocks.3.attention.w_v": 0.8155364990234375, "blocks.3.attention.w_out": 0.843902587890625, "blocks.3.feed_forward.w1": 0.8278045654296875, "blocks.3.feed_forward.w2": 0.8051834106445312, "blocks.3.feed_forward.w3": 0.8188871145248413, "blocks.4.attention.w_q": 0.8312454223632812, "blocks.4.attention.w_k": 0.8224372863769531, "blocks.4.attention.w_v": 0.8175697326660156, "blocks.4.attention.w_out": 0.8392982482910156, "blocks.4.feed_forward.w1": 0.8310419917106628, "blocks.4.feed_forward.w2": 0.8013623952865601, "blocks.4.feed_forward.w3": 0.8215726613998413, "blocks.5.attention.w_q": 0.8178634643554688, "blocks.5.attention.w_k": 0.8073348999023438, "blocks.5.attention.w_v": 0.79962158203125, "blocks.5.attention.w_out": 0.8028526306152344, "blocks.5.feed_forward.w1": 0.8326683044433594, "blocks.5.feed_forward.w2": 0.7892163991928101, "blocks.5.feed_forward.w3": 0.8243217468261719, "blocks.6.attention.w_q": 0.8220977783203125, "blocks.6.attention.w_k": 0.81298828125, "blocks.6.attention.w_v": 0.7896194458007812, "blocks.6.attention.w_out": 0.7626953125, "blocks.6.feed_forward.w1": 0.8360494375228882, "blocks.6.feed_forward.w2": 0.7716993093490601, "blocks.6.feed_forward.w3": 0.8247947692871094, "blocks.7.attention.w_q": 0.8268356323242188, "blocks.7.attention.w_k": 0.81488037109375, "blocks.7.attention.w_v": 0.7932167053222656, "blocks.7.attention.w_out": 0.7612419128417969, "blocks.7.feed_forward.w1": 0.8398298025131226, "blocks.7.feed_forward.w2": 0.7533429861068726, "blocks.7.feed_forward.w3": 0.8279470205307007, "blocks.8.attention.w_q": 0.8297157287597656, "blocks.8.attention.w_k": 0.820068359375, "blocks.8.attention.w_v": 0.7963829040527344, "blocks.8.attention.w_out": 0.7742424011230469, "blocks.8.feed_forward.w1": 0.8432807922363281, "blocks.8.feed_forward.w2": 0.7216936945915222, "blocks.8.feed_forward.w3": 0.8300489187240601}} +{"step": 2300, "changed_frac": 0.8111920805058928, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7854766845703125, "blocks.0.attention.w_k": 0.7877311706542969, "blocks.0.attention.w_v": 0.7933006286621094, "blocks.0.attention.w_out": 0.8174819946289062, "blocks.0.feed_forward.w1": 0.839508056640625, "blocks.0.feed_forward.w2": 0.8141161799430847, "blocks.0.feed_forward.w3": 0.833770751953125, "blocks.1.attention.w_q": 0.8225288391113281, "blocks.1.attention.w_k": 0.8154182434082031, "blocks.1.attention.w_v": 0.840362548828125, "blocks.1.attention.w_out": 0.860260009765625, "blocks.1.feed_forward.w1": 0.8222007751464844, "blocks.1.feed_forward.w2": 0.8039373159408569, "blocks.1.feed_forward.w3": 0.8186671137809753, "blocks.2.attention.w_q": 0.8294029235839844, "blocks.2.attention.w_k": 0.8222846984863281, "blocks.2.attention.w_v": 0.8196945190429688, "blocks.2.attention.w_out": 0.8456153869628906, "blocks.2.feed_forward.w1": 0.8224296569824219, "blocks.2.feed_forward.w2": 0.7983347773551941, "blocks.2.feed_forward.w3": 0.815985381603241, "blocks.3.attention.w_q": 0.815216064453125, "blocks.3.attention.w_k": 0.8076171875, "blocks.3.attention.w_v": 0.811798095703125, "blocks.3.attention.w_out": 0.8402328491210938, "blocks.3.feed_forward.w1": 0.8248710632324219, "blocks.3.feed_forward.w2": 0.8024139404296875, "blocks.3.feed_forward.w3": 0.8156179189682007, "blocks.4.attention.w_q": 0.8280563354492188, "blocks.4.attention.w_k": 0.8170051574707031, "blocks.4.attention.w_v": 0.8143844604492188, "blocks.4.attention.w_out": 0.8337860107421875, "blocks.4.feed_forward.w1": 0.8278605341911316, "blocks.4.feed_forward.w2": 0.7976926565170288, "blocks.4.feed_forward.w3": 0.8188464045524597, "blocks.5.attention.w_q": 0.8138771057128906, "blocks.5.attention.w_k": 0.8032302856445312, "blocks.5.attention.w_v": 0.7955703735351562, "blocks.5.attention.w_out": 0.7979927062988281, "blocks.5.feed_forward.w1": 0.8304011225700378, "blocks.5.feed_forward.w2": 0.7877299189567566, "blocks.5.feed_forward.w3": 0.8211085200309753, "blocks.6.attention.w_q": 0.8189277648925781, "blocks.6.attention.w_k": 0.8086624145507812, "blocks.6.attention.w_v": 0.7845344543457031, "blocks.6.attention.w_out": 0.7591590881347656, "blocks.6.feed_forward.w1": 0.8324839472770691, "blocks.6.feed_forward.w2": 0.7696889638900757, "blocks.6.feed_forward.w3": 0.8220164179801941, "blocks.7.attention.w_q": 0.8240852355957031, "blocks.7.attention.w_k": 0.8119964599609375, "blocks.7.attention.w_v": 0.7899436950683594, "blocks.7.attention.w_out": 0.7565841674804688, "blocks.7.feed_forward.w1": 0.8358116149902344, "blocks.7.feed_forward.w2": 0.7491976618766785, "blocks.7.feed_forward.w3": 0.8238881826400757, "blocks.8.attention.w_q": 0.8267593383789062, "blocks.8.attention.w_k": 0.8163185119628906, "blocks.8.attention.w_v": 0.7942237854003906, "blocks.8.attention.w_out": 0.7677497863769531, "blocks.8.feed_forward.w1": 0.8401629328727722, "blocks.8.feed_forward.w2": 0.7169303894042969, "blocks.8.feed_forward.w3": 0.8250631093978882}} +{"step": 2400, "changed_frac": 0.8080452813042535, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7826423645019531, "blocks.0.attention.w_k": 0.7816925048828125, "blocks.0.attention.w_v": 0.7878913879394531, "blocks.0.attention.w_out": 0.8139152526855469, "blocks.0.feed_forward.w1": 0.8363825678825378, "blocks.0.feed_forward.w2": 0.8100471496582031, "blocks.0.feed_forward.w3": 0.8307062983512878, "blocks.1.attention.w_q": 0.8207321166992188, "blocks.1.attention.w_k": 0.8121299743652344, "blocks.1.attention.w_v": 0.8384666442871094, "blocks.1.attention.w_out": 0.857940673828125, "blocks.1.feed_forward.w1": 0.819000244140625, "blocks.1.feed_forward.w2": 0.8014513850212097, "blocks.1.feed_forward.w3": 0.812744140625, "blocks.2.attention.w_q": 0.8248367309570312, "blocks.2.attention.w_k": 0.8208198547363281, "blocks.2.attention.w_v": 0.8186302185058594, "blocks.2.attention.w_out": 0.844329833984375, "blocks.2.feed_forward.w1": 0.8198140859603882, "blocks.2.feed_forward.w2": 0.7943776845932007, "blocks.2.feed_forward.w3": 0.8132286071777344, "blocks.3.attention.w_q": 0.8122291564941406, "blocks.3.attention.w_k": 0.8053016662597656, "blocks.3.attention.w_v": 0.8087234497070312, "blocks.3.attention.w_out": 0.8379631042480469, "blocks.3.feed_forward.w1": 0.8217506408691406, "blocks.3.feed_forward.w2": 0.8008817434310913, "blocks.3.feed_forward.w3": 0.8133456110954285, "blocks.4.attention.w_q": 0.8249664306640625, "blocks.4.attention.w_k": 0.8120803833007812, "blocks.4.attention.w_v": 0.8123359680175781, "blocks.4.attention.w_out": 0.8315963745117188, "blocks.4.feed_forward.w1": 0.8251953125, "blocks.4.feed_forward.w2": 0.7968584895133972, "blocks.4.feed_forward.w3": 0.8151995539665222, "blocks.5.attention.w_q": 0.8121337890625, "blocks.5.attention.w_k": 0.8019294738769531, "blocks.5.attention.w_v": 0.7908897399902344, "blocks.5.attention.w_out": 0.7946891784667969, "blocks.5.feed_forward.w1": 0.8262825012207031, "blocks.5.feed_forward.w2": 0.7852987051010132, "blocks.5.feed_forward.w3": 0.8162549734115601, "blocks.6.attention.w_q": 0.8158798217773438, "blocks.6.attention.w_k": 0.8066749572753906, "blocks.6.attention.w_v": 0.7792091369628906, "blocks.6.attention.w_out": 0.7567100524902344, "blocks.6.feed_forward.w1": 0.8296089172363281, "blocks.6.feed_forward.w2": 0.7657178640365601, "blocks.6.feed_forward.w3": 0.8199946284294128, "blocks.7.attention.w_q": 0.8211746215820312, "blocks.7.attention.w_k": 0.808319091796875, "blocks.7.attention.w_v": 0.7856063842773438, "blocks.7.attention.w_out": 0.7524375915527344, "blocks.7.feed_forward.w1": 0.8336130976676941, "blocks.7.feed_forward.w2": 0.7444814443588257, "blocks.7.feed_forward.w3": 0.8205592036247253, "blocks.8.attention.w_q": 0.8245124816894531, "blocks.8.attention.w_k": 0.812591552734375, "blocks.8.attention.w_v": 0.7892723083496094, "blocks.8.attention.w_out": 0.7633819580078125, "blocks.8.feed_forward.w1": 0.8387947082519531, "blocks.8.feed_forward.w2": 0.7110837697982788, "blocks.8.feed_forward.w3": 0.8228811025619507}} +{"step": 2500, "changed_frac": 0.8045042315099993, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.777557373046875, "blocks.0.attention.w_k": 0.7746315002441406, "blocks.0.attention.w_v": 0.7837333679199219, "blocks.0.attention.w_out": 0.8092079162597656, "blocks.0.feed_forward.w1": 0.8321762084960938, "blocks.0.feed_forward.w2": 0.8072179555892944, "blocks.0.feed_forward.w3": 0.8271395564079285, "blocks.1.attention.w_q": 0.816314697265625, "blocks.1.attention.w_k": 0.8077049255371094, "blocks.1.attention.w_v": 0.83599853515625, "blocks.1.attention.w_out": 0.8566322326660156, "blocks.1.feed_forward.w1": 0.815222442150116, "blocks.1.feed_forward.w2": 0.7988459467887878, "blocks.1.feed_forward.w3": 0.8094469904899597, "blocks.2.attention.w_q": 0.8214035034179688, "blocks.2.attention.w_k": 0.8155097961425781, "blocks.2.attention.w_v": 0.8133087158203125, "blocks.2.attention.w_out": 0.841583251953125, "blocks.2.feed_forward.w1": 0.8168157339096069, "blocks.2.feed_forward.w2": 0.7928683161735535, "blocks.2.feed_forward.w3": 0.8101412653923035, "blocks.3.attention.w_q": 0.8074378967285156, "blocks.3.attention.w_k": 0.7991447448730469, "blocks.3.attention.w_v": 0.804901123046875, "blocks.3.attention.w_out": 0.8338699340820312, "blocks.3.feed_forward.w1": 0.8186073303222656, "blocks.3.feed_forward.w2": 0.7972120046615601, "blocks.3.feed_forward.w3": 0.8093580007553101, "blocks.4.attention.w_q": 0.8208656311035156, "blocks.4.attention.w_k": 0.8084983825683594, "blocks.4.attention.w_v": 0.8086509704589844, "blocks.4.attention.w_out": 0.8291702270507812, "blocks.4.feed_forward.w1": 0.8216184377670288, "blocks.4.feed_forward.w2": 0.7941017150878906, "blocks.4.feed_forward.w3": 0.8123919367790222, "blocks.5.attention.w_q": 0.8062210083007812, "blocks.5.attention.w_k": 0.7975692749023438, "blocks.5.attention.w_v": 0.7854499816894531, "blocks.5.attention.w_out": 0.7906112670898438, "blocks.5.feed_forward.w1": 0.8236135244369507, "blocks.5.feed_forward.w2": 0.7804667353630066, "blocks.5.feed_forward.w3": 0.8147748708724976, "blocks.6.attention.w_q": 0.8134498596191406, "blocks.6.attention.w_k": 0.8029937744140625, "blocks.6.attention.w_v": 0.7761344909667969, "blocks.6.attention.w_out": 0.7523384094238281, "blocks.6.feed_forward.w1": 0.825164794921875, "blocks.6.feed_forward.w2": 0.7630857229232788, "blocks.6.feed_forward.w3": 0.8154513239860535, "blocks.7.attention.w_q": 0.8172950744628906, "blocks.7.attention.w_k": 0.8041038513183594, "blocks.7.attention.w_v": 0.7781715393066406, "blocks.7.attention.w_out": 0.7454872131347656, "blocks.7.feed_forward.w1": 0.8307406306266785, "blocks.7.feed_forward.w2": 0.7404429316520691, "blocks.7.feed_forward.w3": 0.8184420466423035, "blocks.8.attention.w_q": 0.8217353820800781, "blocks.8.attention.w_k": 0.8118515014648438, "blocks.8.attention.w_v": 0.7851066589355469, "blocks.8.attention.w_out": 0.7584991455078125, "blocks.8.feed_forward.w1": 0.8366636037826538, "blocks.8.feed_forward.w2": 0.7061449885368347, "blocks.8.feed_forward.w3": 0.8197962641716003}} +{"step": 2600, "changed_frac": 0.8007320827907987, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7712020874023438, "blocks.0.attention.w_k": 0.770751953125, "blocks.0.attention.w_v": 0.7780990600585938, "blocks.0.attention.w_out": 0.8069114685058594, "blocks.0.feed_forward.w1": 0.8289210200309753, "blocks.0.feed_forward.w2": 0.8032748103141785, "blocks.0.feed_forward.w3": 0.8239568471908569, "blocks.1.attention.w_q": 0.8137092590332031, "blocks.1.attention.w_k": 0.8035736083984375, "blocks.1.attention.w_v": 0.8325614929199219, "blocks.1.attention.w_out": 0.8531494140625, "blocks.1.feed_forward.w1": 0.8115514516830444, "blocks.1.feed_forward.w2": 0.7949409484863281, "blocks.1.feed_forward.w3": 0.8068618774414062, "blocks.2.attention.w_q": 0.8176231384277344, "blocks.2.attention.w_k": 0.8110237121582031, "blocks.2.attention.w_v": 0.8093719482421875, "blocks.2.attention.w_out": 0.8408470153808594, "blocks.2.feed_forward.w1": 0.8135401606559753, "blocks.2.feed_forward.w2": 0.7892405390739441, "blocks.2.feed_forward.w3": 0.8063825368881226, "blocks.3.attention.w_q": 0.8029060363769531, "blocks.3.attention.w_k": 0.7957267761230469, "blocks.3.attention.w_v": 0.8001785278320312, "blocks.3.attention.w_out": 0.8303146362304688, "blocks.3.feed_forward.w1": 0.8151830434799194, "blocks.3.feed_forward.w2": 0.7947629690170288, "blocks.3.feed_forward.w3": 0.805877685546875, "blocks.4.attention.w_q": 0.8184967041015625, "blocks.4.attention.w_k": 0.8033943176269531, "blocks.4.attention.w_v": 0.8020286560058594, "blocks.4.attention.w_out": 0.8253173828125, "blocks.4.feed_forward.w1": 0.8185564875602722, "blocks.4.feed_forward.w2": 0.7912864685058594, "blocks.4.feed_forward.w3": 0.807684600353241, "blocks.5.attention.w_q": 0.8044853210449219, "blocks.5.attention.w_k": 0.7940521240234375, "blocks.5.attention.w_v": 0.7814064025878906, "blocks.5.attention.w_out": 0.7882118225097656, "blocks.5.feed_forward.w1": 0.8195114135742188, "blocks.5.feed_forward.w2": 0.777557373046875, "blocks.5.feed_forward.w3": 0.8099963068962097, "blocks.6.attention.w_q": 0.809722900390625, "blocks.6.attention.w_k": 0.7978096008300781, "blocks.6.attention.w_v": 0.7705459594726562, "blocks.6.attention.w_out": 0.747406005859375, "blocks.6.feed_forward.w1": 0.821966826915741, "blocks.6.feed_forward.w2": 0.7595037221908569, "blocks.6.feed_forward.w3": 0.8117930293083191, "blocks.7.attention.w_q": 0.8125267028808594, "blocks.7.attention.w_k": 0.8017578125, "blocks.7.attention.w_v": 0.7774391174316406, "blocks.7.attention.w_out": 0.7400093078613281, "blocks.7.feed_forward.w1": 0.8277448415756226, "blocks.7.feed_forward.w2": 0.7356135249137878, "blocks.7.feed_forward.w3": 0.8137321472167969, "blocks.8.attention.w_q": 0.8190536499023438, "blocks.8.attention.w_k": 0.8071327209472656, "blocks.8.attention.w_v": 0.781585693359375, "blocks.8.attention.w_out": 0.7527732849121094, "blocks.8.feed_forward.w1": 0.8327713012695312, "blocks.8.feed_forward.w2": 0.6998494863510132, "blocks.8.feed_forward.w3": 0.8154551386833191}} +{"step": 2700, "changed_frac": 0.7968458191961305, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7676658630371094, "blocks.0.attention.w_k": 0.7650299072265625, "blocks.0.attention.w_v": 0.7744712829589844, "blocks.0.attention.w_out": 0.8033409118652344, "blocks.0.feed_forward.w1": 0.8254737854003906, "blocks.0.feed_forward.w2": 0.7988433837890625, "blocks.0.feed_forward.w3": 0.8199793696403503, "blocks.1.attention.w_q": 0.8117408752441406, "blocks.1.attention.w_k": 0.8012847900390625, "blocks.1.attention.w_v": 0.8288497924804688, "blocks.1.attention.w_out": 0.8520393371582031, "blocks.1.feed_forward.w1": 0.8070869445800781, "blocks.1.feed_forward.w2": 0.7909940481185913, "blocks.1.feed_forward.w3": 0.8032989501953125, "blocks.2.attention.w_q": 0.8144569396972656, "blocks.2.attention.w_k": 0.807861328125, "blocks.2.attention.w_v": 0.8052291870117188, "blocks.2.attention.w_out": 0.8352317810058594, "blocks.2.feed_forward.w1": 0.8101234436035156, "blocks.2.feed_forward.w2": 0.7874603271484375, "blocks.2.feed_forward.w3": 0.8024381399154663, "blocks.3.attention.w_q": 0.7970428466796875, "blocks.3.attention.w_k": 0.792266845703125, "blocks.3.attention.w_v": 0.7969512939453125, "blocks.3.attention.w_out": 0.8270378112792969, "blocks.3.feed_forward.w1": 0.81072998046875, "blocks.3.feed_forward.w2": 0.7913627624511719, "blocks.3.feed_forward.w3": 0.8018264770507812, "blocks.4.attention.w_q": 0.8155441284179688, "blocks.4.attention.w_k": 0.8025970458984375, "blocks.4.attention.w_v": 0.7956123352050781, "blocks.4.attention.w_out": 0.8216590881347656, "blocks.4.feed_forward.w1": 0.81536865234375, "blocks.4.feed_forward.w2": 0.7878939509391785, "blocks.4.feed_forward.w3": 0.8044217824935913, "blocks.5.attention.w_q": 0.8009262084960938, "blocks.5.attention.w_k": 0.7886276245117188, "blocks.5.attention.w_v": 0.7778701782226562, "blocks.5.attention.w_out": 0.7829704284667969, "blocks.5.feed_forward.w1": 0.8166745901107788, "blocks.5.feed_forward.w2": 0.7738698720932007, "blocks.5.feed_forward.w3": 0.8063201904296875, "blocks.6.attention.w_q": 0.8041152954101562, "blocks.6.attention.w_k": 0.7954940795898438, "blocks.6.attention.w_v": 0.7630996704101562, "blocks.6.attention.w_out": 0.7449188232421875, "blocks.6.feed_forward.w1": 0.8200200796127319, "blocks.6.feed_forward.w2": 0.7547251582145691, "blocks.6.feed_forward.w3": 0.8067907094955444, "blocks.7.attention.w_q": 0.8090667724609375, "blocks.7.attention.w_k": 0.7970466613769531, "blocks.7.attention.w_v": 0.7692756652832031, "blocks.7.attention.w_out": 0.736236572265625, "blocks.7.feed_forward.w1": 0.824127197265625, "blocks.7.feed_forward.w2": 0.7316513061523438, "blocks.7.feed_forward.w3": 0.8101361989974976, "blocks.8.attention.w_q": 0.8128852844238281, "blocks.8.attention.w_k": 0.8028144836425781, "blocks.8.attention.w_v": 0.7753219604492188, "blocks.8.attention.w_out": 0.7460746765136719, "blocks.8.feed_forward.w1": 0.8296114802360535, "blocks.8.feed_forward.w2": 0.6932322382926941, "blocks.8.feed_forward.w3": 0.8116404414176941}} +{"step": 2800, "changed_frac": 0.7928775233081263, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7614212036132812, "blocks.0.attention.w_k": 0.7601394653320312, "blocks.0.attention.w_v": 0.7688751220703125, "blocks.0.attention.w_out": 0.7972450256347656, "blocks.0.feed_forward.w1": 0.8211924433708191, "blocks.0.feed_forward.w2": 0.7946815490722656, "blocks.0.feed_forward.w3": 0.8171577453613281, "blocks.1.attention.w_q": 0.8059158325195312, "blocks.1.attention.w_k": 0.7977447509765625, "blocks.1.attention.w_v": 0.8254776000976562, "blocks.1.attention.w_out": 0.8490104675292969, "blocks.1.feed_forward.w1": 0.8047778010368347, "blocks.1.feed_forward.w2": 0.7883046865463257, "blocks.1.feed_forward.w3": 0.7994855642318726, "blocks.2.attention.w_q": 0.8092536926269531, "blocks.2.attention.w_k": 0.8048362731933594, "blocks.2.attention.w_v": 0.80230712890625, "blocks.2.attention.w_out": 0.8336868286132812, "blocks.2.feed_forward.w1": 0.8068428039550781, "blocks.2.feed_forward.w2": 0.7841784358024597, "blocks.2.feed_forward.w3": 0.7993863821029663, "blocks.3.attention.w_q": 0.7919197082519531, "blocks.3.attention.w_k": 0.7860069274902344, "blocks.3.attention.w_v": 0.7909278869628906, "blocks.3.attention.w_out": 0.8243789672851562, "blocks.3.feed_forward.w1": 0.8069547414779663, "blocks.3.feed_forward.w2": 0.7877769470214844, "blocks.3.feed_forward.w3": 0.7985496520996094, "blocks.4.attention.w_q": 0.8116188049316406, "blocks.4.attention.w_k": 0.797698974609375, "blocks.4.attention.w_v": 0.7932472229003906, "blocks.4.attention.w_out": 0.8188552856445312, "blocks.4.feed_forward.w1": 0.8109080195426941, "blocks.4.feed_forward.w2": 0.7845700979232788, "blocks.4.feed_forward.w3": 0.8009185791015625, "blocks.5.attention.w_q": 0.7947845458984375, "blocks.5.attention.w_k": 0.7847213745117188, "blocks.5.attention.w_v": 0.7717514038085938, "blocks.5.attention.w_out": 0.7802810668945312, "blocks.5.feed_forward.w1": 0.8134092092514038, "blocks.5.feed_forward.w2": 0.7694219350814819, "blocks.5.feed_forward.w3": 0.8007024526596069, "blocks.6.attention.w_q": 0.8010902404785156, "blocks.6.attention.w_k": 0.7929344177246094, "blocks.6.attention.w_v": 0.7583541870117188, "blocks.6.attention.w_out": 0.7387886047363281, "blocks.6.feed_forward.w1": 0.8162460327148438, "blocks.6.feed_forward.w2": 0.7508417963981628, "blocks.6.feed_forward.w3": 0.8037440180778503, "blocks.7.attention.w_q": 0.8072586059570312, "blocks.7.attention.w_k": 0.7927665710449219, "blocks.7.attention.w_v": 0.7665252685546875, "blocks.7.attention.w_out": 0.731201171875, "blocks.7.feed_forward.w1": 0.8206939697265625, "blocks.7.feed_forward.w2": 0.7244085073471069, "blocks.7.feed_forward.w3": 0.8064079284667969, "blocks.8.attention.w_q": 0.8078536987304688, "blocks.8.attention.w_k": 0.7996826171875, "blocks.8.attention.w_v": 0.7716865539550781, "blocks.8.attention.w_out": 0.7412605285644531, "blocks.8.feed_forward.w1": 0.8255208730697632, "blocks.8.feed_forward.w2": 0.6870893239974976, "blocks.8.feed_forward.w3": 0.8075498342514038}} +{"step": 2900, "changed_frac": 0.7885196718395265, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7571678161621094, "blocks.0.attention.w_k": 0.7526283264160156, "blocks.0.attention.w_v": 0.7653617858886719, "blocks.0.attention.w_out": 0.7941665649414062, "blocks.0.feed_forward.w1": 0.8167864680290222, "blocks.0.feed_forward.w2": 0.7920557856559753, "blocks.0.feed_forward.w3": 0.8119443655014038, "blocks.1.attention.w_q": 0.8038139343261719, "blocks.1.attention.w_k": 0.7942314147949219, "blocks.1.attention.w_v": 0.8203315734863281, "blocks.1.attention.w_out": 0.8458976745605469, "blocks.1.feed_forward.w1": 0.7996482849121094, "blocks.1.feed_forward.w2": 0.7840220332145691, "blocks.1.feed_forward.w3": 0.7952359914779663, "blocks.2.attention.w_q": 0.8043899536132812, "blocks.2.attention.w_k": 0.7984504699707031, "blocks.2.attention.w_v": 0.7988662719726562, "blocks.2.attention.w_out": 0.8300666809082031, "blocks.2.feed_forward.w1": 0.8026301264762878, "blocks.2.feed_forward.w2": 0.7799415588378906, "blocks.2.feed_forward.w3": 0.7948443293571472, "blocks.3.attention.w_q": 0.7873687744140625, "blocks.3.attention.w_k": 0.7826614379882812, "blocks.3.attention.w_v": 0.7847785949707031, "blocks.3.attention.w_out": 0.8211860656738281, "blocks.3.feed_forward.w1": 0.8029531240463257, "blocks.3.feed_forward.w2": 0.7835413813591003, "blocks.3.feed_forward.w3": 0.7950999140739441, "blocks.4.attention.w_q": 0.8038902282714844, "blocks.4.attention.w_k": 0.7910842895507812, "blocks.4.attention.w_v": 0.7882575988769531, "blocks.4.attention.w_out": 0.8140144348144531, "blocks.4.feed_forward.w1": 0.8071009516716003, "blocks.4.feed_forward.w2": 0.7798754572868347, "blocks.4.feed_forward.w3": 0.7959849238395691, "blocks.5.attention.w_q": 0.7905006408691406, "blocks.5.attention.w_k": 0.78021240234375, "blocks.5.attention.w_v": 0.7672500610351562, "blocks.5.attention.w_out": 0.7758827209472656, "blocks.5.feed_forward.w1": 0.8090095520019531, "blocks.5.feed_forward.w2": 0.7662455439567566, "blocks.5.feed_forward.w3": 0.7966436147689819, "blocks.6.attention.w_q": 0.7962455749511719, "blocks.6.attention.w_k": 0.7879791259765625, "blocks.6.attention.w_v": 0.7523994445800781, "blocks.6.attention.w_out": 0.7347946166992188, "blocks.6.feed_forward.w1": 0.8127250671386719, "blocks.6.feed_forward.w2": 0.7454974055290222, "blocks.6.feed_forward.w3": 0.7997411489486694, "blocks.7.attention.w_q": 0.8022956848144531, "blocks.7.attention.w_k": 0.7879066467285156, "blocks.7.attention.w_v": 0.7601661682128906, "blocks.7.attention.w_out": 0.7249221801757812, "blocks.7.feed_forward.w1": 0.8161532282829285, "blocks.7.feed_forward.w2": 0.720794677734375, "blocks.7.feed_forward.w3": 0.8021901845932007, "blocks.8.attention.w_q": 0.8051834106445312, "blocks.8.attention.w_k": 0.7934913635253906, "blocks.8.attention.w_v": 0.7655525207519531, "blocks.8.attention.w_out": 0.7365989685058594, "blocks.8.feed_forward.w1": 0.8223737478256226, "blocks.8.feed_forward.w2": 0.6813291311264038, "blocks.8.feed_forward.w3": 0.8045679926872253}} +{"step": 3000, "changed_frac": 0.784170884352464, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7474288940429688, "blocks.0.attention.w_k": 0.7464218139648438, "blocks.0.attention.w_v": 0.7584800720214844, "blocks.0.attention.w_out": 0.7911758422851562, "blocks.0.feed_forward.w1": 0.8119863271713257, "blocks.0.feed_forward.w2": 0.7885119318962097, "blocks.0.feed_forward.w3": 0.8075485229492188, "blocks.1.attention.w_q": 0.7999687194824219, "blocks.1.attention.w_k": 0.7900161743164062, "blocks.1.attention.w_v": 0.8171577453613281, "blocks.1.attention.w_out": 0.8430366516113281, "blocks.1.feed_forward.w1": 0.7950083613395691, "blocks.1.feed_forward.w2": 0.7810134887695312, "blocks.1.feed_forward.w3": 0.7904828786849976, "blocks.2.attention.w_q": 0.8010520935058594, "blocks.2.attention.w_k": 0.7943687438964844, "blocks.2.attention.w_v": 0.7925758361816406, "blocks.2.attention.w_out": 0.8277587890625, "blocks.2.feed_forward.w1": 0.7988904714584351, "blocks.2.feed_forward.w2": 0.7756551504135132, "blocks.2.feed_forward.w3": 0.7920736074447632, "blocks.3.attention.w_q": 0.7830276489257812, "blocks.3.attention.w_k": 0.7775650024414062, "blocks.3.attention.w_v": 0.77996826171875, "blocks.3.attention.w_out": 0.8166427612304688, "blocks.3.feed_forward.w1": 0.7985763549804688, "blocks.3.feed_forward.w2": 0.780364990234375, "blocks.3.feed_forward.w3": 0.7903875112533569, "blocks.4.attention.w_q": 0.8006668090820312, "blocks.4.attention.w_k": 0.7872467041015625, "blocks.4.attention.w_v": 0.7820816040039062, "blocks.4.attention.w_out": 0.8098526000976562, "blocks.4.feed_forward.w1": 0.8038355708122253, "blocks.4.feed_forward.w2": 0.7770563960075378, "blocks.4.feed_forward.w3": 0.7918269038200378, "blocks.5.attention.w_q": 0.7870521545410156, "blocks.5.attention.w_k": 0.7769126892089844, "blocks.5.attention.w_v": 0.7624893188476562, "blocks.5.attention.w_out": 0.7726631164550781, "blocks.5.feed_forward.w1": 0.8053156733512878, "blocks.5.feed_forward.w2": 0.7624194025993347, "blocks.5.feed_forward.w3": 0.7931226491928101, "blocks.6.attention.w_q": 0.7920074462890625, "blocks.6.attention.w_k": 0.780914306640625, "blocks.6.attention.w_v": 0.7486610412597656, "blocks.6.attention.w_out": 0.7313346862792969, "blocks.6.feed_forward.w1": 0.8083000183105469, "blocks.6.feed_forward.w2": 0.7404073476791382, "blocks.6.feed_forward.w3": 0.7951304316520691, "blocks.7.attention.w_q": 0.7966079711914062, "blocks.7.attention.w_k": 0.7822380065917969, "blocks.7.attention.w_v": 0.7554512023925781, "blocks.7.attention.w_out": 0.7184906005859375, "blocks.7.feed_forward.w1": 0.813513457775116, "blocks.7.feed_forward.w2": 0.7150382995605469, "blocks.7.feed_forward.w3": 0.7978503108024597, "blocks.8.attention.w_q": 0.79925537109375, "blocks.8.attention.w_k": 0.7899932861328125, "blocks.8.attention.w_v": 0.7609176635742188, "blocks.8.attention.w_out": 0.7284088134765625, "blocks.8.feed_forward.w1": 0.8185666799545288, "blocks.8.feed_forward.w2": 0.6730575561523438, "blocks.8.feed_forward.w3": 0.8000946044921875}} +{"step": 3100, "changed_frac": 0.7791470258663862, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7439498901367188, "blocks.0.attention.w_k": 0.7380905151367188, "blocks.0.attention.w_v": 0.7524795532226562, "blocks.0.attention.w_out": 0.7859268188476562, "blocks.0.feed_forward.w1": 0.8070729970932007, "blocks.0.feed_forward.w2": 0.7837511897087097, "blocks.0.feed_forward.w3": 0.8028678894042969, "blocks.1.attention.w_q": 0.7939109802246094, "blocks.1.attention.w_k": 0.7837066650390625, "blocks.1.attention.w_v": 0.8115653991699219, "blocks.1.attention.w_out": 0.8394889831542969, "blocks.1.feed_forward.w1": 0.7899322509765625, "blocks.1.feed_forward.w2": 0.7759361267089844, "blocks.1.feed_forward.w3": 0.7854385375976562, "blocks.2.attention.w_q": 0.7950859069824219, "blocks.2.attention.w_k": 0.7880592346191406, "blocks.2.attention.w_v": 0.7894134521484375, "blocks.2.attention.w_out": 0.822723388671875, "blocks.2.feed_forward.w1": 0.7944552302360535, "blocks.2.feed_forward.w2": 0.7713038325309753, "blocks.2.feed_forward.w3": 0.7851142883300781, "blocks.3.attention.w_q": 0.7773818969726562, "blocks.3.attention.w_k": 0.7714271545410156, "blocks.3.attention.w_v": 0.7752494812011719, "blocks.3.attention.w_out": 0.8111076354980469, "blocks.3.feed_forward.w1": 0.7939834594726562, "blocks.3.feed_forward.w2": 0.7770856618881226, "blocks.3.feed_forward.w3": 0.7850418090820312, "blocks.4.attention.w_q": 0.795013427734375, "blocks.4.attention.w_k": 0.7823944091796875, "blocks.4.attention.w_v": 0.7778549194335938, "blocks.4.attention.w_out": 0.8062934875488281, "blocks.4.feed_forward.w1": 0.798065185546875, "blocks.4.feed_forward.w2": 0.7707914113998413, "blocks.4.feed_forward.w3": 0.7872047424316406, "blocks.5.attention.w_q": 0.7804069519042969, "blocks.5.attention.w_k": 0.7708892822265625, "blocks.5.attention.w_v": 0.7569198608398438, "blocks.5.attention.w_out": 0.7663078308105469, "blocks.5.feed_forward.w1": 0.8011283874511719, "blocks.5.feed_forward.w2": 0.7576993703842163, "blocks.5.feed_forward.w3": 0.7879905700683594, "blocks.6.attention.w_q": 0.78643798828125, "blocks.6.attention.w_k": 0.778656005859375, "blocks.6.attention.w_v": 0.7414474487304688, "blocks.6.attention.w_out": 0.7290573120117188, "blocks.6.feed_forward.w1": 0.8039728999137878, "blocks.6.feed_forward.w2": 0.7350744009017944, "blocks.6.feed_forward.w3": 0.7897657155990601, "blocks.7.attention.w_q": 0.7918243408203125, "blocks.7.attention.w_k": 0.7779617309570312, "blocks.7.attention.w_v": 0.7514762878417969, "blocks.7.attention.w_out": 0.710418701171875, "blocks.7.feed_forward.w1": 0.8085263967514038, "blocks.7.feed_forward.w2": 0.7095286250114441, "blocks.7.feed_forward.w3": 0.7935931086540222, "blocks.8.attention.w_q": 0.7981948852539062, "blocks.8.attention.w_k": 0.785675048828125, "blocks.8.attention.w_v": 0.7545623779296875, "blocks.8.attention.w_out": 0.7195968627929688, "blocks.8.feed_forward.w1": 0.8157171010971069, "blocks.8.feed_forward.w2": 0.6674283742904663, "blocks.8.feed_forward.w3": 0.7946128845214844}} +{"step": 3200, "changed_frac": 0.7740124107426049, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7374687194824219, "blocks.0.attention.w_k": 0.7334709167480469, "blocks.0.attention.w_v": 0.745208740234375, "blocks.0.attention.w_out": 0.7822189331054688, "blocks.0.feed_forward.w1": 0.8028297424316406, "blocks.0.feed_forward.w2": 0.778631865978241, "blocks.0.feed_forward.w3": 0.7980028986930847, "blocks.1.attention.w_q": 0.7896499633789062, "blocks.1.attention.w_k": 0.7823715209960938, "blocks.1.attention.w_v": 0.8062400817871094, "blocks.1.attention.w_out": 0.8350028991699219, "blocks.1.feed_forward.w1": 0.7853495478630066, "blocks.1.feed_forward.w2": 0.7700996398925781, "blocks.1.feed_forward.w3": 0.7806587219238281, "blocks.2.attention.w_q": 0.7885284423828125, "blocks.2.attention.w_k": 0.7841606140136719, "blocks.2.attention.w_v": 0.7822647094726562, "blocks.2.attention.w_out": 0.8193740844726562, "blocks.2.feed_forward.w1": 0.7898814082145691, "blocks.2.feed_forward.w2": 0.7678489685058594, "blocks.2.feed_forward.w3": 0.7817497253417969, "blocks.3.attention.w_q": 0.7735595703125, "blocks.3.attention.w_k": 0.7651214599609375, "blocks.3.attention.w_v": 0.7693099975585938, "blocks.3.attention.w_out": 0.8087234497070312, "blocks.3.feed_forward.w1": 0.7901713252067566, "blocks.3.feed_forward.w2": 0.7718035578727722, "blocks.3.feed_forward.w3": 0.7807782888412476, "blocks.4.attention.w_q": 0.790374755859375, "blocks.4.attention.w_k": 0.7769126892089844, "blocks.4.attention.w_v": 0.7713661193847656, "blocks.4.attention.w_out": 0.7997360229492188, "blocks.4.feed_forward.w1": 0.7939237356185913, "blocks.4.feed_forward.w2": 0.7667173147201538, "blocks.4.feed_forward.w3": 0.7817370295524597, "blocks.5.attention.w_q": 0.774932861328125, "blocks.5.attention.w_k": 0.7653884887695312, "blocks.5.attention.w_v": 0.7505302429199219, "blocks.5.attention.w_out": 0.7625846862792969, "blocks.5.feed_forward.w1": 0.7960981130599976, "blocks.5.feed_forward.w2": 0.7524083852767944, "blocks.5.feed_forward.w3": 0.7828585505485535, "blocks.6.attention.w_q": 0.7797508239746094, "blocks.6.attention.w_k": 0.7717437744140625, "blocks.6.attention.w_v": 0.7343177795410156, "blocks.6.attention.w_out": 0.720794677734375, "blocks.6.feed_forward.w1": 0.799444854259491, "blocks.6.feed_forward.w2": 0.7295163869857788, "blocks.6.feed_forward.w3": 0.785736083984375, "blocks.7.attention.w_q": 0.7850914001464844, "blocks.7.attention.w_k": 0.7720451354980469, "blocks.7.attention.w_v": 0.7433624267578125, "blocks.7.attention.w_out": 0.7048149108886719, "blocks.7.feed_forward.w1": 0.8051618337631226, "blocks.7.feed_forward.w2": 0.702582061290741, "blocks.7.feed_forward.w3": 0.7884013056755066, "blocks.8.attention.w_q": 0.7908401489257812, "blocks.8.attention.w_k": 0.7793922424316406, "blocks.8.attention.w_v": 0.7464981079101562, "blocks.8.attention.w_out": 0.713623046875, "blocks.8.feed_forward.w1": 0.8101844787597656, "blocks.8.feed_forward.w2": 0.6590360403060913, "blocks.8.feed_forward.w3": 0.7892812490463257}} +{"step": 3300, "changed_frac": 0.7689834659935063, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7291221618652344, "blocks.0.attention.w_k": 0.7266387939453125, "blocks.0.attention.w_v": 0.7404251098632812, "blocks.0.attention.w_out": 0.7782936096191406, "blocks.0.feed_forward.w1": 0.7988077998161316, "blocks.0.feed_forward.w2": 0.7745794057846069, "blocks.0.feed_forward.w3": 0.7937876582145691, "blocks.1.attention.w_q": 0.7845649719238281, "blocks.1.attention.w_k": 0.7729988098144531, "blocks.1.attention.w_v": 0.8018455505371094, "blocks.1.attention.w_out": 0.8324203491210938, "blocks.1.feed_forward.w1": 0.7804756164550781, "blocks.1.feed_forward.w2": 0.765625, "blocks.1.feed_forward.w3": 0.7766914367675781, "blocks.2.attention.w_q": 0.7835464477539062, "blocks.2.attention.w_k": 0.7750244140625, "blocks.2.attention.w_v": 0.7772293090820312, "blocks.2.attention.w_out": 0.8149909973144531, "blocks.2.feed_forward.w1": 0.7843043208122253, "blocks.2.feed_forward.w2": 0.7642682790756226, "blocks.2.feed_forward.w3": 0.7766036987304688, "blocks.3.attention.w_q": 0.7660179138183594, "blocks.3.attention.w_k": 0.7597236633300781, "blocks.3.attention.w_v": 0.7632255554199219, "blocks.3.attention.w_out": 0.8026008605957031, "blocks.3.feed_forward.w1": 0.7857005000114441, "blocks.3.feed_forward.w2": 0.7683690786361694, "blocks.3.feed_forward.w3": 0.7766189575195312, "blocks.4.attention.w_q": 0.7861900329589844, "blocks.4.attention.w_k": 0.7728118896484375, "blocks.4.attention.w_v": 0.7664871215820312, "blocks.4.attention.w_out": 0.794189453125, "blocks.4.feed_forward.w1": 0.7893728017807007, "blocks.4.feed_forward.w2": 0.7612940669059753, "blocks.4.feed_forward.w3": 0.7768110036849976, "blocks.5.attention.w_q": 0.7689476013183594, "blocks.5.attention.w_k": 0.7596702575683594, "blocks.5.attention.w_v": 0.7433547973632812, "blocks.5.attention.w_out": 0.757293701171875, "blocks.5.feed_forward.w1": 0.7908605337142944, "blocks.5.feed_forward.w2": 0.7473424673080444, "blocks.5.feed_forward.w3": 0.7788442373275757, "blocks.6.attention.w_q": 0.7757072448730469, "blocks.6.attention.w_k": 0.7656784057617188, "blocks.6.attention.w_v": 0.7264175415039062, "blocks.6.attention.w_out": 0.7145881652832031, "blocks.6.feed_forward.w1": 0.7944743037223816, "blocks.6.feed_forward.w2": 0.7252184748649597, "blocks.6.feed_forward.w3": 0.7804057002067566, "blocks.7.attention.w_q": 0.7790374755859375, "blocks.7.attention.w_k": 0.766998291015625, "blocks.7.attention.w_v": 0.7366065979003906, "blocks.7.attention.w_out": 0.7003211975097656, "blocks.7.feed_forward.w1": 0.8007291555404663, "blocks.7.feed_forward.w2": 0.6958541870117188, "blocks.7.feed_forward.w3": 0.7834574580192566, "blocks.8.attention.w_q": 0.7864723205566406, "blocks.8.attention.w_k": 0.7745399475097656, "blocks.8.attention.w_v": 0.7432136535644531, "blocks.8.attention.w_out": 0.7055740356445312, "blocks.8.feed_forward.w1": 0.8054835200309753, "blocks.8.feed_forward.w2": 0.6519648432731628, "blocks.8.feed_forward.w3": 0.7848218679428101}} +{"step": 3400, "changed_frac": 0.7631577711838943, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7203140258789062, "blocks.0.attention.w_k": 0.7165946960449219, "blocks.0.attention.w_v": 0.7325553894042969, "blocks.0.attention.w_out": 0.773193359375, "blocks.0.feed_forward.w1": 0.792486846446991, "blocks.0.feed_forward.w2": 0.7684148550033569, "blocks.0.feed_forward.w3": 0.7878049612045288, "blocks.1.attention.w_q": 0.7786102294921875, "blocks.1.attention.w_k": 0.767822265625, "blocks.1.attention.w_v": 0.79638671875, "blocks.1.attention.w_out": 0.8309440612792969, "blocks.1.feed_forward.w1": 0.7745755910873413, "blocks.1.feed_forward.w2": 0.7605807185173035, "blocks.1.feed_forward.w3": 0.7712377309799194, "blocks.2.attention.w_q": 0.7784614562988281, "blocks.2.attention.w_k": 0.7703285217285156, "blocks.2.attention.w_v": 0.7709808349609375, "blocks.2.attention.w_out": 0.8101425170898438, "blocks.2.feed_forward.w1": 0.7788938283920288, "blocks.2.feed_forward.w2": 0.7588615417480469, "blocks.2.feed_forward.w3": 0.7706565856933594, "blocks.3.attention.w_q": 0.7615280151367188, "blocks.3.attention.w_k": 0.7525520324707031, "blocks.3.attention.w_v": 0.7572212219238281, "blocks.3.attention.w_out": 0.7984466552734375, "blocks.3.feed_forward.w1": 0.780859649181366, "blocks.3.feed_forward.w2": 0.7631976008415222, "blocks.3.feed_forward.w3": 0.7715288996696472, "blocks.4.attention.w_q": 0.7786636352539062, "blocks.4.attention.w_k": 0.766632080078125, "blocks.4.attention.w_v": 0.76025390625, "blocks.4.attention.w_out": 0.7895889282226562, "blocks.4.feed_forward.w1": 0.7840118408203125, "blocks.4.feed_forward.w2": 0.7550939321517944, "blocks.4.feed_forward.w3": 0.7718009948730469, "blocks.5.attention.w_q": 0.7635917663574219, "blocks.5.attention.w_k": 0.7519302368164062, "blocks.5.attention.w_v": 0.7358207702636719, "blocks.5.attention.w_out": 0.7507667541503906, "blocks.5.feed_forward.w1": 0.7852325439453125, "blocks.5.feed_forward.w2": 0.7408078908920288, "blocks.5.feed_forward.w3": 0.7733065485954285, "blocks.6.attention.w_q": 0.7673797607421875, "blocks.6.attention.w_k": 0.7583045959472656, "blocks.6.attention.w_v": 0.7190971374511719, "blocks.6.attention.w_out": 0.7085647583007812, "blocks.6.feed_forward.w1": 0.7889289855957031, "blocks.6.feed_forward.w2": 0.7181040644645691, "blocks.6.feed_forward.w3": 0.7744852900505066, "blocks.7.attention.w_q": 0.7756195068359375, "blocks.7.attention.w_k": 0.7604866027832031, "blocks.7.attention.w_v": 0.7297439575195312, "blocks.7.attention.w_out": 0.6915702819824219, "blocks.7.feed_forward.w1": 0.7955793142318726, "blocks.7.feed_forward.w2": 0.6895116567611694, "blocks.7.feed_forward.w3": 0.7786026000976562, "blocks.8.attention.w_q": 0.7811698913574219, "blocks.8.attention.w_k": 0.7677116394042969, "blocks.8.attention.w_v": 0.7372245788574219, "blocks.8.attention.w_out": 0.7004966735839844, "blocks.8.feed_forward.w1": 0.8010368347167969, "blocks.8.feed_forward.w2": 0.6440290212631226, "blocks.8.feed_forward.w3": 0.7799568176269531}} +{"step": 3500, "changed_frac": 0.7573590400891427, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7152023315429688, "blocks.0.attention.w_k": 0.70819091796875, "blocks.0.attention.w_v": 0.7244606018066406, "blocks.0.attention.w_out": 0.7688331604003906, "blocks.0.feed_forward.w1": 0.7873281240463257, "blocks.0.feed_forward.w2": 0.7633082270622253, "blocks.0.feed_forward.w3": 0.7829526662826538, "blocks.1.attention.w_q": 0.7746009826660156, "blocks.1.attention.w_k": 0.7616233825683594, "blocks.1.attention.w_v": 0.7919387817382812, "blocks.1.attention.w_out": 0.8265953063964844, "blocks.1.feed_forward.w1": 0.7690150141716003, "blocks.1.feed_forward.w2": 0.7552503347396851, "blocks.1.feed_forward.w3": 0.7652193903923035, "blocks.2.attention.w_q": 0.7717781066894531, "blocks.2.attention.w_k": 0.7618865966796875, "blocks.2.attention.w_v": 0.7644500732421875, "blocks.2.attention.w_out": 0.8071250915527344, "blocks.2.feed_forward.w1": 0.7736778259277344, "blocks.2.feed_forward.w2": 0.7542877197265625, "blocks.2.feed_forward.w3": 0.76507568359375, "blocks.3.attention.w_q": 0.7547721862792969, "blocks.3.attention.w_k": 0.74346923828125, "blocks.3.attention.w_v": 0.7516517639160156, "blocks.3.attention.w_out": 0.794189453125, "blocks.3.feed_forward.w1": 0.7750943899154663, "blocks.3.feed_forward.w2": 0.7575734853744507, "blocks.3.feed_forward.w3": 0.7657610774040222, "blocks.4.attention.w_q": 0.7757225036621094, "blocks.4.attention.w_k": 0.7576942443847656, "blocks.4.attention.w_v": 0.75482177734375, "blocks.4.attention.w_out": 0.7835464477539062, "blocks.4.feed_forward.w1": 0.7783368825912476, "blocks.4.feed_forward.w2": 0.7505887746810913, "blocks.4.feed_forward.w3": 0.7673110961914062, "blocks.5.attention.w_q": 0.7568016052246094, "blocks.5.attention.w_k": 0.7452507019042969, "blocks.5.attention.w_v": 0.7279739379882812, "blocks.5.attention.w_out": 0.7458839416503906, "blocks.5.feed_forward.w1": 0.7813021540641785, "blocks.5.feed_forward.w2": 0.735260009765625, "blocks.5.feed_forward.w3": 0.768255889415741, "blocks.6.attention.w_q": 0.76239013671875, "blocks.6.attention.w_k": 0.7527847290039062, "blocks.6.attention.w_v": 0.7116966247558594, "blocks.6.attention.w_out": 0.7020492553710938, "blocks.6.feed_forward.w1": 0.7837346792221069, "blocks.6.feed_forward.w2": 0.7111727595329285, "blocks.6.feed_forward.w3": 0.7698275446891785, "blocks.7.attention.w_q": 0.7682113647460938, "blocks.7.attention.w_k": 0.7523193359375, "blocks.7.attention.w_v": 0.7261123657226562, "blocks.7.attention.w_out": 0.68414306640625, "blocks.7.feed_forward.w1": 0.7905858755111694, "blocks.7.feed_forward.w2": 0.6813596487045288, "blocks.7.feed_forward.w3": 0.7724291682243347, "blocks.8.attention.w_q": 0.7738838195800781, "blocks.8.attention.w_k": 0.7594375610351562, "blocks.8.attention.w_v": 0.7308387756347656, "blocks.8.attention.w_out": 0.6887702941894531, "blocks.8.feed_forward.w1": 0.7967974543571472, "blocks.8.feed_forward.w2": 0.6344439387321472, "blocks.8.feed_forward.w3": 0.7740198969841003}} +{"step": 3600, "changed_frac": 0.7510052868443676, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7080841064453125, "blocks.0.attention.w_k": 0.7020988464355469, "blocks.0.attention.w_v": 0.7191543579101562, "blocks.0.attention.w_out": 0.7617530822753906, "blocks.0.feed_forward.w1": 0.780157744884491, "blocks.0.feed_forward.w2": 0.7565116882324219, "blocks.0.feed_forward.w3": 0.7758535146713257, "blocks.1.attention.w_q": 0.7669105529785156, "blocks.1.attention.w_k": 0.7536544799804688, "blocks.1.attention.w_v": 0.7858734130859375, "blocks.1.attention.w_out": 0.821380615234375, "blocks.1.feed_forward.w1": 0.7637456655502319, "blocks.1.feed_forward.w2": 0.7493197321891785, "blocks.1.feed_forward.w3": 0.7584546804428101, "blocks.2.attention.w_q": 0.7662391662597656, "blocks.2.attention.w_k": 0.7542686462402344, "blocks.2.attention.w_v": 0.7599258422851562, "blocks.2.attention.w_out": 0.8026504516601562, "blocks.2.feed_forward.w1": 0.7675933837890625, "blocks.2.feed_forward.w2": 0.7474098205566406, "blocks.2.feed_forward.w3": 0.7590459585189819, "blocks.3.attention.w_q": 0.746124267578125, "blocks.3.attention.w_k": 0.7354736328125, "blocks.3.attention.w_v": 0.7462806701660156, "blocks.3.attention.w_out": 0.7874412536621094, "blocks.3.feed_forward.w1": 0.7702001333236694, "blocks.3.feed_forward.w2": 0.7525380849838257, "blocks.3.feed_forward.w3": 0.7604103088378906, "blocks.4.attention.w_q": 0.7689704895019531, "blocks.4.attention.w_k": 0.7500267028808594, "blocks.4.attention.w_v": 0.745697021484375, "blocks.4.attention.w_out": 0.7779312133789062, "blocks.4.feed_forward.w1": 0.7728831171989441, "blocks.4.feed_forward.w2": 0.7438176870346069, "blocks.4.feed_forward.w3": 0.7603086233139038, "blocks.5.attention.w_q": 0.7489891052246094, "blocks.5.attention.w_k": 0.7377853393554688, "blocks.5.attention.w_v": 0.722564697265625, "blocks.5.attention.w_out": 0.7403182983398438, "blocks.5.feed_forward.w1": 0.7739944458007812, "blocks.5.feed_forward.w2": 0.7299537658691406, "blocks.5.feed_forward.w3": 0.7610219717025757, "blocks.6.attention.w_q": 0.7569122314453125, "blocks.6.attention.w_k": 0.7458267211914062, "blocks.6.attention.w_v": 0.7039756774902344, "blocks.6.attention.w_out": 0.6987533569335938, "blocks.6.feed_forward.w1": 0.7775675654411316, "blocks.6.feed_forward.w2": 0.7070668935775757, "blocks.6.feed_forward.w3": 0.7630285024642944, "blocks.7.attention.w_q": 0.7613067626953125, "blocks.7.attention.w_k": 0.7459449768066406, "blocks.7.attention.w_v": 0.7182121276855469, "blocks.7.attention.w_out": 0.6779060363769531, "blocks.7.feed_forward.w1": 0.7848154902458191, "blocks.7.feed_forward.w2": 0.6746368408203125, "blocks.7.feed_forward.w3": 0.7660713195800781, "blocks.8.attention.w_q": 0.766998291015625, "blocks.8.attention.w_k": 0.7552108764648438, "blocks.8.attention.w_v": 0.7230873107910156, "blocks.8.attention.w_out": 0.6821212768554688, "blocks.8.feed_forward.w1": 0.7913564443588257, "blocks.8.feed_forward.w2": 0.6246973872184753, "blocks.8.feed_forward.w3": 0.7681287527084351}} +{"step": 3700, "changed_frac": 0.7443146664872129, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6978836059570312, "blocks.0.attention.w_k": 0.6939964294433594, "blocks.0.attention.w_v": 0.71026611328125, "blocks.0.attention.w_out": 0.7573471069335938, "blocks.0.feed_forward.w1": 0.7745501399040222, "blocks.0.feed_forward.w2": 0.7489446401596069, "blocks.0.feed_forward.w3": 0.7699725031852722, "blocks.1.attention.w_q": 0.7621345520019531, "blocks.1.attention.w_k": 0.7481880187988281, "blocks.1.attention.w_v": 0.7784271240234375, "blocks.1.attention.w_out": 0.8166732788085938, "blocks.1.feed_forward.w1": 0.7587738037109375, "blocks.1.feed_forward.w2": 0.744177520275116, "blocks.1.feed_forward.w3": 0.7514788508415222, "blocks.2.attention.w_q": 0.7588958740234375, "blocks.2.attention.w_k": 0.7472381591796875, "blocks.2.attention.w_v": 0.7517280578613281, "blocks.2.attention.w_out": 0.7974433898925781, "blocks.2.feed_forward.w1": 0.7616208791732788, "blocks.2.feed_forward.w2": 0.741858184337616, "blocks.2.feed_forward.w3": 0.7525711059570312, "blocks.3.attention.w_q": 0.7401161193847656, "blocks.3.attention.w_k": 0.7288665771484375, "blocks.3.attention.w_v": 0.7379226684570312, "blocks.3.attention.w_out": 0.7827491760253906, "blocks.3.feed_forward.w1": 0.7625885009765625, "blocks.3.feed_forward.w2": 0.7465171813964844, "blocks.3.feed_forward.w3": 0.7532056570053101, "blocks.4.attention.w_q": 0.7631416320800781, "blocks.4.attention.w_k": 0.742950439453125, "blocks.4.attention.w_v": 0.7385673522949219, "blocks.4.attention.w_out": 0.7724456787109375, "blocks.4.feed_forward.w1": 0.7664400935173035, "blocks.4.feed_forward.w2": 0.7378171682357788, "blocks.4.feed_forward.w3": 0.7537638545036316, "blocks.5.attention.w_q": 0.7429924011230469, "blocks.5.attention.w_k": 0.7294731140136719, "blocks.5.attention.w_v": 0.7138862609863281, "blocks.5.attention.w_out": 0.7310676574707031, "blocks.5.feed_forward.w1": 0.7689590454101562, "blocks.5.feed_forward.w2": 0.7228063344955444, "blocks.5.feed_forward.w3": 0.7534345388412476, "blocks.6.attention.w_q": 0.7483406066894531, "blocks.6.attention.w_k": 0.7392959594726562, "blocks.6.attention.w_v": 0.6960678100585938, "blocks.6.attention.w_out": 0.6911048889160156, "blocks.6.feed_forward.w1": 0.7713140249252319, "blocks.6.feed_forward.w2": 0.6991564631462097, "blocks.6.feed_forward.w3": 0.7574590444564819, "blocks.7.attention.w_q": 0.7548370361328125, "blocks.7.attention.w_k": 0.7398147583007812, "blocks.7.attention.w_v": 0.7070045471191406, "blocks.7.attention.w_out": 0.6697235107421875, "blocks.7.feed_forward.w1": 0.7791938781738281, "blocks.7.feed_forward.w2": 0.6659215688705444, "blocks.7.feed_forward.w3": 0.7601217031478882, "blocks.8.attention.w_q": 0.761993408203125, "blocks.8.attention.w_k": 0.7451972961425781, "blocks.8.attention.w_v": 0.7164649963378906, "blocks.8.attention.w_out": 0.6734466552734375, "blocks.8.feed_forward.w1": 0.7856496572494507, "blocks.8.feed_forward.w2": 0.6152877807617188, "blocks.8.feed_forward.w3": 0.7621244192123413}} +{"step": 3800, "changed_frac": 0.7370591774964944, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6859779357910156, "blocks.0.attention.w_k": 0.6837005615234375, "blocks.0.attention.w_v": 0.7018547058105469, "blocks.0.attention.w_out": 0.7503585815429688, "blocks.0.feed_forward.w1": 0.7674268484115601, "blocks.0.feed_forward.w2": 0.7438189387321472, "blocks.0.feed_forward.w3": 0.7630526423454285, "blocks.1.attention.w_q": 0.7535476684570312, "blocks.1.attention.w_k": 0.7396202087402344, "blocks.1.attention.w_v": 0.7744827270507812, "blocks.1.attention.w_out": 0.8139839172363281, "blocks.1.feed_forward.w1": 0.7499224543571472, "blocks.1.feed_forward.w2": 0.7380485534667969, "blocks.1.feed_forward.w3": 0.7451820373535156, "blocks.2.attention.w_q": 0.749298095703125, "blocks.2.attention.w_k": 0.7391586303710938, "blocks.2.attention.w_v": 0.7441787719726562, "blocks.2.attention.w_out": 0.7901763916015625, "blocks.2.feed_forward.w1": 0.7540003657341003, "blocks.2.feed_forward.w2": 0.735565185546875, "blocks.2.feed_forward.w3": 0.7464599609375, "blocks.3.attention.w_q": 0.7315254211425781, "blocks.3.attention.w_k": 0.7210693359375, "blocks.3.attention.w_v": 0.7274971008300781, "blocks.3.attention.w_out": 0.7761726379394531, "blocks.3.feed_forward.w1": 0.7560603022575378, "blocks.3.feed_forward.w2": 0.7393264770507812, "blocks.3.feed_forward.w3": 0.7467269897460938, "blocks.4.attention.w_q": 0.7547378540039062, "blocks.4.attention.w_k": 0.7323570251464844, "blocks.4.attention.w_v": 0.7299995422363281, "blocks.4.attention.w_out": 0.7645950317382812, "blocks.4.feed_forward.w1": 0.7602450251579285, "blocks.4.feed_forward.w2": 0.7316144704818726, "blocks.4.feed_forward.w3": 0.7473335266113281, "blocks.5.attention.w_q": 0.7360649108886719, "blocks.5.attention.w_k": 0.7212104797363281, "blocks.5.attention.w_v": 0.7051811218261719, "blocks.5.attention.w_out": 0.7246818542480469, "blocks.5.feed_forward.w1": 0.7621142268180847, "blocks.5.feed_forward.w2": 0.7163480520248413, "blocks.5.feed_forward.w3": 0.748053252696991, "blocks.6.attention.w_q": 0.7404518127441406, "blocks.6.attention.w_k": 0.73052978515625, "blocks.6.attention.w_v": 0.6865234375, "blocks.6.attention.w_out": 0.682159423828125, "blocks.6.feed_forward.w1": 0.7645963430404663, "blocks.6.feed_forward.w2": 0.6899986267089844, "blocks.6.feed_forward.w3": 0.750830352306366, "blocks.7.attention.w_q": 0.7464256286621094, "blocks.7.attention.w_k": 0.7298698425292969, "blocks.7.attention.w_v": 0.7025871276855469, "blocks.7.attention.w_out": 0.6589012145996094, "blocks.7.feed_forward.w1": 0.7724596858024597, "blocks.7.feed_forward.w2": 0.6575838923454285, "blocks.7.feed_forward.w3": 0.7539838552474976, "blocks.8.attention.w_q": 0.7527046203613281, "blocks.8.attention.w_k": 0.7402305603027344, "blocks.8.attention.w_v": 0.7075920104980469, "blocks.8.attention.w_out": 0.6628990173339844, "blocks.8.feed_forward.w1": 0.7795614004135132, "blocks.8.feed_forward.w2": 0.6055463552474976, "blocks.8.feed_forward.w3": 0.755346953868866}} +{"step": 3900, "changed_frac": 0.7294835147694645, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6789741516113281, "blocks.0.attention.w_k": 0.6733627319335938, "blocks.0.attention.w_v": 0.6921882629394531, "blocks.0.attention.w_out": 0.7434768676757812, "blocks.0.feed_forward.w1": 0.760016143321991, "blocks.0.feed_forward.w2": 0.7362073659896851, "blocks.0.feed_forward.w3": 0.7554258108139038, "blocks.1.attention.w_q": 0.7453269958496094, "blocks.1.attention.w_k": 0.7314262390136719, "blocks.1.attention.w_v": 0.7651863098144531, "blocks.1.attention.w_out": 0.8071823120117188, "blocks.1.feed_forward.w1": 0.7433153986930847, "blocks.1.feed_forward.w2": 0.7306785583496094, "blocks.1.feed_forward.w3": 0.7379583120346069, "blocks.2.attention.w_q": 0.7417144775390625, "blocks.2.attention.w_k": 0.7280998229980469, "blocks.2.attention.w_v": 0.7369918823242188, "blocks.2.attention.w_out": 0.7852096557617188, "blocks.2.feed_forward.w1": 0.7468643188476562, "blocks.2.feed_forward.w2": 0.7284762263298035, "blocks.2.feed_forward.w3": 0.7391408681869507, "blocks.3.attention.w_q": 0.7213935852050781, "blocks.3.attention.w_k": 0.7122001647949219, "blocks.3.attention.w_v": 0.7196807861328125, "blocks.3.attention.w_out": 0.7701034545898438, "blocks.3.feed_forward.w1": 0.7486280202865601, "blocks.3.feed_forward.w2": 0.7326711416244507, "blocks.3.feed_forward.w3": 0.7395502924919128, "blocks.4.attention.w_q": 0.7468490600585938, "blocks.4.attention.w_k": 0.7257499694824219, "blocks.4.attention.w_v": 0.7217445373535156, "blocks.4.attention.w_out": 0.7584152221679688, "blocks.4.feed_forward.w1": 0.7542890310287476, "blocks.4.feed_forward.w2": 0.7233428955078125, "blocks.4.feed_forward.w3": 0.7409185171127319, "blocks.5.attention.w_q": 0.7252044677734375, "blocks.5.attention.w_k": 0.7139015197753906, "blocks.5.attention.w_v": 0.6996345520019531, "blocks.5.attention.w_out": 0.7171134948730469, "blocks.5.feed_forward.w1": 0.7554982900619507, "blocks.5.feed_forward.w2": 0.7082430720329285, "blocks.5.feed_forward.w3": 0.7411308288574219, "blocks.6.attention.w_q": 0.7300376892089844, "blocks.6.attention.w_k": 0.719757080078125, "blocks.6.attention.w_v": 0.6783294677734375, "blocks.6.attention.w_out": 0.6748771667480469, "blocks.6.feed_forward.w1": 0.7581622004508972, "blocks.6.feed_forward.w2": 0.6821848750114441, "blocks.6.feed_forward.w3": 0.7429008483886719, "blocks.7.attention.w_q": 0.7378959655761719, "blocks.7.attention.w_k": 0.7229995727539062, "blocks.7.attention.w_v": 0.694244384765625, "blocks.7.attention.w_out": 0.6508865356445312, "blocks.7.feed_forward.w1": 0.7655588984489441, "blocks.7.feed_forward.w2": 0.6481870412826538, "blocks.7.feed_forward.w3": 0.7470601797103882, "blocks.8.attention.w_q": 0.7457695007324219, "blocks.8.attention.w_k": 0.7308273315429688, "blocks.8.attention.w_v": 0.6981582641601562, "blocks.8.attention.w_out": 0.6539306640625, "blocks.8.feed_forward.w1": 0.7732938528060913, "blocks.8.feed_forward.w2": 0.5951741933822632, "blocks.8.feed_forward.w3": 0.7486991882324219}} +{"step": 4000, "changed_frac": 0.7211534549028445, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6671180725097656, "blocks.0.attention.w_k": 0.6624794006347656, "blocks.0.attention.w_v": 0.6827125549316406, "blocks.0.attention.w_out": 0.7364349365234375, "blocks.0.feed_forward.w1": 0.7533823847770691, "blocks.0.feed_forward.w2": 0.7279803156852722, "blocks.0.feed_forward.w3": 0.7488683462142944, "blocks.1.attention.w_q": 0.7373313903808594, "blocks.1.attention.w_k": 0.7222633361816406, "blocks.1.attention.w_v": 0.7579765319824219, "blocks.1.attention.w_out": 0.8001861572265625, "blocks.1.feed_forward.w1": 0.7355117797851562, "blocks.1.feed_forward.w2": 0.7234319448471069, "blocks.1.feed_forward.w3": 0.7297134399414062, "blocks.2.attention.w_q": 0.7327423095703125, "blocks.2.attention.w_k": 0.7197647094726562, "blocks.2.attention.w_v": 0.7271728515625, "blocks.2.attention.w_out": 0.7781867980957031, "blocks.2.feed_forward.w1": 0.7394091486930847, "blocks.2.feed_forward.w2": 0.7214902639389038, "blocks.2.feed_forward.w3": 0.7310028076171875, "blocks.3.attention.w_q": 0.7131233215332031, "blocks.3.attention.w_k": 0.7024803161621094, "blocks.3.attention.w_v": 0.7104225158691406, "blocks.3.attention.w_out": 0.7641143798828125, "blocks.3.feed_forward.w1": 0.7416877746582031, "blocks.3.feed_forward.w2": 0.7262128591537476, "blocks.3.feed_forward.w3": 0.7308133840560913, "blocks.4.attention.w_q": 0.7383270263671875, "blocks.4.attention.w_k": 0.7149810791015625, "blocks.4.attention.w_v": 0.7106399536132812, "blocks.4.attention.w_out": 0.7515640258789062, "blocks.4.feed_forward.w1": 0.7458750605583191, "blocks.4.feed_forward.w2": 0.7161331176757812, "blocks.4.feed_forward.w3": 0.7325490713119507, "blocks.5.attention.w_q": 0.7159690856933594, "blocks.5.attention.w_k": 0.7036628723144531, "blocks.5.attention.w_v": 0.6858177185058594, "blocks.5.attention.w_out": 0.7101287841796875, "blocks.5.feed_forward.w1": 0.7481791377067566, "blocks.5.feed_forward.w2": 0.6986109614372253, "blocks.5.feed_forward.w3": 0.732580840587616, "blocks.6.attention.w_q": 0.7231864929199219, "blocks.6.attention.w_k": 0.7114143371582031, "blocks.6.attention.w_v": 0.6661643981933594, "blocks.6.attention.w_out": 0.664459228515625, "blocks.6.feed_forward.w1": 0.7510846853256226, "blocks.6.feed_forward.w2": 0.6734060049057007, "blocks.6.feed_forward.w3": 0.7351455688476562, "blocks.7.attention.w_q": 0.7284164428710938, "blocks.7.attention.w_k": 0.71307373046875, "blocks.7.attention.w_v": 0.6837806701660156, "blocks.7.attention.w_out": 0.643463134765625, "blocks.7.feed_forward.w1": 0.7580604553222656, "blocks.7.feed_forward.w2": 0.6398506164550781, "blocks.7.feed_forward.w3": 0.7391637563705444, "blocks.8.attention.w_q": 0.73651123046875, "blocks.8.attention.w_k": 0.7204856872558594, "blocks.8.attention.w_v": 0.6898422241210938, "blocks.8.attention.w_out": 0.6412582397460938, "blocks.8.feed_forward.w1": 0.7663688659667969, "blocks.8.feed_forward.w2": 0.5824432373046875, "blocks.8.feed_forward.w3": 0.7401441335678101}} +{"step": 4100, "changed_frac": 0.7125483781863482, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6558341979980469, "blocks.0.attention.w_k": 0.6506729125976562, "blocks.0.attention.w_v": 0.6731910705566406, "blocks.0.attention.w_out": 0.7299537658691406, "blocks.0.feed_forward.w1": 0.7441698908805847, "blocks.0.feed_forward.w2": 0.7195536494255066, "blocks.0.feed_forward.w3": 0.7390581965446472, "blocks.1.attention.w_q": 0.72723388671875, "blocks.1.attention.w_k": 0.7118415832519531, "blocks.1.attention.w_v": 0.7489662170410156, "blocks.1.attention.w_out": 0.7946052551269531, "blocks.1.feed_forward.w1": 0.7266858816146851, "blocks.1.feed_forward.w2": 0.7165095210075378, "blocks.1.feed_forward.w3": 0.7213834524154663, "blocks.2.attention.w_q": 0.7249298095703125, "blocks.2.attention.w_k": 0.7115936279296875, "blocks.2.attention.w_v": 0.7198486328125, "blocks.2.attention.w_out": 0.7729339599609375, "blocks.2.feed_forward.w1": 0.73095703125, "blocks.2.feed_forward.w2": 0.713446319103241, "blocks.2.feed_forward.w3": 0.7219073176383972, "blocks.3.attention.w_q": 0.7030830383300781, "blocks.3.attention.w_k": 0.6934432983398438, "blocks.3.attention.w_v": 0.6983146667480469, "blocks.3.attention.w_out": 0.755859375, "blocks.3.feed_forward.w1": 0.7328872680664062, "blocks.3.feed_forward.w2": 0.7185758352279663, "blocks.3.feed_forward.w3": 0.7231013178825378, "blocks.4.attention.w_q": 0.7272071838378906, "blocks.4.attention.w_k": 0.7048873901367188, "blocks.4.attention.w_v": 0.7054557800292969, "blocks.4.attention.w_out": 0.7433357238769531, "blocks.4.feed_forward.w1": 0.7384898066520691, "blocks.4.feed_forward.w2": 0.7093557119369507, "blocks.4.feed_forward.w3": 0.7236989736557007, "blocks.5.attention.w_q": 0.7052154541015625, "blocks.5.attention.w_k": 0.695465087890625, "blocks.5.attention.w_v": 0.6769447326660156, "blocks.5.attention.w_out": 0.7023048400878906, "blocks.5.feed_forward.w1": 0.7404505610466003, "blocks.5.feed_forward.w2": 0.688940703868866, "blocks.5.feed_forward.w3": 0.7245127558708191, "blocks.6.attention.w_q": 0.7123146057128906, "blocks.6.attention.w_k": 0.7016143798828125, "blocks.6.attention.w_v": 0.6553916931152344, "blocks.6.attention.w_out": 0.6568946838378906, "blocks.6.feed_forward.w1": 0.7440707087516785, "blocks.6.feed_forward.w2": 0.6633008718490601, "blocks.6.feed_forward.w3": 0.7269719839096069, "blocks.7.attention.w_q": 0.7177772521972656, "blocks.7.attention.w_k": 0.7034263610839844, "blocks.7.attention.w_v": 0.6747093200683594, "blocks.7.attention.w_out": 0.6308860778808594, "blocks.7.feed_forward.w1": 0.7515284419059753, "blocks.7.feed_forward.w2": 0.6287307739257812, "blocks.7.feed_forward.w3": 0.7312660217285156, "blocks.8.attention.w_q": 0.7265472412109375, "blocks.8.attention.w_k": 0.71185302734375, "blocks.8.attention.w_v": 0.6805496215820312, "blocks.8.attention.w_out": 0.6311187744140625, "blocks.8.feed_forward.w1": 0.7581901550292969, "blocks.8.feed_forward.w2": 0.5740839838981628, "blocks.8.feed_forward.w3": 0.7321586608886719}} +{"step": 4200, "changed_frac": 0.7032161288791232, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.64385986328125, "blocks.0.attention.w_k": 0.6360092163085938, "blocks.0.attention.w_v": 0.6623611450195312, "blocks.0.attention.w_out": 0.7214469909667969, "blocks.0.feed_forward.w1": 0.7364909052848816, "blocks.0.feed_forward.w2": 0.7106565237045288, "blocks.0.feed_forward.w3": 0.7306900024414062, "blocks.1.attention.w_q": 0.7192764282226562, "blocks.1.attention.w_k": 0.7022018432617188, "blocks.1.attention.w_v": 0.7407035827636719, "blocks.1.attention.w_out": 0.7896041870117188, "blocks.1.feed_forward.w1": 0.7175369262695312, "blocks.1.feed_forward.w2": 0.7070859670639038, "blocks.1.feed_forward.w3": 0.7121162414550781, "blocks.2.attention.w_q": 0.7149314880371094, "blocks.2.attention.w_k": 0.697784423828125, "blocks.2.attention.w_v": 0.709136962890625, "blocks.2.attention.w_out": 0.7637825012207031, "blocks.2.feed_forward.w1": 0.7220103144645691, "blocks.2.feed_forward.w2": 0.7066510915756226, "blocks.2.feed_forward.w3": 0.713104248046875, "blocks.3.attention.w_q": 0.6926612854003906, "blocks.3.attention.w_k": 0.6821060180664062, "blocks.3.attention.w_v": 0.6906929016113281, "blocks.3.attention.w_out": 0.7486648559570312, "blocks.3.feed_forward.w1": 0.7257639765739441, "blocks.3.feed_forward.w2": 0.7081527709960938, "blocks.3.feed_forward.w3": 0.714263916015625, "blocks.4.attention.w_q": 0.7187271118164062, "blocks.4.attention.w_k": 0.6961135864257812, "blocks.4.attention.w_v": 0.6929779052734375, "blocks.4.attention.w_out": 0.7343101501464844, "blocks.4.feed_forward.w1": 0.7278861999511719, "blocks.4.feed_forward.w2": 0.7005538940429688, "blocks.4.feed_forward.w3": 0.7141189575195312, "blocks.5.attention.w_q": 0.6950263977050781, "blocks.5.attention.w_k": 0.685516357421875, "blocks.5.attention.w_v": 0.6654586791992188, "blocks.5.attention.w_out": 0.6932754516601562, "blocks.5.feed_forward.w1": 0.7329851984977722, "blocks.5.feed_forward.w2": 0.6812121272087097, "blocks.5.feed_forward.w3": 0.7151387929916382, "blocks.6.attention.w_q": 0.7023391723632812, "blocks.6.attention.w_k": 0.6890220642089844, "blocks.6.attention.w_v": 0.6459846496582031, "blocks.6.attention.w_out": 0.6463279724121094, "blocks.6.feed_forward.w1": 0.7347615957260132, "blocks.6.feed_forward.w2": 0.6530876159667969, "blocks.6.feed_forward.w3": 0.71942138671875, "blocks.7.attention.w_q": 0.7111701965332031, "blocks.7.attention.w_k": 0.693756103515625, "blocks.7.attention.w_v": 0.6630401611328125, "blocks.7.attention.w_out": 0.6203651428222656, "blocks.7.feed_forward.w1": 0.7420222163200378, "blocks.7.feed_forward.w2": 0.6180127859115601, "blocks.7.feed_forward.w3": 0.722754180431366, "blocks.8.attention.w_q": 0.7182884216308594, "blocks.8.attention.w_k": 0.7015876770019531, "blocks.8.attention.w_v": 0.6705055236816406, "blocks.8.attention.w_out": 0.6196632385253906, "blocks.8.feed_forward.w1": 0.7500228881835938, "blocks.8.feed_forward.w2": 0.5596199035644531, "blocks.8.feed_forward.w3": 0.7230822443962097}} +{"step": 4300, "changed_frac": 0.6932308816502237, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.63189697265625, "blocks.0.attention.w_k": 0.6243820190429688, "blocks.0.attention.w_v": 0.6527137756347656, "blocks.0.attention.w_out": 0.7131080627441406, "blocks.0.feed_forward.w1": 0.7268359065055847, "blocks.0.feed_forward.w2": 0.7014045715332031, "blocks.0.feed_forward.w3": 0.7208048701286316, "blocks.1.attention.w_q": 0.7100181579589844, "blocks.1.attention.w_k": 0.691375732421875, "blocks.1.attention.w_v": 0.7323684692382812, "blocks.1.attention.w_out": 0.7831344604492188, "blocks.1.feed_forward.w1": 0.708282470703125, "blocks.1.feed_forward.w2": 0.6972643733024597, "blocks.1.feed_forward.w3": 0.7024345397949219, "blocks.2.attention.w_q": 0.7024574279785156, "blocks.2.attention.w_k": 0.68572998046875, "blocks.2.attention.w_v": 0.6977691650390625, "blocks.2.attention.w_out": 0.7564849853515625, "blocks.2.feed_forward.w1": 0.7134997248649597, "blocks.2.feed_forward.w2": 0.697516143321991, "blocks.2.feed_forward.w3": 0.7046178579330444, "blocks.3.attention.w_q": 0.6810340881347656, "blocks.3.attention.w_k": 0.6651268005371094, "blocks.3.attention.w_v": 0.6786842346191406, "blocks.3.attention.w_out": 0.7402572631835938, "blocks.3.feed_forward.w1": 0.7175788879394531, "blocks.3.feed_forward.w2": 0.6987355947494507, "blocks.3.feed_forward.w3": 0.7050043940544128, "blocks.4.attention.w_q": 0.70648193359375, "blocks.4.attention.w_k": 0.68475341796875, "blocks.4.attention.w_v": 0.68133544921875, "blocks.4.attention.w_out": 0.7269783020019531, "blocks.4.feed_forward.w1": 0.7209078669548035, "blocks.4.feed_forward.w2": 0.6915079951286316, "blocks.4.feed_forward.w3": 0.7042312622070312, "blocks.5.attention.w_q": 0.683746337890625, "blocks.5.attention.w_k": 0.6732139587402344, "blocks.5.attention.w_v": 0.65472412109375, "blocks.5.attention.w_out": 0.6815681457519531, "blocks.5.feed_forward.w1": 0.7213847041130066, "blocks.5.feed_forward.w2": 0.671332061290741, "blocks.5.feed_forward.w3": 0.7052015066146851, "blocks.6.attention.w_q": 0.6905059814453125, "blocks.6.attention.w_k": 0.6768226623535156, "blocks.6.attention.w_v": 0.6346969604492188, "blocks.6.attention.w_out": 0.6380653381347656, "blocks.6.feed_forward.w1": 0.726294219493866, "blocks.6.feed_forward.w2": 0.6428095698356628, "blocks.6.feed_forward.w3": 0.7080930471420288, "blocks.7.attention.w_q": 0.6981849670410156, "blocks.7.attention.w_k": 0.6803932189941406, "blocks.7.attention.w_v": 0.6538200378417969, "blocks.7.attention.w_out": 0.6101760864257812, "blocks.7.feed_forward.w1": 0.7332713007926941, "blocks.7.feed_forward.w2": 0.6053301692008972, "blocks.7.feed_forward.w3": 0.7131080627441406, "blocks.8.attention.w_q": 0.7047920227050781, "blocks.8.attention.w_k": 0.6906280517578125, "blocks.8.attention.w_v": 0.6601943969726562, "blocks.8.attention.w_out": 0.6085929870605469, "blocks.8.feed_forward.w1": 0.7417182922363281, "blocks.8.feed_forward.w2": 0.5469602346420288, "blocks.8.feed_forward.w3": 0.7144699096679688}} +{"step": 4400, "changed_frac": 0.6826209043845152, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6167640686035156, "blocks.0.attention.w_k": 0.6103744506835938, "blocks.0.attention.w_v": 0.641082763671875, "blocks.0.attention.w_out": 0.7026939392089844, "blocks.0.feed_forward.w1": 0.7162399291992188, "blocks.0.feed_forward.w2": 0.6920318603515625, "blocks.0.feed_forward.w3": 0.7117640376091003, "blocks.1.attention.w_q": 0.6976470947265625, "blocks.1.attention.w_k": 0.6783065795898438, "blocks.1.attention.w_v": 0.7233428955078125, "blocks.1.attention.w_out": 0.7756881713867188, "blocks.1.feed_forward.w1": 0.6983960866928101, "blocks.1.feed_forward.w2": 0.688788115978241, "blocks.1.feed_forward.w3": 0.6919505000114441, "blocks.2.attention.w_q": 0.6937637329101562, "blocks.2.attention.w_k": 0.6736640930175781, "blocks.2.attention.w_v": 0.6876907348632812, "blocks.2.attention.w_out": 0.7486419677734375, "blocks.2.feed_forward.w1": 0.7030080556869507, "blocks.2.feed_forward.w2": 0.6877047419548035, "blocks.2.feed_forward.w3": 0.6942698359489441, "blocks.3.attention.w_q": 0.6682891845703125, "blocks.3.attention.w_k": 0.6541290283203125, "blocks.3.attention.w_v": 0.6668586730957031, "blocks.3.attention.w_out": 0.7285308837890625, "blocks.3.feed_forward.w1": 0.7058245539665222, "blocks.3.feed_forward.w2": 0.6896044611930847, "blocks.3.feed_forward.w3": 0.6955465078353882, "blocks.4.attention.w_q": 0.6955947875976562, "blocks.4.attention.w_k": 0.6747474670410156, "blocks.4.attention.w_v": 0.6707267761230469, "blocks.4.attention.w_out": 0.7160415649414062, "blocks.4.feed_forward.w1": 0.7104060053825378, "blocks.4.feed_forward.w2": 0.6812833547592163, "blocks.4.feed_forward.w3": 0.6937078237533569, "blocks.5.attention.w_q": 0.671295166015625, "blocks.5.attention.w_k": 0.6592903137207031, "blocks.5.attention.w_v": 0.6405487060546875, "blocks.5.attention.w_out": 0.6709136962890625, "blocks.5.feed_forward.w1": 0.7125562429428101, "blocks.5.feed_forward.w2": 0.6584345698356628, "blocks.5.feed_forward.w3": 0.6944402456283569, "blocks.6.attention.w_q": 0.6826286315917969, "blocks.6.attention.w_k": 0.6646766662597656, "blocks.6.attention.w_v": 0.6215934753417969, "blocks.6.attention.w_out": 0.6251144409179688, "blocks.6.feed_forward.w1": 0.7159652709960938, "blocks.6.feed_forward.w2": 0.631018340587616, "blocks.6.feed_forward.w3": 0.6991183161735535, "blocks.7.attention.w_q": 0.6847381591796875, "blocks.7.attention.w_k": 0.6655349731445312, "blocks.7.attention.w_v": 0.643646240234375, "blocks.7.attention.w_out": 0.5992202758789062, "blocks.7.feed_forward.w1": 0.7241681814193726, "blocks.7.feed_forward.w2": 0.5925623774528503, "blocks.7.feed_forward.w3": 0.7040113210678101, "blocks.8.attention.w_q": 0.6955146789550781, "blocks.8.attention.w_k": 0.6772727966308594, "blocks.8.attention.w_v": 0.6492233276367188, "blocks.8.attention.w_out": 0.5959739685058594, "blocks.8.feed_forward.w1": 0.7324714660644531, "blocks.8.feed_forward.w2": 0.5348180532455444, "blocks.8.feed_forward.w3": 0.704870879650116}} +{"step": 4500, "changed_frac": 0.6715037680079794, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6044845581054688, "blocks.0.attention.w_k": 0.5933685302734375, "blocks.0.attention.w_v": 0.6297416687011719, "blocks.0.attention.w_out": 0.6941261291503906, "blocks.0.feed_forward.w1": 0.7057749629020691, "blocks.0.feed_forward.w2": 0.6817169189453125, "blocks.0.feed_forward.w3": 0.7004966735839844, "blocks.1.attention.w_q": 0.6858634948730469, "blocks.1.attention.w_k": 0.6714973449707031, "blocks.1.attention.w_v": 0.7093238830566406, "blocks.1.attention.w_out": 0.767333984375, "blocks.1.feed_forward.w1": 0.6873639822006226, "blocks.1.feed_forward.w2": 0.6800817251205444, "blocks.1.feed_forward.w3": 0.682037353515625, "blocks.2.attention.w_q": 0.6774940490722656, "blocks.2.attention.w_k": 0.6614799499511719, "blocks.2.attention.w_v": 0.6772918701171875, "blocks.2.attention.w_out": 0.7414779663085938, "blocks.2.feed_forward.w1": 0.6933645009994507, "blocks.2.feed_forward.w2": 0.6770464777946472, "blocks.2.feed_forward.w3": 0.6833877563476562, "blocks.3.attention.w_q": 0.6556854248046875, "blocks.3.attention.w_k": 0.641754150390625, "blocks.3.attention.w_v": 0.654876708984375, "blocks.3.attention.w_out": 0.7201766967773438, "blocks.3.feed_forward.w1": 0.6970024108886719, "blocks.3.feed_forward.w2": 0.6787008047103882, "blocks.3.feed_forward.w3": 0.6838315725326538, "blocks.4.attention.w_q": 0.6829681396484375, "blocks.4.attention.w_k": 0.6587295532226562, "blocks.4.attention.w_v": 0.6586265563964844, "blocks.4.attention.w_out": 0.7059516906738281, "blocks.4.feed_forward.w1": 0.6999855041503906, "blocks.4.feed_forward.w2": 0.6698595881462097, "blocks.4.feed_forward.w3": 0.6837615966796875, "blocks.5.attention.w_q": 0.6578598022460938, "blocks.5.attention.w_k": 0.6484298706054688, "blocks.5.attention.w_v": 0.6265182495117188, "blocks.5.attention.w_out": 0.6608543395996094, "blocks.5.feed_forward.w1": 0.7030372619628906, "blocks.5.feed_forward.w2": 0.6471074819564819, "blocks.5.feed_forward.w3": 0.685187041759491, "blocks.6.attention.w_q": 0.6668624877929688, "blocks.6.attention.w_k": 0.6512413024902344, "blocks.6.attention.w_v": 0.6092109680175781, "blocks.6.attention.w_out": 0.6140708923339844, "blocks.6.feed_forward.w1": 0.7062658071517944, "blocks.6.feed_forward.w2": 0.6182467341423035, "blocks.6.feed_forward.w3": 0.6879145503044128, "blocks.7.attention.w_q": 0.6741180419921875, "blocks.7.attention.w_k": 0.6550712585449219, "blocks.7.attention.w_v": 0.6327629089355469, "blocks.7.attention.w_out": 0.5873374938964844, "blocks.7.feed_forward.w1": 0.7137769460678101, "blocks.7.feed_forward.w2": 0.5786234736442566, "blocks.7.feed_forward.w3": 0.6929804682731628, "blocks.8.attention.w_q": 0.6829681396484375, "blocks.8.attention.w_k": 0.6633834838867188, "blocks.8.attention.w_v": 0.638153076171875, "blocks.8.attention.w_out": 0.5836753845214844, "blocks.8.feed_forward.w1": 0.7236976623535156, "blocks.8.feed_forward.w2": 0.5185903310775757, "blocks.8.feed_forward.w3": 0.693884551525116}} +{"step": 4600, "changed_frac": 0.6592280722071981, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5876617431640625, "blocks.0.attention.w_k": 0.5793914794921875, "blocks.0.attention.w_v": 0.6167869567871094, "blocks.0.attention.w_out": 0.682281494140625, "blocks.0.feed_forward.w1": 0.6945292353630066, "blocks.0.feed_forward.w2": 0.6715800166130066, "blocks.0.feed_forward.w3": 0.6879361867904663, "blocks.1.attention.w_q": 0.6732444763183594, "blocks.1.attention.w_k": 0.6553916931152344, "blocks.1.attention.w_v": 0.70147705078125, "blocks.1.attention.w_out": 0.7566070556640625, "blocks.1.feed_forward.w1": 0.6756566762924194, "blocks.1.feed_forward.w2": 0.6668078303337097, "blocks.1.feed_forward.w3": 0.6700172424316406, "blocks.2.attention.w_q": 0.6644935607910156, "blocks.2.attention.w_k": 0.6453895568847656, "blocks.2.attention.w_v": 0.6655654907226562, "blocks.2.attention.w_out": 0.7311744689941406, "blocks.2.feed_forward.w1": 0.6813952326774597, "blocks.2.feed_forward.w2": 0.6667594909667969, "blocks.2.feed_forward.w3": 0.6709836721420288, "blocks.3.attention.w_q": 0.6400794982910156, "blocks.3.attention.w_k": 0.6289863586425781, "blocks.3.attention.w_v": 0.6413536071777344, "blocks.3.attention.w_out": 0.7107505798339844, "blocks.3.feed_forward.w1": 0.6848462820053101, "blocks.3.feed_forward.w2": 0.6678492426872253, "blocks.3.feed_forward.w3": 0.6730397939682007, "blocks.4.attention.w_q": 0.6706085205078125, "blocks.4.attention.w_k": 0.6468658447265625, "blocks.4.attention.w_v": 0.6469993591308594, "blocks.4.attention.w_out": 0.6942863464355469, "blocks.4.feed_forward.w1": 0.6895332336425781, "blocks.4.feed_forward.w2": 0.6590284109115601, "blocks.4.feed_forward.w3": 0.6715075373649597, "blocks.5.attention.w_q": 0.6445465087890625, "blocks.5.attention.w_k": 0.6338081359863281, "blocks.5.attention.w_v": 0.6159782409667969, "blocks.5.attention.w_out": 0.649078369140625, "blocks.5.feed_forward.w1": 0.6905403137207031, "blocks.5.feed_forward.w2": 0.6352755427360535, "blocks.5.feed_forward.w3": 0.6729342341423035, "blocks.6.attention.w_q": 0.6517982482910156, "blocks.6.attention.w_k": 0.6373252868652344, "blocks.6.attention.w_v": 0.5958938598632812, "blocks.6.attention.w_out": 0.6007843017578125, "blocks.6.feed_forward.w1": 0.6958478689193726, "blocks.6.feed_forward.w2": 0.6046231985092163, "blocks.6.feed_forward.w3": 0.6763814687728882, "blocks.7.attention.w_q": 0.6597747802734375, "blocks.7.attention.w_k": 0.6427459716796875, "blocks.7.attention.w_v": 0.6180191040039062, "blocks.7.attention.w_out": 0.5692329406738281, "blocks.7.feed_forward.w1": 0.7029203176498413, "blocks.7.feed_forward.w2": 0.5637385249137878, "blocks.7.feed_forward.w3": 0.6812846064567566, "blocks.8.attention.w_q": 0.6694564819335938, "blocks.8.attention.w_k": 0.6510696411132812, "blocks.8.attention.w_v": 0.6225357055664062, "blocks.8.attention.w_out": 0.567413330078125, "blocks.8.feed_forward.w1": 0.7130393981933594, "blocks.8.feed_forward.w2": 0.5034040212631226, "blocks.8.feed_forward.w3": 0.6821505427360535}} +{"step": 4700, "changed_frac": 0.6461742107684796, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5725212097167969, "blocks.0.attention.w_k": 0.563873291015625, "blocks.0.attention.w_v": 0.6019363403320312, "blocks.0.attention.w_out": 0.6718330383300781, "blocks.0.feed_forward.w1": 0.6823527216911316, "blocks.0.feed_forward.w2": 0.6583175659179688, "blocks.0.feed_forward.w3": 0.6755460500717163, "blocks.1.attention.w_q": 0.6597442626953125, "blocks.1.attention.w_k": 0.6405258178710938, "blocks.1.attention.w_v": 0.686248779296875, "blocks.1.attention.w_out": 0.7489128112792969, "blocks.1.feed_forward.w1": 0.6630579829216003, "blocks.1.feed_forward.w2": 0.6542638540267944, "blocks.1.feed_forward.w3": 0.6574046015739441, "blocks.2.attention.w_q": 0.649078369140625, "blocks.2.attention.w_k": 0.631103515625, "blocks.2.attention.w_v": 0.6495742797851562, "blocks.2.attention.w_out": 0.7203178405761719, "blocks.2.feed_forward.w1": 0.6701545715332031, "blocks.2.feed_forward.w2": 0.6539993286132812, "blocks.2.feed_forward.w3": 0.6598294973373413, "blocks.3.attention.w_q": 0.6240997314453125, "blocks.3.attention.w_k": 0.6136131286621094, "blocks.3.attention.w_v": 0.6262245178222656, "blocks.3.attention.w_out": 0.6973152160644531, "blocks.3.feed_forward.w1": 0.6721280813217163, "blocks.3.feed_forward.w2": 0.6549758911132812, "blocks.3.feed_forward.w3": 0.660003662109375, "blocks.4.attention.w_q": 0.6572990417480469, "blocks.4.attention.w_k": 0.6307868957519531, "blocks.4.attention.w_v": 0.6319313049316406, "blocks.4.attention.w_out": 0.6840896606445312, "blocks.4.feed_forward.w1": 0.6774533987045288, "blocks.4.feed_forward.w2": 0.6450119018554688, "blocks.4.feed_forward.w3": 0.6601041555404663, "blocks.5.attention.w_q": 0.6299476623535156, "blocks.5.attention.w_k": 0.6187934875488281, "blocks.5.attention.w_v": 0.5999183654785156, "blocks.5.attention.w_out": 0.6365852355957031, "blocks.5.feed_forward.w1": 0.6780611872673035, "blocks.5.feed_forward.w2": 0.6232694387435913, "blocks.5.feed_forward.w3": 0.6604499816894531, "blocks.6.attention.w_q": 0.6377754211425781, "blocks.6.attention.w_k": 0.6198234558105469, "blocks.6.attention.w_v": 0.5813484191894531, "blocks.6.attention.w_out": 0.5909461975097656, "blocks.6.feed_forward.w1": 0.6819051504135132, "blocks.6.feed_forward.w2": 0.5925649404525757, "blocks.6.feed_forward.w3": 0.6630300283432007, "blocks.7.attention.w_q": 0.6426582336425781, "blocks.7.attention.w_k": 0.6273231506347656, "blocks.7.attention.w_v": 0.602142333984375, "blocks.7.attention.w_out": 0.5592002868652344, "blocks.7.feed_forward.w1": 0.6930758357048035, "blocks.7.feed_forward.w2": 0.5484886169433594, "blocks.7.feed_forward.w3": 0.6693433523178101, "blocks.8.attention.w_q": 0.6546478271484375, "blocks.8.attention.w_k": 0.6346664428710938, "blocks.8.attention.w_v": 0.61309814453125, "blocks.8.attention.w_out": 0.5509109497070312, "blocks.8.feed_forward.w1": 0.7006582021713257, "blocks.8.feed_forward.w2": 0.4877828061580658, "blocks.8.feed_forward.w3": 0.670623779296875}} +{"step": 4800, "changed_frac": 0.6324123154338609, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5545883178710938, "blocks.0.attention.w_k": 0.5453453063964844, "blocks.0.attention.w_v": 0.5861091613769531, "blocks.0.attention.w_out": 0.6618423461914062, "blocks.0.feed_forward.w1": 0.6670672297477722, "blocks.0.feed_forward.w2": 0.6474965810775757, "blocks.0.feed_forward.w3": 0.6623014211654663, "blocks.1.attention.w_q": 0.64569091796875, "blocks.1.attention.w_k": 0.625152587890625, "blocks.1.attention.w_v": 0.6746139526367188, "blocks.1.attention.w_out": 0.7380294799804688, "blocks.1.feed_forward.w1": 0.6491216421127319, "blocks.1.feed_forward.w2": 0.6418800354003906, "blocks.1.feed_forward.w3": 0.6442998647689819, "blocks.2.attention.w_q": 0.6330337524414062, "blocks.2.attention.w_k": 0.6144523620605469, "blocks.2.attention.w_v": 0.6345710754394531, "blocks.2.attention.w_out": 0.7076339721679688, "blocks.2.feed_forward.w1": 0.6555633544921875, "blocks.2.feed_forward.w2": 0.6436818838119507, "blocks.2.feed_forward.w3": 0.6469027400016785, "blocks.3.attention.w_q": 0.6105117797851562, "blocks.3.attention.w_k": 0.5967216491699219, "blocks.3.attention.w_v": 0.6109123229980469, "blocks.3.attention.w_out": 0.6857833862304688, "blocks.3.feed_forward.w1": 0.6593106985092163, "blocks.3.feed_forward.w2": 0.6428718566894531, "blocks.3.feed_forward.w3": 0.6461703181266785, "blocks.4.attention.w_q": 0.6416664123535156, "blocks.4.attention.w_k": 0.6127128601074219, "blocks.4.attention.w_v": 0.6174430847167969, "blocks.4.attention.w_out": 0.6695480346679688, "blocks.4.feed_forward.w1": 0.6634089350700378, "blocks.4.feed_forward.w2": 0.6318957209587097, "blocks.4.feed_forward.w3": 0.6470082998275757, "blocks.5.attention.w_q": 0.6119308471679688, "blocks.5.attention.w_k": 0.6018409729003906, "blocks.5.attention.w_v": 0.5859909057617188, "blocks.5.attention.w_out": 0.6215248107910156, "blocks.5.feed_forward.w1": 0.6655451655387878, "blocks.5.feed_forward.w2": 0.6083641052246094, "blocks.5.feed_forward.w3": 0.6463102102279663, "blocks.6.attention.w_q": 0.6234397888183594, "blocks.6.attention.w_k": 0.6070175170898438, "blocks.6.attention.w_v": 0.5652961730957031, "blocks.6.attention.w_out": 0.57598876953125, "blocks.6.feed_forward.w1": 0.6698888540267944, "blocks.6.feed_forward.w2": 0.5757064819335938, "blocks.6.feed_forward.w3": 0.6507644653320312, "blocks.7.attention.w_q": 0.630157470703125, "blocks.7.attention.w_k": 0.61181640625, "blocks.7.attention.w_v": 0.5883598327636719, "blocks.7.attention.w_out": 0.5416145324707031, "blocks.7.feed_forward.w1": 0.6789321899414062, "blocks.7.feed_forward.w2": 0.5350354909896851, "blocks.7.feed_forward.w3": 0.6566365957260132, "blocks.8.attention.w_q": 0.6403312683105469, "blocks.8.attention.w_k": 0.6204566955566406, "blocks.8.attention.w_v": 0.5962944030761719, "blocks.8.attention.w_out": 0.5379829406738281, "blocks.8.feed_forward.w1": 0.6905860900878906, "blocks.8.feed_forward.w2": 0.4713427424430847, "blocks.8.feed_forward.w3": 0.6571859121322632}} +{"step": 4900, "changed_frac": 0.6178768842648237, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5380287170410156, "blocks.0.attention.w_k": 0.5295867919921875, "blocks.0.attention.w_v": 0.5725860595703125, "blocks.0.attention.w_out": 0.64764404296875, "blocks.0.feed_forward.w1": 0.6540718078613281, "blocks.0.feed_forward.w2": 0.6327349543571472, "blocks.0.feed_forward.w3": 0.6480115652084351, "blocks.1.attention.w_q": 0.6310615539550781, "blocks.1.attention.w_k": 0.6116867065429688, "blocks.1.attention.w_v": 0.6589012145996094, "blocks.1.attention.w_out": 0.7267379760742188, "blocks.1.feed_forward.w1": 0.6351839900016785, "blocks.1.feed_forward.w2": 0.6278063654899597, "blocks.1.feed_forward.w3": 0.6302961111068726, "blocks.2.attention.w_q": 0.6200904846191406, "blocks.2.attention.w_k": 0.6007652282714844, "blocks.2.attention.w_v": 0.6192588806152344, "blocks.2.attention.w_out": 0.697479248046875, "blocks.2.feed_forward.w1": 0.6423848867416382, "blocks.2.feed_forward.w2": 0.6263504028320312, "blocks.2.feed_forward.w3": 0.6320292353630066, "blocks.3.attention.w_q": 0.596099853515625, "blocks.3.attention.w_k": 0.5803871154785156, "blocks.3.attention.w_v": 0.5957221984863281, "blocks.3.attention.w_out": 0.6725616455078125, "blocks.3.feed_forward.w1": 0.6450538635253906, "blocks.3.feed_forward.w2": 0.6285070180892944, "blocks.3.feed_forward.w3": 0.6318385004997253, "blocks.4.attention.w_q": 0.6262321472167969, "blocks.4.attention.w_k": 0.5991363525390625, "blocks.4.attention.w_v": 0.603363037109375, "blocks.4.attention.w_out": 0.6584396362304688, "blocks.4.feed_forward.w1": 0.6509717702865601, "blocks.4.feed_forward.w2": 0.6195449829101562, "blocks.4.feed_forward.w3": 0.6323420405387878, "blocks.5.attention.w_q": 0.5959358215332031, "blocks.5.attention.w_k": 0.58551025390625, "blocks.5.attention.w_v": 0.5682411193847656, "blocks.5.attention.w_out": 0.6084098815917969, "blocks.5.feed_forward.w1": 0.6516367793083191, "blocks.5.feed_forward.w2": 0.5936330556869507, "blocks.5.feed_forward.w3": 0.632781982421875, "blocks.6.attention.w_q": 0.6082992553710938, "blocks.6.attention.w_k": 0.5899505615234375, "blocks.6.attention.w_v": 0.546905517578125, "blocks.6.attention.w_out": 0.5614471435546875, "blocks.6.feed_forward.w1": 0.6546542048454285, "blocks.6.feed_forward.w2": 0.5613746643066406, "blocks.6.feed_forward.w3": 0.6351852416992188, "blocks.7.attention.w_q": 0.61279296875, "blocks.7.attention.w_k": 0.5942649841308594, "blocks.7.attention.w_v": 0.5753211975097656, "blocks.7.attention.w_out": 0.5282745361328125, "blocks.7.feed_forward.w1": 0.6652972102165222, "blocks.7.feed_forward.w2": 0.5178057551383972, "blocks.7.feed_forward.w3": 0.6421941518783569, "blocks.8.attention.w_q": 0.6240501403808594, "blocks.8.attention.w_k": 0.6047897338867188, "blocks.8.attention.w_v": 0.5808334350585938, "blocks.8.attention.w_out": 0.5197868347167969, "blocks.8.feed_forward.w1": 0.6762034296989441, "blocks.8.feed_forward.w2": 0.4549802243709564, "blocks.8.feed_forward.w3": 0.6441307067871094}} +{"step": 5000, "changed_frac": 0.6025374648917434, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5190544128417969, "blocks.0.attention.w_k": 0.5093231201171875, "blocks.0.attention.w_v": 0.5586318969726562, "blocks.0.attention.w_out": 0.6352424621582031, "blocks.0.feed_forward.w1": 0.6390203237533569, "blocks.0.feed_forward.w2": 0.6179110407829285, "blocks.0.feed_forward.w3": 0.6336581110954285, "blocks.1.attention.w_q": 0.6141281127929688, "blocks.1.attention.w_k": 0.5898628234863281, "blocks.1.attention.w_v": 0.643402099609375, "blocks.1.attention.w_out": 0.7159309387207031, "blocks.1.feed_forward.w1": 0.6211535334587097, "blocks.1.feed_forward.w2": 0.6132494807243347, "blocks.1.feed_forward.w3": 0.6147651672363281, "blocks.2.attention.w_q": 0.6012001037597656, "blocks.2.attention.w_k": 0.5830192565917969, "blocks.2.attention.w_v": 0.6061782836914062, "blocks.2.attention.w_out": 0.684967041015625, "blocks.2.feed_forward.w1": 0.6264127492904663, "blocks.2.feed_forward.w2": 0.6132990717887878, "blocks.2.feed_forward.w3": 0.6174583435058594, "blocks.3.attention.w_q": 0.5761032104492188, "blocks.3.attention.w_k": 0.5592231750488281, "blocks.3.attention.w_v": 0.5799751281738281, "blocks.3.attention.w_out": 0.6584739685058594, "blocks.3.feed_forward.w1": 0.6312128901481628, "blocks.3.feed_forward.w2": 0.6147537231445312, "blocks.3.feed_forward.w3": 0.6172803640365601, "blocks.4.attention.w_q": 0.6093902587890625, "blocks.4.attention.w_k": 0.58184814453125, "blocks.4.attention.w_v": 0.5877265930175781, "blocks.4.attention.w_out": 0.6412582397460938, "blocks.4.feed_forward.w1": 0.636077880859375, "blocks.4.feed_forward.w2": 0.604010283946991, "blocks.4.feed_forward.w3": 0.6168124079704285, "blocks.5.attention.w_q": 0.5800933837890625, "blocks.5.attention.w_k": 0.5692214965820312, "blocks.5.attention.w_v": 0.5551071166992188, "blocks.5.attention.w_out": 0.5933647155761719, "blocks.5.feed_forward.w1": 0.638525664806366, "blocks.5.feed_forward.w2": 0.5790011286735535, "blocks.5.feed_forward.w3": 0.6176478266716003, "blocks.6.attention.w_q": 0.5899009704589844, "blocks.6.attention.w_k": 0.5713272094726562, "blocks.6.attention.w_v": 0.5291099548339844, "blocks.6.attention.w_out": 0.5445823669433594, "blocks.6.feed_forward.w1": 0.6415176391601562, "blocks.6.feed_forward.w2": 0.5453872680664062, "blocks.6.feed_forward.w3": 0.6203295588493347, "blocks.7.attention.w_q": 0.5960731506347656, "blocks.7.attention.w_k": 0.5778465270996094, "blocks.7.attention.w_v": 0.5553932189941406, "blocks.7.attention.w_out": 0.5083847045898438, "blocks.7.feed_forward.w1": 0.6506068110466003, "blocks.7.feed_forward.w2": 0.5008558034896851, "blocks.7.feed_forward.w3": 0.6275914907455444, "blocks.8.attention.w_q": 0.6056632995605469, "blocks.8.attention.w_k": 0.5877532958984375, "blocks.8.attention.w_v": 0.566162109375, "blocks.8.attention.w_out": 0.5027618408203125, "blocks.8.feed_forward.w1": 0.6622263789176941, "blocks.8.feed_forward.w2": 0.4389737546443939, "blocks.8.feed_forward.w3": 0.6299947500228882}} +{"step": 5100, "changed_frac": 0.58606935158754, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5006561279296875, "blocks.0.attention.w_k": 0.4904632568359375, "blocks.0.attention.w_v": 0.5400199890136719, "blocks.0.attention.w_out": 0.6221046447753906, "blocks.0.feed_forward.w1": 0.6234983205795288, "blocks.0.feed_forward.w2": 0.6035398244857788, "blocks.0.feed_forward.w3": 0.6185315847396851, "blocks.1.attention.w_q": 0.5988922119140625, "blocks.1.attention.w_k": 0.5732536315917969, "blocks.1.attention.w_v": 0.6305122375488281, "blocks.1.attention.w_out": 0.7026405334472656, "blocks.1.feed_forward.w1": 0.6039835810661316, "blocks.1.feed_forward.w2": 0.5975748896598816, "blocks.1.feed_forward.w3": 0.5984052419662476, "blocks.2.attention.w_q": 0.5850181579589844, "blocks.2.attention.w_k": 0.5625267028808594, "blocks.2.attention.w_v": 0.5870552062988281, "blocks.2.attention.w_out": 0.6687469482421875, "blocks.2.feed_forward.w1": 0.6107419729232788, "blocks.2.feed_forward.w2": 0.5979665517807007, "blocks.2.feed_forward.w3": 0.6001002192497253, "blocks.3.attention.w_q": 0.5584602355957031, "blocks.3.attention.w_k": 0.5408248901367188, "blocks.3.attention.w_v": 0.5624008178710938, "blocks.3.attention.w_out": 0.6464920043945312, "blocks.3.feed_forward.w1": 0.615423858165741, "blocks.3.feed_forward.w2": 0.5990486145019531, "blocks.3.feed_forward.w3": 0.6006431579589844, "blocks.4.attention.w_q": 0.5935134887695312, "blocks.4.attention.w_k": 0.5599746704101562, "blocks.4.attention.w_v": 0.5687599182128906, "blocks.4.attention.w_out": 0.6266326904296875, "blocks.4.feed_forward.w1": 0.618445098400116, "blocks.4.feed_forward.w2": 0.5874481201171875, "blocks.4.feed_forward.w3": 0.6004270315170288, "blocks.5.attention.w_q": 0.560028076171875, "blocks.5.attention.w_k": 0.5478096008300781, "blocks.5.attention.w_v": 0.5322036743164062, "blocks.5.attention.w_out": 0.5785064697265625, "blocks.5.feed_forward.w1": 0.6233901977539062, "blocks.5.feed_forward.w2": 0.5634142756462097, "blocks.5.feed_forward.w3": 0.6019262075424194, "blocks.6.attention.w_q": 0.570281982421875, "blocks.6.attention.w_k": 0.5549545288085938, "blocks.6.attention.w_v": 0.5123100280761719, "blocks.6.attention.w_out": 0.5271339416503906, "blocks.6.feed_forward.w1": 0.6264063715934753, "blocks.6.feed_forward.w2": 0.5280036926269531, "blocks.6.feed_forward.w3": 0.6038653254508972, "blocks.7.attention.w_q": 0.5782585144042969, "blocks.7.attention.w_k": 0.5585441589355469, "blocks.7.attention.w_v": 0.5401458740234375, "blocks.7.attention.w_out": 0.4929695129394531, "blocks.7.feed_forward.w1": 0.6351115107536316, "blocks.7.feed_forward.w2": 0.4837303161621094, "blocks.7.feed_forward.w3": 0.6111335754394531, "blocks.8.attention.w_q": 0.5888137817382812, "blocks.8.attention.w_k": 0.5690650939941406, "blocks.8.attention.w_v": 0.54901123046875, "blocks.8.attention.w_out": 0.487396240234375, "blocks.8.feed_forward.w1": 0.6481603384017944, "blocks.8.feed_forward.w2": 0.42015838623046875, "blocks.8.feed_forward.w3": 0.6134999990463257}} +{"step": 5200, "changed_frac": 0.5689885392148271, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4828605651855469, "blocks.0.attention.w_k": 0.47345733642578125, "blocks.0.attention.w_v": 0.5235939025878906, "blocks.0.attention.w_out": 0.6064186096191406, "blocks.0.feed_forward.w1": 0.6070951223373413, "blocks.0.feed_forward.w2": 0.5863978266716003, "blocks.0.feed_forward.w3": 0.6010169982910156, "blocks.1.attention.w_q": 0.5787467956542969, "blocks.1.attention.w_k": 0.5542068481445312, "blocks.1.attention.w_v": 0.6092300415039062, "blocks.1.attention.w_out": 0.6903762817382812, "blocks.1.feed_forward.w1": 0.5876401662826538, "blocks.1.feed_forward.w2": 0.5816663503646851, "blocks.1.feed_forward.w3": 0.5820579528808594, "blocks.2.attention.w_q": 0.5641517639160156, "blocks.2.attention.w_k": 0.543212890625, "blocks.2.attention.w_v": 0.571319580078125, "blocks.2.attention.w_out": 0.6534042358398438, "blocks.2.feed_forward.w1": 0.5946617126464844, "blocks.2.feed_forward.w2": 0.5824025869369507, "blocks.2.feed_forward.w3": 0.5834325551986694, "blocks.3.attention.w_q": 0.5403213500976562, "blocks.3.attention.w_k": 0.5237083435058594, "blocks.3.attention.w_v": 0.54461669921875, "blocks.3.attention.w_out": 0.6302375793457031, "blocks.3.feed_forward.w1": 0.5985056757926941, "blocks.3.feed_forward.w2": 0.5813369750976562, "blocks.3.feed_forward.w3": 0.5847905874252319, "blocks.4.attention.w_q": 0.5743789672851562, "blocks.4.attention.w_k": 0.5419425964355469, "blocks.4.attention.w_v": 0.5519218444824219, "blocks.4.attention.w_out": 0.6121330261230469, "blocks.4.feed_forward.w1": 0.6023241877555847, "blocks.4.feed_forward.w2": 0.5715383291244507, "blocks.4.feed_forward.w3": 0.5856501460075378, "blocks.5.attention.w_q": 0.5423202514648438, "blocks.5.attention.w_k": 0.532806396484375, "blocks.5.attention.w_v": 0.5152816772460938, "blocks.5.attention.w_out": 0.5596427917480469, "blocks.5.feed_forward.w1": 0.6052958369255066, "blocks.5.feed_forward.w2": 0.5430170893669128, "blocks.5.feed_forward.w3": 0.5847384333610535, "blocks.6.attention.w_q": 0.5520477294921875, "blocks.6.attention.w_k": 0.5349388122558594, "blocks.6.attention.w_v": 0.495452880859375, "blocks.6.attention.w_out": 0.5110282897949219, "blocks.6.feed_forward.w1": 0.6096343994140625, "blocks.6.feed_forward.w2": 0.5100669860839844, "blocks.6.feed_forward.w3": 0.5870196223258972, "blocks.7.attention.w_q": 0.5572853088378906, "blocks.7.attention.w_k": 0.5384674072265625, "blocks.7.attention.w_v": 0.5217170715332031, "blocks.7.attention.w_out": 0.4744377136230469, "blocks.7.feed_forward.w1": 0.6187515258789062, "blocks.7.feed_forward.w2": 0.46628063917160034, "blocks.7.feed_forward.w3": 0.5953330993652344, "blocks.8.attention.w_q": 0.5706405639648438, "blocks.8.attention.w_k": 0.5525245666503906, "blocks.8.attention.w_v": 0.5303573608398438, "blocks.8.attention.w_out": 0.4657707214355469, "blocks.8.feed_forward.w1": 0.6327425837516785, "blocks.8.feed_forward.w2": 0.40184149146080017, "blocks.8.feed_forward.w3": 0.5969950556755066}} +{"step": 5300, "changed_frac": 0.5511400271684695, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4610023498535156, "blocks.0.attention.w_k": 0.45088958740234375, "blocks.0.attention.w_v": 0.50311279296875, "blocks.0.attention.w_out": 0.5904426574707031, "blocks.0.feed_forward.w1": 0.587805449962616, "blocks.0.feed_forward.w2": 0.5671882629394531, "blocks.0.feed_forward.w3": 0.5825831294059753, "blocks.1.attention.w_q": 0.5603790283203125, "blocks.1.attention.w_k": 0.5352630615234375, "blocks.1.attention.w_v": 0.5924606323242188, "blocks.1.attention.w_out": 0.6787605285644531, "blocks.1.feed_forward.w1": 0.5688565969467163, "blocks.1.feed_forward.w2": 0.566321074962616, "blocks.1.feed_forward.w3": 0.5634078979492188, "blocks.2.attention.w_q": 0.5442924499511719, "blocks.2.attention.w_k": 0.5234375, "blocks.2.attention.w_v": 0.5504684448242188, "blocks.2.attention.w_out": 0.640716552734375, "blocks.2.feed_forward.w1": 0.5759188532829285, "blocks.2.feed_forward.w2": 0.5648549795150757, "blocks.2.feed_forward.w3": 0.5659739375114441, "blocks.3.attention.w_q": 0.5188789367675781, "blocks.3.attention.w_k": 0.5007286071777344, "blocks.3.attention.w_v": 0.5238380432128906, "blocks.3.attention.w_out": 0.6138534545898438, "blocks.3.feed_forward.w1": 0.5798238515853882, "blocks.3.feed_forward.w2": 0.5648460388183594, "blocks.3.feed_forward.w3": 0.5676892995834351, "blocks.4.attention.w_q": 0.5501060485839844, "blocks.4.attention.w_k": 0.5212364196777344, "blocks.4.attention.w_v": 0.5320549011230469, "blocks.4.attention.w_out": 0.5933685302734375, "blocks.4.feed_forward.w1": 0.5860837697982788, "blocks.4.feed_forward.w2": 0.5546442866325378, "blocks.4.feed_forward.w3": 0.5676994323730469, "blocks.5.attention.w_q": 0.52349853515625, "blocks.5.attention.w_k": 0.5108375549316406, "blocks.5.attention.w_v": 0.4982032775878906, "blocks.5.attention.w_out": 0.54205322265625, "blocks.5.feed_forward.w1": 0.5895971059799194, "blocks.5.feed_forward.w2": 0.5271390676498413, "blocks.5.feed_forward.w3": 0.566931426525116, "blocks.6.attention.w_q": 0.5307540893554688, "blocks.6.attention.w_k": 0.5131149291992188, "blocks.6.attention.w_v": 0.4748382568359375, "blocks.6.attention.w_out": 0.49332427978515625, "blocks.6.feed_forward.w1": 0.5924581289291382, "blocks.6.feed_forward.w2": 0.4929339289665222, "blocks.6.feed_forward.w3": 0.5709304809570312, "blocks.7.attention.w_q": 0.5433349609375, "blocks.7.attention.w_k": 0.5181922912597656, "blocks.7.attention.w_v": 0.5046310424804688, "blocks.7.attention.w_out": 0.4547271728515625, "blocks.7.feed_forward.w1": 0.6034927368164062, "blocks.7.feed_forward.w2": 0.4482167661190033, "blocks.7.feed_forward.w3": 0.5779533386230469, "blocks.8.attention.w_q": 0.5507850646972656, "blocks.8.attention.w_k": 0.5322990417480469, "blocks.8.attention.w_v": 0.5115966796875, "blocks.8.attention.w_out": 0.448150634765625, "blocks.8.feed_forward.w1": 0.6175016164779663, "blocks.8.feed_forward.w2": 0.3843841552734375, "blocks.8.feed_forward.w3": 0.5806821584701538}} +{"step": 5400, "changed_frac": 0.5319723145574586, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4437980651855469, "blocks.0.attention.w_k": 0.4303436279296875, "blocks.0.attention.w_v": 0.48468780517578125, "blocks.0.attention.w_out": 0.5739707946777344, "blocks.0.feed_forward.w1": 0.5710589289665222, "blocks.0.feed_forward.w2": 0.55059814453125, "blocks.0.feed_forward.w3": 0.5635083913803101, "blocks.1.attention.w_q": 0.5390892028808594, "blocks.1.attention.w_k": 0.5161094665527344, "blocks.1.attention.w_v": 0.5744972229003906, "blocks.1.attention.w_out": 0.6608009338378906, "blocks.1.feed_forward.w1": 0.5504150390625, "blocks.1.feed_forward.w2": 0.5464299917221069, "blocks.1.feed_forward.w3": 0.5458590388298035, "blocks.2.attention.w_q": 0.5223464965820312, "blocks.2.attention.w_k": 0.5008926391601562, "blocks.2.attention.w_v": 0.529449462890625, "blocks.2.attention.w_out": 0.6236457824707031, "blocks.2.feed_forward.w1": 0.5574557185173035, "blocks.2.feed_forward.w2": 0.5466219782829285, "blocks.2.feed_forward.w3": 0.5471814870834351, "blocks.3.attention.w_q": 0.4996299743652344, "blocks.3.attention.w_k": 0.4824714660644531, "blocks.3.attention.w_v": 0.5052757263183594, "blocks.3.attention.w_out": 0.5945053100585938, "blocks.3.feed_forward.w1": 0.5626131892204285, "blocks.3.feed_forward.w2": 0.5461235046386719, "blocks.3.feed_forward.w3": 0.548803985118866, "blocks.4.attention.w_q": 0.5328102111816406, "blocks.4.attention.w_k": 0.5013847351074219, "blocks.4.attention.w_v": 0.5153007507324219, "blocks.4.attention.w_out": 0.5736770629882812, "blocks.4.feed_forward.w1": 0.5679194331169128, "blocks.4.feed_forward.w2": 0.5341784358024597, "blocks.4.feed_forward.w3": 0.5477676391601562, "blocks.5.attention.w_q": 0.5013885498046875, "blocks.5.attention.w_k": 0.4907798767089844, "blocks.5.attention.w_v": 0.47772216796875, "blocks.5.attention.w_out": 0.5233421325683594, "blocks.5.feed_forward.w1": 0.5705757141113281, "blocks.5.feed_forward.w2": 0.5071424245834351, "blocks.5.feed_forward.w3": 0.547217071056366, "blocks.6.attention.w_q": 0.5110359191894531, "blocks.6.attention.w_k": 0.49399566650390625, "blocks.6.attention.w_v": 0.45497894287109375, "blocks.6.attention.w_out": 0.4741325378417969, "blocks.6.feed_forward.w1": 0.5732676386833191, "blocks.6.feed_forward.w2": 0.4715830683708191, "blocks.6.feed_forward.w3": 0.5519371032714844, "blocks.7.attention.w_q": 0.5214920043945312, "blocks.7.attention.w_k": 0.496826171875, "blocks.7.attention.w_v": 0.48241424560546875, "blocks.7.attention.w_out": 0.4391937255859375, "blocks.7.feed_forward.w1": 0.5842323303222656, "blocks.7.feed_forward.w2": 0.4284477233886719, "blocks.7.feed_forward.w3": 0.5581640005111694, "blocks.8.attention.w_q": 0.5293312072753906, "blocks.8.attention.w_k": 0.5091934204101562, "blocks.8.attention.w_v": 0.494110107421875, "blocks.8.attention.w_out": 0.42981719970703125, "blocks.8.feed_forward.w1": 0.598419189453125, "blocks.8.feed_forward.w2": 0.36374664306640625, "blocks.8.feed_forward.w3": 0.5608393549919128}} +{"step": 5500, "changed_frac": 0.5124428251869658, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.41756439208984375, "blocks.0.attention.w_k": 0.40804290771484375, "blocks.0.attention.w_v": 0.4666481018066406, "blocks.0.attention.w_out": 0.5584068298339844, "blocks.0.feed_forward.w1": 0.5491142272949219, "blocks.0.feed_forward.w2": 0.5331815481185913, "blocks.0.feed_forward.w3": 0.5452868342399597, "blocks.1.attention.w_q": 0.5191078186035156, "blocks.1.attention.w_k": 0.4908485412597656, "blocks.1.attention.w_v": 0.5521469116210938, "blocks.1.attention.w_out": 0.6430702209472656, "blocks.1.feed_forward.w1": 0.529913604259491, "blocks.1.feed_forward.w2": 0.5284423828125, "blocks.1.feed_forward.w3": 0.5243136286735535, "blocks.2.attention.w_q": 0.5012054443359375, "blocks.2.attention.w_k": 0.47832489013671875, "blocks.2.attention.w_v": 0.5107307434082031, "blocks.2.attention.w_out": 0.6051406860351562, "blocks.2.feed_forward.w1": 0.5380872488021851, "blocks.2.feed_forward.w2": 0.5299009084701538, "blocks.2.feed_forward.w3": 0.527069091796875, "blocks.3.attention.w_q": 0.47589874267578125, "blocks.3.attention.w_k": 0.4614219665527344, "blocks.3.attention.w_v": 0.48586273193359375, "blocks.3.attention.w_out": 0.5755424499511719, "blocks.3.feed_forward.w1": 0.5425249934196472, "blocks.3.feed_forward.w2": 0.5279426574707031, "blocks.3.feed_forward.w3": 0.528448760509491, "blocks.4.attention.w_q": 0.5113334655761719, "blocks.4.attention.w_k": 0.4776573181152344, "blocks.4.attention.w_v": 0.4920234680175781, "blocks.4.attention.w_out": 0.5574874877929688, "blocks.4.feed_forward.w1": 0.5471700429916382, "blocks.4.feed_forward.w2": 0.5157827138900757, "blocks.4.feed_forward.w3": 0.5273005366325378, "blocks.5.attention.w_q": 0.48291015625, "blocks.5.attention.w_k": 0.4724922180175781, "blocks.5.attention.w_v": 0.4598236083984375, "blocks.5.attention.w_out": 0.5035324096679688, "blocks.5.feed_forward.w1": 0.5511335134506226, "blocks.5.feed_forward.w2": 0.487762451171875, "blocks.5.feed_forward.w3": 0.528204619884491, "blocks.6.attention.w_q": 0.4910125732421875, "blocks.6.attention.w_k": 0.47168731689453125, "blocks.6.attention.w_v": 0.4337158203125, "blocks.6.attention.w_out": 0.4550895690917969, "blocks.6.feed_forward.w1": 0.5563087463378906, "blocks.6.feed_forward.w2": 0.4523671567440033, "blocks.6.feed_forward.w3": 0.5325114130973816, "blocks.7.attention.w_q": 0.5002174377441406, "blocks.7.attention.w_k": 0.47602081298828125, "blocks.7.attention.w_v": 0.4632110595703125, "blocks.7.attention.w_out": 0.4185791015625, "blocks.7.feed_forward.w1": 0.566992461681366, "blocks.7.feed_forward.w2": 0.4087766110897064, "blocks.7.feed_forward.w3": 0.5393422842025757, "blocks.8.attention.w_q": 0.5075645446777344, "blocks.8.attention.w_k": 0.4901084899902344, "blocks.8.attention.w_v": 0.4735450744628906, "blocks.8.attention.w_out": 0.4053153991699219, "blocks.8.feed_forward.w1": 0.5811297297477722, "blocks.8.feed_forward.w2": 0.3462321162223816, "blocks.8.feed_forward.w3": 0.542266845703125}} +{"step": 5600, "changed_frac": 0.4923385880951189, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.39867401123046875, "blocks.0.attention.w_k": 0.38860321044921875, "blocks.0.attention.w_v": 0.4471778869628906, "blocks.0.attention.w_out": 0.5427589416503906, "blocks.0.feed_forward.w1": 0.5309473872184753, "blocks.0.feed_forward.w2": 0.5133565664291382, "blocks.0.feed_forward.w3": 0.5245615839958191, "blocks.1.attention.w_q": 0.49609375, "blocks.1.attention.w_k": 0.47113037109375, "blocks.1.attention.w_v": 0.5324859619140625, "blocks.1.attention.w_out": 0.6263542175292969, "blocks.1.feed_forward.w1": 0.5101598501205444, "blocks.1.feed_forward.w2": 0.5112406611442566, "blocks.1.feed_forward.w3": 0.5052668452262878, "blocks.2.attention.w_q": 0.4803276062011719, "blocks.2.attention.w_k": 0.45737457275390625, "blocks.2.attention.w_v": 0.4876670837402344, "blocks.2.attention.w_out": 0.5862655639648438, "blocks.2.feed_forward.w1": 0.5176734924316406, "blocks.2.feed_forward.w2": 0.5098559260368347, "blocks.2.feed_forward.w3": 0.5074259638786316, "blocks.3.attention.w_q": 0.452178955078125, "blocks.3.attention.w_k": 0.43772125244140625, "blocks.3.attention.w_v": 0.4630699157714844, "blocks.3.attention.w_out": 0.5562515258789062, "blocks.3.feed_forward.w1": 0.5235583186149597, "blocks.3.feed_forward.w2": 0.5081138610839844, "blocks.3.feed_forward.w3": 0.5092849731445312, "blocks.4.attention.w_q": 0.49047088623046875, "blocks.4.attention.w_k": 0.4577903747558594, "blocks.4.attention.w_v": 0.4718208312988281, "blocks.4.attention.w_out": 0.536956787109375, "blocks.4.feed_forward.w1": 0.5270398855209351, "blocks.4.feed_forward.w2": 0.4966977536678314, "blocks.4.feed_forward.w3": 0.5084609985351562, "blocks.5.attention.w_q": 0.4599037170410156, "blocks.5.attention.w_k": 0.4481620788574219, "blocks.5.attention.w_v": 0.4349174499511719, "blocks.5.attention.w_out": 0.48265838623046875, "blocks.5.feed_forward.w1": 0.5299581289291382, "blocks.5.feed_forward.w2": 0.4669087827205658, "blocks.5.feed_forward.w3": 0.5080413818359375, "blocks.6.attention.w_q": 0.4666328430175781, "blocks.6.attention.w_k": 0.4514198303222656, "blocks.6.attention.w_v": 0.415191650390625, "blocks.6.attention.w_out": 0.434967041015625, "blocks.6.feed_forward.w1": 0.5347684621810913, "blocks.6.feed_forward.w2": 0.43347930908203125, "blocks.6.feed_forward.w3": 0.5124601125717163, "blocks.7.attention.w_q": 0.4776191711425781, "blocks.7.attention.w_k": 0.455780029296875, "blocks.7.attention.w_v": 0.4420814514160156, "blocks.7.attention.w_out": 0.39916229248046875, "blocks.7.feed_forward.w1": 0.5458564758300781, "blocks.7.feed_forward.w2": 0.3894551694393158, "blocks.7.feed_forward.w3": 0.5188407897949219, "blocks.8.attention.w_q": 0.48371124267578125, "blocks.8.attention.w_k": 0.4684104919433594, "blocks.8.attention.w_v": 0.4535789489746094, "blocks.8.attention.w_out": 0.38642120361328125, "blocks.8.feed_forward.w1": 0.5612754821777344, "blocks.8.feed_forward.w2": 0.3276761472225189, "blocks.8.feed_forward.w3": 0.5215772390365601}} +{"step": 5700, "changed_frac": 0.4720690474550948, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.37743377685546875, "blocks.0.attention.w_k": 0.36763763427734375, "blocks.0.attention.w_v": 0.4242210388183594, "blocks.0.attention.w_out": 0.5223655700683594, "blocks.0.feed_forward.w1": 0.5089187622070312, "blocks.0.feed_forward.w2": 0.49421438574790955, "blocks.0.feed_forward.w3": 0.5041300654411316, "blocks.1.attention.w_q": 0.47629547119140625, "blocks.1.attention.w_k": 0.4490928649902344, "blocks.1.attention.w_v": 0.5120925903320312, "blocks.1.attention.w_out": 0.6112174987792969, "blocks.1.feed_forward.w1": 0.48913320899009705, "blocks.1.feed_forward.w2": 0.4900563657283783, "blocks.1.feed_forward.w3": 0.4854469299316406, "blocks.2.attention.w_q": 0.454742431640625, "blocks.2.attention.w_k": 0.4354667663574219, "blocks.2.attention.w_v": 0.46883392333984375, "blocks.2.attention.w_out": 0.5669898986816406, "blocks.2.feed_forward.w1": 0.4977162778377533, "blocks.2.feed_forward.w2": 0.4926503598690033, "blocks.2.feed_forward.w3": 0.48788198828697205, "blocks.3.attention.w_q": 0.4296531677246094, "blocks.3.attention.w_k": 0.4184532165527344, "blocks.3.attention.w_v": 0.4415245056152344, "blocks.3.attention.w_out": 0.5383529663085938, "blocks.3.feed_forward.w1": 0.5031356811523438, "blocks.3.feed_forward.w2": 0.4892209470272064, "blocks.3.feed_forward.w3": 0.4880790710449219, "blocks.4.attention.w_q": 0.4669380187988281, "blocks.4.attention.w_k": 0.4337120056152344, "blocks.4.attention.w_v": 0.4515953063964844, "blocks.4.attention.w_out": 0.5155258178710938, "blocks.4.feed_forward.w1": 0.5074297785758972, "blocks.4.feed_forward.w2": 0.4768028259277344, "blocks.4.feed_forward.w3": 0.48728054761886597, "blocks.5.attention.w_q": 0.4404487609863281, "blocks.5.attention.w_k": 0.42859649658203125, "blocks.5.attention.w_v": 0.4169464111328125, "blocks.5.attention.w_out": 0.4626922607421875, "blocks.5.feed_forward.w1": 0.5105463862419128, "blocks.5.feed_forward.w2": 0.44775137305259705, "blocks.5.feed_forward.w3": 0.4872080683708191, "blocks.6.attention.w_q": 0.4461517333984375, "blocks.6.attention.w_k": 0.4277496337890625, "blocks.6.attention.w_v": 0.39371490478515625, "blocks.6.attention.w_out": 0.4135780334472656, "blocks.6.feed_forward.w1": 0.5141398310661316, "blocks.6.feed_forward.w2": 0.4135271906852722, "blocks.6.feed_forward.w3": 0.4922282099723816, "blocks.7.attention.w_q": 0.456329345703125, "blocks.7.attention.w_k": 0.4302520751953125, "blocks.7.attention.w_v": 0.42415618896484375, "blocks.7.attention.w_out": 0.3802909851074219, "blocks.7.feed_forward.w1": 0.524542510509491, "blocks.7.feed_forward.w2": 0.36890411376953125, "blocks.7.feed_forward.w3": 0.4996833801269531, "blocks.8.attention.w_q": 0.4649200439453125, "blocks.8.attention.w_k": 0.4449577331542969, "blocks.8.attention.w_v": 0.4339332580566406, "blocks.8.attention.w_out": 0.36759185791015625, "blocks.8.feed_forward.w1": 0.5395584106445312, "blocks.8.feed_forward.w2": 0.3120104670524597, "blocks.8.feed_forward.w3": 0.5003446340560913}} +{"step": 5800, "changed_frac": 0.45113529303135014, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.35640716552734375, "blocks.0.attention.w_k": 0.3491172790527344, "blocks.0.attention.w_v": 0.4101715087890625, "blocks.0.attention.w_out": 0.5030708312988281, "blocks.0.feed_forward.w1": 0.48614630103111267, "blocks.0.feed_forward.w2": 0.47329458594322205, "blocks.0.feed_forward.w3": 0.48002880811691284, "blocks.1.attention.w_q": 0.4533729553222656, "blocks.1.attention.w_k": 0.42694854736328125, "blocks.1.attention.w_v": 0.48956298828125, "blocks.1.attention.w_out": 0.5886917114257812, "blocks.1.feed_forward.w1": 0.4695383906364441, "blocks.1.feed_forward.w2": 0.4691886901855469, "blocks.1.feed_forward.w3": 0.46329373121261597, "blocks.2.attention.w_q": 0.43425750732421875, "blocks.2.attention.w_k": 0.4104270935058594, "blocks.2.attention.w_v": 0.44757843017578125, "blocks.2.attention.w_out": 0.548004150390625, "blocks.2.feed_forward.w1": 0.4764506220817566, "blocks.2.feed_forward.w2": 0.4720700681209564, "blocks.2.feed_forward.w3": 0.46642178297042847, "blocks.3.attention.w_q": 0.4116058349609375, "blocks.3.attention.w_k": 0.3947181701660156, "blocks.3.attention.w_v": 0.4193916320800781, "blocks.3.attention.w_out": 0.5175399780273438, "blocks.3.feed_forward.w1": 0.48169073462486267, "blocks.3.feed_forward.w2": 0.46725592017173767, "blocks.3.feed_forward.w3": 0.4666392207145691, "blocks.4.attention.w_q": 0.442718505859375, "blocks.4.attention.w_k": 0.4117317199707031, "blocks.4.attention.w_v": 0.4307823181152344, "blocks.4.attention.w_out": 0.4954109191894531, "blocks.4.feed_forward.w1": 0.48468655347824097, "blocks.4.feed_forward.w2": 0.4550285339355469, "blocks.4.feed_forward.w3": 0.4671592712402344, "blocks.5.attention.w_q": 0.417449951171875, "blocks.5.attention.w_k": 0.4068450927734375, "blocks.5.attention.w_v": 0.39614105224609375, "blocks.5.attention.w_out": 0.4428062438964844, "blocks.5.feed_forward.w1": 0.490936279296875, "blocks.5.feed_forward.w2": 0.4262326657772064, "blocks.5.feed_forward.w3": 0.46661633253097534, "blocks.6.attention.w_q": 0.4246826171875, "blocks.6.attention.w_k": 0.4085502624511719, "blocks.6.attention.w_v": 0.3761177062988281, "blocks.6.attention.w_out": 0.3969268798828125, "blocks.6.feed_forward.w1": 0.49270376563072205, "blocks.6.feed_forward.w2": 0.3920949399471283, "blocks.6.feed_forward.w3": 0.4710845947265625, "blocks.7.attention.w_q": 0.4345283508300781, "blocks.7.attention.w_k": 0.409027099609375, "blocks.7.attention.w_v": 0.40384674072265625, "blocks.7.attention.w_out": 0.36004638671875, "blocks.7.feed_forward.w1": 0.5053571462631226, "blocks.7.feed_forward.w2": 0.350128173828125, "blocks.7.feed_forward.w3": 0.4787229001522064, "blocks.8.attention.w_q": 0.43973541259765625, "blocks.8.attention.w_k": 0.4240455627441406, "blocks.8.attention.w_v": 0.4119071960449219, "blocks.8.attention.w_out": 0.3484382629394531, "blocks.8.feed_forward.w1": 0.5195999145507812, "blocks.8.feed_forward.w2": 0.2939555048942566, "blocks.8.feed_forward.w3": 0.4804166257381439}} +{"step": 5900, "changed_frac": 0.4298851469643096, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3350257873535156, "blocks.0.attention.w_k": 0.3260765075683594, "blocks.0.attention.w_v": 0.3872337341308594, "blocks.0.attention.w_out": 0.4839439392089844, "blocks.0.feed_forward.w1": 0.4642690122127533, "blocks.0.feed_forward.w2": 0.4525413513183594, "blocks.0.feed_forward.w3": 0.4584897458553314, "blocks.1.attention.w_q": 0.42928314208984375, "blocks.1.attention.w_k": 0.403045654296875, "blocks.1.attention.w_v": 0.4667091369628906, "blocks.1.attention.w_out": 0.569305419921875, "blocks.1.feed_forward.w1": 0.4463233947753906, "blocks.1.feed_forward.w2": 0.4479154050350189, "blocks.1.feed_forward.w3": 0.4410133361816406, "blocks.2.attention.w_q": 0.41056060791015625, "blocks.2.attention.w_k": 0.389190673828125, "blocks.2.attention.w_v": 0.4243011474609375, "blocks.2.attention.w_out": 0.527496337890625, "blocks.2.feed_forward.w1": 0.45489501953125, "blocks.2.feed_forward.w2": 0.44827908277511597, "blocks.2.feed_forward.w3": 0.4447898864746094, "blocks.3.attention.w_q": 0.38764190673828125, "blocks.3.attention.w_k": 0.37369537353515625, "blocks.3.attention.w_v": 0.3991241455078125, "blocks.3.attention.w_out": 0.49741363525390625, "blocks.3.feed_forward.w1": 0.4598070979118347, "blocks.3.feed_forward.w2": 0.4472503662109375, "blocks.3.feed_forward.w3": 0.4462432861328125, "blocks.4.attention.w_q": 0.4214019775390625, "blocks.4.attention.w_k": 0.3885002136230469, "blocks.4.attention.w_v": 0.4109687805175781, "blocks.4.attention.w_out": 0.47460174560546875, "blocks.4.feed_forward.w1": 0.465728759765625, "blocks.4.feed_forward.w2": 0.4357541501522064, "blocks.4.feed_forward.w3": 0.4442405700683594, "blocks.5.attention.w_q": 0.394012451171875, "blocks.5.attention.w_k": 0.385284423828125, "blocks.5.attention.w_v": 0.37226104736328125, "blocks.5.attention.w_out": 0.422271728515625, "blocks.5.feed_forward.w1": 0.46942901611328125, "blocks.5.feed_forward.w2": 0.40464529395103455, "blocks.5.feed_forward.w3": 0.44488653540611267, "blocks.6.attention.w_q": 0.39933013916015625, "blocks.6.attention.w_k": 0.3864479064941406, "blocks.6.attention.w_v": 0.3549156188964844, "blocks.6.attention.w_out": 0.3766899108886719, "blocks.6.feed_forward.w1": 0.4717559814453125, "blocks.6.feed_forward.w2": 0.37275442481040955, "blocks.6.feed_forward.w3": 0.44894537329673767, "blocks.7.attention.w_q": 0.4124488830566406, "blocks.7.attention.w_k": 0.3899078369140625, "blocks.7.attention.w_v": 0.3832740783691406, "blocks.7.attention.w_out": 0.34011077880859375, "blocks.7.feed_forward.w1": 0.4838193356990814, "blocks.7.feed_forward.w2": 0.3311042785644531, "blocks.7.feed_forward.w3": 0.4568367004394531, "blocks.8.attention.w_q": 0.4166831970214844, "blocks.8.attention.w_k": 0.4019317626953125, "blocks.8.attention.w_v": 0.3942832946777344, "blocks.8.attention.w_out": 0.3289299011230469, "blocks.8.feed_forward.w1": 0.49915823340415955, "blocks.8.feed_forward.w2": 0.27754467725753784, "blocks.8.feed_forward.w3": 0.4589996337890625}} +{"step": 6000, "changed_frac": 0.40946676792242587, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3145027160644531, "blocks.0.attention.w_k": 0.3076515197753906, "blocks.0.attention.w_v": 0.3712615966796875, "blocks.0.attention.w_out": 0.4673309326171875, "blocks.0.feed_forward.w1": 0.4420153498649597, "blocks.0.feed_forward.w2": 0.4327380061149597, "blocks.0.feed_forward.w3": 0.43763861060142517, "blocks.1.attention.w_q": 0.40622711181640625, "blocks.1.attention.w_k": 0.3817596435546875, "blocks.1.attention.w_v": 0.4470863342285156, "blocks.1.attention.w_out": 0.5511512756347656, "blocks.1.feed_forward.w1": 0.42497381567955017, "blocks.1.feed_forward.w2": 0.4291064143180847, "blocks.1.feed_forward.w3": 0.42002105712890625, "blocks.2.attention.w_q": 0.3891754150390625, "blocks.2.attention.w_k": 0.3663215637207031, "blocks.2.attention.w_v": 0.4041404724121094, "blocks.2.attention.w_out": 0.5055007934570312, "blocks.2.feed_forward.w1": 0.4333000183105469, "blocks.2.feed_forward.w2": 0.4293721616268158, "blocks.2.feed_forward.w3": 0.4252586364746094, "blocks.3.attention.w_q": 0.364501953125, "blocks.3.attention.w_k": 0.35117340087890625, "blocks.3.attention.w_v": 0.3770790100097656, "blocks.3.attention.w_out": 0.4756584167480469, "blocks.3.feed_forward.w1": 0.43847784399986267, "blocks.3.feed_forward.w2": 0.4257443845272064, "blocks.3.feed_forward.w3": 0.4238840937614441, "blocks.4.attention.w_q": 0.3985328674316406, "blocks.4.attention.w_k": 0.3664283752441406, "blocks.4.attention.w_v": 0.3886451721191406, "blocks.4.attention.w_out": 0.4552879333496094, "blocks.4.feed_forward.w1": 0.4440142512321472, "blocks.4.feed_forward.w2": 0.4167836606502533, "blocks.4.feed_forward.w3": 0.42380523681640625, "blocks.5.attention.w_q": 0.3729820251464844, "blocks.5.attention.w_k": 0.3661537170410156, "blocks.5.attention.w_v": 0.3533134460449219, "blocks.5.attention.w_out": 0.40252685546875, "blocks.5.feed_forward.w1": 0.44824856519699097, "blocks.5.feed_forward.w2": 0.38572439551353455, "blocks.5.feed_forward.w3": 0.4238573908805847, "blocks.6.attention.w_q": 0.38106536865234375, "blocks.6.attention.w_k": 0.3667869567871094, "blocks.6.attention.w_v": 0.33618927001953125, "blocks.6.attention.w_out": 0.3573493957519531, "blocks.6.feed_forward.w1": 0.4496498107910156, "blocks.6.feed_forward.w2": 0.35293707251548767, "blocks.6.feed_forward.w3": 0.42699307203292847, "blocks.7.attention.w_q": 0.3894996643066406, "blocks.7.attention.w_k": 0.3682594299316406, "blocks.7.attention.w_v": 0.3642921447753906, "blocks.7.attention.w_out": 0.3228340148925781, "blocks.7.feed_forward.w1": 0.46260708570480347, "blocks.7.feed_forward.w2": 0.31492871046066284, "blocks.7.feed_forward.w3": 0.4353269040584564, "blocks.8.attention.w_q": 0.3968162536621094, "blocks.8.attention.w_k": 0.380706787109375, "blocks.8.attention.w_v": 0.3709220886230469, "blocks.8.attention.w_out": 0.3129463195800781, "blocks.8.feed_forward.w1": 0.47842663526535034, "blocks.8.feed_forward.w2": 0.2618255615234375, "blocks.8.feed_forward.w3": 0.4375254511833191}} +{"step": 6100, "changed_frac": 0.3893757029476329, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2981986999511719, "blocks.0.attention.w_k": 0.2894859313964844, "blocks.0.attention.w_v": 0.3507575988769531, "blocks.0.attention.w_out": 0.44692230224609375, "blocks.0.feed_forward.w1": 0.4222514033317566, "blocks.0.feed_forward.w2": 0.4136289060115814, "blocks.0.feed_forward.w3": 0.416473388671875, "blocks.1.attention.w_q": 0.3840904235839844, "blocks.1.attention.w_k": 0.36119842529296875, "blocks.1.attention.w_v": 0.4227752685546875, "blocks.1.attention.w_out": 0.5316009521484375, "blocks.1.feed_forward.w1": 0.4038200378417969, "blocks.1.feed_forward.w2": 0.40875244140625, "blocks.1.feed_forward.w3": 0.3995921015739441, "blocks.2.attention.w_q": 0.36727142333984375, "blocks.2.attention.w_k": 0.3473396301269531, "blocks.2.attention.w_v": 0.3842353820800781, "blocks.2.attention.w_out": 0.48699951171875, "blocks.2.feed_forward.w1": 0.41193389892578125, "blocks.2.feed_forward.w2": 0.4081929624080658, "blocks.2.feed_forward.w3": 0.40319952368736267, "blocks.3.attention.w_q": 0.3486061096191406, "blocks.3.attention.w_k": 0.33260345458984375, "blocks.3.attention.w_v": 0.3596153259277344, "blocks.3.attention.w_out": 0.4556465148925781, "blocks.3.feed_forward.w1": 0.4187571406364441, "blocks.3.feed_forward.w2": 0.40572232007980347, "blocks.3.feed_forward.w3": 0.4054514765739441, "blocks.4.attention.w_q": 0.3764152526855469, "blocks.4.attention.w_k": 0.3449554443359375, "blocks.4.attention.w_v": 0.3702888488769531, "blocks.4.attention.w_out": 0.433349609375, "blocks.4.feed_forward.w1": 0.42397183179855347, "blocks.4.feed_forward.w2": 0.3928489685058594, "blocks.4.feed_forward.w3": 0.4029477536678314, "blocks.5.attention.w_q": 0.3534584045410156, "blocks.5.attention.w_k": 0.3471488952636719, "blocks.5.attention.w_v": 0.33557891845703125, "blocks.5.attention.w_out": 0.38323974609375, "blocks.5.feed_forward.w1": 0.42700958251953125, "blocks.5.feed_forward.w2": 0.3662872314453125, "blocks.5.feed_forward.w3": 0.40428799390792847, "blocks.6.attention.w_q": 0.3580131530761719, "blocks.6.attention.w_k": 0.3454132080078125, "blocks.6.attention.w_v": 0.3162956237792969, "blocks.6.attention.w_out": 0.33856201171875, "blocks.6.feed_forward.w1": 0.42936834692955017, "blocks.6.feed_forward.w2": 0.33554205298423767, "blocks.6.feed_forward.w3": 0.4066416621208191, "blocks.7.attention.w_q": 0.36775970458984375, "blocks.7.attention.w_k": 0.3465919494628906, "blocks.7.attention.w_v": 0.345428466796875, "blocks.7.attention.w_out": 0.307373046875, "blocks.7.feed_forward.w1": 0.4417063593864441, "blocks.7.feed_forward.w2": 0.29628628492355347, "blocks.7.feed_forward.w3": 0.41389593482017517, "blocks.8.attention.w_q": 0.3759269714355469, "blocks.8.attention.w_k": 0.3597908020019531, "blocks.8.attention.w_v": 0.3521270751953125, "blocks.8.attention.w_out": 0.2937278747558594, "blocks.8.feed_forward.w1": 0.4573313593864441, "blocks.8.feed_forward.w2": 0.2464141845703125, "blocks.8.feed_forward.w3": 0.4170735776424408}} +{"step": 6200, "changed_frac": 0.3703057704827724, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.282928466796875, "blocks.0.attention.w_k": 0.2743186950683594, "blocks.0.attention.w_v": 0.3329811096191406, "blocks.0.attention.w_out": 0.4293937683105469, "blocks.0.feed_forward.w1": 0.4014434814453125, "blocks.0.feed_forward.w2": 0.39217376708984375, "blocks.0.feed_forward.w3": 0.3960406184196472, "blocks.1.attention.w_q": 0.3652534484863281, "blocks.1.attention.w_k": 0.3400154113769531, "blocks.1.attention.w_v": 0.4009819030761719, "blocks.1.attention.w_out": 0.5101356506347656, "blocks.1.feed_forward.w1": 0.3849830627441406, "blocks.1.feed_forward.w2": 0.388214111328125, "blocks.1.feed_forward.w3": 0.3820699155330658, "blocks.2.attention.w_q": 0.3474311828613281, "blocks.2.attention.w_k": 0.3258934020996094, "blocks.2.attention.w_v": 0.3603553771972656, "blocks.2.attention.w_out": 0.4662017822265625, "blocks.2.feed_forward.w1": 0.39387768507003784, "blocks.2.feed_forward.w2": 0.3885002136230469, "blocks.2.feed_forward.w3": 0.3855031430721283, "blocks.3.attention.w_q": 0.325103759765625, "blocks.3.attention.w_k": 0.31334686279296875, "blocks.3.attention.w_v": 0.3415031433105469, "blocks.3.attention.w_out": 0.43499755859375, "blocks.3.feed_forward.w1": 0.39831799268722534, "blocks.3.feed_forward.w2": 0.38622793555259705, "blocks.3.feed_forward.w3": 0.38534674048423767, "blocks.4.attention.w_q": 0.3550872802734375, "blocks.4.attention.w_k": 0.3256034851074219, "blocks.4.attention.w_v": 0.3481178283691406, "blocks.4.attention.w_out": 0.4141082763671875, "blocks.4.feed_forward.w1": 0.4036089777946472, "blocks.4.feed_forward.w2": 0.37593841552734375, "blocks.4.feed_forward.w3": 0.3844718933105469, "blocks.5.attention.w_q": 0.3344154357910156, "blocks.5.attention.w_k": 0.3258857727050781, "blocks.5.attention.w_v": 0.3162841796875, "blocks.5.attention.w_out": 0.3658485412597656, "blocks.5.feed_forward.w1": 0.4057553708553314, "blocks.5.feed_forward.w2": 0.34850186109542847, "blocks.5.feed_forward.w3": 0.38433584570884705, "blocks.6.attention.w_q": 0.33991241455078125, "blocks.6.attention.w_k": 0.3277778625488281, "blocks.6.attention.w_v": 0.30072021484375, "blocks.6.attention.w_out": 0.3231086730957031, "blocks.6.feed_forward.w1": 0.40906399488449097, "blocks.6.feed_forward.w2": 0.3176968991756439, "blocks.6.feed_forward.w3": 0.38828787207603455, "blocks.7.attention.w_q": 0.3490791320800781, "blocks.7.attention.w_k": 0.326416015625, "blocks.7.attention.w_v": 0.3274955749511719, "blocks.7.attention.w_out": 0.29046630859375, "blocks.7.feed_forward.w1": 0.42064666748046875, "blocks.7.feed_forward.w2": 0.27998098731040955, "blocks.7.feed_forward.w3": 0.3957761228084564, "blocks.8.attention.w_q": 0.35466766357421875, "blocks.8.attention.w_k": 0.3426475524902344, "blocks.8.attention.w_v": 0.3341636657714844, "blocks.8.attention.w_out": 0.280059814453125, "blocks.8.feed_forward.w1": 0.43639883399009705, "blocks.8.feed_forward.w2": 0.23385748267173767, "blocks.8.feed_forward.w3": 0.39733633399009705}} +{"step": 6300, "changed_frac": 0.3526136610243056, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.26674652099609375, "blocks.0.attention.w_k": 0.2599525451660156, "blocks.0.attention.w_v": 0.3182563781738281, "blocks.0.attention.w_out": 0.4136009216308594, "blocks.0.feed_forward.w1": 0.38065212965011597, "blocks.0.feed_forward.w2": 0.3772010803222656, "blocks.0.feed_forward.w3": 0.3776664733886719, "blocks.1.attention.w_q": 0.3445396423339844, "blocks.1.attention.w_k": 0.3224296569824219, "blocks.1.attention.w_v": 0.3834877014160156, "blocks.1.attention.w_out": 0.493194580078125, "blocks.1.feed_forward.w1": 0.36496609449386597, "blocks.1.feed_forward.w2": 0.3707568049430847, "blocks.1.feed_forward.w3": 0.36140698194503784, "blocks.2.attention.w_q": 0.3285942077636719, "blocks.2.attention.w_k": 0.3092231750488281, "blocks.2.attention.w_v": 0.34317779541015625, "blocks.2.attention.w_out": 0.44610595703125, "blocks.2.feed_forward.w1": 0.3756446838378906, "blocks.2.feed_forward.w2": 0.3708941340446472, "blocks.2.feed_forward.w3": 0.36540985107421875, "blocks.3.attention.w_q": 0.30965423583984375, "blocks.3.attention.w_k": 0.29679107666015625, "blocks.3.attention.w_v": 0.3235969543457031, "blocks.3.attention.w_out": 0.41719818115234375, "blocks.3.feed_forward.w1": 0.37970098853111267, "blocks.3.feed_forward.w2": 0.36854809522628784, "blocks.3.feed_forward.w3": 0.3662249445915222, "blocks.4.attention.w_q": 0.33940887451171875, "blocks.4.attention.w_k": 0.30695343017578125, "blocks.4.attention.w_v": 0.3348388671875, "blocks.4.attention.w_out": 0.3939628601074219, "blocks.4.feed_forward.w1": 0.3856773376464844, "blocks.4.feed_forward.w2": 0.3586718440055847, "blocks.4.feed_forward.w3": 0.3657124936580658, "blocks.5.attention.w_q": 0.31536865234375, "blocks.5.attention.w_k": 0.30986785888671875, "blocks.5.attention.w_v": 0.3009300231933594, "blocks.5.attention.w_out": 0.34485626220703125, "blocks.5.feed_forward.w1": 0.38956961035728455, "blocks.5.feed_forward.w2": 0.3302510678768158, "blocks.5.feed_forward.w3": 0.3662656247615814, "blocks.6.attention.w_q": 0.3218574523925781, "blocks.6.attention.w_k": 0.3080863952636719, "blocks.6.attention.w_v": 0.2845497131347656, "blocks.6.attention.w_out": 0.3069610595703125, "blocks.6.feed_forward.w1": 0.3900044858455658, "blocks.6.feed_forward.w2": 0.30120849609375, "blocks.6.feed_forward.w3": 0.3697751462459564, "blocks.7.attention.w_q": 0.33046722412109375, "blocks.7.attention.w_k": 0.31232452392578125, "blocks.7.attention.w_v": 0.3112983703613281, "blocks.7.attention.w_out": 0.2783164978027344, "blocks.7.feed_forward.w1": 0.40254339575767517, "blocks.7.feed_forward.w2": 0.26559704542160034, "blocks.7.feed_forward.w3": 0.3754259943962097, "blocks.8.attention.w_q": 0.333984375, "blocks.8.attention.w_k": 0.3209381103515625, "blocks.8.attention.w_v": 0.3173027038574219, "blocks.8.attention.w_out": 0.2663002014160156, "blocks.8.feed_forward.w1": 0.4180488586425781, "blocks.8.feed_forward.w2": 0.22158432006835938, "blocks.8.feed_forward.w3": 0.3808174133300781}} +{"step": 6400, "changed_frac": 0.3363110142895299, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2523651123046875, "blocks.0.attention.w_k": 0.24558258056640625, "blocks.0.attention.w_v": 0.30233001708984375, "blocks.0.attention.w_out": 0.3952903747558594, "blocks.0.feed_forward.w1": 0.36476263403892517, "blocks.0.feed_forward.w2": 0.3601277768611908, "blocks.0.feed_forward.w3": 0.35978955030441284, "blocks.1.attention.w_q": 0.3276786804199219, "blocks.1.attention.w_k": 0.304443359375, "blocks.1.attention.w_v": 0.3641510009765625, "blocks.1.attention.w_out": 0.47528839111328125, "blocks.1.feed_forward.w1": 0.34886425733566284, "blocks.1.feed_forward.w2": 0.35417303442955017, "blocks.1.feed_forward.w3": 0.3452402949333191, "blocks.2.attention.w_q": 0.3110771179199219, "blocks.2.attention.w_k": 0.2923393249511719, "blocks.2.attention.w_v": 0.3285942077636719, "blocks.2.attention.w_out": 0.4275703430175781, "blocks.2.feed_forward.w1": 0.35628002882003784, "blocks.2.feed_forward.w2": 0.35393399000167847, "blocks.2.feed_forward.w3": 0.34748587012290955, "blocks.3.attention.w_q": 0.2941932678222656, "blocks.3.attention.w_k": 0.281463623046875, "blocks.3.attention.w_v": 0.3064537048339844, "blocks.3.attention.w_out": 0.3988838195800781, "blocks.3.feed_forward.w1": 0.3621610105037689, "blocks.3.feed_forward.w2": 0.35269293189048767, "blocks.3.feed_forward.w3": 0.34880194067955017, "blocks.4.attention.w_q": 0.32053375244140625, "blocks.4.attention.w_k": 0.2936439514160156, "blocks.4.attention.w_v": 0.3192787170410156, "blocks.4.attention.w_out": 0.3772773742675781, "blocks.4.feed_forward.w1": 0.3680572509765625, "blocks.4.feed_forward.w2": 0.3422482907772064, "blocks.4.feed_forward.w3": 0.3484128415584564, "blocks.5.attention.w_q": 0.30089569091796875, "blocks.5.attention.w_k": 0.29407501220703125, "blocks.5.attention.w_v": 0.2882881164550781, "blocks.5.attention.w_out": 0.3304710388183594, "blocks.5.feed_forward.w1": 0.37080511450767517, "blocks.5.feed_forward.w2": 0.3155568540096283, "blocks.5.feed_forward.w3": 0.3484153747558594, "blocks.6.attention.w_q": 0.3075218200683594, "blocks.6.attention.w_k": 0.29283905029296875, "blocks.6.attention.w_v": 0.27266693115234375, "blocks.6.attention.w_out": 0.2904548645019531, "blocks.6.feed_forward.w1": 0.3726336359977722, "blocks.6.feed_forward.w2": 0.2872060239315033, "blocks.6.feed_forward.w3": 0.35218557715415955, "blocks.7.attention.w_q": 0.31366729736328125, "blocks.7.attention.w_k": 0.2944793701171875, "blocks.7.attention.w_v": 0.2936668395996094, "blocks.7.attention.w_out": 0.2633552551269531, "blocks.7.feed_forward.w1": 0.3856773376464844, "blocks.7.feed_forward.w2": 0.2538134455680847, "blocks.7.feed_forward.w3": 0.3603477478027344, "blocks.8.attention.w_q": 0.3175392150878906, "blocks.8.attention.w_k": 0.3082466125488281, "blocks.8.attention.w_v": 0.3041419982910156, "blocks.8.attention.w_out": 0.2516288757324219, "blocks.8.feed_forward.w1": 0.39965057373046875, "blocks.8.feed_forward.w2": 0.21142959594726562, "blocks.8.feed_forward.w3": 0.36458462476730347}} +{"step": 6500, "changed_frac": 0.32173218686356503, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.239715576171875, "blocks.0.attention.w_k": 0.23251724243164062, "blocks.0.attention.w_v": 0.2900428771972656, "blocks.0.attention.w_out": 0.3809089660644531, "blocks.0.feed_forward.w1": 0.34778594970703125, "blocks.0.feed_forward.w2": 0.3448728024959564, "blocks.0.feed_forward.w3": 0.3440946042537689, "blocks.1.attention.w_q": 0.313934326171875, "blocks.1.attention.w_k": 0.29097747802734375, "blocks.1.attention.w_v": 0.3499183654785156, "blocks.1.attention.w_out": 0.4563179016113281, "blocks.1.feed_forward.w1": 0.3344408869743347, "blocks.1.feed_forward.w2": 0.33679327368736267, "blocks.1.feed_forward.w3": 0.3310279846191406, "blocks.2.attention.w_q": 0.2963218688964844, "blocks.2.attention.w_k": 0.2782745361328125, "blocks.2.attention.w_v": 0.3121070861816406, "blocks.2.attention.w_out": 0.4128837585449219, "blocks.2.feed_forward.w1": 0.342529296875, "blocks.2.feed_forward.w2": 0.3404045104980469, "blocks.2.feed_forward.w3": 0.33481472730636597, "blocks.3.attention.w_q": 0.279327392578125, "blocks.3.attention.w_k": 0.2675323486328125, "blocks.3.attention.w_v": 0.294158935546875, "blocks.3.attention.w_out": 0.3862152099609375, "blocks.3.feed_forward.w1": 0.34712982177734375, "blocks.3.feed_forward.w2": 0.33705776929855347, "blocks.3.feed_forward.w3": 0.3336842954158783, "blocks.4.attention.w_q": 0.3049201965332031, "blocks.4.attention.w_k": 0.278106689453125, "blocks.4.attention.w_v": 0.3039054870605469, "blocks.4.attention.w_out": 0.36225128173828125, "blocks.4.feed_forward.w1": 0.3514048457145691, "blocks.4.feed_forward.w2": 0.3271484375, "blocks.4.feed_forward.w3": 0.33576712012290955, "blocks.5.attention.w_q": 0.2844505310058594, "blocks.5.attention.w_k": 0.27973175048828125, "blocks.5.attention.w_v": 0.2739601135253906, "blocks.5.attention.w_out": 0.31633758544921875, "blocks.5.feed_forward.w1": 0.35568365454673767, "blocks.5.feed_forward.w2": 0.3016141355037689, "blocks.5.feed_forward.w3": 0.3334299921989441, "blocks.6.attention.w_q": 0.2909889221191406, "blocks.6.attention.w_k": 0.27950286865234375, "blocks.6.attention.w_v": 0.25824737548828125, "blocks.6.attention.w_out": 0.279266357421875, "blocks.6.feed_forward.w1": 0.3569933772087097, "blocks.6.feed_forward.w2": 0.27280935645103455, "blocks.6.feed_forward.w3": 0.3386472165584564, "blocks.7.attention.w_q": 0.299346923828125, "blocks.7.attention.w_k": 0.2805976867675781, "blocks.7.attention.w_v": 0.283538818359375, "blocks.7.attention.w_out": 0.2521247863769531, "blocks.7.feed_forward.w1": 0.3686714172363281, "blocks.7.feed_forward.w2": 0.2416788786649704, "blocks.7.feed_forward.w3": 0.3439750671386719, "blocks.8.attention.w_q": 0.3031883239746094, "blocks.8.attention.w_k": 0.2942047119140625, "blocks.8.attention.w_v": 0.2880516052246094, "blocks.8.attention.w_out": 0.24114227294921875, "blocks.8.feed_forward.w1": 0.3843371272087097, "blocks.8.feed_forward.w2": 0.20178350806236267, "blocks.8.feed_forward.w3": 0.34730276465415955}} +{"step": 6600, "changed_frac": 0.30951273339426416, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2278289794921875, "blocks.0.attention.w_k": 0.2239227294921875, "blocks.0.attention.w_v": 0.2806205749511719, "blocks.0.attention.w_out": 0.3699188232421875, "blocks.0.feed_forward.w1": 0.3342539668083191, "blocks.0.feed_forward.w2": 0.33174771070480347, "blocks.0.feed_forward.w3": 0.3303616940975189, "blocks.1.attention.w_q": 0.3008995056152344, "blocks.1.attention.w_k": 0.27866363525390625, "blocks.1.attention.w_v": 0.3328208923339844, "blocks.1.attention.w_out": 0.4422340393066406, "blocks.1.feed_forward.w1": 0.32018154859542847, "blocks.1.feed_forward.w2": 0.32588323950767517, "blocks.1.feed_forward.w3": 0.3185157775878906, "blocks.2.attention.w_q": 0.28295135498046875, "blocks.2.attention.w_k": 0.267059326171875, "blocks.2.attention.w_v": 0.3022308349609375, "blocks.2.attention.w_out": 0.39752960205078125, "blocks.2.feed_forward.w1": 0.32857006788253784, "blocks.2.feed_forward.w2": 0.3264007568359375, "blocks.2.feed_forward.w3": 0.32108816504478455, "blocks.3.attention.w_q": 0.2697181701660156, "blocks.3.attention.w_k": 0.2566490173339844, "blocks.3.attention.w_v": 0.28192138671875, "blocks.3.attention.w_out": 0.3702850341796875, "blocks.3.feed_forward.w1": 0.33457058668136597, "blocks.3.feed_forward.w2": 0.32526907324790955, "blocks.3.feed_forward.w3": 0.3210894465446472, "blocks.4.attention.w_q": 0.2930946350097656, "blocks.4.attention.w_k": 0.2672843933105469, "blocks.4.attention.w_v": 0.29218292236328125, "blocks.4.attention.w_out": 0.3499183654785156, "blocks.4.feed_forward.w1": 0.33920416235923767, "blocks.4.feed_forward.w2": 0.3151702880859375, "blocks.4.feed_forward.w3": 0.3216056823730469, "blocks.5.attention.w_q": 0.2727470397949219, "blocks.5.attention.w_k": 0.2690086364746094, "blocks.5.attention.w_v": 0.2618141174316406, "blocks.5.attention.w_out": 0.30381011962890625, "blocks.5.feed_forward.w1": 0.3423296809196472, "blocks.5.feed_forward.w2": 0.29110464453697205, "blocks.5.feed_forward.w3": 0.32211431860923767, "blocks.6.attention.w_q": 0.2805938720703125, "blocks.6.attention.w_k": 0.2661933898925781, "blocks.6.attention.w_v": 0.24851608276367188, "blocks.6.attention.w_out": 0.2672538757324219, "blocks.6.feed_forward.w1": 0.34433621168136597, "blocks.6.feed_forward.w2": 0.26346588134765625, "blocks.6.feed_forward.w3": 0.32489269971847534, "blocks.7.attention.w_q": 0.28554534912109375, "blocks.7.attention.w_k": 0.268707275390625, "blocks.7.attention.w_v": 0.2736701965332031, "blocks.7.attention.w_out": 0.24179458618164062, "blocks.7.feed_forward.w1": 0.3563168942928314, "blocks.7.feed_forward.w2": 0.2327524870634079, "blocks.7.feed_forward.w3": 0.33145269751548767, "blocks.8.attention.w_q": 0.28923797607421875, "blocks.8.attention.w_k": 0.28026580810546875, "blocks.8.attention.w_v": 0.27918243408203125, "blocks.8.attention.w_out": 0.23152542114257812, "blocks.8.feed_forward.w1": 0.3702812194824219, "blocks.8.feed_forward.w2": 0.1950937956571579, "blocks.8.feed_forward.w3": 0.3337440490722656}} +{"step": 6700, "changed_frac": 0.2993337843153212, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.22118377685546875, "blocks.0.attention.w_k": 0.21380615234375, "blocks.0.attention.w_v": 0.27016448974609375, "blocks.0.attention.w_out": 0.3590850830078125, "blocks.0.feed_forward.w1": 0.3239848017692566, "blocks.0.feed_forward.w2": 0.32254156470298767, "blocks.0.feed_forward.w3": 0.3205820918083191, "blocks.1.attention.w_q": 0.290313720703125, "blocks.1.attention.w_k": 0.2700958251953125, "blocks.1.attention.w_v": 0.3227119445800781, "blocks.1.attention.w_out": 0.4293098449707031, "blocks.1.feed_forward.w1": 0.3113822937011719, "blocks.1.feed_forward.w2": 0.315673828125, "blocks.1.feed_forward.w3": 0.3072662353515625, "blocks.2.attention.w_q": 0.2721138000488281, "blocks.2.attention.w_k": 0.2556343078613281, "blocks.2.attention.w_v": 0.2905921936035156, "blocks.2.attention.w_out": 0.3858489990234375, "blocks.2.feed_forward.w1": 0.3184611201286316, "blocks.2.feed_forward.w2": 0.3142852783203125, "blocks.2.feed_forward.w3": 0.30969876050949097, "blocks.3.attention.w_q": 0.25998687744140625, "blocks.3.attention.w_k": 0.246551513671875, "blocks.3.attention.w_v": 0.2730560302734375, "blocks.3.attention.w_out": 0.3606834411621094, "blocks.3.feed_forward.w1": 0.323760986328125, "blocks.3.feed_forward.w2": 0.31440991163253784, "blocks.3.feed_forward.w3": 0.3114738464355469, "blocks.4.attention.w_q": 0.283477783203125, "blocks.4.attention.w_k": 0.2548103332519531, "blocks.4.attention.w_v": 0.2826042175292969, "blocks.4.attention.w_out": 0.3386077880859375, "blocks.4.feed_forward.w1": 0.3268267512321472, "blocks.4.feed_forward.w2": 0.3047688901424408, "blocks.4.feed_forward.w3": 0.3115895688533783, "blocks.5.attention.w_q": 0.2634696960449219, "blocks.5.attention.w_k": 0.2600593566894531, "blocks.5.attention.w_v": 0.255157470703125, "blocks.5.attention.w_out": 0.2945404052734375, "blocks.5.feed_forward.w1": 0.3316904902458191, "blocks.5.feed_forward.w2": 0.28070068359375, "blocks.5.feed_forward.w3": 0.3114356994628906, "blocks.6.attention.w_q": 0.2683601379394531, "blocks.6.attention.w_k": 0.2569618225097656, "blocks.6.attention.w_v": 0.23910903930664062, "blocks.6.attention.w_out": 0.25980377197265625, "blocks.6.feed_forward.w1": 0.3328793942928314, "blocks.6.feed_forward.w2": 0.2559191584587097, "blocks.6.feed_forward.w3": 0.31468456983566284, "blocks.7.attention.w_q": 0.2736244201660156, "blocks.7.attention.w_k": 0.2571144104003906, "blocks.7.attention.w_v": 0.26213836669921875, "blocks.7.attention.w_out": 0.23341751098632812, "blocks.7.feed_forward.w1": 0.34392422437667847, "blocks.7.feed_forward.w2": 0.2247772216796875, "blocks.7.feed_forward.w3": 0.32056301832199097, "blocks.8.attention.w_q": 0.27846527099609375, "blocks.8.attention.w_k": 0.27042388916015625, "blocks.8.attention.w_v": 0.2691917419433594, "blocks.8.attention.w_out": 0.22208404541015625, "blocks.8.feed_forward.w1": 0.35951870679855347, "blocks.8.feed_forward.w2": 0.1878611296415329, "blocks.8.feed_forward.w3": 0.32517117261886597}} +{"step": 6800, "changed_frac": 0.29166503645416, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.21593856811523438, "blocks.0.attention.w_k": 0.2098388671875, "blocks.0.attention.w_v": 0.2653236389160156, "blocks.0.attention.w_out": 0.34942626953125, "blocks.0.feed_forward.w1": 0.3145179748535156, "blocks.0.feed_forward.w2": 0.3157157897949219, "blocks.0.feed_forward.w3": 0.3106269836425781, "blocks.1.attention.w_q": 0.28509521484375, "blocks.1.attention.w_k": 0.2594642639160156, "blocks.1.attention.w_v": 0.31450653076171875, "blocks.1.attention.w_out": 0.4213066101074219, "blocks.1.feed_forward.w1": 0.30170950293540955, "blocks.1.feed_forward.w2": 0.30795544385910034, "blocks.1.feed_forward.w3": 0.2995287775993347, "blocks.2.attention.w_q": 0.26488494873046875, "blocks.2.attention.w_k": 0.24814987182617188, "blocks.2.attention.w_v": 0.2845497131347656, "blocks.2.attention.w_out": 0.37592315673828125, "blocks.2.feed_forward.w1": 0.31069692969322205, "blocks.2.feed_forward.w2": 0.3087361752986908, "blocks.2.feed_forward.w3": 0.30104827880859375, "blocks.3.attention.w_q": 0.2513923645019531, "blocks.3.attention.w_k": 0.2403717041015625, "blocks.3.attention.w_v": 0.26508331298828125, "blocks.3.attention.w_out": 0.3511924743652344, "blocks.3.feed_forward.w1": 0.3148028254508972, "blocks.3.feed_forward.w2": 0.3084615170955658, "blocks.3.feed_forward.w3": 0.3018455505371094, "blocks.4.attention.w_q": 0.2735748291015625, "blocks.4.attention.w_k": 0.24791336059570312, "blocks.4.attention.w_v": 0.2775306701660156, "blocks.4.attention.w_out": 0.330841064453125, "blocks.4.feed_forward.w1": 0.3202107846736908, "blocks.4.feed_forward.w2": 0.29663342237472534, "blocks.4.feed_forward.w3": 0.3029022216796875, "blocks.5.attention.w_q": 0.2557563781738281, "blocks.5.attention.w_k": 0.2505035400390625, "blocks.5.attention.w_v": 0.24887847900390625, "blocks.5.attention.w_out": 0.2862434387207031, "blocks.5.feed_forward.w1": 0.3236287534236908, "blocks.5.feed_forward.w2": 0.27418264746665955, "blocks.5.feed_forward.w3": 0.3019549250602722, "blocks.6.attention.w_q": 0.2611045837402344, "blocks.6.attention.w_k": 0.249847412109375, "blocks.6.attention.w_v": 0.23475265502929688, "blocks.6.attention.w_out": 0.2536811828613281, "blocks.6.feed_forward.w1": 0.3252906799316406, "blocks.6.feed_forward.w2": 0.24840164184570312, "blocks.6.feed_forward.w3": 0.306915283203125, "blocks.7.attention.w_q": 0.26743316650390625, "blocks.7.attention.w_k": 0.24808883666992188, "blocks.7.attention.w_v": 0.2583961486816406, "blocks.7.attention.w_out": 0.22993850708007812, "blocks.7.feed_forward.w1": 0.3358256220817566, "blocks.7.feed_forward.w2": 0.21869787573814392, "blocks.7.feed_forward.w3": 0.3122596740722656, "blocks.8.attention.w_q": 0.271575927734375, "blocks.8.attention.w_k": 0.262725830078125, "blocks.8.attention.w_v": 0.2634010314941406, "blocks.8.attention.w_out": 0.21613311767578125, "blocks.8.feed_forward.w1": 0.35072964429855347, "blocks.8.feed_forward.w2": 0.1818084716796875, "blocks.8.feed_forward.w3": 0.31626003980636597}} +{"step": 6900, "changed_frac": 0.2861067617041433, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.21222305297851562, "blocks.0.attention.w_k": 0.20557022094726562, "blocks.0.attention.w_v": 0.2613792419433594, "blocks.0.attention.w_out": 0.34491729736328125, "blocks.0.feed_forward.w1": 0.3107045590877533, "blocks.0.feed_forward.w2": 0.3091545104980469, "blocks.0.feed_forward.w3": 0.3056437373161316, "blocks.1.attention.w_q": 0.2764472961425781, "blocks.1.attention.w_k": 0.25489044189453125, "blocks.1.attention.w_v": 0.30554962158203125, "blocks.1.attention.w_out": 0.41291046142578125, "blocks.1.feed_forward.w1": 0.2953681945800781, "blocks.1.feed_forward.w2": 0.3031094968318939, "blocks.1.feed_forward.w3": 0.29426321387290955, "blocks.2.attention.w_q": 0.2594032287597656, "blocks.2.attention.w_k": 0.24259567260742188, "blocks.2.attention.w_v": 0.2758293151855469, "blocks.2.attention.w_out": 0.3698539733886719, "blocks.2.feed_forward.w1": 0.30496472120285034, "blocks.2.feed_forward.w2": 0.30180996656417847, "blocks.2.feed_forward.w3": 0.2959403991699219, "blocks.3.attention.w_q": 0.244873046875, "blocks.3.attention.w_k": 0.23235702514648438, "blocks.3.attention.w_v": 0.2596473693847656, "blocks.3.attention.w_out": 0.342132568359375, "blocks.3.feed_forward.w1": 0.30950164794921875, "blocks.3.feed_forward.w2": 0.3007609248161316, "blocks.3.feed_forward.w3": 0.29778164625167847, "blocks.4.attention.w_q": 0.2686958312988281, "blocks.4.attention.w_k": 0.24244308471679688, "blocks.4.attention.w_v": 0.2682151794433594, "blocks.4.attention.w_out": 0.3215980529785156, "blocks.4.feed_forward.w1": 0.3138694763183594, "blocks.4.feed_forward.w2": 0.2911008298397064, "blocks.4.feed_forward.w3": 0.29766845703125, "blocks.5.attention.w_q": 0.25054168701171875, "blocks.5.attention.w_k": 0.24657440185546875, "blocks.5.attention.w_v": 0.24252700805664062, "blocks.5.attention.w_out": 0.2813606262207031, "blocks.5.feed_forward.w1": 0.31683349609375, "blocks.5.feed_forward.w2": 0.26936087012290955, "blocks.5.feed_forward.w3": 0.29708099365234375, "blocks.6.attention.w_q": 0.2574043273925781, "blocks.6.attention.w_k": 0.24574661254882812, "blocks.6.attention.w_v": 0.22888946533203125, "blocks.6.attention.w_out": 0.2519989013671875, "blocks.6.feed_forward.w1": 0.31763333082199097, "blocks.6.feed_forward.w2": 0.24437586963176727, "blocks.6.feed_forward.w3": 0.30060577392578125, "blocks.7.attention.w_q": 0.260467529296875, "blocks.7.attention.w_k": 0.24345779418945312, "blocks.7.attention.w_v": 0.2523612976074219, "blocks.7.attention.w_out": 0.22423934936523438, "blocks.7.feed_forward.w1": 0.32989501953125, "blocks.7.feed_forward.w2": 0.21411769092082977, "blocks.7.feed_forward.w3": 0.3078816831111908, "blocks.8.attention.w_q": 0.26470947265625, "blocks.8.attention.w_k": 0.2572975158691406, "blocks.8.attention.w_v": 0.2589263916015625, "blocks.8.attention.w_out": 0.21462249755859375, "blocks.8.feed_forward.w1": 0.3440348505973816, "blocks.8.feed_forward.w2": 0.17977523803710938, "blocks.8.feed_forward.w3": 0.3107083737850189}} +{"step": 7000, "changed_frac": 0.2839681837293837, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.208709716796875, "blocks.0.attention.w_k": 0.20327377319335938, "blocks.0.attention.w_v": 0.25859832763671875, "blocks.0.attention.w_out": 0.3432121276855469, "blocks.0.feed_forward.w1": 0.30540212988853455, "blocks.0.feed_forward.w2": 0.30728405714035034, "blocks.0.feed_forward.w3": 0.30310314893722534, "blocks.1.attention.w_q": 0.2757225036621094, "blocks.1.attention.w_k": 0.2529335021972656, "blocks.1.attention.w_v": 0.3038902282714844, "blocks.1.attention.w_out": 0.4119682312011719, "blocks.1.feed_forward.w1": 0.2942466735839844, "blocks.1.feed_forward.w2": 0.30193328857421875, "blocks.1.feed_forward.w3": 0.2932014465332031, "blocks.2.attention.w_q": 0.25569915771484375, "blocks.2.attention.w_k": 0.23989486694335938, "blocks.2.attention.w_v": 0.2746620178222656, "blocks.2.attention.w_out": 0.3667716979980469, "blocks.2.feed_forward.w1": 0.3012275695800781, "blocks.2.feed_forward.w2": 0.30131277441978455, "blocks.2.feed_forward.w3": 0.29480236768722534, "blocks.3.attention.w_q": 0.24361801147460938, "blocks.3.attention.w_k": 0.23236846923828125, "blocks.3.attention.w_v": 0.2588157653808594, "blocks.3.attention.w_out": 0.342742919921875, "blocks.3.feed_forward.w1": 0.30725225806236267, "blocks.3.feed_forward.w2": 0.29902905225753784, "blocks.3.feed_forward.w3": 0.2955220639705658, "blocks.4.attention.w_q": 0.2647743225097656, "blocks.4.attention.w_k": 0.24061965942382812, "blocks.4.attention.w_v": 0.2666358947753906, "blocks.4.attention.w_out": 0.32117462158203125, "blocks.4.feed_forward.w1": 0.31217193603515625, "blocks.4.feed_forward.w2": 0.2894083857536316, "blocks.4.feed_forward.w3": 0.29468536376953125, "blocks.5.attention.w_q": 0.247833251953125, "blocks.5.attention.w_k": 0.24391555786132812, "blocks.5.attention.w_v": 0.240081787109375, "blocks.5.attention.w_out": 0.28148651123046875, "blocks.5.feed_forward.w1": 0.3154042661190033, "blocks.5.feed_forward.w2": 0.2669944763183594, "blocks.5.feed_forward.w3": 0.2956695556640625, "blocks.6.attention.w_q": 0.2542686462402344, "blocks.6.attention.w_k": 0.23987960815429688, "blocks.6.attention.w_v": 0.22716522216796875, "blocks.6.attention.w_out": 0.24728775024414062, "blocks.6.feed_forward.w1": 0.3174247741699219, "blocks.6.feed_forward.w2": 0.24212265014648438, "blocks.6.feed_forward.w3": 0.2980308532714844, "blocks.7.attention.w_q": 0.2589225769042969, "blocks.7.attention.w_k": 0.2412109375, "blocks.7.attention.w_v": 0.2510948181152344, "blocks.7.attention.w_out": 0.22347640991210938, "blocks.7.feed_forward.w1": 0.3257598876953125, "blocks.7.feed_forward.w2": 0.21302413940429688, "blocks.7.feed_forward.w3": 0.30467861890792847, "blocks.8.attention.w_q": 0.2630767822265625, "blocks.8.attention.w_k": 0.2538948059082031, "blocks.8.attention.w_v": 0.25632476806640625, "blocks.8.attention.w_out": 0.21089553833007812, "blocks.8.feed_forward.w1": 0.3406028747558594, "blocks.8.feed_forward.w2": 0.17910894751548767, "blocks.8.feed_forward.w3": 0.30638885498046875}} diff --git a/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..c7483182844be5dc0775eccef9d45d02ce95a931 --- /dev/null +++ b/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6C-muon-lr5.0e-03-alr2.4e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03374125435948372, + "blocks.0.attention.w_out.weight": 0.017394229769706726, + "blocks.0.attention.w_q.weight": 0.03317985311150551, + "blocks.0.attention.w_v.weight": 0.026263536885380745, + "blocks.0.feed_forward.w1.weight": 0.024344122037291527, + "blocks.0.feed_forward.w2.weight": 0.011206123977899551, + "blocks.0.feed_forward.w3.weight": 0.024652931839227676, + "blocks.1.attention.w_k.weight": 0.026402069255709648, + "blocks.1.attention.w_out.weight": 0.014104348607361317, + "blocks.1.attention.w_q.weight": 0.023821039125323296, + "blocks.1.attention.w_v.weight": 0.02094453014433384, + "blocks.1.feed_forward.w1.weight": 0.02458595111966133, + "blocks.1.feed_forward.w2.weight": 0.011814985424280167, + "blocks.1.feed_forward.w3.weight": 0.02455621026456356, + "blocks.2.attention.w_k.weight": 0.028968648985028267, + "blocks.2.attention.w_out.weight": 0.016764895990490913, + "blocks.2.attention.w_q.weight": 0.026901163160800934, + "blocks.2.attention.w_v.weight": 0.02347755990922451, + "blocks.2.feed_forward.w1.weight": 0.024022642523050308, + "blocks.2.feed_forward.w2.weight": 0.01216219738125801, + "blocks.2.feed_forward.w3.weight": 0.024543974548578262, + "blocks.3.attention.w_k.weight": 0.03164133429527283, + "blocks.3.attention.w_out.weight": 0.018422555178403854, + "blocks.3.attention.w_q.weight": 0.029934294521808624, + "blocks.3.attention.w_v.weight": 0.025081804022192955, + "blocks.3.feed_forward.w1.weight": 0.02378128282725811, + "blocks.3.feed_forward.w2.weight": 0.012583448551595211, + "blocks.3.feed_forward.w3.weight": 0.024553921073675156, + "blocks.4.attention.w_k.weight": 0.029081791639328003, + "blocks.4.attention.w_out.weight": 0.01952558383345604, + "blocks.4.attention.w_q.weight": 0.026170069351792336, + "blocks.4.attention.w_v.weight": 0.023879699409008026, + "blocks.4.feed_forward.w1.weight": 0.02329259365797043, + "blocks.4.feed_forward.w2.weight": 0.013250941410660744, + "blocks.4.feed_forward.w3.weight": 0.024505294859409332, + "blocks.5.attention.w_k.weight": 0.031023316085338593, + "blocks.5.attention.w_out.weight": 0.023192288354039192, + "blocks.5.attention.w_q.weight": 0.03111259452998638, + "blocks.5.attention.w_v.weight": 0.02695506066083908, + "blocks.5.feed_forward.w1.weight": 0.02366304211318493, + "blocks.5.feed_forward.w2.weight": 0.014528370462357998, + "blocks.5.feed_forward.w3.weight": 0.024949010461568832, + "blocks.6.attention.w_k.weight": 0.030626974999904633, + "blocks.6.attention.w_out.weight": 0.025394974276423454, + "blocks.6.attention.w_q.weight": 0.029888534918427467, + "blocks.6.attention.w_v.weight": 0.02883816510438919, + "blocks.6.feed_forward.w1.weight": 0.023631572723388672, + "blocks.6.feed_forward.w2.weight": 0.0165004450827837, + "blocks.6.feed_forward.w3.weight": 0.02496545948088169, + "blocks.7.attention.w_k.weight": 0.030607029795646667, + "blocks.7.attention.w_out.weight": 0.02945161797106266, + "blocks.7.attention.w_q.weight": 0.02897716499865055, + "blocks.7.attention.w_v.weight": 0.026108313351869583, + "blocks.7.feed_forward.w1.weight": 0.0230444073677063, + "blocks.7.feed_forward.w2.weight": 0.019285865128040314, + "blocks.7.feed_forward.w3.weight": 0.02466016262769699, + "blocks.8.attention.w_k.weight": 0.02949461340904236, + "blocks.8.attention.w_out.weight": 0.032729774713516235, + "blocks.8.attention.w_q.weight": 0.028002003207802773, + "blocks.8.attention.w_v.weight": 0.025902429595589638, + "blocks.8.feed_forward.w1.weight": 0.021963126957416534, + "blocks.8.feed_forward.w2.weight": 0.023389209061861038, + "blocks.8.feed_forward.w3.weight": 0.024073218926787376 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.77718177596004 + }, + "base_loss": 3.77718177596004, + "loss": { + "0.005": [ + 3.777504953312541, + 3.7774433538038537, + 3.777360502562477 + ], + "0.01": [ + 3.777823521949816, + 3.777388272223491, + 3.777599467185572 + ], + "0.02": [ + 3.778442189152565, + 3.778372976265414, + 3.7786784464488887 + ], + "0.05": [ + 3.7891684065544724, + 3.787226580340188, + 3.7887021360276147 + ], + "0.1": [ + 3.8300688293231095, + 3.823862180979881, + 3.825551228162358 + ], + "0.15": [ + 3.907008184514867, + 3.8932372221375076, + 3.8947257902076524 + ], + "0.2": [ + 4.036798560584778, + 4.010163742878948, + 4.009202321064793 + ], + "0.3": [ + 4.555328604368981, + 4.474308806955896, + 4.466747646253235 + ], + "0.4": [ + 5.598485333612767, + 5.369444471569876, + 5.425180880726888 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.777504953312541, + 3.7774433538038537, + 3.777360502562477 + ], + "0.01": [ + 3.777823521949816, + 3.777388272223491, + 3.777599467185572 + ], + "0.02": [ + 3.778442189152565, + 3.778372976265414, + 3.7786784464488887 + ], + "0.05": [ + 3.7891684065544724, + 3.787226580340188, + 3.7887021360276147 + ], + "0.1": [ + 3.8300688293231095, + 3.823862180979881, + 3.825551228162358 + ], + "0.15": [ + 3.907008184514867, + 3.8932372221375076, + 3.8947257902076524 + ], + "0.2": [ + 4.036798560584778, + 4.010163742878948, + 4.009202321064793 + ], + "0.3": [ + 4.555328604368981, + 4.474308806955896, + 4.466747646253235 + ], + "0.4": [ + 5.598485333612767, + 5.369444471569876, + 5.425180880726888 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.00025449393291721023, + "0.01": 0.0004219778262530838, + "0.02": 0.0013160946622488012, + "0.05": 0.01118393168071874, + "0.1": 0.04931230352840954, + "0.15": 0.12114195632663582, + "0.2": 0.241539765549466, + "0.3": 0.7216132432326647, + "0.4": 1.6871884526764704 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.00025449393291721023, + "0.01": 0.0004219778262530838, + "0.02": 0.0013160946622488012, + "0.05": 0.01118393168071874, + "0.1": 0.04931230352840954, + "0.15": 0.12114195632663582, + "0.2": 0.241539765549466, + "0.3": 0.7216132432326647, + "0.4": 1.6871884526764704 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 94.9 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/bf16_start.json b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..c101e78786068cb2a7a0357557d9f9e661272f90 --- /dev/null +++ b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.868069196517996, + "heldout_cc_en_tail_4M": 4.450294431703181 + }, + "loss_bf16_master": 3.868069196517996, + "losses_eval_model": { + "val_dolma_cc": 3.929202462064875 + }, + "bf16_minus_quant": -0.061133265546879034, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-t5-32", + "gpu": "NVIDIA L40S", + "t_s": 18.2 +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/finalize.json b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bb5c60ddfa73c3f6dbc62ce3692bd9ccf1c8e1 --- /dev/null +++ b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/finalize.json @@ -0,0 +1,100 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101", + "arm": "D", + "trust_mode": "quest", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.929202462064875 + }, + "losses_eval_model": { + "val_dolma_cc": 3.929202462064875 + }, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.030829403549432755, + "blocks.0.attention.w_k": 0.03138097748160362, + "blocks.0.attention.w_v": 0.03173093497753143, + "blocks.0.attention.w_out": 0.035128153860569, + "blocks.0.feed_forward.w1": 0.03041580319404602, + "blocks.0.feed_forward.w2": 0.03616543114185333, + "blocks.0.feed_forward.w3": 0.030544783920049667, + "blocks.1.attention.w_q": 0.030408073216676712, + "blocks.1.attention.w_k": 0.03193686157464981, + "blocks.1.attention.w_v": 0.03291736915707588, + "blocks.1.attention.w_out": 0.03405354171991348, + "blocks.1.feed_forward.w1": 0.03065265715122223, + "blocks.1.feed_forward.w2": 0.03504412621259689, + "blocks.1.feed_forward.w3": 0.030772464349865913, + "blocks.2.attention.w_q": 0.0304478257894516, + "blocks.2.attention.w_k": 0.033165160566568375, + "blocks.2.attention.w_v": 0.03176955506205559, + "blocks.2.attention.w_out": 0.036036714911460876, + "blocks.2.feed_forward.w1": 0.03044271469116211, + "blocks.2.feed_forward.w2": 0.034600503742694855, + "blocks.2.feed_forward.w3": 0.030636915937066078, + "blocks.3.attention.w_q": 0.03080570138990879, + "blocks.3.attention.w_k": 0.034662649035453796, + "blocks.3.attention.w_v": 0.03191298246383667, + "blocks.3.attention.w_out": 0.03563730791211128, + "blocks.3.feed_forward.w1": 0.03042546659708023, + "blocks.3.feed_forward.w2": 0.034708838909864426, + "blocks.3.feed_forward.w3": 0.030459171161055565, + "blocks.4.attention.w_q": 0.030570467934012413, + "blocks.4.attention.w_k": 0.03665788844227791, + "blocks.4.attention.w_v": 0.03221665695309639, + "blocks.4.attention.w_out": 0.030736306682229042, + "blocks.4.feed_forward.w1": 0.030367029830813408, + "blocks.4.feed_forward.w2": 0.035368699580430984, + "blocks.4.feed_forward.w3": 0.03075707145035267, + "blocks.5.attention.w_q": 0.030523749068379402, + "blocks.5.attention.w_k": 0.036643169820308685, + "blocks.5.attention.w_v": 0.03178601339459419, + "blocks.5.attention.w_out": 0.03139742091298103, + "blocks.5.feed_forward.w1": 0.03046942874789238, + "blocks.5.feed_forward.w2": 0.034888070076704025, + "blocks.5.feed_forward.w3": 0.030534150078892708, + "blocks.6.attention.w_q": 0.030377991497516632, + "blocks.6.attention.w_k": 0.03384949266910553, + "blocks.6.attention.w_v": 0.03119940496981144, + "blocks.6.attention.w_out": 0.03862449899315834, + "blocks.6.feed_forward.w1": 0.030582811683416367, + "blocks.6.feed_forward.w2": 0.034286051988601685, + "blocks.6.feed_forward.w3": 0.030554648488759995, + "blocks.7.attention.w_q": 0.03031197190284729, + "blocks.7.attention.w_k": 0.03283331170678139, + "blocks.7.attention.w_v": 0.03150441497564316, + "blocks.7.attention.w_out": 0.03711240738630295, + "blocks.7.feed_forward.w1": 0.030440548434853554, + "blocks.7.feed_forward.w2": 0.03637996315956116, + "blocks.7.feed_forward.w3": 0.030583521351218224, + "blocks.8.attention.w_q": 0.030546970665454865, + "blocks.8.attention.w_k": 0.03139302879571915, + "blocks.8.attention.w_v": 0.03195328637957573, + "blocks.8.attention.w_out": 0.03745830059051514, + "blocks.8.feed_forward.w1": 0.030681632459163666, + "blocks.8.feed_forward.w2": 0.03934292495250702, + "blocks.8.feed_forward.w3": 0.030482586473226547 + }, + "rel_quant_noise_mean": 0.03255726955831051, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 1.0702641010284424, + "loss_eval_model": 3.929202462064875, + "checks": { + "snap_reproduces_fakequant": true, + "weights_on_arm_grid": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/launch.yaml b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c689d4f04c202f00208163639414224092885098 --- /dev/null +++ b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/launch.yaml @@ -0,0 +1,126 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 101 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.quest_train_module.QuestTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + fq_arm: D + fq_trust_mode: quest + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: olmo_core.optim.AdamWConfig + lr: 0.0012 + betas: + - 0.9 + - 0.95 + eps: 1.0e-15 + weight_decay: 0.1 + fused: true + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 701 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1007019 + save_async: false + osc_probe: + _CLASS_: precq.quest_callbacks.OscillationProbeCallback + interval: 100 + bits: 6 + arm: D + trust_mode: quest +wandb: + enabled: false +init_seed: 101 +n_tokens: 1839988736 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/osc_probe.jsonl b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/osc_probe.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..06a04b10f854c81a7b79a30ee738dc2ca750bdc9 --- /dev/null +++ b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/osc_probe.jsonl @@ -0,0 +1,72 @@ +{"start": true, "step": 0, "arm": "D", "bits": 6, "interval": 100} +{"step": 0} +{"step": 100, "changed_frac": 0.4817906241131644, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7852210998535156, "blocks.0.attention.w_k": 0.7768058776855469, "blocks.0.attention.w_v": 0.4585762023925781, "blocks.0.attention.w_out": 0.4543304443359375, "blocks.0.feed_forward.w1": 0.49022039771080017, "blocks.0.feed_forward.w2": 0.4890391230583191, "blocks.0.feed_forward.w3": 0.4933980405330658, "blocks.1.attention.w_q": 0.5579566955566406, "blocks.1.attention.w_k": 0.5486831665039062, "blocks.1.attention.w_v": 0.3992042541503906, "blocks.1.attention.w_out": 0.34156036376953125, "blocks.1.feed_forward.w1": 0.4749082028865814, "blocks.1.feed_forward.w2": 0.4763438105583191, "blocks.1.feed_forward.w3": 0.4754066467285156, "blocks.2.attention.w_q": 0.5386810302734375, "blocks.2.attention.w_k": 0.5375251770019531, "blocks.2.attention.w_v": 0.381866455078125, "blocks.2.attention.w_out": 0.34928131103515625, "blocks.2.feed_forward.w1": 0.473724365234375, "blocks.2.feed_forward.w2": 0.47306570410728455, "blocks.2.feed_forward.w3": 0.47480902075767517, "blocks.3.attention.w_q": 0.5502471923828125, "blocks.3.attention.w_k": 0.5286865234375, "blocks.3.attention.w_v": 0.3705558776855469, "blocks.3.attention.w_out": 0.37223052978515625, "blocks.3.feed_forward.w1": 0.4740956723690033, "blocks.3.feed_forward.w2": 0.480926513671875, "blocks.3.feed_forward.w3": 0.47579193115234375, "blocks.4.attention.w_q": 0.5368080139160156, "blocks.4.attention.w_k": 0.52874755859375, "blocks.4.attention.w_v": 0.3753242492675781, "blocks.4.attention.w_out": 0.39603424072265625, "blocks.4.feed_forward.w1": 0.4760068356990814, "blocks.4.feed_forward.w2": 0.4777514338493347, "blocks.4.feed_forward.w3": 0.47440338134765625, "blocks.5.attention.w_q": 0.545318603515625, "blocks.5.attention.w_k": 0.5422248840332031, "blocks.5.attention.w_v": 0.38057708740234375, "blocks.5.attention.w_out": 0.40692901611328125, "blocks.5.feed_forward.w1": 0.4816551208496094, "blocks.5.feed_forward.w2": 0.48330816626548767, "blocks.5.feed_forward.w3": 0.4746233820915222, "blocks.6.attention.w_q": 0.5472908020019531, "blocks.6.attention.w_k": 0.5527458190917969, "blocks.6.attention.w_v": 0.4025306701660156, "blocks.6.attention.w_out": 0.424591064453125, "blocks.6.feed_forward.w1": 0.4777323603630066, "blocks.6.feed_forward.w2": 0.49318188428878784, "blocks.6.feed_forward.w3": 0.47686895728111267, "blocks.7.attention.w_q": 0.6289215087890625, "blocks.7.attention.w_k": 0.6086349487304688, "blocks.7.attention.w_v": 0.41342926025390625, "blocks.7.attention.w_out": 0.4410552978515625, "blocks.7.feed_forward.w1": 0.47140631079673767, "blocks.7.feed_forward.w2": 0.4868215024471283, "blocks.7.feed_forward.w3": 0.47702154517173767, "blocks.8.attention.w_q": 0.5434379577636719, "blocks.8.attention.w_k": 0.5541458129882812, "blocks.8.attention.w_v": 0.40721893310546875, "blocks.8.attention.w_out": 0.44317626953125, "blocks.8.feed_forward.w1": 0.4704132080078125, "blocks.8.feed_forward.w2": 0.4804980158805847, "blocks.8.feed_forward.w3": 0.45956167578697205}} +{"step": 200, "changed_frac": 0.650450192964994, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8074760437011719, "blocks.0.attention.w_k": 0.7748489379882812, "blocks.0.attention.w_v": 0.5116844177246094, "blocks.0.attention.w_out": 0.4582328796386719, "blocks.0.feed_forward.w1": 0.5544548034667969, "blocks.0.feed_forward.w2": 0.536780059337616, "blocks.0.feed_forward.w3": 0.5393308401107788, "blocks.1.attention.w_q": 0.69720458984375, "blocks.1.attention.w_k": 0.6709442138671875, "blocks.1.attention.w_v": 0.5091934204101562, "blocks.1.attention.w_out": 0.4625434875488281, "blocks.1.feed_forward.w1": 0.5833066701889038, "blocks.1.feed_forward.w2": 0.5690053701400757, "blocks.1.feed_forward.w3": 0.5749320983886719, "blocks.2.attention.w_q": 0.7023696899414062, "blocks.2.attention.w_k": 0.6763496398925781, "blocks.2.attention.w_v": 0.5372657775878906, "blocks.2.attention.w_out": 0.5072441101074219, "blocks.2.feed_forward.w1": 0.6246833801269531, "blocks.2.feed_forward.w2": 0.60272216796875, "blocks.2.feed_forward.w3": 0.6094894409179688, "blocks.3.attention.w_q": 0.7560577392578125, "blocks.3.attention.w_k": 0.7121543884277344, "blocks.3.attention.w_v": 0.5608291625976562, "blocks.3.attention.w_out": 0.5733833312988281, "blocks.3.feed_forward.w1": 0.6553434133529663, "blocks.3.feed_forward.w2": 0.6346092224121094, "blocks.3.feed_forward.w3": 0.6407026052474976, "blocks.4.attention.w_q": 0.767242431640625, "blocks.4.attention.w_k": 0.7250747680664062, "blocks.4.attention.w_v": 0.5468597412109375, "blocks.4.attention.w_out": 0.5781822204589844, "blocks.4.feed_forward.w1": 0.6680984497070312, "blocks.4.feed_forward.w2": 0.6566518545150757, "blocks.4.feed_forward.w3": 0.651800811290741, "blocks.5.attention.w_q": 0.8098793029785156, "blocks.5.attention.w_k": 0.7699317932128906, "blocks.5.attention.w_v": 0.61260986328125, "blocks.5.attention.w_out": 0.65399169921875, "blocks.5.feed_forward.w1": 0.6774024963378906, "blocks.5.feed_forward.w2": 0.6682167053222656, "blocks.5.feed_forward.w3": 0.6651560664176941, "blocks.6.attention.w_q": 0.8161582946777344, "blocks.6.attention.w_k": 0.773223876953125, "blocks.6.attention.w_v": 0.6495819091796875, "blocks.6.attention.w_out": 0.7482261657714844, "blocks.6.feed_forward.w1": 0.6930376887321472, "blocks.6.feed_forward.w2": 0.698523223400116, "blocks.6.feed_forward.w3": 0.6734288930892944, "blocks.7.attention.w_q": 0.8043746948242188, "blocks.7.attention.w_k": 0.7541427612304688, "blocks.7.attention.w_v": 0.6548881530761719, "blocks.7.attention.w_out": 0.74176025390625, "blocks.7.feed_forward.w1": 0.6942622065544128, "blocks.7.feed_forward.w2": 0.7095350027084351, "blocks.7.feed_forward.w3": 0.6807836294174194, "blocks.8.attention.w_q": 0.7969856262207031, "blocks.8.attention.w_k": 0.7538833618164062, "blocks.8.attention.w_v": 0.5974349975585938, "blocks.8.attention.w_out": 0.6699943542480469, "blocks.8.feed_forward.w1": 0.6859105825424194, "blocks.8.feed_forward.w2": 0.7053439021110535, "blocks.8.feed_forward.w3": 0.6666437983512878}} +{"step": 300, "changed_frac": 0.6700374896709735, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7520980834960938, "blocks.0.attention.w_k": 0.6642303466796875, "blocks.0.attention.w_v": 0.44173431396484375, "blocks.0.attention.w_out": 0.4169883728027344, "blocks.0.feed_forward.w1": 0.5064913630485535, "blocks.0.feed_forward.w2": 0.5123800039291382, "blocks.0.feed_forward.w3": 0.48447293043136597, "blocks.1.attention.w_q": 0.6540908813476562, "blocks.1.attention.w_k": 0.6188430786132812, "blocks.1.attention.w_v": 0.5363922119140625, "blocks.1.attention.w_out": 0.5160675048828125, "blocks.1.feed_forward.w1": 0.5920817255973816, "blocks.1.feed_forward.w2": 0.5966415405273438, "blocks.1.feed_forward.w3": 0.578253448009491, "blocks.2.attention.w_q": 0.7043533325195312, "blocks.2.attention.w_k": 0.6632499694824219, "blocks.2.attention.w_v": 0.5677413940429688, "blocks.2.attention.w_out": 0.5848464965820312, "blocks.2.feed_forward.w1": 0.6384710073471069, "blocks.2.feed_forward.w2": 0.6250216364860535, "blocks.2.feed_forward.w3": 0.6209144592285156, "blocks.3.attention.w_q": 0.7287445068359375, "blocks.3.attention.w_k": 0.6639823913574219, "blocks.3.attention.w_v": 0.5963935852050781, "blocks.3.attention.w_out": 0.6724319458007812, "blocks.3.feed_forward.w1": 0.6821556091308594, "blocks.3.feed_forward.w2": 0.677581787109375, "blocks.3.feed_forward.w3": 0.6686884760856628, "blocks.4.attention.w_q": 0.7543449401855469, "blocks.4.attention.w_k": 0.6882743835449219, "blocks.4.attention.w_v": 0.5789642333984375, "blocks.4.attention.w_out": 0.6953811645507812, "blocks.4.feed_forward.w1": 0.6993917226791382, "blocks.4.feed_forward.w2": 0.6882743835449219, "blocks.4.feed_forward.w3": 0.6852442622184753, "blocks.5.attention.w_q": 0.7701225280761719, "blocks.5.attention.w_k": 0.6952018737792969, "blocks.5.attention.w_v": 0.5737686157226562, "blocks.5.attention.w_out": 0.7495079040527344, "blocks.5.feed_forward.w1": 0.7158699035644531, "blocks.5.feed_forward.w2": 0.7151578664779663, "blocks.5.feed_forward.w3": 0.7059008479118347, "blocks.6.attention.w_q": 0.7846450805664062, "blocks.6.attention.w_k": 0.7069473266601562, "blocks.6.attention.w_v": 0.5729713439941406, "blocks.6.attention.w_out": 0.8059577941894531, "blocks.6.feed_forward.w1": 0.7353922724723816, "blocks.6.feed_forward.w2": 0.7528470754623413, "blocks.6.feed_forward.w3": 0.7232984304428101, "blocks.7.attention.w_q": 0.7761039733886719, "blocks.7.attention.w_k": 0.6890792846679688, "blocks.7.attention.w_v": 0.6042022705078125, "blocks.7.attention.w_out": 0.8150444030761719, "blocks.7.feed_forward.w1": 0.7513834834098816, "blocks.7.feed_forward.w2": 0.786041259765625, "blocks.7.feed_forward.w3": 0.7421087026596069, "blocks.8.attention.w_q": 0.7798309326171875, "blocks.8.attention.w_k": 0.7012481689453125, "blocks.8.attention.w_v": 0.6162071228027344, "blocks.8.attention.w_out": 0.7932586669921875, "blocks.8.feed_forward.w1": 0.7494227290153503, "blocks.8.feed_forward.w2": 0.7812601923942566, "blocks.8.feed_forward.w3": 0.7389653921127319}} +{"step": 400, "changed_frac": 0.7143164414625901, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7518501281738281, "blocks.0.attention.w_k": 0.6444816589355469, "blocks.0.attention.w_v": 0.4642906188964844, "blocks.0.attention.w_out": 0.4570960998535156, "blocks.0.feed_forward.w1": 0.5512657165527344, "blocks.0.feed_forward.w2": 0.5915209650993347, "blocks.0.feed_forward.w3": 0.5312322378158569, "blocks.1.attention.w_q": 0.6928634643554688, "blocks.1.attention.w_k": 0.6452369689941406, "blocks.1.attention.w_v": 0.5826339721679688, "blocks.1.attention.w_out": 0.6154136657714844, "blocks.1.feed_forward.w1": 0.6608098745346069, "blocks.1.feed_forward.w2": 0.6731351613998413, "blocks.1.feed_forward.w3": 0.6464487910270691, "blocks.2.attention.w_q": 0.7308731079101562, "blocks.2.attention.w_k": 0.6857757568359375, "blocks.2.attention.w_v": 0.6117706298828125, "blocks.2.attention.w_out": 0.660980224609375, "blocks.2.feed_forward.w1": 0.6990687251091003, "blocks.2.feed_forward.w2": 0.7019755244255066, "blocks.2.feed_forward.w3": 0.683844268321991, "blocks.3.attention.w_q": 0.7705001831054688, "blocks.3.attention.w_k": 0.7047195434570312, "blocks.3.attention.w_v": 0.6217384338378906, "blocks.3.attention.w_out": 0.7552108764648438, "blocks.3.feed_forward.w1": 0.7340850830078125, "blocks.3.feed_forward.w2": 0.7427546381950378, "blocks.3.feed_forward.w3": 0.7264035940170288, "blocks.4.attention.w_q": 0.7893829345703125, "blocks.4.attention.w_k": 0.72021484375, "blocks.4.attention.w_v": 0.6118850708007812, "blocks.4.attention.w_out": 0.7591056823730469, "blocks.4.feed_forward.w1": 0.7482885122299194, "blocks.4.feed_forward.w2": 0.7485911250114441, "blocks.4.feed_forward.w3": 0.7370275259017944, "blocks.5.attention.w_q": 0.8009834289550781, "blocks.5.attention.w_k": 0.7344703674316406, "blocks.5.attention.w_v": 0.6127700805664062, "blocks.5.attention.w_out": 0.7880668640136719, "blocks.5.feed_forward.w1": 0.7623723745346069, "blocks.5.feed_forward.w2": 0.7718073725700378, "blocks.5.feed_forward.w3": 0.7570292353630066, "blocks.6.attention.w_q": 0.8025779724121094, "blocks.6.attention.w_k": 0.7412376403808594, "blocks.6.attention.w_v": 0.6064643859863281, "blocks.6.attention.w_out": 0.8023223876953125, "blocks.6.feed_forward.w1": 0.7682113647460938, "blocks.6.feed_forward.w2": 0.7879600524902344, "blocks.6.feed_forward.w3": 0.7606099843978882, "blocks.7.attention.w_q": 0.7997817993164062, "blocks.7.attention.w_k": 0.7195396423339844, "blocks.7.attention.w_v": 0.6176872253417969, "blocks.7.attention.w_out": 0.8233146667480469, "blocks.7.feed_forward.w1": 0.7794520258903503, "blocks.7.feed_forward.w2": 0.8099238276481628, "blocks.7.feed_forward.w3": 0.7703183889389038, "blocks.8.attention.w_q": 0.8166732788085938, "blocks.8.attention.w_k": 0.7405815124511719, "blocks.8.attention.w_v": 0.6326675415039062, "blocks.8.attention.w_out": 0.8219261169433594, "blocks.8.feed_forward.w1": 0.7739028930664062, "blocks.8.feed_forward.w2": 0.7950605154037476, "blocks.8.feed_forward.w3": 0.7662124633789062}} +{"step": 500, "changed_frac": 0.7514794830583099, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7608108520507812, "blocks.0.attention.w_k": 0.6778182983398438, "blocks.0.attention.w_v": 0.5093574523925781, "blocks.0.attention.w_out": 0.5150260925292969, "blocks.0.feed_forward.w1": 0.6065623164176941, "blocks.0.feed_forward.w2": 0.6706784963607788, "blocks.0.feed_forward.w3": 0.5864664912223816, "blocks.1.attention.w_q": 0.7393074035644531, "blocks.1.attention.w_k": 0.6826820373535156, "blocks.1.attention.w_v": 0.6242218017578125, "blocks.1.attention.w_out": 0.686279296875, "blocks.1.feed_forward.w1": 0.7175674438476562, "blocks.1.feed_forward.w2": 0.7396634817123413, "blocks.1.feed_forward.w3": 0.7041587829589844, "blocks.2.attention.w_q": 0.7608680725097656, "blocks.2.attention.w_k": 0.7144622802734375, "blocks.2.attention.w_v": 0.6452102661132812, "blocks.2.attention.w_out": 0.7164192199707031, "blocks.2.feed_forward.w1": 0.7488607168197632, "blocks.2.feed_forward.w2": 0.7631785273551941, "blocks.2.feed_forward.w3": 0.734527587890625, "blocks.3.attention.w_q": 0.7937393188476562, "blocks.3.attention.w_k": 0.7289619445800781, "blocks.3.attention.w_v": 0.65802001953125, "blocks.3.attention.w_out": 0.782440185546875, "blocks.3.feed_forward.w1": 0.7741686701774597, "blocks.3.feed_forward.w2": 0.7876065969467163, "blocks.3.feed_forward.w3": 0.7682317495346069, "blocks.4.attention.w_q": 0.8160972595214844, "blocks.4.attention.w_k": 0.7527885437011719, "blocks.4.attention.w_v": 0.64739990234375, "blocks.4.attention.w_out": 0.7818565368652344, "blocks.4.feed_forward.w1": 0.7869135737419128, "blocks.4.feed_forward.w2": 0.791968047618866, "blocks.4.feed_forward.w3": 0.7804946899414062, "blocks.5.attention.w_q": 0.82000732421875, "blocks.5.attention.w_k": 0.7517585754394531, "blocks.5.attention.w_v": 0.6461143493652344, "blocks.5.attention.w_out": 0.8015708923339844, "blocks.5.feed_forward.w1": 0.8000208735466003, "blocks.5.feed_forward.w2": 0.8061510920524597, "blocks.5.feed_forward.w3": 0.795928955078125, "blocks.6.attention.w_q": 0.8242607116699219, "blocks.6.attention.w_k": 0.7694587707519531, "blocks.6.attention.w_v": 0.6640968322753906, "blocks.6.attention.w_out": 0.8033790588378906, "blocks.6.feed_forward.w1": 0.7999992370605469, "blocks.6.feed_forward.w2": 0.8095411062240601, "blocks.6.feed_forward.w3": 0.7942136526107788, "blocks.7.attention.w_q": 0.8220672607421875, "blocks.7.attention.w_k": 0.7481422424316406, "blocks.7.attention.w_v": 0.6568794250488281, "blocks.7.attention.w_out": 0.8203392028808594, "blocks.7.feed_forward.w1": 0.8067347407341003, "blocks.7.feed_forward.w2": 0.8150660395622253, "blocks.7.feed_forward.w3": 0.801947295665741, "blocks.8.attention.w_q": 0.8349227905273438, "blocks.8.attention.w_k": 0.7688827514648438, "blocks.8.attention.w_v": 0.6739654541015625, "blocks.8.attention.w_out": 0.8319015502929688, "blocks.8.feed_forward.w1": 0.7952232360839844, "blocks.8.feed_forward.w2": 0.7887420654296875, "blocks.8.feed_forward.w3": 0.7892469167709351}} +{"step": 600, "changed_frac": 0.7837889255621494, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7727699279785156, "blocks.0.attention.w_k": 0.7026138305664062, "blocks.0.attention.w_v": 0.5875129699707031, "blocks.0.attention.w_out": 0.5929756164550781, "blocks.0.feed_forward.w1": 0.6737403869628906, "blocks.0.feed_forward.w2": 0.7436015009880066, "blocks.0.feed_forward.w3": 0.6577402949333191, "blocks.1.attention.w_q": 0.7790756225585938, "blocks.1.attention.w_k": 0.7237930297851562, "blocks.1.attention.w_v": 0.68304443359375, "blocks.1.attention.w_out": 0.7547607421875, "blocks.1.feed_forward.w1": 0.7680231928825378, "blocks.1.feed_forward.w2": 0.7938448786735535, "blocks.1.feed_forward.w3": 0.7575531005859375, "blocks.2.attention.w_q": 0.7900390625, "blocks.2.attention.w_k": 0.7403907775878906, "blocks.2.attention.w_v": 0.6926078796386719, "blocks.2.attention.w_out": 0.7629470825195312, "blocks.2.feed_forward.w1": 0.7885996699333191, "blocks.2.feed_forward.w2": 0.8064092397689819, "blocks.2.feed_forward.w3": 0.778479278087616, "blocks.3.attention.w_q": 0.8119888305664062, "blocks.3.attention.w_k": 0.7533721923828125, "blocks.3.attention.w_v": 0.7012939453125, "blocks.3.attention.w_out": 0.7968635559082031, "blocks.3.feed_forward.w1": 0.806036651134491, "blocks.3.feed_forward.w2": 0.8188972473144531, "blocks.3.feed_forward.w3": 0.8006681203842163, "blocks.4.attention.w_q": 0.828125, "blocks.4.attention.w_k": 0.7671394348144531, "blocks.4.attention.w_v": 0.6982803344726562, "blocks.4.attention.w_out": 0.79681396484375, "blocks.4.feed_forward.w1": 0.8147417902946472, "blocks.4.feed_forward.w2": 0.8169479370117188, "blocks.4.feed_forward.w3": 0.8092575073242188, "blocks.5.attention.w_q": 0.8305473327636719, "blocks.5.attention.w_k": 0.7692794799804688, "blocks.5.attention.w_v": 0.6887741088867188, "blocks.5.attention.w_out": 0.8096733093261719, "blocks.5.feed_forward.w1": 0.8251330256462097, "blocks.5.feed_forward.w2": 0.8261464834213257, "blocks.5.feed_forward.w3": 0.8220062255859375, "blocks.6.attention.w_q": 0.8372039794921875, "blocks.6.attention.w_k": 0.7907333374023438, "blocks.6.attention.w_v": 0.7102546691894531, "blocks.6.attention.w_out": 0.8117828369140625, "blocks.6.feed_forward.w1": 0.8236478567123413, "blocks.6.feed_forward.w2": 0.8244590759277344, "blocks.6.feed_forward.w3": 0.8205668330192566, "blocks.7.attention.w_q": 0.838470458984375, "blocks.7.attention.w_k": 0.7768020629882812, "blocks.7.attention.w_v": 0.7025680541992188, "blocks.7.attention.w_out": 0.8169898986816406, "blocks.7.feed_forward.w1": 0.8278173208236694, "blocks.7.feed_forward.w2": 0.8223928213119507, "blocks.7.feed_forward.w3": 0.8244056701660156, "blocks.8.attention.w_q": 0.8468666076660156, "blocks.8.attention.w_k": 0.7936439514160156, "blocks.8.attention.w_v": 0.7073631286621094, "blocks.8.attention.w_out": 0.8298072814941406, "blocks.8.feed_forward.w1": 0.8165206909179688, "blocks.8.feed_forward.w2": 0.7889569997787476, "blocks.8.feed_forward.w3": 0.8121185302734375}} +{"step": 700, "changed_frac": 0.8086752280210837, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8050460815429688, "blocks.0.attention.w_k": 0.7451438903808594, "blocks.0.attention.w_v": 0.6591911315917969, "blocks.0.attention.w_out": 0.6809272766113281, "blocks.0.feed_forward.w1": 0.7269389033317566, "blocks.0.feed_forward.w2": 0.7971267700195312, "blocks.0.feed_forward.w3": 0.719793975353241, "blocks.1.attention.w_q": 0.8137779235839844, "blocks.1.attention.w_k": 0.7627906799316406, "blocks.1.attention.w_v": 0.7074737548828125, "blocks.1.attention.w_out": 0.7966346740722656, "blocks.1.feed_forward.w1": 0.8000272512435913, "blocks.1.feed_forward.w2": 0.83056640625, "blocks.1.feed_forward.w3": 0.7934837341308594, "blocks.2.attention.w_q": 0.8208694458007812, "blocks.2.attention.w_k": 0.77471923828125, "blocks.2.attention.w_v": 0.7207489013671875, "blocks.2.attention.w_out": 0.8005142211914062, "blocks.2.feed_forward.w1": 0.8163846731185913, "blocks.2.feed_forward.w2": 0.8370450735092163, "blocks.2.feed_forward.w3": 0.8105367422103882, "blocks.3.attention.w_q": 0.8356361389160156, "blocks.3.attention.w_k": 0.7912788391113281, "blocks.3.attention.w_v": 0.7438545227050781, "blocks.3.attention.w_out": 0.80877685546875, "blocks.3.feed_forward.w1": 0.8272501826286316, "blocks.3.feed_forward.w2": 0.8403918147087097, "blocks.3.feed_forward.w3": 0.82452392578125, "blocks.4.attention.w_q": 0.839569091796875, "blocks.4.attention.w_k": 0.7872276306152344, "blocks.4.attention.w_v": 0.7252082824707031, "blocks.4.attention.w_out": 0.8079910278320312, "blocks.4.feed_forward.w1": 0.8338343501091003, "blocks.4.feed_forward.w2": 0.8363279104232788, "blocks.4.feed_forward.w3": 0.8305003046989441, "blocks.5.attention.w_q": 0.8420333862304688, "blocks.5.attention.w_k": 0.7871665954589844, "blocks.5.attention.w_v": 0.7245368957519531, "blocks.5.attention.w_out": 0.8148193359375, "blocks.5.feed_forward.w1": 0.8429425954818726, "blocks.5.feed_forward.w2": 0.8410657644271851, "blocks.5.feed_forward.w3": 0.8416099548339844, "blocks.6.attention.w_q": 0.8487396240234375, "blocks.6.attention.w_k": 0.8104209899902344, "blocks.6.attention.w_v": 0.750946044921875, "blocks.6.attention.w_out": 0.8195037841796875, "blocks.6.feed_forward.w1": 0.8396949768066406, "blocks.6.feed_forward.w2": 0.8355077505111694, "blocks.6.feed_forward.w3": 0.8381932973861694, "blocks.7.attention.w_q": 0.8469352722167969, "blocks.7.attention.w_k": 0.7980461120605469, "blocks.7.attention.w_v": 0.74395751953125, "blocks.7.attention.w_out": 0.8150634765625, "blocks.7.feed_forward.w1": 0.8425318598747253, "blocks.7.feed_forward.w2": 0.8261833190917969, "blocks.7.feed_forward.w3": 0.8403841853141785, "blocks.8.attention.w_q": 0.8563003540039062, "blocks.8.attention.w_k": 0.8142204284667969, "blocks.8.attention.w_v": 0.7372665405273438, "blocks.8.attention.w_out": 0.82843017578125, "blocks.8.feed_forward.w1": 0.8294614553451538, "blocks.8.feed_forward.w2": 0.7877184748649597, "blocks.8.feed_forward.w3": 0.8263868093490601}} +{"step": 800, "changed_frac": 0.8204574911003439, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8245582580566406, "blocks.0.attention.w_k": 0.7728843688964844, "blocks.0.attention.w_v": 0.6919021606445312, "blocks.0.attention.w_out": 0.7234840393066406, "blocks.0.feed_forward.w1": 0.7573649287223816, "blocks.0.feed_forward.w2": 0.8221282958984375, "blocks.0.feed_forward.w3": 0.7561416625976562, "blocks.1.attention.w_q": 0.8410110473632812, "blocks.1.attention.w_k": 0.7945632934570312, "blocks.1.attention.w_v": 0.73834228515625, "blocks.1.attention.w_out": 0.8242683410644531, "blocks.1.feed_forward.w1": 0.8163897395133972, "blocks.1.feed_forward.w2": 0.8480517268180847, "blocks.1.feed_forward.w3": 0.8119939565658569, "blocks.2.attention.w_q": 0.8349838256835938, "blocks.2.attention.w_k": 0.7932891845703125, "blocks.2.attention.w_v": 0.7407875061035156, "blocks.2.attention.w_out": 0.8153762817382812, "blocks.2.feed_forward.w1": 0.8286730647087097, "blocks.2.feed_forward.w2": 0.8510297536849976, "blocks.2.feed_forward.w3": 0.8247464895248413, "blocks.3.attention.w_q": 0.8489913940429688, "blocks.3.attention.w_k": 0.812469482421875, "blocks.3.attention.w_v": 0.7661094665527344, "blocks.3.attention.w_out": 0.8147964477539062, "blocks.3.feed_forward.w1": 0.8373146057128906, "blocks.3.feed_forward.w2": 0.8493169546127319, "blocks.3.feed_forward.w3": 0.8352317810058594, "blocks.4.attention.w_q": 0.8465309143066406, "blocks.4.attention.w_k": 0.8033409118652344, "blocks.4.attention.w_v": 0.7366256713867188, "blocks.4.attention.w_out": 0.8040084838867188, "blocks.4.feed_forward.w1": 0.8413798213005066, "blocks.4.feed_forward.w2": 0.8441187739372253, "blocks.4.feed_forward.w3": 0.8388456106185913, "blocks.5.attention.w_q": 0.8469696044921875, "blocks.5.attention.w_k": 0.7956123352050781, "blocks.5.attention.w_v": 0.741973876953125, "blocks.5.attention.w_out": 0.8104324340820312, "blocks.5.feed_forward.w1": 0.8488197326660156, "blocks.5.feed_forward.w2": 0.8460986018180847, "blocks.5.feed_forward.w3": 0.8476181030273438, "blocks.6.attention.w_q": 0.8589210510253906, "blocks.6.attention.w_k": 0.8201637268066406, "blocks.6.attention.w_v": 0.7621994018554688, "blocks.6.attention.w_out": 0.822662353515625, "blocks.6.feed_forward.w1": 0.8467242121696472, "blocks.6.feed_forward.w2": 0.8378143310546875, "blocks.6.feed_forward.w3": 0.8455111384391785, "blocks.7.attention.w_q": 0.8541069030761719, "blocks.7.attention.w_k": 0.8118362426757812, "blocks.7.attention.w_v": 0.764923095703125, "blocks.7.attention.w_out": 0.804840087890625, "blocks.7.feed_forward.w1": 0.8481063842773438, "blocks.7.feed_forward.w2": 0.8246091604232788, "blocks.7.feed_forward.w3": 0.8457769155502319, "blocks.8.attention.w_q": 0.858062744140625, "blocks.8.attention.w_k": 0.8230781555175781, "blocks.8.attention.w_v": 0.753570556640625, "blocks.8.attention.w_out": 0.8121795654296875, "blocks.8.feed_forward.w1": 0.8360926508903503, "blocks.8.feed_forward.w2": 0.7849159240722656, "blocks.8.feed_forward.w3": 0.8330764770507812}} +{"step": 900, "changed_frac": 0.8170519119653946, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8276290893554688, "blocks.0.attention.w_k": 0.7927474975585938, "blocks.0.attention.w_v": 0.6760139465332031, "blocks.0.attention.w_out": 0.7254104614257812, "blocks.0.feed_forward.w1": 0.7612876892089844, "blocks.0.feed_forward.w2": 0.8277842402458191, "blocks.0.feed_forward.w3": 0.7595660090446472, "blocks.1.attention.w_q": 0.8500900268554688, "blocks.1.attention.w_k": 0.807220458984375, "blocks.1.attention.w_v": 0.7412528991699219, "blocks.1.attention.w_out": 0.8291053771972656, "blocks.1.feed_forward.w1": 0.8161697387695312, "blocks.1.feed_forward.w2": 0.8512319326400757, "blocks.1.feed_forward.w3": 0.8121351003646851, "blocks.2.attention.w_q": 0.83526611328125, "blocks.2.attention.w_k": 0.7957344055175781, "blocks.2.attention.w_v": 0.7371025085449219, "blocks.2.attention.w_out": 0.8060646057128906, "blocks.2.feed_forward.w1": 0.8289909362792969, "blocks.2.feed_forward.w2": 0.8527654409408569, "blocks.2.feed_forward.w3": 0.8253034353256226, "blocks.3.attention.w_q": 0.8466453552246094, "blocks.3.attention.w_k": 0.8083381652832031, "blocks.3.attention.w_v": 0.76129150390625, "blocks.3.attention.w_out": 0.7992897033691406, "blocks.3.feed_forward.w1": 0.8347181081771851, "blocks.3.feed_forward.w2": 0.8495661616325378, "blocks.3.feed_forward.w3": 0.8324763178825378, "blocks.4.attention.w_q": 0.8399810791015625, "blocks.4.attention.w_k": 0.8037567138671875, "blocks.4.attention.w_v": 0.7343215942382812, "blocks.4.attention.w_out": 0.7906646728515625, "blocks.4.feed_forward.w1": 0.8377100825309753, "blocks.4.feed_forward.w2": 0.8419825434684753, "blocks.4.feed_forward.w3": 0.8351529836654663, "blocks.5.attention.w_q": 0.8417205810546875, "blocks.5.attention.w_k": 0.7990341186523438, "blocks.5.attention.w_v": 0.7370796203613281, "blocks.5.attention.w_out": 0.7921409606933594, "blocks.5.feed_forward.w1": 0.8428065180778503, "blocks.5.feed_forward.w2": 0.8419291377067566, "blocks.5.feed_forward.w3": 0.8419443964958191, "blocks.6.attention.w_q": 0.8484878540039062, "blocks.6.attention.w_k": 0.8171195983886719, "blocks.6.attention.w_v": 0.7547492980957031, "blocks.6.attention.w_out": 0.7665672302246094, "blocks.6.feed_forward.w1": 0.8438314199447632, "blocks.6.feed_forward.w2": 0.8326276540756226, "blocks.6.feed_forward.w3": 0.8411521911621094, "blocks.7.attention.w_q": 0.8479881286621094, "blocks.7.attention.w_k": 0.8126029968261719, "blocks.7.attention.w_v": 0.7611122131347656, "blocks.7.attention.w_out": 0.7865638732910156, "blocks.7.feed_forward.w1": 0.8434677124023438, "blocks.7.feed_forward.w2": 0.8161951899528503, "blocks.7.feed_forward.w3": 0.8417689204216003, "blocks.8.attention.w_q": 0.8497505187988281, "blocks.8.attention.w_k": 0.8209381103515625, "blocks.8.attention.w_v": 0.7463645935058594, "blocks.8.attention.w_out": 0.7620353698730469, "blocks.8.feed_forward.w1": 0.8328081965446472, "blocks.8.feed_forward.w2": 0.7742182612419128, "blocks.8.feed_forward.w3": 0.8280410766601562}} +{"step": 1000, "changed_frac": 0.8098423623631144, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8199577331542969, "blocks.0.attention.w_k": 0.7981071472167969, "blocks.0.attention.w_v": 0.6638069152832031, "blocks.0.attention.w_out": 0.705963134765625, "blocks.0.feed_forward.w1": 0.7635892629623413, "blocks.0.feed_forward.w2": 0.8302726745605469, "blocks.0.feed_forward.w3": 0.7583694458007812, "blocks.1.attention.w_q": 0.8426284790039062, "blocks.1.attention.w_k": 0.8014068603515625, "blocks.1.attention.w_v": 0.7262039184570312, "blocks.1.attention.w_out": 0.8181953430175781, "blocks.1.feed_forward.w1": 0.8106765747070312, "blocks.1.feed_forward.w2": 0.8518092036247253, "blocks.1.feed_forward.w3": 0.8094317317008972, "blocks.2.attention.w_q": 0.8338890075683594, "blocks.2.attention.w_k": 0.7951126098632812, "blocks.2.attention.w_v": 0.7311973571777344, "blocks.2.attention.w_out": 0.787445068359375, "blocks.2.feed_forward.w1": 0.8221206665039062, "blocks.2.feed_forward.w2": 0.8513717651367188, "blocks.2.feed_forward.w3": 0.8213615417480469, "blocks.3.attention.w_q": 0.8388862609863281, "blocks.3.attention.w_k": 0.8035125732421875, "blocks.3.attention.w_v": 0.7497406005859375, "blocks.3.attention.w_out": 0.7814559936523438, "blocks.3.feed_forward.w1": 0.8280271291732788, "blocks.3.feed_forward.w2": 0.8448664546012878, "blocks.3.feed_forward.w3": 0.82598876953125, "blocks.4.attention.w_q": 0.8357925415039062, "blocks.4.attention.w_k": 0.8033790588378906, "blocks.4.attention.w_v": 0.7259178161621094, "blocks.4.attention.w_out": 0.7817573547363281, "blocks.4.feed_forward.w1": 0.8307508230209351, "blocks.4.feed_forward.w2": 0.8380610346794128, "blocks.4.feed_forward.w3": 0.8284887075424194, "blocks.5.attention.w_q": 0.8364067077636719, "blocks.5.attention.w_k": 0.7954063415527344, "blocks.5.attention.w_v": 0.7224807739257812, "blocks.5.attention.w_out": 0.769989013671875, "blocks.5.feed_forward.w1": 0.8359082937240601, "blocks.5.feed_forward.w2": 0.8360913991928101, "blocks.5.feed_forward.w3": 0.8351084589958191, "blocks.6.attention.w_q": 0.8402786254882812, "blocks.6.attention.w_k": 0.8098831176757812, "blocks.6.attention.w_v": 0.748046875, "blocks.6.attention.w_out": 0.7119369506835938, "blocks.6.feed_forward.w1": 0.8361562490463257, "blocks.6.feed_forward.w2": 0.8251228332519531, "blocks.6.feed_forward.w3": 0.8343900442123413, "blocks.7.attention.w_q": 0.8421897888183594, "blocks.7.attention.w_k": 0.8080368041992188, "blocks.7.attention.w_v": 0.7518386840820312, "blocks.7.attention.w_out": 0.7547607421875, "blocks.7.feed_forward.w1": 0.8368949890136719, "blocks.7.feed_forward.w2": 0.8042716979980469, "blocks.7.feed_forward.w3": 0.8346163630485535, "blocks.8.attention.w_q": 0.8402557373046875, "blocks.8.attention.w_k": 0.8128738403320312, "blocks.8.attention.w_v": 0.7297477722167969, "blocks.8.attention.w_out": 0.711944580078125, "blocks.8.feed_forward.w1": 0.8286463618278503, "blocks.8.feed_forward.w2": 0.7611821889877319, "blocks.8.feed_forward.w3": 0.8234672546386719}} +{"step": 1100, "changed_frac": 0.8053157839000734, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.827117919921875, "blocks.0.attention.w_k": 0.8081321716308594, "blocks.0.attention.w_v": 0.6720924377441406, "blocks.0.attention.w_out": 0.6875, "blocks.0.feed_forward.w1": 0.7683194875717163, "blocks.0.feed_forward.w2": 0.8327560424804688, "blocks.0.feed_forward.w3": 0.7625160217285156, "blocks.1.attention.w_q": 0.8340110778808594, "blocks.1.attention.w_k": 0.7946548461914062, "blocks.1.attention.w_v": 0.7117767333984375, "blocks.1.attention.w_out": 0.8064804077148438, "blocks.1.feed_forward.w1": 0.8068415522575378, "blocks.1.feed_forward.w2": 0.8516578674316406, "blocks.1.feed_forward.w3": 0.8054059743881226, "blocks.2.attention.w_q": 0.8331565856933594, "blocks.2.attention.w_k": 0.7988052368164062, "blocks.2.attention.w_v": 0.7274360656738281, "blocks.2.attention.w_out": 0.7777252197265625, "blocks.2.feed_forward.w1": 0.8196131587028503, "blocks.2.feed_forward.w2": 0.8507932424545288, "blocks.2.feed_forward.w3": 0.8181177973747253, "blocks.3.attention.w_q": 0.8333053588867188, "blocks.3.attention.w_k": 0.8016891479492188, "blocks.3.attention.w_v": 0.7407989501953125, "blocks.3.attention.w_out": 0.7697639465332031, "blocks.3.feed_forward.w1": 0.8256568908691406, "blocks.3.feed_forward.w2": 0.8431129455566406, "blocks.3.feed_forward.w3": 0.8225746154785156, "blocks.4.attention.w_q": 0.8327560424804688, "blocks.4.attention.w_k": 0.8034820556640625, "blocks.4.attention.w_v": 0.719451904296875, "blocks.4.attention.w_out": 0.7730941772460938, "blocks.4.feed_forward.w1": 0.8277613520622253, "blocks.4.feed_forward.w2": 0.8345260620117188, "blocks.4.feed_forward.w3": 0.8247197866439819, "blocks.5.attention.w_q": 0.8342857360839844, "blocks.5.attention.w_k": 0.797088623046875, "blocks.5.attention.w_v": 0.7178192138671875, "blocks.5.attention.w_out": 0.7558021545410156, "blocks.5.feed_forward.w1": 0.8318163752555847, "blocks.5.feed_forward.w2": 0.8312162160873413, "blocks.5.feed_forward.w3": 0.8292312622070312, "blocks.6.attention.w_q": 0.8343353271484375, "blocks.6.attention.w_k": 0.8072395324707031, "blocks.6.attention.w_v": 0.7409248352050781, "blocks.6.attention.w_out": 0.6852760314941406, "blocks.6.feed_forward.w1": 0.8325335383415222, "blocks.6.feed_forward.w2": 0.8181127309799194, "blocks.6.feed_forward.w3": 0.8298110961914062, "blocks.7.attention.w_q": 0.8357162475585938, "blocks.7.attention.w_k": 0.8057403564453125, "blocks.7.attention.w_v": 0.7469673156738281, "blocks.7.attention.w_out": 0.7269477844238281, "blocks.7.feed_forward.w1": 0.8324317932128906, "blocks.7.feed_forward.w2": 0.7951850891113281, "blocks.7.feed_forward.w3": 0.8295402526855469, "blocks.8.attention.w_q": 0.8331794738769531, "blocks.8.attention.w_k": 0.812408447265625, "blocks.8.attention.w_v": 0.7197608947753906, "blocks.8.attention.w_out": 0.67401123046875, "blocks.8.feed_forward.w1": 0.8247808218002319, "blocks.8.feed_forward.w2": 0.7469037771224976, "blocks.8.feed_forward.w3": 0.8178024291992188}} +{"step": 1200, "changed_frac": 0.7988356565817808, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8257179260253906, "blocks.0.attention.w_k": 0.8041496276855469, "blocks.0.attention.w_v": 0.6631202697753906, "blocks.0.attention.w_out": 0.6651573181152344, "blocks.0.feed_forward.w1": 0.7650794982910156, "blocks.0.feed_forward.w2": 0.8313586115837097, "blocks.0.feed_forward.w3": 0.7584432363510132, "blocks.1.attention.w_q": 0.8260459899902344, "blocks.1.attention.w_k": 0.7883071899414062, "blocks.1.attention.w_v": 0.6971588134765625, "blocks.1.attention.w_out": 0.7986869812011719, "blocks.1.feed_forward.w1": 0.8025182485580444, "blocks.1.feed_forward.w2": 0.8508237600326538, "blocks.1.feed_forward.w3": 0.8010444641113281, "blocks.2.attention.w_q": 0.8313636779785156, "blocks.2.attention.w_k": 0.8007011413574219, "blocks.2.attention.w_v": 0.7166824340820312, "blocks.2.attention.w_out": 0.7711601257324219, "blocks.2.feed_forward.w1": 0.8132883906364441, "blocks.2.feed_forward.w2": 0.8495330810546875, "blocks.2.feed_forward.w3": 0.8129056692123413, "blocks.3.attention.w_q": 0.8281784057617188, "blocks.3.attention.w_k": 0.8009986877441406, "blocks.3.attention.w_v": 0.7327423095703125, "blocks.3.attention.w_out": 0.7598037719726562, "blocks.3.feed_forward.w1": 0.8190931081771851, "blocks.3.feed_forward.w2": 0.8394864797592163, "blocks.3.feed_forward.w3": 0.8159307241439819, "blocks.4.attention.w_q": 0.8303565979003906, "blocks.4.attention.w_k": 0.8007278442382812, "blocks.4.attention.w_v": 0.70562744140625, "blocks.4.attention.w_out": 0.7630386352539062, "blocks.4.feed_forward.w1": 0.8213221430778503, "blocks.4.feed_forward.w2": 0.8293546438217163, "blocks.4.feed_forward.w3": 0.8183072805404663, "blocks.5.attention.w_q": 0.8301429748535156, "blocks.5.attention.w_k": 0.7941398620605469, "blocks.5.attention.w_v": 0.7098922729492188, "blocks.5.attention.w_out": 0.7378959655761719, "blocks.5.feed_forward.w1": 0.8262621760368347, "blocks.5.feed_forward.w2": 0.8236274719238281, "blocks.5.feed_forward.w3": 0.8220393061637878, "blocks.6.attention.w_q": 0.831329345703125, "blocks.6.attention.w_k": 0.8082199096679688, "blocks.6.attention.w_v": 0.7324714660644531, "blocks.6.attention.w_out": 0.6546592712402344, "blocks.6.feed_forward.w1": 0.8270098567008972, "blocks.6.feed_forward.w2": 0.810455322265625, "blocks.6.feed_forward.w3": 0.8231086730957031, "blocks.7.attention.w_q": 0.8317108154296875, "blocks.7.attention.w_k": 0.8026809692382812, "blocks.7.attention.w_v": 0.7394943237304688, "blocks.7.attention.w_out": 0.6955299377441406, "blocks.7.feed_forward.w1": 0.8273735046386719, "blocks.7.feed_forward.w2": 0.7867139577865601, "blocks.7.feed_forward.w3": 0.8231722712516785, "blocks.8.attention.w_q": 0.828643798828125, "blocks.8.attention.w_k": 0.8095054626464844, "blocks.8.attention.w_v": 0.7053565979003906, "blocks.8.attention.w_out": 0.6384963989257812, "blocks.8.feed_forward.w1": 0.8202921748161316, "blocks.8.feed_forward.w2": 0.7359441518783569, "blocks.8.feed_forward.w3": 0.8134714961051941}} +{"step": 1300, "changed_frac": 0.7938763545109675, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.82421875, "blocks.0.attention.w_k": 0.8020057678222656, "blocks.0.attention.w_v": 0.6610565185546875, "blocks.0.attention.w_out": 0.6467437744140625, "blocks.0.feed_forward.w1": 0.7686208486557007, "blocks.0.feed_forward.w2": 0.8300145864486694, "blocks.0.feed_forward.w3": 0.7594566345214844, "blocks.1.attention.w_q": 0.8222122192382812, "blocks.1.attention.w_k": 0.78887939453125, "blocks.1.attention.w_v": 0.6883201599121094, "blocks.1.attention.w_out": 0.7881126403808594, "blocks.1.feed_forward.w1": 0.7978617548942566, "blocks.1.feed_forward.w2": 0.8495941162109375, "blocks.1.feed_forward.w3": 0.7968482971191406, "blocks.2.attention.w_q": 0.8322944641113281, "blocks.2.attention.w_k": 0.7979164123535156, "blocks.2.attention.w_v": 0.7082748413085938, "blocks.2.attention.w_out": 0.7645072937011719, "blocks.2.feed_forward.w1": 0.8097941279411316, "blocks.2.feed_forward.w2": 0.8480212092399597, "blocks.2.feed_forward.w3": 0.8082504272460938, "blocks.3.attention.w_q": 0.8266067504882812, "blocks.3.attention.w_k": 0.7992591857910156, "blocks.3.attention.w_v": 0.7257232666015625, "blocks.3.attention.w_out": 0.7518272399902344, "blocks.3.feed_forward.w1": 0.8146311640739441, "blocks.3.feed_forward.w2": 0.8341090083122253, "blocks.3.feed_forward.w3": 0.8096110224723816, "blocks.4.attention.w_q": 0.8300285339355469, "blocks.4.attention.w_k": 0.7999725341796875, "blocks.4.attention.w_v": 0.7002983093261719, "blocks.4.attention.w_out": 0.75732421875, "blocks.4.feed_forward.w1": 0.817047119140625, "blocks.4.feed_forward.w2": 0.826904296875, "blocks.4.feed_forward.w3": 0.812109649181366, "blocks.5.attention.w_q": 0.8292579650878906, "blocks.5.attention.w_k": 0.7936935424804688, "blocks.5.attention.w_v": 0.7030754089355469, "blocks.5.attention.w_out": 0.7271537780761719, "blocks.5.feed_forward.w1": 0.8207855224609375, "blocks.5.feed_forward.w2": 0.8192214965820312, "blocks.5.feed_forward.w3": 0.8165969848632812, "blocks.6.attention.w_q": 0.82684326171875, "blocks.6.attention.w_k": 0.8045730590820312, "blocks.6.attention.w_v": 0.7272224426269531, "blocks.6.attention.w_out": 0.6356925964355469, "blocks.6.feed_forward.w1": 0.8218345642089844, "blocks.6.feed_forward.w2": 0.8044624328613281, "blocks.6.feed_forward.w3": 0.8177668452262878, "blocks.7.attention.w_q": 0.8288955688476562, "blocks.7.attention.w_k": 0.8019866943359375, "blocks.7.attention.w_v": 0.7347679138183594, "blocks.7.attention.w_out": 0.6717948913574219, "blocks.7.feed_forward.w1": 0.8224601745605469, "blocks.7.feed_forward.w2": 0.7754987478256226, "blocks.7.feed_forward.w3": 0.8169428706169128, "blocks.8.attention.w_q": 0.8247718811035156, "blocks.8.attention.w_k": 0.8050994873046875, "blocks.8.attention.w_v": 0.697357177734375, "blocks.8.attention.w_out": 0.6071434020996094, "blocks.8.feed_forward.w1": 0.8166288137435913, "blocks.8.feed_forward.w2": 0.7257105708122253, "blocks.8.feed_forward.w3": 0.8087577819824219}} +{"step": 1400, "changed_frac": 0.7892588427943042, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.82379150390625, "blocks.0.attention.w_k": 0.8039970397949219, "blocks.0.attention.w_v": 0.6586570739746094, "blocks.0.attention.w_out": 0.6330070495605469, "blocks.0.feed_forward.w1": 0.7698415517807007, "blocks.0.feed_forward.w2": 0.8317121267318726, "blocks.0.feed_forward.w3": 0.7583541870117188, "blocks.1.attention.w_q": 0.8211441040039062, "blocks.1.attention.w_k": 0.7858695983886719, "blocks.1.attention.w_v": 0.6787223815917969, "blocks.1.attention.w_out": 0.7823219299316406, "blocks.1.feed_forward.w1": 0.7949129939079285, "blocks.1.feed_forward.w2": 0.8464800715446472, "blocks.1.feed_forward.w3": 0.7936961054801941, "blocks.2.attention.w_q": 0.8287467956542969, "blocks.2.attention.w_k": 0.7988662719726562, "blocks.2.attention.w_v": 0.705718994140625, "blocks.2.attention.w_out": 0.7595558166503906, "blocks.2.feed_forward.w1": 0.8063697814941406, "blocks.2.feed_forward.w2": 0.8447698354721069, "blocks.2.feed_forward.w3": 0.8033727407455444, "blocks.3.attention.w_q": 0.8265724182128906, "blocks.3.attention.w_k": 0.7983589172363281, "blocks.3.attention.w_v": 0.7211685180664062, "blocks.3.attention.w_out": 0.7425270080566406, "blocks.3.feed_forward.w1": 0.8100738525390625, "blocks.3.feed_forward.w2": 0.8321737051010132, "blocks.3.feed_forward.w3": 0.8053029775619507, "blocks.4.attention.w_q": 0.8259010314941406, "blocks.4.attention.w_k": 0.7975807189941406, "blocks.4.attention.w_v": 0.6939888000488281, "blocks.4.attention.w_out": 0.7519378662109375, "blocks.4.feed_forward.w1": 0.8135274648666382, "blocks.4.feed_forward.w2": 0.8217290639877319, "blocks.4.feed_forward.w3": 0.8084195852279663, "blocks.5.attention.w_q": 0.8280754089355469, "blocks.5.attention.w_k": 0.7917823791503906, "blocks.5.attention.w_v": 0.7006454467773438, "blocks.5.attention.w_out": 0.7141189575195312, "blocks.5.feed_forward.w1": 0.8162638545036316, "blocks.5.feed_forward.w2": 0.8121058344841003, "blocks.5.feed_forward.w3": 0.8103040456771851, "blocks.6.attention.w_q": 0.8221473693847656, "blocks.6.attention.w_k": 0.8013648986816406, "blocks.6.attention.w_v": 0.7254295349121094, "blocks.6.attention.w_out": 0.6175079345703125, "blocks.6.feed_forward.w1": 0.816717803478241, "blocks.6.feed_forward.w2": 0.797278106212616, "blocks.6.feed_forward.w3": 0.812231719493866, "blocks.7.attention.w_q": 0.8243370056152344, "blocks.7.attention.w_k": 0.8006172180175781, "blocks.7.attention.w_v": 0.7297401428222656, "blocks.7.attention.w_out": 0.6521224975585938, "blocks.7.feed_forward.w1": 0.8177236318588257, "blocks.7.feed_forward.w2": 0.7667376399040222, "blocks.7.feed_forward.w3": 0.8097356557846069, "blocks.8.attention.w_q": 0.81988525390625, "blocks.8.attention.w_k": 0.80731201171875, "blocks.8.attention.w_v": 0.68975830078125, "blocks.8.attention.w_out": 0.5818901062011719, "blocks.8.feed_forward.w1": 0.8144149780273438, "blocks.8.feed_forward.w2": 0.7144368886947632, "blocks.8.feed_forward.w3": 0.8040199279785156}} +{"step": 1500, "changed_frac": 0.7844421843178252, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8259429931640625, "blocks.0.attention.w_k": 0.8034858703613281, "blocks.0.attention.w_v": 0.6485328674316406, "blocks.0.attention.w_out": 0.6210136413574219, "blocks.0.feed_forward.w1": 0.7707227468490601, "blocks.0.feed_forward.w2": 0.8275909423828125, "blocks.0.feed_forward.w3": 0.7547963857650757, "blocks.1.attention.w_q": 0.8165206909179688, "blocks.1.attention.w_k": 0.7838287353515625, "blocks.1.attention.w_v": 0.6694488525390625, "blocks.1.attention.w_out": 0.7765426635742188, "blocks.1.feed_forward.w1": 0.7902666926383972, "blocks.1.feed_forward.w2": 0.8454093933105469, "blocks.1.feed_forward.w3": 0.7896804809570312, "blocks.2.attention.w_q": 0.8293380737304688, "blocks.2.attention.w_k": 0.7989845275878906, "blocks.2.attention.w_v": 0.7001571655273438, "blocks.2.attention.w_out": 0.7521553039550781, "blocks.2.feed_forward.w1": 0.8013242483139038, "blocks.2.feed_forward.w2": 0.8426641225814819, "blocks.2.feed_forward.w3": 0.7993265986442566, "blocks.3.attention.w_q": 0.8239364624023438, "blocks.3.attention.w_k": 0.7959175109863281, "blocks.3.attention.w_v": 0.7148513793945312, "blocks.3.attention.w_out": 0.7332077026367188, "blocks.3.feed_forward.w1": 0.8065478205680847, "blocks.3.feed_forward.w2": 0.8287264704704285, "blocks.3.feed_forward.w3": 0.8001289367675781, "blocks.4.attention.w_q": 0.8247222900390625, "blocks.4.attention.w_k": 0.7960433959960938, "blocks.4.attention.w_v": 0.6834602355957031, "blocks.4.attention.w_out": 0.7473258972167969, "blocks.4.feed_forward.w1": 0.8065109252929688, "blocks.4.feed_forward.w2": 0.8161544799804688, "blocks.4.feed_forward.w3": 0.8027598261833191, "blocks.5.attention.w_q": 0.8265342712402344, "blocks.5.attention.w_k": 0.790924072265625, "blocks.5.attention.w_v": 0.6946525573730469, "blocks.5.attention.w_out": 0.7043685913085938, "blocks.5.feed_forward.w1": 0.8135401606559753, "blocks.5.feed_forward.w2": 0.8071517944335938, "blocks.5.feed_forward.w3": 0.8048223257064819, "blocks.6.attention.w_q": 0.8192291259765625, "blocks.6.attention.w_k": 0.80072021484375, "blocks.6.attention.w_v": 0.7189979553222656, "blocks.6.attention.w_out": 0.6021690368652344, "blocks.6.feed_forward.w1": 0.8143908381462097, "blocks.6.feed_forward.w2": 0.7915522456169128, "blocks.6.feed_forward.w3": 0.8071314692497253, "blocks.7.attention.w_q": 0.8237037658691406, "blocks.7.attention.w_k": 0.8006134033203125, "blocks.7.attention.w_v": 0.7237930297851562, "blocks.7.attention.w_out": 0.6347923278808594, "blocks.7.feed_forward.w1": 0.8136419057846069, "blocks.7.feed_forward.w2": 0.7586479187011719, "blocks.7.feed_forward.w3": 0.8045031428337097, "blocks.8.attention.w_q": 0.8189125061035156, "blocks.8.attention.w_k": 0.8042411804199219, "blocks.8.attention.w_v": 0.6792068481445312, "blocks.8.attention.w_out": 0.5602188110351562, "blocks.8.feed_forward.w1": 0.8103243708610535, "blocks.8.feed_forward.w2": 0.7033576965332031, "blocks.8.feed_forward.w3": 0.7987403869628906}} +{"step": 1600, "changed_frac": 0.7799079764602531, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8192253112792969, "blocks.0.attention.w_k": 0.8040351867675781, "blocks.0.attention.w_v": 0.6445121765136719, "blocks.0.attention.w_out": 0.6105155944824219, "blocks.0.feed_forward.w1": 0.769439697265625, "blocks.0.feed_forward.w2": 0.828399658203125, "blocks.0.feed_forward.w3": 0.7543691396713257, "blocks.1.attention.w_q": 0.8171195983886719, "blocks.1.attention.w_k": 0.7856178283691406, "blocks.1.attention.w_v": 0.6644973754882812, "blocks.1.attention.w_out": 0.7717018127441406, "blocks.1.feed_forward.w1": 0.7875760793685913, "blocks.1.feed_forward.w2": 0.8425788879394531, "blocks.1.feed_forward.w3": 0.7855504751205444, "blocks.2.attention.w_q": 0.82757568359375, "blocks.2.attention.w_k": 0.8000411987304688, "blocks.2.attention.w_v": 0.6938629150390625, "blocks.2.attention.w_out": 0.7451515197753906, "blocks.2.feed_forward.w1": 0.7970174551010132, "blocks.2.feed_forward.w2": 0.8387934565544128, "blocks.2.feed_forward.w3": 0.7943840026855469, "blocks.3.attention.w_q": 0.8217391967773438, "blocks.3.attention.w_k": 0.7957534790039062, "blocks.3.attention.w_v": 0.7095069885253906, "blocks.3.attention.w_out": 0.7241630554199219, "blocks.3.feed_forward.w1": 0.8017451167106628, "blocks.3.feed_forward.w2": 0.8256543874740601, "blocks.3.feed_forward.w3": 0.7964324951171875, "blocks.4.attention.w_q": 0.820953369140625, "blocks.4.attention.w_k": 0.7946968078613281, "blocks.4.attention.w_v": 0.6738204956054688, "blocks.4.attention.w_out": 0.7418212890625, "blocks.4.feed_forward.w1": 0.8045463562011719, "blocks.4.feed_forward.w2": 0.8130162954330444, "blocks.4.feed_forward.w3": 0.7963333129882812, "blocks.5.attention.w_q": 0.8242416381835938, "blocks.5.attention.w_k": 0.7899589538574219, "blocks.5.attention.w_v": 0.693450927734375, "blocks.5.attention.w_out": 0.6938095092773438, "blocks.5.feed_forward.w1": 0.8075269460678101, "blocks.5.feed_forward.w2": 0.8017565608024597, "blocks.5.feed_forward.w3": 0.7996177673339844, "blocks.6.attention.w_q": 0.8180694580078125, "blocks.6.attention.w_k": 0.8000717163085938, "blocks.6.attention.w_v": 0.7132911682128906, "blocks.6.attention.w_out": 0.5898628234863281, "blocks.6.feed_forward.w1": 0.8088849782943726, "blocks.6.feed_forward.w2": 0.7852210998535156, "blocks.6.feed_forward.w3": 0.8012619018554688, "blocks.7.attention.w_q": 0.8214988708496094, "blocks.7.attention.w_k": 0.7989845275878906, "blocks.7.attention.w_v": 0.72186279296875, "blocks.7.attention.w_out": 0.6163673400878906, "blocks.7.feed_forward.w1": 0.810034453868866, "blocks.7.feed_forward.w2": 0.7507578730583191, "blocks.7.feed_forward.w3": 0.7984809875488281, "blocks.8.attention.w_q": 0.8152923583984375, "blocks.8.attention.w_k": 0.8043251037597656, "blocks.8.attention.w_v": 0.6726188659667969, "blocks.8.attention.w_out": 0.5393142700195312, "blocks.8.feed_forward.w1": 0.8065032958984375, "blocks.8.feed_forward.w2": 0.6911888122558594, "blocks.8.feed_forward.w3": 0.7928962707519531}} +{"step": 1700, "changed_frac": 0.7744166056315104, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.820159912109375, "blocks.0.attention.w_k": 0.8011207580566406, "blocks.0.attention.w_v": 0.6310272216796875, "blocks.0.attention.w_out": 0.5969810485839844, "blocks.0.feed_forward.w1": 0.7673581838607788, "blocks.0.feed_forward.w2": 0.8264516592025757, "blocks.0.feed_forward.w3": 0.7507667541503906, "blocks.1.attention.w_q": 0.8131866455078125, "blocks.1.attention.w_k": 0.7847061157226562, "blocks.1.attention.w_v": 0.6526031494140625, "blocks.1.attention.w_out": 0.7653083801269531, "blocks.1.feed_forward.w1": 0.7823740839958191, "blocks.1.feed_forward.w2": 0.8394648432731628, "blocks.1.feed_forward.w3": 0.7802518606185913, "blocks.2.attention.w_q": 0.8247795104980469, "blocks.2.attention.w_k": 0.79693603515625, "blocks.2.attention.w_v": 0.6888694763183594, "blocks.2.attention.w_out": 0.7364540100097656, "blocks.2.feed_forward.w1": 0.7922286987304688, "blocks.2.feed_forward.w2": 0.8350791931152344, "blocks.2.feed_forward.w3": 0.7884521484375, "blocks.3.attention.w_q": 0.8209419250488281, "blocks.3.attention.w_k": 0.7931365966796875, "blocks.3.attention.w_v": 0.7006874084472656, "blocks.3.attention.w_out": 0.7161178588867188, "blocks.3.feed_forward.w1": 0.7959913015365601, "blocks.3.feed_forward.w2": 0.8212191462516785, "blocks.3.feed_forward.w3": 0.7891477346420288, "blocks.4.attention.w_q": 0.8174400329589844, "blocks.4.attention.w_k": 0.7937850952148438, "blocks.4.attention.w_v": 0.6633377075195312, "blocks.4.attention.w_out": 0.7385482788085938, "blocks.4.feed_forward.w1": 0.7988790273666382, "blocks.4.feed_forward.w2": 0.8065618276596069, "blocks.4.feed_forward.w3": 0.7902819514274597, "blocks.5.attention.w_q": 0.822357177734375, "blocks.5.attention.w_k": 0.7887840270996094, "blocks.5.attention.w_v": 0.6846847534179688, "blocks.5.attention.w_out": 0.6812095642089844, "blocks.5.feed_forward.w1": 0.8027064204216003, "blocks.5.feed_forward.w2": 0.7950922846794128, "blocks.5.feed_forward.w3": 0.7924537658691406, "blocks.6.attention.w_q": 0.8145637512207031, "blocks.6.attention.w_k": 0.7973136901855469, "blocks.6.attention.w_v": 0.7078514099121094, "blocks.6.attention.w_out": 0.5741958618164062, "blocks.6.feed_forward.w1": 0.8054670095443726, "blocks.6.feed_forward.w2": 0.7800242304801941, "blocks.6.feed_forward.w3": 0.7959302663803101, "blocks.7.attention.w_q": 0.82025146484375, "blocks.7.attention.w_k": 0.7965202331542969, "blocks.7.attention.w_v": 0.711944580078125, "blocks.7.attention.w_out": 0.5961532592773438, "blocks.7.feed_forward.w1": 0.8056488037109375, "blocks.7.feed_forward.w2": 0.7419751882553101, "blocks.7.feed_forward.w3": 0.7921320796012878, "blocks.8.attention.w_q": 0.8132781982421875, "blocks.8.attention.w_k": 0.8048667907714844, "blocks.8.attention.w_v": 0.6624107360839844, "blocks.8.attention.w_out": 0.5201034545898438, "blocks.8.feed_forward.w1": 0.8044599294662476, "blocks.8.feed_forward.w2": 0.6821696162223816, "blocks.8.feed_forward.w3": 0.7888082265853882}} +{"step": 1800, "changed_frac": 0.7705173003367889, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8184051513671875, "blocks.0.attention.w_k": 0.803436279296875, "blocks.0.attention.w_v": 0.6311607360839844, "blocks.0.attention.w_out": 0.5919685363769531, "blocks.0.feed_forward.w1": 0.7668495178222656, "blocks.0.feed_forward.w2": 0.8241513967514038, "blocks.0.feed_forward.w3": 0.7487818598747253, "blocks.1.attention.w_q": 0.8139381408691406, "blocks.1.attention.w_k": 0.7855491638183594, "blocks.1.attention.w_v": 0.6532173156738281, "blocks.1.attention.w_out": 0.7616043090820312, "blocks.1.feed_forward.w1": 0.7803624868392944, "blocks.1.feed_forward.w2": 0.8374418020248413, "blocks.1.feed_forward.w3": 0.77667236328125, "blocks.2.attention.w_q": 0.8221168518066406, "blocks.2.attention.w_k": 0.797760009765625, "blocks.2.attention.w_v": 0.6849594116210938, "blocks.2.attention.w_out": 0.7323188781738281, "blocks.2.feed_forward.w1": 0.7890739440917969, "blocks.2.feed_forward.w2": 0.8325678706169128, "blocks.2.feed_forward.w3": 0.7852859497070312, "blocks.3.attention.w_q": 0.8193244934082031, "blocks.3.attention.w_k": 0.7923202514648438, "blocks.3.attention.w_v": 0.6977615356445312, "blocks.3.attention.w_out": 0.71221923828125, "blocks.3.feed_forward.w1": 0.7919031977653503, "blocks.3.feed_forward.w2": 0.8180160522460938, "blocks.3.feed_forward.w3": 0.784918487071991, "blocks.4.attention.w_q": 0.8152656555175781, "blocks.4.attention.w_k": 0.7907829284667969, "blocks.4.attention.w_v": 0.65875244140625, "blocks.4.attention.w_out": 0.7326126098632812, "blocks.4.feed_forward.w1": 0.7938665151596069, "blocks.4.feed_forward.w2": 0.8023898005485535, "blocks.4.feed_forward.w3": 0.7865206599235535, "blocks.5.attention.w_q": 0.8203239440917969, "blocks.5.attention.w_k": 0.7893753051757812, "blocks.5.attention.w_v": 0.6804733276367188, "blocks.5.attention.w_out": 0.6742286682128906, "blocks.5.feed_forward.w1": 0.7987798452377319, "blocks.5.feed_forward.w2": 0.7905680537223816, "blocks.5.feed_forward.w3": 0.7884458303451538, "blocks.6.attention.w_q": 0.8120193481445312, "blocks.6.attention.w_k": 0.7954330444335938, "blocks.6.attention.w_v": 0.7024078369140625, "blocks.6.attention.w_out": 0.5663070678710938, "blocks.6.feed_forward.w1": 0.8018710017204285, "blocks.6.feed_forward.w2": 0.7738279104232788, "blocks.6.feed_forward.w3": 0.7899169921875, "blocks.7.attention.w_q": 0.8178939819335938, "blocks.7.attention.w_k": 0.797576904296875, "blocks.7.attention.w_v": 0.7114295959472656, "blocks.7.attention.w_out": 0.5869369506835938, "blocks.7.feed_forward.w1": 0.8000819087028503, "blocks.7.feed_forward.w2": 0.7334696650505066, "blocks.7.feed_forward.w3": 0.786834716796875, "blocks.8.attention.w_q": 0.8101463317871094, "blocks.8.attention.w_k": 0.8029747009277344, "blocks.8.attention.w_v": 0.65985107421875, "blocks.8.attention.w_out": 0.501800537109375, "blocks.8.feed_forward.w1": 0.7998937368392944, "blocks.8.feed_forward.w2": 0.6694221496582031, "blocks.8.feed_forward.w3": 0.7833773493766785}} +{"step": 1900, "changed_frac": 0.7652420304779314, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8117332458496094, "blocks.0.attention.w_k": 0.7969245910644531, "blocks.0.attention.w_v": 0.6158447265625, "blocks.0.attention.w_out": 0.5841255187988281, "blocks.0.feed_forward.w1": 0.764319121837616, "blocks.0.feed_forward.w2": 0.8225797414779663, "blocks.0.feed_forward.w3": 0.7452481985092163, "blocks.1.attention.w_q": 0.8073616027832031, "blocks.1.attention.w_k": 0.7816009521484375, "blocks.1.attention.w_v": 0.6483497619628906, "blocks.1.attention.w_out": 0.7564620971679688, "blocks.1.feed_forward.w1": 0.7757720947265625, "blocks.1.feed_forward.w2": 0.8350843191146851, "blocks.1.feed_forward.w3": 0.7722384333610535, "blocks.2.attention.w_q": 0.820892333984375, "blocks.2.attention.w_k": 0.7921485900878906, "blocks.2.attention.w_v": 0.6799087524414062, "blocks.2.attention.w_out": 0.7286720275878906, "blocks.2.feed_forward.w1": 0.7856916189193726, "blocks.2.feed_forward.w2": 0.8295211791992188, "blocks.2.feed_forward.w3": 0.7786712646484375, "blocks.3.attention.w_q": 0.81695556640625, "blocks.3.attention.w_k": 0.7908096313476562, "blocks.3.attention.w_v": 0.6914329528808594, "blocks.3.attention.w_out": 0.703338623046875, "blocks.3.feed_forward.w1": 0.7878138422966003, "blocks.3.feed_forward.w2": 0.8134994506835938, "blocks.3.feed_forward.w3": 0.7786178588867188, "blocks.4.attention.w_q": 0.8127479553222656, "blocks.4.attention.w_k": 0.7881736755371094, "blocks.4.attention.w_v": 0.6449394226074219, "blocks.4.attention.w_out": 0.7267036437988281, "blocks.4.feed_forward.w1": 0.789862334728241, "blocks.4.feed_forward.w2": 0.7961744070053101, "blocks.4.feed_forward.w3": 0.7794723510742188, "blocks.5.attention.w_q": 0.8193588256835938, "blocks.5.attention.w_k": 0.7841644287109375, "blocks.5.attention.w_v": 0.6727294921875, "blocks.5.attention.w_out": 0.6632423400878906, "blocks.5.feed_forward.w1": 0.7944437861442566, "blocks.5.feed_forward.w2": 0.7847887873649597, "blocks.5.feed_forward.w3": 0.7818629145622253, "blocks.6.attention.w_q": 0.8097114562988281, "blocks.6.attention.w_k": 0.7924880981445312, "blocks.6.attention.w_v": 0.6965446472167969, "blocks.6.attention.w_out": 0.5578117370605469, "blocks.6.feed_forward.w1": 0.7969551086425781, "blocks.6.feed_forward.w2": 0.7684783935546875, "blocks.6.feed_forward.w3": 0.7846629023551941, "blocks.7.attention.w_q": 0.815521240234375, "blocks.7.attention.w_k": 0.7944297790527344, "blocks.7.attention.w_v": 0.7028160095214844, "blocks.7.attention.w_out": 0.5687789916992188, "blocks.7.feed_forward.w1": 0.797125518321991, "blocks.7.feed_forward.w2": 0.7259661555290222, "blocks.7.feed_forward.w3": 0.7819595336914062, "blocks.8.attention.w_q": 0.8082466125488281, "blocks.8.attention.w_k": 0.7986984252929688, "blocks.8.attention.w_v": 0.6481208801269531, "blocks.8.attention.w_out": 0.4888458251953125, "blocks.8.feed_forward.w1": 0.7963155508041382, "blocks.8.feed_forward.w2": 0.6594251394271851, "blocks.8.feed_forward.w3": 0.7776781916618347}} +{"step": 2000, "changed_frac": 0.7597202724880643, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8128013610839844, "blocks.0.attention.w_k": 0.7920036315917969, "blocks.0.attention.w_v": 0.6070976257324219, "blocks.0.attention.w_out": 0.5707550048828125, "blocks.0.feed_forward.w1": 0.7623342275619507, "blocks.0.feed_forward.w2": 0.8192176818847656, "blocks.0.feed_forward.w3": 0.7405726313591003, "blocks.1.attention.w_q": 0.8078231811523438, "blocks.1.attention.w_k": 0.7761993408203125, "blocks.1.attention.w_v": 0.635589599609375, "blocks.1.attention.w_out": 0.7509880065917969, "blocks.1.feed_forward.w1": 0.7707570791244507, "blocks.1.feed_forward.w2": 0.8321864008903503, "blocks.1.feed_forward.w3": 0.7647514343261719, "blocks.2.attention.w_q": 0.8188705444335938, "blocks.2.attention.w_k": 0.7895736694335938, "blocks.2.attention.w_v": 0.6718368530273438, "blocks.2.attention.w_out": 0.7212715148925781, "blocks.2.feed_forward.w1": 0.7791455984115601, "blocks.2.feed_forward.w2": 0.8256391286849976, "blocks.2.feed_forward.w3": 0.7737363576889038, "blocks.3.attention.w_q": 0.8143997192382812, "blocks.3.attention.w_k": 0.7866592407226562, "blocks.3.attention.w_v": 0.6826553344726562, "blocks.3.attention.w_out": 0.6942214965820312, "blocks.3.feed_forward.w1": 0.7824796438217163, "blocks.3.feed_forward.w2": 0.8095359802246094, "blocks.3.feed_forward.w3": 0.7724800109863281, "blocks.4.attention.w_q": 0.8100242614746094, "blocks.4.attention.w_k": 0.7840957641601562, "blocks.4.attention.w_v": 0.6324577331542969, "blocks.4.attention.w_out": 0.7192840576171875, "blocks.4.feed_forward.w1": 0.7856712341308594, "blocks.4.feed_forward.w2": 0.7911364436149597, "blocks.4.feed_forward.w3": 0.7736638784408569, "blocks.5.attention.w_q": 0.8160858154296875, "blocks.5.attention.w_k": 0.78240966796875, "blocks.5.attention.w_v": 0.6701622009277344, "blocks.5.attention.w_out": 0.6534652709960938, "blocks.5.feed_forward.w1": 0.7895724177360535, "blocks.5.feed_forward.w2": 0.7791048884391785, "blocks.5.feed_forward.w3": 0.7754923701286316, "blocks.6.attention.w_q": 0.8090629577636719, "blocks.6.attention.w_k": 0.7906150817871094, "blocks.6.attention.w_v": 0.692596435546875, "blocks.6.attention.w_out": 0.5446548461914062, "blocks.6.feed_forward.w1": 0.7919425964355469, "blocks.6.feed_forward.w2": 0.7609469294548035, "blocks.6.feed_forward.w3": 0.7778104543685913, "blocks.7.attention.w_q": 0.8132057189941406, "blocks.7.attention.w_k": 0.7912139892578125, "blocks.7.attention.w_v": 0.6992073059082031, "blocks.7.attention.w_out": 0.5560951232910156, "blocks.7.feed_forward.w1": 0.7919120788574219, "blocks.7.feed_forward.w2": 0.7175649404525757, "blocks.7.feed_forward.w3": 0.7747167348861694, "blocks.8.attention.w_q": 0.8076400756835938, "blocks.8.attention.w_k": 0.7967758178710938, "blocks.8.attention.w_v": 0.6393661499023438, "blocks.8.attention.w_out": 0.4743003845214844, "blocks.8.feed_forward.w1": 0.793060302734375, "blocks.8.feed_forward.w2": 0.6512998342514038, "blocks.8.feed_forward.w3": 0.770538330078125}} +{"step": 2100, "changed_frac": 0.7545463692428719, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8057937622070312, "blocks.0.attention.w_k": 0.7873725891113281, "blocks.0.attention.w_v": 0.5951576232910156, "blocks.0.attention.w_out": 0.5615577697753906, "blocks.0.feed_forward.w1": 0.7593269348144531, "blocks.0.feed_forward.w2": 0.8176867365837097, "blocks.0.feed_forward.w3": 0.7362747192382812, "blocks.1.attention.w_q": 0.8043594360351562, "blocks.1.attention.w_k": 0.7783355712890625, "blocks.1.attention.w_v": 0.6315803527832031, "blocks.1.attention.w_out": 0.7467308044433594, "blocks.1.feed_forward.w1": 0.7657585144042969, "blocks.1.feed_forward.w2": 0.8284658193588257, "blocks.1.feed_forward.w3": 0.7593981623649597, "blocks.2.attention.w_q": 0.8160476684570312, "blocks.2.attention.w_k": 0.7901420593261719, "blocks.2.attention.w_v": 0.6668548583984375, "blocks.2.attention.w_out": 0.7161941528320312, "blocks.2.feed_forward.w1": 0.7741190791130066, "blocks.2.feed_forward.w2": 0.8218663930892944, "blocks.2.feed_forward.w3": 0.7672958374023438, "blocks.3.attention.w_q": 0.8117866516113281, "blocks.3.attention.w_k": 0.7862586975097656, "blocks.3.attention.w_v": 0.6775360107421875, "blocks.3.attention.w_out": 0.6864089965820312, "blocks.3.feed_forward.w1": 0.7774060964584351, "blocks.3.feed_forward.w2": 0.8051656484603882, "blocks.3.feed_forward.w3": 0.7672615051269531, "blocks.4.attention.w_q": 0.8067054748535156, "blocks.4.attention.w_k": 0.7822685241699219, "blocks.4.attention.w_v": 0.6182632446289062, "blocks.4.attention.w_out": 0.7167701721191406, "blocks.4.feed_forward.w1": 0.780364990234375, "blocks.4.feed_forward.w2": 0.7876790761947632, "blocks.4.feed_forward.w3": 0.7675260305404663, "blocks.5.attention.w_q": 0.8150596618652344, "blocks.5.attention.w_k": 0.7812919616699219, "blocks.5.attention.w_v": 0.6656875610351562, "blocks.5.attention.w_out": 0.6446266174316406, "blocks.5.feed_forward.w1": 0.7855479121208191, "blocks.5.feed_forward.w2": 0.77288818359375, "blocks.5.feed_forward.w3": 0.7707774043083191, "blocks.6.attention.w_q": 0.8046150207519531, "blocks.6.attention.w_k": 0.7848129272460938, "blocks.6.attention.w_v": 0.687744140625, "blocks.6.attention.w_out": 0.53546142578125, "blocks.6.feed_forward.w1": 0.78765869140625, "blocks.6.feed_forward.w2": 0.7544898986816406, "blocks.6.feed_forward.w3": 0.7721595764160156, "blocks.7.attention.w_q": 0.8099174499511719, "blocks.7.attention.w_k": 0.7887306213378906, "blocks.7.attention.w_v": 0.6933021545410156, "blocks.7.attention.w_out": 0.5431060791015625, "blocks.7.feed_forward.w1": 0.7884445190429688, "blocks.7.feed_forward.w2": 0.7085813283920288, "blocks.7.feed_forward.w3": 0.7684835195541382, "blocks.8.attention.w_q": 0.8036231994628906, "blocks.8.attention.w_k": 0.7950859069824219, "blocks.8.attention.w_v": 0.6309127807617188, "blocks.8.attention.w_out": 0.4610252380371094, "blocks.8.feed_forward.w1": 0.7885259389877319, "blocks.8.feed_forward.w2": 0.640350341796875, "blocks.8.feed_forward.w3": 0.7634302973747253}} +{"step": 2200, "changed_frac": 0.7485435681465344, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8008003234863281, "blocks.0.attention.w_k": 0.7847862243652344, "blocks.0.attention.w_v": 0.5844192504882812, "blocks.0.attention.w_out": 0.5537834167480469, "blocks.0.feed_forward.w1": 0.7555631399154663, "blocks.0.feed_forward.w2": 0.8139775991439819, "blocks.0.feed_forward.w3": 0.7327448725700378, "blocks.1.attention.w_q": 0.8014106750488281, "blocks.1.attention.w_k": 0.7735328674316406, "blocks.1.attention.w_v": 0.6186981201171875, "blocks.1.attention.w_out": 0.7392120361328125, "blocks.1.feed_forward.w1": 0.7621536254882812, "blocks.1.feed_forward.w2": 0.8253962397575378, "blocks.1.feed_forward.w3": 0.7541275024414062, "blocks.2.attention.w_q": 0.8130760192871094, "blocks.2.attention.w_k": 0.7819671630859375, "blocks.2.attention.w_v": 0.65545654296875, "blocks.2.attention.w_out": 0.7086601257324219, "blocks.2.feed_forward.w1": 0.7692362666130066, "blocks.2.feed_forward.w2": 0.8172098994255066, "blocks.2.feed_forward.w3": 0.7625389099121094, "blocks.3.attention.w_q": 0.8071060180664062, "blocks.3.attention.w_k": 0.7813034057617188, "blocks.3.attention.w_v": 0.6664237976074219, "blocks.3.attention.w_out": 0.677459716796875, "blocks.3.feed_forward.w1": 0.772192656993866, "blocks.3.feed_forward.w2": 0.8000094294548035, "blocks.3.feed_forward.w3": 0.7612470388412476, "blocks.4.attention.w_q": 0.8038177490234375, "blocks.4.attention.w_k": 0.7787590026855469, "blocks.4.attention.w_v": 0.6059417724609375, "blocks.4.attention.w_out": 0.7117500305175781, "blocks.4.feed_forward.w1": 0.7752291560173035, "blocks.4.feed_forward.w2": 0.7818311452865601, "blocks.4.feed_forward.w3": 0.7612228393554688, "blocks.5.attention.w_q": 0.8097114562988281, "blocks.5.attention.w_k": 0.7768287658691406, "blocks.5.attention.w_v": 0.6568450927734375, "blocks.5.attention.w_out": 0.6340141296386719, "blocks.5.feed_forward.w1": 0.7802645564079285, "blocks.5.feed_forward.w2": 0.7666308283805847, "blocks.5.feed_forward.w3": 0.7625338435173035, "blocks.6.attention.w_q": 0.8028335571289062, "blocks.6.attention.w_k": 0.7817306518554688, "blocks.6.attention.w_v": 0.6778755187988281, "blocks.6.attention.w_out": 0.5249366760253906, "blocks.6.feed_forward.w1": 0.7835426330566406, "blocks.6.feed_forward.w2": 0.7474988698959351, "blocks.6.feed_forward.w3": 0.7651380300521851, "blocks.7.attention.w_q": 0.8083686828613281, "blocks.7.attention.w_k": 0.7839317321777344, "blocks.7.attention.w_v": 0.6842422485351562, "blocks.7.attention.w_out": 0.5280799865722656, "blocks.7.feed_forward.w1": 0.7830645442008972, "blocks.7.feed_forward.w2": 0.6994349360466003, "blocks.7.feed_forward.w3": 0.7624499201774597, "blocks.8.attention.w_q": 0.7997627258300781, "blocks.8.attention.w_k": 0.7938766479492188, "blocks.8.attention.w_v": 0.6232452392578125, "blocks.8.attention.w_out": 0.4463081359863281, "blocks.8.feed_forward.w1": 0.7847506403923035, "blocks.8.feed_forward.w2": 0.628180205821991, "blocks.8.feed_forward.w3": 0.7580451965332031}} +{"step": 2300, "changed_frac": 0.7441246489174346, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.8036041259765625, "blocks.0.attention.w_k": 0.7899513244628906, "blocks.0.attention.w_v": 0.5801429748535156, "blocks.0.attention.w_out": 0.5465126037597656, "blocks.0.feed_forward.w1": 0.7538427114486694, "blocks.0.feed_forward.w2": 0.8118438720703125, "blocks.0.feed_forward.w3": 0.7287648916244507, "blocks.1.attention.w_q": 0.7977981567382812, "blocks.1.attention.w_k": 0.7724189758300781, "blocks.1.attention.w_v": 0.6161727905273438, "blocks.1.attention.w_out": 0.7353973388671875, "blocks.1.feed_forward.w1": 0.7575492858886719, "blocks.1.feed_forward.w2": 0.8205007314682007, "blocks.1.feed_forward.w3": 0.7491645812988281, "blocks.2.attention.w_q": 0.8089103698730469, "blocks.2.attention.w_k": 0.7821083068847656, "blocks.2.attention.w_v": 0.6494331359863281, "blocks.2.attention.w_out": 0.705596923828125, "blocks.2.feed_forward.w1": 0.7638282775878906, "blocks.2.feed_forward.w2": 0.8139063715934753, "blocks.2.feed_forward.w3": 0.7568499445915222, "blocks.3.attention.w_q": 0.8049392700195312, "blocks.3.attention.w_k": 0.7770919799804688, "blocks.3.attention.w_v": 0.6611785888671875, "blocks.3.attention.w_out": 0.6740570068359375, "blocks.3.feed_forward.w1": 0.7687250971794128, "blocks.3.feed_forward.w2": 0.7952842712402344, "blocks.3.feed_forward.w3": 0.7556585073471069, "blocks.4.attention.w_q": 0.7999420166015625, "blocks.4.attention.w_k": 0.773956298828125, "blocks.4.attention.w_v": 0.5974502563476562, "blocks.4.attention.w_out": 0.706085205078125, "blocks.4.feed_forward.w1": 0.7711524963378906, "blocks.4.feed_forward.w2": 0.7764905691146851, "blocks.4.feed_forward.w3": 0.7559382319450378, "blocks.5.attention.w_q": 0.8083419799804688, "blocks.5.attention.w_k": 0.7744903564453125, "blocks.5.attention.w_v": 0.6525306701660156, "blocks.5.attention.w_out": 0.6264839172363281, "blocks.5.feed_forward.w1": 0.7757415771484375, "blocks.5.feed_forward.w2": 0.7600606679916382, "blocks.5.feed_forward.w3": 0.7570317983627319, "blocks.6.attention.w_q": 0.7993087768554688, "blocks.6.attention.w_k": 0.779022216796875, "blocks.6.attention.w_v": 0.6771926879882812, "blocks.6.attention.w_out": 0.5145339965820312, "blocks.6.feed_forward.w1": 0.7791595458984375, "blocks.6.feed_forward.w2": 0.7416661977767944, "blocks.6.feed_forward.w3": 0.7591743469238281, "blocks.7.attention.w_q": 0.8066062927246094, "blocks.7.attention.w_k": 0.7812347412109375, "blocks.7.attention.w_v": 0.6804618835449219, "blocks.7.attention.w_out": 0.519439697265625, "blocks.7.feed_forward.w1": 0.7790374755859375, "blocks.7.feed_forward.w2": 0.6926104426383972, "blocks.7.feed_forward.w3": 0.7562484741210938, "blocks.8.attention.w_q": 0.7975578308105469, "blocks.8.attention.w_k": 0.7893791198730469, "blocks.8.attention.w_v": 0.619049072265625, "blocks.8.attention.w_out": 0.4388847351074219, "blocks.8.feed_forward.w1": 0.782867431640625, "blocks.8.feed_forward.w2": 0.6216405630111694, "blocks.8.feed_forward.w3": 0.7537015676498413}} +{"step": 2400, "changed_frac": 0.7373801663390591, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7873611450195312, "blocks.0.attention.w_k": 0.7744522094726562, "blocks.0.attention.w_v": 0.5626029968261719, "blocks.0.attention.w_out": 0.5339775085449219, "blocks.0.feed_forward.w1": 0.7481269836425781, "blocks.0.feed_forward.w2": 0.8075637817382812, "blocks.0.feed_forward.w3": 0.7201411128044128, "blocks.1.attention.w_q": 0.7976417541503906, "blocks.1.attention.w_k": 0.7662429809570312, "blocks.1.attention.w_v": 0.60491943359375, "blocks.1.attention.w_out": 0.7261199951171875, "blocks.1.feed_forward.w1": 0.7522290945053101, "blocks.1.feed_forward.w2": 0.8181241750717163, "blocks.1.feed_forward.w3": 0.7410163879394531, "blocks.2.attention.w_q": 0.80419921875, "blocks.2.attention.w_k": 0.7766609191894531, "blocks.2.attention.w_v": 0.6410293579101562, "blocks.2.attention.w_out": 0.6992950439453125, "blocks.2.feed_forward.w1": 0.7586683034896851, "blocks.2.feed_forward.w2": 0.8098678588867188, "blocks.2.feed_forward.w3": 0.75048828125, "blocks.3.attention.w_q": 0.8019065856933594, "blocks.3.attention.w_k": 0.7729988098144531, "blocks.3.attention.w_v": 0.6565170288085938, "blocks.3.attention.w_out": 0.6618766784667969, "blocks.3.feed_forward.w1": 0.7634353637695312, "blocks.3.feed_forward.w2": 0.7913450002670288, "blocks.3.feed_forward.w3": 0.7504565119743347, "blocks.4.attention.w_q": 0.7958526611328125, "blocks.4.attention.w_k": 0.7698783874511719, "blocks.4.attention.w_v": 0.5841293334960938, "blocks.4.attention.w_out": 0.6999588012695312, "blocks.4.feed_forward.w1": 0.7656835317611694, "blocks.4.feed_forward.w2": 0.7716115713119507, "blocks.4.feed_forward.w3": 0.7485733032226562, "blocks.5.attention.w_q": 0.8058700561523438, "blocks.5.attention.w_k": 0.7685890197753906, "blocks.5.attention.w_v": 0.644500732421875, "blocks.5.attention.w_out": 0.6154365539550781, "blocks.5.feed_forward.w1": 0.770941436290741, "blocks.5.feed_forward.w2": 0.7546641230583191, "blocks.5.feed_forward.w3": 0.7506002187728882, "blocks.6.attention.w_q": 0.7960052490234375, "blocks.6.attention.w_k": 0.777099609375, "blocks.6.attention.w_v": 0.6694297790527344, "blocks.6.attention.w_out": 0.5026397705078125, "blocks.6.feed_forward.w1": 0.7735163569450378, "blocks.6.feed_forward.w2": 0.7351888418197632, "blocks.6.feed_forward.w3": 0.7513898611068726, "blocks.7.attention.w_q": 0.8017921447753906, "blocks.7.attention.w_k": 0.77728271484375, "blocks.7.attention.w_v": 0.6726341247558594, "blocks.7.attention.w_out": 0.5071563720703125, "blocks.7.feed_forward.w1": 0.7739881277084351, "blocks.7.feed_forward.w2": 0.6823743581771851, "blocks.7.feed_forward.w3": 0.7496656179428101, "blocks.8.attention.w_q": 0.796295166015625, "blocks.8.attention.w_k": 0.7869720458984375, "blocks.8.attention.w_v": 0.6075439453125, "blocks.8.attention.w_out": 0.42189788818359375, "blocks.8.feed_forward.w1": 0.7762489318847656, "blocks.8.feed_forward.w2": 0.606451690196991, "blocks.8.feed_forward.w3": 0.7458776235580444}} +{"step": 2500, "changed_frac": 0.7312529800284622, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7901649475097656, "blocks.0.attention.w_k": 0.7767181396484375, "blocks.0.attention.w_v": 0.5519256591796875, "blocks.0.attention.w_out": 0.5223236083984375, "blocks.0.feed_forward.w1": 0.7432975769042969, "blocks.0.feed_forward.w2": 0.8047218322753906, "blocks.0.feed_forward.w3": 0.715972900390625, "blocks.1.attention.w_q": 0.7925148010253906, "blocks.1.attention.w_k": 0.7627601623535156, "blocks.1.attention.w_v": 0.5939559936523438, "blocks.1.attention.w_out": 0.7198219299316406, "blocks.1.feed_forward.w1": 0.7471224665641785, "blocks.1.feed_forward.w2": 0.8129705190658569, "blocks.1.feed_forward.w3": 0.7344602346420288, "blocks.2.attention.w_q": 0.8012046813964844, "blocks.2.attention.w_k": 0.7754440307617188, "blocks.2.attention.w_v": 0.6302452087402344, "blocks.2.attention.w_out": 0.6927986145019531, "blocks.2.feed_forward.w1": 0.752020537853241, "blocks.2.feed_forward.w2": 0.8054453730583191, "blocks.2.feed_forward.w3": 0.7439651489257812, "blocks.3.attention.w_q": 0.7979316711425781, "blocks.3.attention.w_k": 0.7692146301269531, "blocks.3.attention.w_v": 0.6462135314941406, "blocks.3.attention.w_out": 0.6554832458496094, "blocks.3.feed_forward.w1": 0.7571678161621094, "blocks.3.feed_forward.w2": 0.7854830622673035, "blocks.3.feed_forward.w3": 0.7428767085075378, "blocks.4.attention.w_q": 0.7945556640625, "blocks.4.attention.w_k": 0.7645912170410156, "blocks.4.attention.w_v": 0.5709342956542969, "blocks.4.attention.w_out": 0.6948127746582031, "blocks.4.feed_forward.w1": 0.7597732543945312, "blocks.4.feed_forward.w2": 0.7647285461425781, "blocks.4.feed_forward.w3": 0.7420603632926941, "blocks.5.attention.w_q": 0.8029594421386719, "blocks.5.attention.w_k": 0.7663536071777344, "blocks.5.attention.w_v": 0.6380729675292969, "blocks.5.attention.w_out": 0.6067733764648438, "blocks.5.feed_forward.w1": 0.7639452815055847, "blocks.5.feed_forward.w2": 0.7480863332748413, "blocks.5.feed_forward.w3": 0.7447446584701538, "blocks.6.attention.w_q": 0.7904319763183594, "blocks.6.attention.w_k": 0.7684440612792969, "blocks.6.attention.w_v": 0.6593399047851562, "blocks.6.attention.w_out": 0.4936103820800781, "blocks.6.feed_forward.w1": 0.7685712575912476, "blocks.6.feed_forward.w2": 0.7284431457519531, "blocks.6.feed_forward.w3": 0.7450701594352722, "blocks.7.attention.w_q": 0.7986373901367188, "blocks.7.attention.w_k": 0.7727317810058594, "blocks.7.attention.w_v": 0.6634101867675781, "blocks.7.attention.w_out": 0.4968719482421875, "blocks.7.feed_forward.w1": 0.7691205739974976, "blocks.7.feed_forward.w2": 0.6727320551872253, "blocks.7.feed_forward.w3": 0.7428728938102722, "blocks.8.attention.w_q": 0.7910194396972656, "blocks.8.attention.w_k": 0.784820556640625, "blocks.8.attention.w_v": 0.5987319946289062, "blocks.8.attention.w_out": 0.41107177734375, "blocks.8.feed_forward.w1": 0.7725194692611694, "blocks.8.feed_forward.w2": 0.5950241088867188, "blocks.8.feed_forward.w3": 0.7400385737419128}} +{"step": 2600, "changed_frac": 0.7252780229617388, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7827987670898438, "blocks.0.attention.w_k": 0.7683792114257812, "blocks.0.attention.w_v": 0.5405464172363281, "blocks.0.attention.w_out": 0.5172653198242188, "blocks.0.feed_forward.w1": 0.739532470703125, "blocks.0.feed_forward.w2": 0.8028793334960938, "blocks.0.feed_forward.w3": 0.7110519409179688, "blocks.1.attention.w_q": 0.7885246276855469, "blocks.1.attention.w_k": 0.7583770751953125, "blocks.1.attention.w_v": 0.5886878967285156, "blocks.1.attention.w_out": 0.7108955383300781, "blocks.1.feed_forward.w1": 0.7400360107421875, "blocks.1.feed_forward.w2": 0.810370147228241, "blocks.1.feed_forward.w3": 0.7287394404411316, "blocks.2.attention.w_q": 0.7968711853027344, "blocks.2.attention.w_k": 0.7710723876953125, "blocks.2.attention.w_v": 0.6185188293457031, "blocks.2.attention.w_out": 0.6870689392089844, "blocks.2.feed_forward.w1": 0.7472763061523438, "blocks.2.feed_forward.w2": 0.8009490966796875, "blocks.2.feed_forward.w3": 0.7378578186035156, "blocks.3.attention.w_q": 0.7980422973632812, "blocks.3.attention.w_k": 0.7648849487304688, "blocks.3.attention.w_v": 0.6389083862304688, "blocks.3.attention.w_out": 0.6460456848144531, "blocks.3.feed_forward.w1": 0.7527784109115601, "blocks.3.feed_forward.w2": 0.7803268432617188, "blocks.3.feed_forward.w3": 0.7362136840820312, "blocks.4.attention.w_q": 0.7888374328613281, "blocks.4.attention.w_k": 0.759918212890625, "blocks.4.attention.w_v": 0.5560340881347656, "blocks.4.attention.w_out": 0.6879119873046875, "blocks.4.feed_forward.w1": 0.7549616694450378, "blocks.4.feed_forward.w2": 0.7585347890853882, "blocks.4.feed_forward.w3": 0.7344322204589844, "blocks.5.attention.w_q": 0.7994613647460938, "blocks.5.attention.w_k": 0.7630348205566406, "blocks.5.attention.w_v": 0.632476806640625, "blocks.5.attention.w_out": 0.5987014770507812, "blocks.5.feed_forward.w1": 0.7598559260368347, "blocks.5.feed_forward.w2": 0.7415822744369507, "blocks.5.feed_forward.w3": 0.7357737421989441, "blocks.6.attention.w_q": 0.7882652282714844, "blocks.6.attention.w_k": 0.7668190002441406, "blocks.6.attention.w_v": 0.6535491943359375, "blocks.6.attention.w_out": 0.4857940673828125, "blocks.6.feed_forward.w1": 0.7632598876953125, "blocks.6.feed_forward.w2": 0.7211278676986694, "blocks.6.feed_forward.w3": 0.7387797236442566, "blocks.7.attention.w_q": 0.7940635681152344, "blocks.7.attention.w_k": 0.7691497802734375, "blocks.7.attention.w_v": 0.6549072265625, "blocks.7.attention.w_out": 0.4847145080566406, "blocks.7.feed_forward.w1": 0.7628161311149597, "blocks.7.feed_forward.w2": 0.6649360656738281, "blocks.7.feed_forward.w3": 0.7368087768554688, "blocks.8.attention.w_q": 0.7900428771972656, "blocks.8.attention.w_k": 0.7819328308105469, "blocks.8.attention.w_v": 0.5876235961914062, "blocks.8.attention.w_out": 0.3996391296386719, "blocks.8.feed_forward.w1": 0.7678006887435913, "blocks.8.feed_forward.w2": 0.5843162536621094, "blocks.8.feed_forward.w3": 0.7329241633415222}} +{"step": 2700, "changed_frac": 0.7189093043661525, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7782821655273438, "blocks.0.attention.w_k": 0.7640838623046875, "blocks.0.attention.w_v": 0.5328903198242188, "blocks.0.attention.w_out": 0.5086669921875, "blocks.0.feed_forward.w1": 0.7333628535270691, "blocks.0.feed_forward.w2": 0.7991320490837097, "blocks.0.feed_forward.w3": 0.7036883234977722, "blocks.1.attention.w_q": 0.7861595153808594, "blocks.1.attention.w_k": 0.757293701171875, "blocks.1.attention.w_v": 0.5828933715820312, "blocks.1.attention.w_out": 0.7059974670410156, "blocks.1.feed_forward.w1": 0.7346929311752319, "blocks.1.feed_forward.w2": 0.8064702749252319, "blocks.1.feed_forward.w3": 0.7218500971794128, "blocks.2.attention.w_q": 0.79364013671875, "blocks.2.attention.w_k": 0.7649459838867188, "blocks.2.attention.w_v": 0.6117897033691406, "blocks.2.attention.w_out": 0.682098388671875, "blocks.2.feed_forward.w1": 0.7408968806266785, "blocks.2.feed_forward.w2": 0.7965482473373413, "blocks.2.feed_forward.w3": 0.73101806640625, "blocks.3.attention.w_q": 0.7927970886230469, "blocks.3.attention.w_k": 0.7603263854980469, "blocks.3.attention.w_v": 0.627838134765625, "blocks.3.attention.w_out": 0.6402244567871094, "blocks.3.feed_forward.w1": 0.7449366450309753, "blocks.3.feed_forward.w2": 0.7756678462028503, "blocks.3.feed_forward.w3": 0.7288398742675781, "blocks.4.attention.w_q": 0.7850875854492188, "blocks.4.attention.w_k": 0.7574081420898438, "blocks.4.attention.w_v": 0.54339599609375, "blocks.4.attention.w_out": 0.6818656921386719, "blocks.4.feed_forward.w1": 0.7489115595817566, "blocks.4.feed_forward.w2": 0.751715362071991, "blocks.4.feed_forward.w3": 0.7279803156852722, "blocks.5.attention.w_q": 0.7960433959960938, "blocks.5.attention.w_k": 0.756683349609375, "blocks.5.attention.w_v": 0.6260337829589844, "blocks.5.attention.w_out": 0.5879554748535156, "blocks.5.feed_forward.w1": 0.7549654841423035, "blocks.5.feed_forward.w2": 0.735107421875, "blocks.5.feed_forward.w3": 0.7296524047851562, "blocks.6.attention.w_q": 0.7856941223144531, "blocks.6.attention.w_k": 0.7618255615234375, "blocks.6.attention.w_v": 0.6446495056152344, "blocks.6.attention.w_out": 0.4744377136230469, "blocks.6.feed_forward.w1": 0.7562167048454285, "blocks.6.feed_forward.w2": 0.7119979858398438, "blocks.6.feed_forward.w3": 0.7312736511230469, "blocks.7.attention.w_q": 0.7919235229492188, "blocks.7.attention.w_k": 0.7669525146484375, "blocks.7.attention.w_v": 0.6500587463378906, "blocks.7.attention.w_out": 0.4721488952636719, "blocks.7.feed_forward.w1": 0.760016143321991, "blocks.7.feed_forward.w2": 0.6552543640136719, "blocks.7.feed_forward.w3": 0.7280845642089844, "blocks.8.attention.w_q": 0.7866630554199219, "blocks.8.attention.w_k": 0.77734375, "blocks.8.attention.w_v": 0.5788917541503906, "blocks.8.attention.w_out": 0.390472412109375, "blocks.8.feed_forward.w1": 0.7637342214584351, "blocks.8.feed_forward.w2": 0.5716196894645691, "blocks.8.feed_forward.w3": 0.725341796875}} +{"step": 2800, "changed_frac": 0.7132756159855769, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7818870544433594, "blocks.0.attention.w_k": 0.7664451599121094, "blocks.0.attention.w_v": 0.5201606750488281, "blocks.0.attention.w_out": 0.5022735595703125, "blocks.0.feed_forward.w1": 0.7284177541732788, "blocks.0.feed_forward.w2": 0.7954114675521851, "blocks.0.feed_forward.w3": 0.6992454528808594, "blocks.1.attention.w_q": 0.7846107482910156, "blocks.1.attention.w_k": 0.7537612915039062, "blocks.1.attention.w_v": 0.5754318237304688, "blocks.1.attention.w_out": 0.7007637023925781, "blocks.1.feed_forward.w1": 0.7280718684196472, "blocks.1.feed_forward.w2": 0.802130401134491, "blocks.1.feed_forward.w3": 0.714636504650116, "blocks.2.attention.w_q": 0.7894020080566406, "blocks.2.attention.w_k": 0.7619056701660156, "blocks.2.attention.w_v": 0.6067543029785156, "blocks.2.attention.w_out": 0.6747169494628906, "blocks.2.feed_forward.w1": 0.7349370718002319, "blocks.2.feed_forward.w2": 0.7919133901596069, "blocks.2.feed_forward.w3": 0.7240537405014038, "blocks.3.attention.w_q": 0.7891349792480469, "blocks.3.attention.w_k": 0.7571525573730469, "blocks.3.attention.w_v": 0.6245346069335938, "blocks.3.attention.w_out": 0.6297187805175781, "blocks.3.feed_forward.w1": 0.7403475642204285, "blocks.3.feed_forward.w2": 0.7698987722396851, "blocks.3.feed_forward.w3": 0.7220395803451538, "blocks.4.attention.w_q": 0.7812309265136719, "blocks.4.attention.w_k": 0.75384521484375, "blocks.4.attention.w_v": 0.5345611572265625, "blocks.4.attention.w_out": 0.6778984069824219, "blocks.4.feed_forward.w1": 0.7431628108024597, "blocks.4.feed_forward.w2": 0.7465858459472656, "blocks.4.feed_forward.w3": 0.7219938039779663, "blocks.5.attention.w_q": 0.7932701110839844, "blocks.5.attention.w_k": 0.7535514831542969, "blocks.5.attention.w_v": 0.6226387023925781, "blocks.5.attention.w_out": 0.5796394348144531, "blocks.5.feed_forward.w1": 0.7495193481445312, "blocks.5.feed_forward.w2": 0.7282397150993347, "blocks.5.feed_forward.w3": 0.7230339050292969, "blocks.6.attention.w_q": 0.7829666137695312, "blocks.6.attention.w_k": 0.7556381225585938, "blocks.6.attention.w_v": 0.6411590576171875, "blocks.6.attention.w_out": 0.4687614440917969, "blocks.6.feed_forward.w1": 0.753119170665741, "blocks.6.feed_forward.w2": 0.7046076655387878, "blocks.6.feed_forward.w3": 0.7248331904411316, "blocks.7.attention.w_q": 0.7890586853027344, "blocks.7.attention.w_k": 0.7640190124511719, "blocks.7.attention.w_v": 0.6421318054199219, "blocks.7.attention.w_out": 0.46202850341796875, "blocks.7.feed_forward.w1": 0.7538871765136719, "blocks.7.feed_forward.w2": 0.6470273733139038, "blocks.7.feed_forward.w3": 0.7210769653320312, "blocks.8.attention.w_q": 0.7826957702636719, "blocks.8.attention.w_k": 0.7729682922363281, "blocks.8.attention.w_v": 0.5713005065917969, "blocks.8.attention.w_out": 0.3768653869628906, "blocks.8.feed_forward.w1": 0.7595316767692566, "blocks.8.feed_forward.w2": 0.5623029470443726, "blocks.8.feed_forward.w3": 0.7194303274154663}} +{"step": 2900, "changed_frac": 0.7055994995638856, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7667617797851562, "blocks.0.attention.w_k": 0.7478561401367188, "blocks.0.attention.w_v": 0.5029449462890625, "blocks.0.attention.w_out": 0.4914207458496094, "blocks.0.feed_forward.w1": 0.7230377197265625, "blocks.0.feed_forward.w2": 0.7905439138412476, "blocks.0.feed_forward.w3": 0.6909612417221069, "blocks.1.attention.w_q": 0.7793617248535156, "blocks.1.attention.w_k": 0.7462348937988281, "blocks.1.attention.w_v": 0.5668258666992188, "blocks.1.attention.w_out": 0.6932258605957031, "blocks.1.feed_forward.w1": 0.7209205627441406, "blocks.1.feed_forward.w2": 0.7970085144042969, "blocks.1.feed_forward.w3": 0.7071546316146851, "blocks.2.attention.w_q": 0.7833366394042969, "blocks.2.attention.w_k": 0.7559890747070312, "blocks.2.attention.w_v": 0.5918998718261719, "blocks.2.attention.w_out": 0.6668701171875, "blocks.2.feed_forward.w1": 0.7279523611068726, "blocks.2.feed_forward.w2": 0.7875162959098816, "blocks.2.feed_forward.w3": 0.71539306640625, "blocks.3.attention.w_q": 0.7831764221191406, "blocks.3.attention.w_k": 0.7511940002441406, "blocks.3.attention.w_v": 0.6135749816894531, "blocks.3.attention.w_out": 0.620574951171875, "blocks.3.feed_forward.w1": 0.7338282465934753, "blocks.3.feed_forward.w2": 0.7645429372787476, "blocks.3.feed_forward.w3": 0.714874267578125, "blocks.4.attention.w_q": 0.7761383056640625, "blocks.4.attention.w_k": 0.7471504211425781, "blocks.4.attention.w_v": 0.5165863037109375, "blocks.4.attention.w_out": 0.6693878173828125, "blocks.4.feed_forward.w1": 0.7386792898178101, "blocks.4.feed_forward.w2": 0.7399470210075378, "blocks.4.feed_forward.w3": 0.7134997248649597, "blocks.5.attention.w_q": 0.7884712219238281, "blocks.5.attention.w_k": 0.7477684020996094, "blocks.5.attention.w_v": 0.6121940612792969, "blocks.5.attention.w_out": 0.5692329406738281, "blocks.5.feed_forward.w1": 0.7428983449935913, "blocks.5.feed_forward.w2": 0.7219009399414062, "blocks.5.feed_forward.w3": 0.7151438593864441, "blocks.6.attention.w_q": 0.7762336730957031, "blocks.6.attention.w_k": 0.7495269775390625, "blocks.6.attention.w_v": 0.6326560974121094, "blocks.6.attention.w_out": 0.4552574157714844, "blocks.6.feed_forward.w1": 0.7475687861442566, "blocks.6.feed_forward.w2": 0.6967430114746094, "blocks.6.feed_forward.w3": 0.7173271179199219, "blocks.7.attention.w_q": 0.7832183837890625, "blocks.7.attention.w_k": 0.7558441162109375, "blocks.7.attention.w_v": 0.6307716369628906, "blocks.7.attention.w_out": 0.44837188720703125, "blocks.7.feed_forward.w1": 0.7466773986816406, "blocks.7.feed_forward.w2": 0.6345698237419128, "blocks.7.feed_forward.w3": 0.7139066457748413, "blocks.8.attention.w_q": 0.77813720703125, "blocks.8.attention.w_k": 0.7710723876953125, "blocks.8.attention.w_v": 0.5626373291015625, "blocks.8.attention.w_out": 0.3656578063964844, "blocks.8.feed_forward.w1": 0.7532730102539062, "blocks.8.feed_forward.w2": 0.551483154296875, "blocks.8.feed_forward.w3": 0.7118415832519531}} +{"step": 3000, "changed_frac": 0.6983351748213809, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7607421875, "blocks.0.attention.w_k": 0.7420845031738281, "blocks.0.attention.w_v": 0.4943885803222656, "blocks.0.attention.w_out": 0.4840240478515625, "blocks.0.feed_forward.w1": 0.7167345881462097, "blocks.0.feed_forward.w2": 0.7863451838493347, "blocks.0.feed_forward.w3": 0.6846542358398438, "blocks.1.attention.w_q": 0.7742729187011719, "blocks.1.attention.w_k": 0.7406120300292969, "blocks.1.attention.w_v": 0.5538444519042969, "blocks.1.attention.w_out": 0.6826057434082031, "blocks.1.feed_forward.w1": 0.7150179743766785, "blocks.1.feed_forward.w2": 0.7911517024040222, "blocks.1.feed_forward.w3": 0.6998113393783569, "blocks.2.attention.w_q": 0.7781600952148438, "blocks.2.attention.w_k": 0.7489471435546875, "blocks.2.attention.w_v": 0.5810775756835938, "blocks.2.attention.w_out": 0.6594581604003906, "blocks.2.feed_forward.w1": 0.7211202383041382, "blocks.2.feed_forward.w2": 0.7809613943099976, "blocks.2.feed_forward.w3": 0.7085177302360535, "blocks.3.attention.w_q": 0.7809791564941406, "blocks.3.attention.w_k": 0.7446556091308594, "blocks.3.attention.w_v": 0.6026496887207031, "blocks.3.attention.w_out": 0.6111221313476562, "blocks.3.feed_forward.w1": 0.7265853881835938, "blocks.3.feed_forward.w2": 0.7581049799919128, "blocks.3.feed_forward.w3": 0.7083371877670288, "blocks.4.attention.w_q": 0.77166748046875, "blocks.4.attention.w_k": 0.7418251037597656, "blocks.4.attention.w_v": 0.5023651123046875, "blocks.4.attention.w_out": 0.6609954833984375, "blocks.4.feed_forward.w1": 0.7321307063102722, "blocks.4.feed_forward.w2": 0.7331289052963257, "blocks.4.feed_forward.w3": 0.7052358388900757, "blocks.5.attention.w_q": 0.7860870361328125, "blocks.5.attention.w_k": 0.7425537109375, "blocks.5.attention.w_v": 0.6059951782226562, "blocks.5.attention.w_out": 0.5586509704589844, "blocks.5.feed_forward.w1": 0.7364057302474976, "blocks.5.feed_forward.w2": 0.7137731313705444, "blocks.5.feed_forward.w3": 0.7075920104980469, "blocks.6.attention.w_q": 0.769378662109375, "blocks.6.attention.w_k": 0.7450141906738281, "blocks.6.attention.w_v": 0.625091552734375, "blocks.6.attention.w_out": 0.44696807861328125, "blocks.6.feed_forward.w1": 0.7409452199935913, "blocks.6.feed_forward.w2": 0.6878598928451538, "blocks.6.feed_forward.w3": 0.7084528803825378, "blocks.7.attention.w_q": 0.7789573669433594, "blocks.7.attention.w_k": 0.7496604919433594, "blocks.7.attention.w_v": 0.6236648559570312, "blocks.7.attention.w_out": 0.43865203857421875, "blocks.7.feed_forward.w1": 0.7415987849235535, "blocks.7.feed_forward.w2": 0.6252301931381226, "blocks.7.feed_forward.w3": 0.7052116394042969, "blocks.8.attention.w_q": 0.7743415832519531, "blocks.8.attention.w_k": 0.7634696960449219, "blocks.8.attention.w_v": 0.5513153076171875, "blocks.8.attention.w_out": 0.3545417785644531, "blocks.8.feed_forward.w1": 0.7471873164176941, "blocks.8.feed_forward.w2": 0.5392036437988281, "blocks.8.feed_forward.w3": 0.7035014033317566}} +{"step": 3100, "changed_frac": 0.6903912796933427, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7554168701171875, "blocks.0.attention.w_k": 0.7391929626464844, "blocks.0.attention.w_v": 0.48181915283203125, "blocks.0.attention.w_out": 0.47199249267578125, "blocks.0.feed_forward.w1": 0.7099635004997253, "blocks.0.feed_forward.w2": 0.7819328308105469, "blocks.0.feed_forward.w3": 0.6776263117790222, "blocks.1.attention.w_q": 0.7732086181640625, "blocks.1.attention.w_k": 0.7360687255859375, "blocks.1.attention.w_v": 0.5466041564941406, "blocks.1.attention.w_out": 0.6755714416503906, "blocks.1.feed_forward.w1": 0.7079951167106628, "blocks.1.feed_forward.w2": 0.7859522700309753, "blocks.1.feed_forward.w3": 0.6907157897949219, "blocks.2.attention.w_q": 0.7733650207519531, "blocks.2.attention.w_k": 0.7418251037597656, "blocks.2.attention.w_v": 0.5702247619628906, "blocks.2.attention.w_out": 0.6540184020996094, "blocks.2.feed_forward.w1": 0.7136917114257812, "blocks.2.feed_forward.w2": 0.7760874629020691, "blocks.2.feed_forward.w3": 0.7001635432243347, "blocks.3.attention.w_q": 0.775848388671875, "blocks.3.attention.w_k": 0.7377281188964844, "blocks.3.attention.w_v": 0.5944747924804688, "blocks.3.attention.w_out": 0.6013526916503906, "blocks.3.feed_forward.w1": 0.7197062373161316, "blocks.3.feed_forward.w2": 0.7510147094726562, "blocks.3.feed_forward.w3": 0.6994222402572632, "blocks.4.attention.w_q": 0.7663841247558594, "blocks.4.attention.w_k": 0.7358779907226562, "blocks.4.attention.w_v": 0.4915313720703125, "blocks.4.attention.w_out": 0.6544303894042969, "blocks.4.feed_forward.w1": 0.7245368957519531, "blocks.4.feed_forward.w2": 0.7233874201774597, "blocks.4.feed_forward.w3": 0.6979993581771851, "blocks.5.attention.w_q": 0.7827110290527344, "blocks.5.attention.w_k": 0.7326698303222656, "blocks.5.attention.w_v": 0.5962600708007812, "blocks.5.attention.w_out": 0.5473213195800781, "blocks.5.feed_forward.w1": 0.7302805781364441, "blocks.5.feed_forward.w2": 0.704437255859375, "blocks.5.feed_forward.w3": 0.6988767385482788, "blocks.6.attention.w_q": 0.7668037414550781, "blocks.6.attention.w_k": 0.7367286682128906, "blocks.6.attention.w_v": 0.6122169494628906, "blocks.6.attention.w_out": 0.4355621337890625, "blocks.6.feed_forward.w1": 0.7336082458496094, "blocks.6.feed_forward.w2": 0.6783485412597656, "blocks.6.feed_forward.w3": 0.7000694274902344, "blocks.7.attention.w_q": 0.7743873596191406, "blocks.7.attention.w_k": 0.7429580688476562, "blocks.7.attention.w_v": 0.6104965209960938, "blocks.7.attention.w_out": 0.4255943298339844, "blocks.7.feed_forward.w1": 0.7331759333610535, "blocks.7.feed_forward.w2": 0.6129150390625, "blocks.7.feed_forward.w3": 0.6974906921386719, "blocks.8.attention.w_q": 0.7690620422363281, "blocks.8.attention.w_k": 0.7577590942382812, "blocks.8.attention.w_v": 0.5368995666503906, "blocks.8.attention.w_out": 0.3407936096191406, "blocks.8.feed_forward.w1": 0.7410876154899597, "blocks.8.feed_forward.w2": 0.525238037109375, "blocks.8.feed_forward.w3": 0.6944834589958191}} +{"step": 3200, "changed_frac": 0.6828444065191807, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7453041076660156, "blocks.0.attention.w_k": 0.7308692932128906, "blocks.0.attention.w_v": 0.4658164978027344, "blocks.0.attention.w_out": 0.46579742431640625, "blocks.0.feed_forward.w1": 0.703338623046875, "blocks.0.feed_forward.w2": 0.7787590026855469, "blocks.0.feed_forward.w3": 0.6703872680664062, "blocks.1.attention.w_q": 0.7657852172851562, "blocks.1.attention.w_k": 0.729644775390625, "blocks.1.attention.w_v": 0.5371170043945312, "blocks.1.attention.w_out": 0.6687164306640625, "blocks.1.feed_forward.w1": 0.7013791799545288, "blocks.1.feed_forward.w2": 0.7810478210449219, "blocks.1.feed_forward.w3": 0.6818491816520691, "blocks.2.attention.w_q": 0.7690773010253906, "blocks.2.attention.w_k": 0.7367019653320312, "blocks.2.attention.w_v": 0.5589141845703125, "blocks.2.attention.w_out": 0.6465568542480469, "blocks.2.feed_forward.w1": 0.7058194875717163, "blocks.2.feed_forward.w2": 0.7707557678222656, "blocks.2.feed_forward.w3": 0.6908404231071472, "blocks.3.attention.w_q": 0.7700538635253906, "blocks.3.attention.w_k": 0.7315101623535156, "blocks.3.attention.w_v": 0.5820388793945312, "blocks.3.attention.w_out": 0.5912895202636719, "blocks.3.feed_forward.w1": 0.7131690979003906, "blocks.3.feed_forward.w2": 0.7431589961051941, "blocks.3.feed_forward.w3": 0.6916122436523438, "blocks.4.attention.w_q": 0.7597846984863281, "blocks.4.attention.w_k": 0.7278480529785156, "blocks.4.attention.w_v": 0.472381591796875, "blocks.4.attention.w_out": 0.6449775695800781, "blocks.4.feed_forward.w1": 0.7178421020507812, "blocks.4.feed_forward.w2": 0.717535674571991, "blocks.4.feed_forward.w3": 0.6875330805778503, "blocks.5.attention.w_q": 0.7771072387695312, "blocks.5.attention.w_k": 0.7292098999023438, "blocks.5.attention.w_v": 0.5883522033691406, "blocks.5.attention.w_out": 0.5378265380859375, "blocks.5.feed_forward.w1": 0.7229945063591003, "blocks.5.feed_forward.w2": 0.6970443725585938, "blocks.5.feed_forward.w3": 0.6903547048568726, "blocks.6.attention.w_q": 0.7602996826171875, "blocks.6.attention.w_k": 0.7337760925292969, "blocks.6.attention.w_v": 0.6047477722167969, "blocks.6.attention.w_out": 0.425689697265625, "blocks.6.feed_forward.w1": 0.7266362905502319, "blocks.6.feed_forward.w2": 0.6719207763671875, "blocks.6.feed_forward.w3": 0.6923205256462097, "blocks.7.attention.w_q": 0.7695388793945312, "blocks.7.attention.w_k": 0.7360038757324219, "blocks.7.attention.w_v": 0.601959228515625, "blocks.7.attention.w_out": 0.41448974609375, "blocks.7.feed_forward.w1": 0.7261505126953125, "blocks.7.feed_forward.w2": 0.6044324636459351, "blocks.7.feed_forward.w3": 0.6891085505485535, "blocks.8.attention.w_q": 0.7643508911132812, "blocks.8.attention.w_k": 0.7540817260742188, "blocks.8.attention.w_v": 0.529296875, "blocks.8.attention.w_out": 0.33113861083984375, "blocks.8.feed_forward.w1": 0.7349433898925781, "blocks.8.feed_forward.w2": 0.5135511159896851, "blocks.8.feed_forward.w3": 0.6870956420898438}} +{"step": 3300, "changed_frac": 0.674875732161041, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7409477233886719, "blocks.0.attention.w_k": 0.7253532409667969, "blocks.0.attention.w_v": 0.4573211669921875, "blocks.0.attention.w_out": 0.453369140625, "blocks.0.feed_forward.w1": 0.6962127685546875, "blocks.0.feed_forward.w2": 0.7734655141830444, "blocks.0.feed_forward.w3": 0.6620979309082031, "blocks.1.attention.w_q": 0.7584991455078125, "blocks.1.attention.w_k": 0.7235336303710938, "blocks.1.attention.w_v": 0.5296669006347656, "blocks.1.attention.w_out": 0.65899658203125, "blocks.1.feed_forward.w1": 0.6927680969238281, "blocks.1.feed_forward.w2": 0.7754796743392944, "blocks.1.feed_forward.w3": 0.6730537414550781, "blocks.2.attention.w_q": 0.7622108459472656, "blocks.2.attention.w_k": 0.7282638549804688, "blocks.2.attention.w_v": 0.5488929748535156, "blocks.2.attention.w_out": 0.6371078491210938, "blocks.2.feed_forward.w1": 0.698065459728241, "blocks.2.feed_forward.w2": 0.7628555297851562, "blocks.2.feed_forward.w3": 0.683197021484375, "blocks.3.attention.w_q": 0.7656135559082031, "blocks.3.attention.w_k": 0.7270889282226562, "blocks.3.attention.w_v": 0.5726051330566406, "blocks.3.attention.w_out": 0.5848731994628906, "blocks.3.feed_forward.w1": 0.7055460810661316, "blocks.3.feed_forward.w2": 0.7361564636230469, "blocks.3.feed_forward.w3": 0.6837081909179688, "blocks.4.attention.w_q": 0.7533111572265625, "blocks.4.attention.w_k": 0.7219734191894531, "blocks.4.attention.w_v": 0.4625358581542969, "blocks.4.attention.w_out": 0.63836669921875, "blocks.4.feed_forward.w1": 0.7103627920150757, "blocks.4.feed_forward.w2": 0.7102267146110535, "blocks.4.feed_forward.w3": 0.6798515319824219, "blocks.5.attention.w_q": 0.769622802734375, "blocks.5.attention.w_k": 0.7249526977539062, "blocks.5.attention.w_v": 0.5806503295898438, "blocks.5.attention.w_out": 0.5244598388671875, "blocks.5.feed_forward.w1": 0.7156524658203125, "blocks.5.feed_forward.w2": 0.6878318786621094, "blocks.5.feed_forward.w3": 0.682037353515625, "blocks.6.attention.w_q": 0.7552413940429688, "blocks.6.attention.w_k": 0.7249183654785156, "blocks.6.attention.w_v": 0.5971298217773438, "blocks.6.attention.w_out": 0.4163360595703125, "blocks.6.feed_forward.w1": 0.7197176814079285, "blocks.6.feed_forward.w2": 0.6613337397575378, "blocks.6.feed_forward.w3": 0.6833648681640625, "blocks.7.attention.w_q": 0.7635231018066406, "blocks.7.attention.w_k": 0.7322044372558594, "blocks.7.attention.w_v": 0.5935821533203125, "blocks.7.attention.w_out": 0.403228759765625, "blocks.7.feed_forward.w1": 0.7198804616928101, "blocks.7.feed_forward.w2": 0.5950368642807007, "blocks.7.feed_forward.w3": 0.6805928945541382, "blocks.8.attention.w_q": 0.7605514526367188, "blocks.8.attention.w_k": 0.7481040954589844, "blocks.8.attention.w_v": 0.5188484191894531, "blocks.8.attention.w_out": 0.3198356628417969, "blocks.8.feed_forward.w1": 0.7282193899154663, "blocks.8.feed_forward.w2": 0.49860256910324097, "blocks.8.feed_forward.w3": 0.6769294738769531}} +{"step": 3400, "changed_frac": 0.6675831721379206, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7301445007324219, "blocks.0.attention.w_k": 0.7159652709960938, "blocks.0.attention.w_v": 0.4454307556152344, "blocks.0.attention.w_out": 0.4493408203125, "blocks.0.feed_forward.w1": 0.6901372671127319, "blocks.0.feed_forward.w2": 0.7672538757324219, "blocks.0.feed_forward.w3": 0.6536814570426941, "blocks.1.attention.w_q": 0.753265380859375, "blocks.1.attention.w_k": 0.7201995849609375, "blocks.1.attention.w_v": 0.524383544921875, "blocks.1.attention.w_out": 0.6530113220214844, "blocks.1.feed_forward.w1": 0.6858648061752319, "blocks.1.feed_forward.w2": 0.7692502737045288, "blocks.1.feed_forward.w3": 0.6657981872558594, "blocks.2.attention.w_q": 0.7567863464355469, "blocks.2.attention.w_k": 0.7254295349121094, "blocks.2.attention.w_v": 0.5405654907226562, "blocks.2.attention.w_out": 0.6306571960449219, "blocks.2.feed_forward.w1": 0.6910807490348816, "blocks.2.feed_forward.w2": 0.7567952871322632, "blocks.2.feed_forward.w3": 0.6738433837890625, "blocks.3.attention.w_q": 0.7590370178222656, "blocks.3.attention.w_k": 0.7207260131835938, "blocks.3.attention.w_v": 0.5625572204589844, "blocks.3.attention.w_out": 0.5731277465820312, "blocks.3.feed_forward.w1": 0.6990305781364441, "blocks.3.feed_forward.w2": 0.7296435236930847, "blocks.3.feed_forward.w3": 0.6734034419059753, "blocks.4.attention.w_q": 0.7486610412597656, "blocks.4.attention.w_k": 0.7142333984375, "blocks.4.attention.w_v": 0.45188140869140625, "blocks.4.attention.w_out": 0.6309738159179688, "blocks.4.feed_forward.w1": 0.7034009695053101, "blocks.4.feed_forward.w2": 0.7021624445915222, "blocks.4.feed_forward.w3": 0.6709327697753906, "blocks.5.attention.w_q": 0.7683219909667969, "blocks.5.attention.w_k": 0.7155647277832031, "blocks.5.attention.w_v": 0.5749588012695312, "blocks.5.attention.w_out": 0.5162467956542969, "blocks.5.feed_forward.w1": 0.7100474238395691, "blocks.5.feed_forward.w2": 0.6775462031364441, "blocks.5.feed_forward.w3": 0.6725375056266785, "blocks.6.attention.w_q": 0.7517929077148438, "blocks.6.attention.w_k": 0.7186203002929688, "blocks.6.attention.w_v": 0.5907135009765625, "blocks.6.attention.w_out": 0.40964508056640625, "blocks.6.feed_forward.w1": 0.7138010859489441, "blocks.6.feed_forward.w2": 0.652533233165741, "blocks.6.feed_forward.w3": 0.6756376028060913, "blocks.7.attention.w_q": 0.7606239318847656, "blocks.7.attention.w_k": 0.7278633117675781, "blocks.7.attention.w_v": 0.5873489379882812, "blocks.7.attention.w_out": 0.39579010009765625, "blocks.7.feed_forward.w1": 0.7138875722885132, "blocks.7.feed_forward.w2": 0.5849100947380066, "blocks.7.feed_forward.w3": 0.6720848083496094, "blocks.8.attention.w_q": 0.7547760009765625, "blocks.8.attention.w_k": 0.7412796020507812, "blocks.8.attention.w_v": 0.5120201110839844, "blocks.8.attention.w_out": 0.3148841857910156, "blocks.8.feed_forward.w1": 0.7217153310775757, "blocks.8.feed_forward.w2": 0.49063873291015625, "blocks.8.feed_forward.w3": 0.6691831350326538}} +{"step": 3500, "changed_frac": 0.6573326893341846, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.72296142578125, "blocks.0.attention.w_k": 0.708038330078125, "blocks.0.attention.w_v": 0.4316062927246094, "blocks.0.attention.w_out": 0.4335899353027344, "blocks.0.feed_forward.w1": 0.6809527277946472, "blocks.0.feed_forward.w2": 0.7613691091537476, "blocks.0.feed_forward.w3": 0.6429405212402344, "blocks.1.attention.w_q": 0.7484283447265625, "blocks.1.attention.w_k": 0.7080001831054688, "blocks.1.attention.w_v": 0.5077667236328125, "blocks.1.attention.w_out": 0.6459083557128906, "blocks.1.feed_forward.w1": 0.6751124262809753, "blocks.1.feed_forward.w2": 0.7625707387924194, "blocks.1.feed_forward.w3": 0.6559079885482788, "blocks.2.attention.w_q": 0.7484130859375, "blocks.2.attention.w_k": 0.7157669067382812, "blocks.2.attention.w_v": 0.52105712890625, "blocks.2.attention.w_out": 0.6199798583984375, "blocks.2.feed_forward.w1": 0.680639922618866, "blocks.2.feed_forward.w2": 0.7522825002670288, "blocks.2.feed_forward.w3": 0.6640090942382812, "blocks.3.attention.w_q": 0.7526473999023438, "blocks.3.attention.w_k": 0.7118301391601562, "blocks.3.attention.w_v": 0.5497360229492188, "blocks.3.attention.w_out": 0.5603828430175781, "blocks.3.feed_forward.w1": 0.6883811950683594, "blocks.3.feed_forward.w2": 0.7202631831169128, "blocks.3.feed_forward.w3": 0.6640294790267944, "blocks.4.attention.w_q": 0.7431640625, "blocks.4.attention.w_k": 0.7044525146484375, "blocks.4.attention.w_v": 0.4312591552734375, "blocks.4.attention.w_out": 0.6191444396972656, "blocks.4.feed_forward.w1": 0.6929550170898438, "blocks.4.feed_forward.w2": 0.692267119884491, "blocks.4.feed_forward.w3": 0.6595446467399597, "blocks.5.attention.w_q": 0.7596054077148438, "blocks.5.attention.w_k": 0.7093048095703125, "blocks.5.attention.w_v": 0.5618515014648438, "blocks.5.attention.w_out": 0.5030670166015625, "blocks.5.feed_forward.w1": 0.7000249624252319, "blocks.5.feed_forward.w2": 0.6695455312728882, "blocks.5.feed_forward.w3": 0.6625404357910156, "blocks.6.attention.w_q": 0.7450714111328125, "blocks.6.attention.w_k": 0.7121849060058594, "blocks.6.attention.w_v": 0.5760040283203125, "blocks.6.attention.w_out": 0.39171600341796875, "blocks.6.feed_forward.w1": 0.7048810720443726, "blocks.6.feed_forward.w2": 0.6407598257064819, "blocks.6.feed_forward.w3": 0.665716826915741, "blocks.7.attention.w_q": 0.7523918151855469, "blocks.7.attention.w_k": 0.7183723449707031, "blocks.7.attention.w_v": 0.57196044921875, "blocks.7.attention.w_out": 0.3834114074707031, "blocks.7.feed_forward.w1": 0.7047094106674194, "blocks.7.feed_forward.w2": 0.569830596446991, "blocks.7.feed_forward.w3": 0.6624132990837097, "blocks.8.attention.w_q": 0.7491226196289062, "blocks.8.attention.w_k": 0.7310714721679688, "blocks.8.attention.w_v": 0.4978179931640625, "blocks.8.attention.w_out": 0.29961395263671875, "blocks.8.feed_forward.w1": 0.7133306264877319, "blocks.8.feed_forward.w2": 0.4755363464355469, "blocks.8.feed_forward.w3": 0.6578941345214844}} +{"step": 3600, "changed_frac": 0.6486264090252738, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7212562561035156, "blocks.0.attention.w_k": 0.7091751098632812, "blocks.0.attention.w_v": 0.4327430725097656, "blocks.0.attention.w_out": 0.4295768737792969, "blocks.0.feed_forward.w1": 0.6751632690429688, "blocks.0.feed_forward.w2": 0.75634765625, "blocks.0.feed_forward.w3": 0.6368192434310913, "blocks.1.attention.w_q": 0.7427177429199219, "blocks.1.attention.w_k": 0.7026214599609375, "blocks.1.attention.w_v": 0.4974784851074219, "blocks.1.attention.w_out": 0.6356849670410156, "blocks.1.feed_forward.w1": 0.6653480529785156, "blocks.1.feed_forward.w2": 0.7567520141601562, "blocks.1.feed_forward.w3": 0.6464208364486694, "blocks.2.attention.w_q": 0.7443122863769531, "blocks.2.attention.w_k": 0.7109451293945312, "blocks.2.attention.w_v": 0.5085029602050781, "blocks.2.attention.w_out": 0.6121253967285156, "blocks.2.feed_forward.w1": 0.6719144582748413, "blocks.2.feed_forward.w2": 0.7435290217399597, "blocks.2.feed_forward.w3": 0.651739776134491, "blocks.3.attention.w_q": 0.746429443359375, "blocks.3.attention.w_k": 0.7034034729003906, "blocks.3.attention.w_v": 0.5387229919433594, "blocks.3.attention.w_out": 0.5503654479980469, "blocks.3.feed_forward.w1": 0.6804542541503906, "blocks.3.feed_forward.w2": 0.7135810852050781, "blocks.3.feed_forward.w3": 0.6539561152458191, "blocks.4.attention.w_q": 0.7343406677246094, "blocks.4.attention.w_k": 0.697784423828125, "blocks.4.attention.w_v": 0.4162864685058594, "blocks.4.attention.w_out": 0.6091728210449219, "blocks.4.feed_forward.w1": 0.6843211054801941, "blocks.4.feed_forward.w2": 0.6844533681869507, "blocks.4.feed_forward.w3": 0.6496620178222656, "blocks.5.attention.w_q": 0.7527351379394531, "blocks.5.attention.w_k": 0.6980628967285156, "blocks.5.attention.w_v": 0.5527381896972656, "blocks.5.attention.w_out": 0.4923362731933594, "blocks.5.feed_forward.w1": 0.691199004650116, "blocks.5.feed_forward.w2": 0.6580798029899597, "blocks.5.feed_forward.w3": 0.6519508361816406, "blocks.6.attention.w_q": 0.7365951538085938, "blocks.6.attention.w_k": 0.7018547058105469, "blocks.6.attention.w_v": 0.5632400512695312, "blocks.6.attention.w_out": 0.3824462890625, "blocks.6.feed_forward.w1": 0.6965205073356628, "blocks.6.feed_forward.w2": 0.6306533813476562, "blocks.6.feed_forward.w3": 0.6552505493164062, "blocks.7.attention.w_q": 0.7471389770507812, "blocks.7.attention.w_k": 0.7121238708496094, "blocks.7.attention.w_v": 0.5603752136230469, "blocks.7.attention.w_out": 0.36669158935546875, "blocks.7.feed_forward.w1": 0.6970609426498413, "blocks.7.feed_forward.w2": 0.5576934814453125, "blocks.7.feed_forward.w3": 0.65118408203125, "blocks.8.attention.w_q": 0.7427101135253906, "blocks.8.attention.w_k": 0.7242202758789062, "blocks.8.attention.w_v": 0.4835548400878906, "blocks.8.attention.w_out": 0.2915763854980469, "blocks.8.feed_forward.w1": 0.7069715261459351, "blocks.8.feed_forward.w2": 0.46292752027511597, "blocks.8.feed_forward.w3": 0.6491279602050781}} +{"step": 3700, "changed_frac": 0.6387174557416867, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.7047386169433594, "blocks.0.attention.w_k": 0.69140625, "blocks.0.attention.w_v": 0.4085197448730469, "blocks.0.attention.w_out": 0.41913604736328125, "blocks.0.feed_forward.w1": 0.6639493703842163, "blocks.0.feed_forward.w2": 0.7501411437988281, "blocks.0.feed_forward.w3": 0.6244494318962097, "blocks.1.attention.w_q": 0.7321014404296875, "blocks.1.attention.w_k": 0.6955108642578125, "blocks.1.attention.w_v": 0.4891357421875, "blocks.1.attention.w_out": 0.6261482238769531, "blocks.1.feed_forward.w1": 0.6568590998649597, "blocks.1.feed_forward.w2": 0.7499542236328125, "blocks.1.feed_forward.w3": 0.6364492177963257, "blocks.2.attention.w_q": 0.7362174987792969, "blocks.2.attention.w_k": 0.7014846801757812, "blocks.2.attention.w_v": 0.4938011169433594, "blocks.2.attention.w_out": 0.6014556884765625, "blocks.2.feed_forward.w1": 0.6633313894271851, "blocks.2.feed_forward.w2": 0.7354647517204285, "blocks.2.feed_forward.w3": 0.6420542597770691, "blocks.3.attention.w_q": 0.7379493713378906, "blocks.3.attention.w_k": 0.6926231384277344, "blocks.3.attention.w_v": 0.5228157043457031, "blocks.3.attention.w_out": 0.5412445068359375, "blocks.3.feed_forward.w1": 0.6708704829216003, "blocks.3.feed_forward.w2": 0.7047208547592163, "blocks.3.feed_forward.w3": 0.6434364318847656, "blocks.4.attention.w_q": 0.7269973754882812, "blocks.4.attention.w_k": 0.6876220703125, "blocks.4.attention.w_v": 0.4030418395996094, "blocks.4.attention.w_out": 0.6018486022949219, "blocks.4.feed_forward.w1": 0.6745097041130066, "blocks.4.feed_forward.w2": 0.6737899780273438, "blocks.4.feed_forward.w3": 0.6387316584587097, "blocks.5.attention.w_q": 0.74822998046875, "blocks.5.attention.w_k": 0.6943855285644531, "blocks.5.attention.w_v": 0.5417366027832031, "blocks.5.attention.w_out": 0.4796714782714844, "blocks.5.feed_forward.w1": 0.6833152770996094, "blocks.5.feed_forward.w2": 0.649511992931366, "blocks.5.feed_forward.w3": 0.6426429748535156, "blocks.6.attention.w_q": 0.7315292358398438, "blocks.6.attention.w_k": 0.6953773498535156, "blocks.6.attention.w_v": 0.551483154296875, "blocks.6.attention.w_out": 0.3735847473144531, "blocks.6.feed_forward.w1": 0.6855265498161316, "blocks.6.feed_forward.w2": 0.6198921203613281, "blocks.6.feed_forward.w3": 0.6439031362533569, "blocks.7.attention.w_q": 0.7404403686523438, "blocks.7.attention.w_k": 0.7009773254394531, "blocks.7.attention.w_v": 0.5499687194824219, "blocks.7.attention.w_out": 0.35997772216796875, "blocks.7.feed_forward.w1": 0.6886495351791382, "blocks.7.feed_forward.w2": 0.5438003540039062, "blocks.7.feed_forward.w3": 0.6404800415039062, "blocks.8.attention.w_q": 0.7361640930175781, "blocks.8.attention.w_k": 0.7196540832519531, "blocks.8.attention.w_v": 0.4729194641113281, "blocks.8.attention.w_out": 0.2800140380859375, "blocks.8.feed_forward.w1": 0.6983503103256226, "blocks.8.feed_forward.w2": 0.45085906982421875, "blocks.8.feed_forward.w3": 0.6377004384994507}} +{"step": 3800, "changed_frac": 0.6287978408683059, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6933097839355469, "blocks.0.attention.w_k": 0.6814384460449219, "blocks.0.attention.w_v": 0.3967475891113281, "blocks.0.attention.w_out": 0.4092369079589844, "blocks.0.feed_forward.w1": 0.6535632014274597, "blocks.0.feed_forward.w2": 0.7429224848747253, "blocks.0.feed_forward.w3": 0.6128883361816406, "blocks.1.attention.w_q": 0.7272186279296875, "blocks.1.attention.w_k": 0.6856117248535156, "blocks.1.attention.w_v": 0.4797630310058594, "blocks.1.attention.w_out": 0.6178550720214844, "blocks.1.feed_forward.w1": 0.6452802419662476, "blocks.1.feed_forward.w2": 0.7423032522201538, "blocks.1.feed_forward.w3": 0.6236152648925781, "blocks.2.attention.w_q": 0.7280120849609375, "blocks.2.attention.w_k": 0.6932868957519531, "blocks.2.attention.w_v": 0.4801750183105469, "blocks.2.attention.w_out": 0.5929031372070312, "blocks.2.feed_forward.w1": 0.6531512141227722, "blocks.2.feed_forward.w2": 0.7283312678337097, "blocks.2.feed_forward.w3": 0.6322301626205444, "blocks.3.attention.w_q": 0.7320213317871094, "blocks.3.attention.w_k": 0.6835861206054688, "blocks.3.attention.w_v": 0.5116615295410156, "blocks.3.attention.w_out": 0.5281639099121094, "blocks.3.feed_forward.w1": 0.6608988642692566, "blocks.3.feed_forward.w2": 0.6947708129882812, "blocks.3.feed_forward.w3": 0.6338094472885132, "blocks.4.attention.w_q": 0.7206192016601562, "blocks.4.attention.w_k": 0.6801490783691406, "blocks.4.attention.w_v": 0.3902435302734375, "blocks.4.attention.w_out": 0.589935302734375, "blocks.4.feed_forward.w1": 0.6666196584701538, "blocks.4.feed_forward.w2": 0.6640052795410156, "blocks.4.feed_forward.w3": 0.6284815669059753, "blocks.5.attention.w_q": 0.7412452697753906, "blocks.5.attention.w_k": 0.6846504211425781, "blocks.5.attention.w_v": 0.533782958984375, "blocks.5.attention.w_out": 0.47021484375, "blocks.5.feed_forward.w1": 0.6720556020736694, "blocks.5.feed_forward.w2": 0.6375147700309753, "blocks.5.feed_forward.w3": 0.6318804621696472, "blocks.6.attention.w_q": 0.7231597900390625, "blocks.6.attention.w_k": 0.6821861267089844, "blocks.6.attention.w_v": 0.5432472229003906, "blocks.6.attention.w_out": 0.3640632629394531, "blocks.6.feed_forward.w1": 0.6772016286849976, "blocks.6.feed_forward.w2": 0.6084213256835938, "blocks.6.feed_forward.w3": 0.6329168081283569, "blocks.7.attention.w_q": 0.7331390380859375, "blocks.7.attention.w_k": 0.6959419250488281, "blocks.7.attention.w_v": 0.5394783020019531, "blocks.7.attention.w_out": 0.3475227355957031, "blocks.7.feed_forward.w1": 0.6789716482162476, "blocks.7.feed_forward.w2": 0.531042754650116, "blocks.7.feed_forward.w3": 0.6316909790039062, "blocks.8.attention.w_q": 0.7275619506835938, "blocks.8.attention.w_k": 0.7117118835449219, "blocks.8.attention.w_v": 0.4639549255371094, "blocks.8.attention.w_out": 0.2711067199707031, "blocks.8.feed_forward.w1": 0.6887321472167969, "blocks.8.feed_forward.w2": 0.4365069270133972, "blocks.8.feed_forward.w3": 0.6283416748046875}} +{"step": 3900, "changed_frac": 0.6178300971658821, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.68536376953125, "blocks.0.attention.w_k": 0.6725540161132812, "blocks.0.attention.w_v": 0.3846244812011719, "blocks.0.attention.w_out": 0.3969078063964844, "blocks.0.feed_forward.w1": 0.6460647583007812, "blocks.0.feed_forward.w2": 0.7355753779411316, "blocks.0.feed_forward.w3": 0.6030451655387878, "blocks.1.attention.w_q": 0.7177085876464844, "blocks.1.attention.w_k": 0.6763877868652344, "blocks.1.attention.w_v": 0.46523284912109375, "blocks.1.attention.w_out": 0.6061134338378906, "blocks.1.feed_forward.w1": 0.6359037160873413, "blocks.1.feed_forward.w2": 0.7359479665756226, "blocks.1.feed_forward.w3": 0.6126785278320312, "blocks.2.attention.w_q": 0.7163810729980469, "blocks.2.attention.w_k": 0.6840553283691406, "blocks.2.attention.w_v": 0.46761322021484375, "blocks.2.attention.w_out": 0.5793228149414062, "blocks.2.feed_forward.w1": 0.6415519714355469, "blocks.2.feed_forward.w2": 0.7206255793571472, "blocks.2.feed_forward.w3": 0.6201032400131226, "blocks.3.attention.w_q": 0.7231597900390625, "blocks.3.attention.w_k": 0.674560546875, "blocks.3.attention.w_v": 0.4992408752441406, "blocks.3.attention.w_out": 0.5173759460449219, "blocks.3.feed_forward.w1": 0.6480941772460938, "blocks.3.feed_forward.w2": 0.6863505244255066, "blocks.3.feed_forward.w3": 0.6212844848632812, "blocks.4.attention.w_q": 0.7092933654785156, "blocks.4.attention.w_k": 0.6683387756347656, "blocks.4.attention.w_v": 0.37445068359375, "blocks.4.attention.w_out": 0.5789794921875, "blocks.4.feed_forward.w1": 0.6557058095932007, "blocks.4.feed_forward.w2": 0.6513277888298035, "blocks.4.feed_forward.w3": 0.6148096919059753, "blocks.5.attention.w_q": 0.7351303100585938, "blocks.5.attention.w_k": 0.6742668151855469, "blocks.5.attention.w_v": 0.5191268920898438, "blocks.5.attention.w_out": 0.4518852233886719, "blocks.5.feed_forward.w1": 0.6622085571289062, "blocks.5.feed_forward.w2": 0.6251144409179688, "blocks.5.feed_forward.w3": 0.6197929382324219, "blocks.6.attention.w_q": 0.7152824401855469, "blocks.6.attention.w_k": 0.6723709106445312, "blocks.6.attention.w_v": 0.5310249328613281, "blocks.6.attention.w_out": 0.3500785827636719, "blocks.6.feed_forward.w1": 0.6677882075309753, "blocks.6.feed_forward.w2": 0.5959612727165222, "blocks.6.feed_forward.w3": 0.6211942434310913, "blocks.7.attention.w_q": 0.7227668762207031, "blocks.7.attention.w_k": 0.6842765808105469, "blocks.7.attention.w_v": 0.5220985412597656, "blocks.7.attention.w_out": 0.33293914794921875, "blocks.7.feed_forward.w1": 0.6706466674804688, "blocks.7.feed_forward.w2": 0.5184898376464844, "blocks.7.feed_forward.w3": 0.6200816035270691, "blocks.8.attention.w_q": 0.7191009521484375, "blocks.8.attention.w_k": 0.7019233703613281, "blocks.8.attention.w_v": 0.446563720703125, "blocks.8.attention.w_out": 0.2606620788574219, "blocks.8.feed_forward.w1": 0.6813761591911316, "blocks.8.feed_forward.w2": 0.4216906428337097, "blocks.8.feed_forward.w3": 0.6162402033805847}} +{"step": 4000, "changed_frac": 0.6063121241382045, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6738357543945312, "blocks.0.attention.w_k": 0.6589584350585938, "blocks.0.attention.w_v": 0.37151336669921875, "blocks.0.attention.w_out": 0.3869171142578125, "blocks.0.feed_forward.w1": 0.6333529353141785, "blocks.0.feed_forward.w2": 0.7284024953842163, "blocks.0.feed_forward.w3": 0.5916519165039062, "blocks.1.attention.w_q": 0.70745849609375, "blocks.1.attention.w_k": 0.6695709228515625, "blocks.1.attention.w_v": 0.4529304504394531, "blocks.1.attention.w_out": 0.5947265625, "blocks.1.feed_forward.w1": 0.6240272521972656, "blocks.1.feed_forward.w2": 0.7276878356933594, "blocks.1.feed_forward.w3": 0.6004893183708191, "blocks.2.attention.w_q": 0.7078437805175781, "blocks.2.attention.w_k": 0.6735877990722656, "blocks.2.attention.w_v": 0.45304107666015625, "blocks.2.attention.w_out": 0.5716094970703125, "blocks.2.feed_forward.w1": 0.6305529475212097, "blocks.2.feed_forward.w2": 0.7108396291732788, "blocks.2.feed_forward.w3": 0.6076736450195312, "blocks.3.attention.w_q": 0.7132606506347656, "blocks.3.attention.w_k": 0.6630401611328125, "blocks.3.attention.w_v": 0.4844017028808594, "blocks.3.attention.w_out": 0.5018424987792969, "blocks.3.feed_forward.w1": 0.6380844116210938, "blocks.3.feed_forward.w2": 0.6742998957633972, "blocks.3.feed_forward.w3": 0.6091537475585938, "blocks.4.attention.w_q": 0.7005882263183594, "blocks.4.attention.w_k": 0.6602897644042969, "blocks.4.attention.w_v": 0.35709381103515625, "blocks.4.attention.w_out": 0.5676803588867188, "blocks.4.feed_forward.w1": 0.6444346308708191, "blocks.4.feed_forward.w2": 0.6404864192008972, "blocks.4.feed_forward.w3": 0.6042137145996094, "blocks.5.attention.w_q": 0.7266387939453125, "blocks.5.attention.w_k": 0.6628875732421875, "blocks.5.attention.w_v": 0.506195068359375, "blocks.5.attention.w_out": 0.4422149658203125, "blocks.5.feed_forward.w1": 0.651434600353241, "blocks.5.feed_forward.w2": 0.6129913330078125, "blocks.5.feed_forward.w3": 0.6056658625602722, "blocks.6.attention.w_q": 0.7062492370605469, "blocks.6.attention.w_k": 0.6648941040039062, "blocks.6.attention.w_v": 0.5190505981445312, "blocks.6.attention.w_out": 0.3336448669433594, "blocks.6.feed_forward.w1": 0.6572316884994507, "blocks.6.feed_forward.w2": 0.5829544067382812, "blocks.6.feed_forward.w3": 0.6080030202865601, "blocks.7.attention.w_q": 0.7114295959472656, "blocks.7.attention.w_k": 0.6762924194335938, "blocks.7.attention.w_v": 0.5079345703125, "blocks.7.attention.w_out": 0.3230438232421875, "blocks.7.feed_forward.w1": 0.6591288447380066, "blocks.7.feed_forward.w2": 0.5021909475326538, "blocks.7.feed_forward.w3": 0.6071929931640625, "blocks.8.attention.w_q": 0.7110557556152344, "blocks.8.attention.w_k": 0.692352294921875, "blocks.8.attention.w_v": 0.4340400695800781, "blocks.8.attention.w_out": 0.2494354248046875, "blocks.8.feed_forward.w1": 0.6695899963378906, "blocks.8.feed_forward.w2": 0.4083341062068939, "blocks.8.feed_forward.w3": 0.6035881042480469}} +{"step": 4100, "changed_frac": 0.5954394870334201, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6617660522460938, "blocks.0.attention.w_k": 0.650543212890625, "blocks.0.attention.w_v": 0.3807220458984375, "blocks.0.attention.w_out": 0.3751029968261719, "blocks.0.feed_forward.w1": 0.6248245239257812, "blocks.0.feed_forward.w2": 0.7204475402832031, "blocks.0.feed_forward.w3": 0.5831820368766785, "blocks.1.attention.w_q": 0.7025070190429688, "blocks.1.attention.w_k": 0.6592483520507812, "blocks.1.attention.w_v": 0.44420623779296875, "blocks.1.attention.w_out": 0.5845146179199219, "blocks.1.feed_forward.w1": 0.612463653087616, "blocks.1.feed_forward.w2": 0.7187004089355469, "blocks.1.feed_forward.w3": 0.5895309448242188, "blocks.2.attention.w_q": 0.70037841796875, "blocks.2.attention.w_k": 0.6637687683105469, "blocks.2.attention.w_v": 0.4386482238769531, "blocks.2.attention.w_out": 0.5608139038085938, "blocks.2.feed_forward.w1": 0.6192881464958191, "blocks.2.feed_forward.w2": 0.701171875, "blocks.2.feed_forward.w3": 0.5951423645019531, "blocks.3.attention.w_q": 0.7058563232421875, "blocks.3.attention.w_k": 0.6553001403808594, "blocks.3.attention.w_v": 0.4715728759765625, "blocks.3.attention.w_out": 0.49010467529296875, "blocks.3.feed_forward.w1": 0.6271209716796875, "blocks.3.feed_forward.w2": 0.6637624502182007, "blocks.3.feed_forward.w3": 0.5978431701660156, "blocks.4.attention.w_q": 0.6916084289550781, "blocks.4.attention.w_k": 0.6493873596191406, "blocks.4.attention.w_v": 0.347991943359375, "blocks.4.attention.w_out": 0.5552749633789062, "blocks.4.feed_forward.w1": 0.6324882507324219, "blocks.4.feed_forward.w2": 0.629553496837616, "blocks.4.feed_forward.w3": 0.5906867980957031, "blocks.5.attention.w_q": 0.7183723449707031, "blocks.5.attention.w_k": 0.6527519226074219, "blocks.5.attention.w_v": 0.4977264404296875, "blocks.5.attention.w_out": 0.4275321960449219, "blocks.5.feed_forward.w1": 0.641058623790741, "blocks.5.feed_forward.w2": 0.5996360778808594, "blocks.5.feed_forward.w3": 0.5937640070915222, "blocks.6.attention.w_q": 0.6951179504394531, "blocks.6.attention.w_k": 0.6523017883300781, "blocks.6.attention.w_v": 0.5046844482421875, "blocks.6.attention.w_out": 0.3256187438964844, "blocks.6.feed_forward.w1": 0.6460215449333191, "blocks.6.feed_forward.w2": 0.5693385004997253, "blocks.6.feed_forward.w3": 0.5952606201171875, "blocks.7.attention.w_q": 0.7053337097167969, "blocks.7.attention.w_k": 0.6637802124023438, "blocks.7.attention.w_v": 0.4977302551269531, "blocks.7.attention.w_out": 0.309417724609375, "blocks.7.feed_forward.w1": 0.6478437185287476, "blocks.7.feed_forward.w2": 0.48862457275390625, "blocks.7.feed_forward.w3": 0.5954755544662476, "blocks.8.attention.w_q": 0.7031784057617188, "blocks.8.attention.w_k": 0.6846694946289062, "blocks.8.attention.w_v": 0.4223976135253906, "blocks.8.attention.w_out": 0.23737716674804688, "blocks.8.feed_forward.w1": 0.6597379446029663, "blocks.8.feed_forward.w2": 0.3929328918457031, "blocks.8.feed_forward.w3": 0.5904706716537476}} +{"step": 4200, "changed_frac": 0.5824568006727431, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6479301452636719, "blocks.0.attention.w_k": 0.6362113952636719, "blocks.0.attention.w_v": 0.3442955017089844, "blocks.0.attention.w_out": 0.3633880615234375, "blocks.0.feed_forward.w1": 0.6109861135482788, "blocks.0.feed_forward.w2": 0.7125040888786316, "blocks.0.feed_forward.w3": 0.5677007436752319, "blocks.1.attention.w_q": 0.6914901733398438, "blocks.1.attention.w_k": 0.6475448608398438, "blocks.1.attention.w_v": 0.4286346435546875, "blocks.1.attention.w_out": 0.569488525390625, "blocks.1.feed_forward.w1": 0.5998255610466003, "blocks.1.feed_forward.w2": 0.7079684138298035, "blocks.1.feed_forward.w3": 0.5752614736557007, "blocks.2.attention.w_q": 0.6879005432128906, "blocks.2.attention.w_k": 0.6523818969726562, "blocks.2.attention.w_v": 0.4205741882324219, "blocks.2.attention.w_out": 0.5482673645019531, "blocks.2.feed_forward.w1": 0.606726348400116, "blocks.2.feed_forward.w2": 0.6912791132926941, "blocks.2.feed_forward.w3": 0.5814170837402344, "blocks.3.attention.w_q": 0.6959495544433594, "blocks.3.attention.w_k": 0.6421318054199219, "blocks.3.attention.w_v": 0.4577369689941406, "blocks.3.attention.w_out": 0.4754371643066406, "blocks.3.feed_forward.w1": 0.6138865351676941, "blocks.3.feed_forward.w2": 0.6531626582145691, "blocks.3.feed_forward.w3": 0.5818557739257812, "blocks.4.attention.w_q": 0.6789665222167969, "blocks.4.attention.w_k": 0.6349716186523438, "blocks.4.attention.w_v": 0.3279266357421875, "blocks.4.attention.w_out": 0.541412353515625, "blocks.4.feed_forward.w1": 0.6209055781364441, "blocks.4.feed_forward.w2": 0.6154683828353882, "blocks.4.feed_forward.w3": 0.5792338252067566, "blocks.5.attention.w_q": 0.7084846496582031, "blocks.5.attention.w_k": 0.6398391723632812, "blocks.5.attention.w_v": 0.4847831726074219, "blocks.5.attention.w_out": 0.4146537780761719, "blocks.5.feed_forward.w1": 0.6281217336654663, "blocks.5.feed_forward.w2": 0.587225615978241, "blocks.5.feed_forward.w3": 0.5798810720443726, "blocks.6.attention.w_q": 0.6867599487304688, "blocks.6.attention.w_k": 0.642974853515625, "blocks.6.attention.w_v": 0.4915580749511719, "blocks.6.attention.w_out": 0.3138236999511719, "blocks.6.feed_forward.w1": 0.6334991455078125, "blocks.6.feed_forward.w2": 0.5532544851303101, "blocks.6.feed_forward.w3": 0.5826886892318726, "blocks.7.attention.w_q": 0.6961288452148438, "blocks.7.attention.w_k": 0.6518898010253906, "blocks.7.attention.w_v": 0.4825439453125, "blocks.7.attention.w_out": 0.299072265625, "blocks.7.feed_forward.w1": 0.6358363032341003, "blocks.7.feed_forward.w2": 0.47527822852134705, "blocks.7.feed_forward.w3": 0.5799407958984375, "blocks.8.attention.w_q": 0.6940040588378906, "blocks.8.attention.w_k": 0.67230224609375, "blocks.8.attention.w_v": 0.4082183837890625, "blocks.8.attention.w_out": 0.22946548461914062, "blocks.8.feed_forward.w1": 0.6480547785758972, "blocks.8.feed_forward.w2": 0.3784332275390625, "blocks.8.feed_forward.w3": 0.5790392756462097}} +{"step": 4300, "changed_frac": 0.5695035559499365, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6329307556152344, "blocks.0.attention.w_k": 0.6215934753417969, "blocks.0.attention.w_v": 0.3301887512207031, "blocks.0.attention.w_out": 0.35137939453125, "blocks.0.feed_forward.w1": 0.5994377136230469, "blocks.0.feed_forward.w2": 0.7025846242904663, "blocks.0.feed_forward.w3": 0.5538368225097656, "blocks.1.attention.w_q": 0.6830101013183594, "blocks.1.attention.w_k": 0.6351318359375, "blocks.1.attention.w_v": 0.41474151611328125, "blocks.1.attention.w_out": 0.560516357421875, "blocks.1.feed_forward.w1": 0.5856794118881226, "blocks.1.feed_forward.w2": 0.6994006037712097, "blocks.1.feed_forward.w3": 0.5605735778808594, "blocks.2.attention.w_q": 0.6746826171875, "blocks.2.attention.w_k": 0.6369781494140625, "blocks.2.attention.w_v": 0.40924072265625, "blocks.2.attention.w_out": 0.5346298217773438, "blocks.2.feed_forward.w1": 0.5917066335678101, "blocks.2.feed_forward.w2": 0.6808459162712097, "blocks.2.feed_forward.w3": 0.5676130056381226, "blocks.3.attention.w_q": 0.6842308044433594, "blocks.3.attention.w_k": 0.6297111511230469, "blocks.3.attention.w_v": 0.4441680908203125, "blocks.3.attention.w_out": 0.4625968933105469, "blocks.3.feed_forward.w1": 0.6012433767318726, "blocks.3.feed_forward.w2": 0.6411985158920288, "blocks.3.feed_forward.w3": 0.5694796442985535, "blocks.4.attention.w_q": 0.6673812866210938, "blocks.4.attention.w_k": 0.6238784790039062, "blocks.4.attention.w_v": 0.3137397766113281, "blocks.4.attention.w_out": 0.5324821472167969, "blocks.4.feed_forward.w1": 0.6072731018066406, "blocks.4.feed_forward.w2": 0.6025251150131226, "blocks.4.feed_forward.w3": 0.5635592341423035, "blocks.5.attention.w_q": 0.6974525451660156, "blocks.5.attention.w_k": 0.6276512145996094, "blocks.5.attention.w_v": 0.46962738037109375, "blocks.5.attention.w_out": 0.4019432067871094, "blocks.5.feed_forward.w1": 0.6160405874252319, "blocks.5.feed_forward.w2": 0.5727132558822632, "blocks.5.feed_forward.w3": 0.5650596618652344, "blocks.6.attention.w_q": 0.6754302978515625, "blocks.6.attention.w_k": 0.626861572265625, "blocks.6.attention.w_v": 0.4771919250488281, "blocks.6.attention.w_out": 0.3022117614746094, "blocks.6.feed_forward.w1": 0.6213735342025757, "blocks.6.feed_forward.w2": 0.5398534536361694, "blocks.6.feed_forward.w3": 0.5688985586166382, "blocks.7.attention.w_q": 0.6860618591308594, "blocks.7.attention.w_k": 0.6392784118652344, "blocks.7.attention.w_v": 0.4690132141113281, "blocks.7.attention.w_out": 0.2858161926269531, "blocks.7.feed_forward.w1": 0.6239001154899597, "blocks.7.feed_forward.w2": 0.45884960889816284, "blocks.7.feed_forward.w3": 0.5663401484489441, "blocks.8.attention.w_q": 0.6846351623535156, "blocks.8.attention.w_k": 0.65863037109375, "blocks.8.attention.w_v": 0.3926544189453125, "blocks.8.attention.w_out": 0.2205963134765625, "blocks.8.feed_forward.w1": 0.636511504650116, "blocks.8.feed_forward.w2": 0.3633308410644531, "blocks.8.feed_forward.w3": 0.5647214651107788}} +{"step": 4400, "changed_frac": 0.5563056163298779, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6240348815917969, "blocks.0.attention.w_k": 0.6111602783203125, "blocks.0.attention.w_v": 0.3191490173339844, "blocks.0.attention.w_out": 0.3416328430175781, "blocks.0.feed_forward.w1": 0.5845884084701538, "blocks.0.feed_forward.w2": 0.6940028071403503, "blocks.0.feed_forward.w3": 0.5405439138412476, "blocks.1.attention.w_q": 0.671356201171875, "blocks.1.attention.w_k": 0.6241378784179688, "blocks.1.attention.w_v": 0.40198516845703125, "blocks.1.attention.w_out": 0.5480575561523438, "blocks.1.feed_forward.w1": 0.5729624629020691, "blocks.1.feed_forward.w2": 0.6884078979492188, "blocks.1.feed_forward.w3": 0.5458234548568726, "blocks.2.attention.w_q": 0.6646575927734375, "blocks.2.attention.w_k": 0.6282768249511719, "blocks.2.attention.w_v": 0.3951683044433594, "blocks.2.attention.w_out": 0.5206069946289062, "blocks.2.feed_forward.w1": 0.5788091421127319, "blocks.2.feed_forward.w2": 0.6692390441894531, "blocks.2.feed_forward.w3": 0.5544586181640625, "blocks.3.attention.w_q": 0.6745223999023438, "blocks.3.attention.w_k": 0.61724853515625, "blocks.3.attention.w_v": 0.4266242980957031, "blocks.3.attention.w_out": 0.4477119445800781, "blocks.3.feed_forward.w1": 0.5870043635368347, "blocks.3.feed_forward.w2": 0.6278508901596069, "blocks.3.feed_forward.w3": 0.55523681640625, "blocks.4.attention.w_q": 0.6545295715332031, "blocks.4.attention.w_k": 0.6121597290039062, "blocks.4.attention.w_v": 0.300048828125, "blocks.4.attention.w_out": 0.5156784057617188, "blocks.4.feed_forward.w1": 0.5938135981559753, "blocks.4.feed_forward.w2": 0.5881831049919128, "blocks.4.feed_forward.w3": 0.5496673583984375, "blocks.5.attention.w_q": 0.6857681274414062, "blocks.5.attention.w_k": 0.6156272888183594, "blocks.5.attention.w_v": 0.4574546813964844, "blocks.5.attention.w_out": 0.38462066650390625, "blocks.5.feed_forward.w1": 0.6025174856185913, "blocks.5.feed_forward.w2": 0.5566673278808594, "blocks.5.feed_forward.w3": 0.5515035390853882, "blocks.6.attention.w_q": 0.6624717712402344, "blocks.6.attention.w_k": 0.615997314453125, "blocks.6.attention.w_v": 0.4632301330566406, "blocks.6.attention.w_out": 0.29273223876953125, "blocks.6.feed_forward.w1": 0.6076876521110535, "blocks.6.feed_forward.w2": 0.5255203247070312, "blocks.6.feed_forward.w3": 0.5541788935661316, "blocks.7.attention.w_q": 0.6736984252929688, "blocks.7.attention.w_k": 0.6301116943359375, "blocks.7.attention.w_v": 0.4542121887207031, "blocks.7.attention.w_out": 0.2736930847167969, "blocks.7.feed_forward.w1": 0.6103299856185913, "blocks.7.feed_forward.w2": 0.44252142310142517, "blocks.7.feed_forward.w3": 0.5505956411361694, "blocks.8.attention.w_q": 0.6721153259277344, "blocks.8.attention.w_k": 0.6508712768554688, "blocks.8.attention.w_v": 0.38135528564453125, "blocks.8.attention.w_out": 0.20996856689453125, "blocks.8.feed_forward.w1": 0.6236000061035156, "blocks.8.feed_forward.w2": 0.35052618384361267, "blocks.8.feed_forward.w3": 0.5487874746322632}} +{"step": 4500, "changed_frac": 0.5429023025382278, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.6198768615722656, "blocks.0.attention.w_k": 0.6116600036621094, "blocks.0.attention.w_v": 0.3126373291015625, "blocks.0.attention.w_out": 0.3307762145996094, "blocks.0.feed_forward.w1": 0.5749855041503906, "blocks.0.feed_forward.w2": 0.6829707026481628, "blocks.0.feed_forward.w3": 0.528082549571991, "blocks.1.attention.w_q": 0.6606292724609375, "blocks.1.attention.w_k": 0.6095924377441406, "blocks.1.attention.w_v": 0.3948631286621094, "blocks.1.attention.w_out": 0.5354690551757812, "blocks.1.feed_forward.w1": 0.5591659545898438, "blocks.1.feed_forward.w2": 0.6760584712028503, "blocks.1.feed_forward.w3": 0.5305938720703125, "blocks.2.attention.w_q": 0.6509132385253906, "blocks.2.attention.w_k": 0.6130294799804688, "blocks.2.attention.w_v": 0.3812751770019531, "blocks.2.attention.w_out": 0.5074386596679688, "blocks.2.feed_forward.w1": 0.5641632080078125, "blocks.2.feed_forward.w2": 0.6547088623046875, "blocks.2.feed_forward.w3": 0.5384801626205444, "blocks.3.attention.w_q": 0.6631278991699219, "blocks.3.attention.w_k": 0.6052093505859375, "blocks.3.attention.w_v": 0.4159088134765625, "blocks.3.attention.w_out": 0.4321861267089844, "blocks.3.feed_forward.w1": 0.5734202265739441, "blocks.3.feed_forward.w2": 0.6143239736557007, "blocks.3.feed_forward.w3": 0.5376726984977722, "blocks.4.attention.w_q": 0.6403312683105469, "blocks.4.attention.w_k": 0.5969581604003906, "blocks.4.attention.w_v": 0.2850074768066406, "blocks.4.attention.w_out": 0.5000190734863281, "blocks.4.feed_forward.w1": 0.5806897878646851, "blocks.4.feed_forward.w2": 0.5727640986442566, "blocks.4.feed_forward.w3": 0.5336660146713257, "blocks.5.attention.w_q": 0.6747703552246094, "blocks.5.attention.w_k": 0.6022605895996094, "blocks.5.attention.w_v": 0.4425926208496094, "blocks.5.attention.w_out": 0.37235260009765625, "blocks.5.feed_forward.w1": 0.5878614187240601, "blocks.5.feed_forward.w2": 0.5455576777458191, "blocks.5.feed_forward.w3": 0.5357717275619507, "blocks.6.attention.w_q": 0.6515693664550781, "blocks.6.attention.w_k": 0.6017875671386719, "blocks.6.attention.w_v": 0.4482460021972656, "blocks.6.attention.w_out": 0.28302001953125, "blocks.6.feed_forward.w1": 0.5936063528060913, "blocks.6.feed_forward.w2": 0.5091120600700378, "blocks.6.feed_forward.w3": 0.5389976501464844, "blocks.7.attention.w_q": 0.6643486022949219, "blocks.7.attention.w_k": 0.6166038513183594, "blocks.7.attention.w_v": 0.4398918151855469, "blocks.7.attention.w_out": 0.26570892333984375, "blocks.7.feed_forward.w1": 0.5980936884880066, "blocks.7.feed_forward.w2": 0.42604193091392517, "blocks.7.feed_forward.w3": 0.5347480773925781, "blocks.8.attention.w_q": 0.6623725891113281, "blocks.8.attention.w_k": 0.6388587951660156, "blocks.8.attention.w_v": 0.368408203125, "blocks.8.attention.w_out": 0.20015335083007812, "blocks.8.feed_forward.w1": 0.6108576655387878, "blocks.8.feed_forward.w2": 0.33590444922447205, "blocks.8.feed_forward.w3": 0.5349401235580444}} +{"step": 4600, "changed_frac": 0.5280438936673678, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5841293334960938, "blocks.0.attention.w_k": 0.578643798828125, "blocks.0.attention.w_v": 0.2999534606933594, "blocks.0.attention.w_out": 0.3175239562988281, "blocks.0.feed_forward.w1": 0.5578550100326538, "blocks.0.feed_forward.w2": 0.6724408864974976, "blocks.0.feed_forward.w3": 0.5093562006950378, "blocks.1.attention.w_q": 0.6479949951171875, "blocks.1.attention.w_k": 0.5981407165527344, "blocks.1.attention.w_v": 0.3787994384765625, "blocks.1.attention.w_out": 0.5229415893554688, "blocks.1.feed_forward.w1": 0.5427869558334351, "blocks.1.feed_forward.w2": 0.66522216796875, "blocks.1.feed_forward.w3": 0.5169207453727722, "blocks.2.attention.w_q": 0.6373023986816406, "blocks.2.attention.w_k": 0.6001930236816406, "blocks.2.attention.w_v": 0.368133544921875, "blocks.2.attention.w_out": 0.4926567077636719, "blocks.2.feed_forward.w1": 0.5481466054916382, "blocks.2.feed_forward.w2": 0.6430740356445312, "blocks.2.feed_forward.w3": 0.5210825800895691, "blocks.3.attention.w_q": 0.6506805419921875, "blocks.3.attention.w_k": 0.5918731689453125, "blocks.3.attention.w_v": 0.3980140686035156, "blocks.3.attention.w_out": 0.41785430908203125, "blocks.3.feed_forward.w1": 0.5582415461540222, "blocks.3.feed_forward.w2": 0.5990180969238281, "blocks.3.feed_forward.w3": 0.5233319997787476, "blocks.4.attention.w_q": 0.632476806640625, "blocks.4.attention.w_k": 0.58184814453125, "blocks.4.attention.w_v": 0.2740364074707031, "blocks.4.attention.w_out": 0.4899749755859375, "blocks.4.feed_forward.w1": 0.5667737722396851, "blocks.4.feed_forward.w2": 0.5585569143295288, "blocks.4.feed_forward.w3": 0.5179672241210938, "blocks.5.attention.w_q": 0.6625938415527344, "blocks.5.attention.w_k": 0.5888938903808594, "blocks.5.attention.w_v": 0.43071746826171875, "blocks.5.attention.w_out": 0.35611724853515625, "blocks.5.feed_forward.w1": 0.5734850764274597, "blocks.5.feed_forward.w2": 0.5266202688217163, "blocks.5.feed_forward.w3": 0.5210037231445312, "blocks.6.attention.w_q": 0.6368446350097656, "blocks.6.attention.w_k": 0.5895843505859375, "blocks.6.attention.w_v": 0.432861328125, "blocks.6.attention.w_out": 0.2709770202636719, "blocks.6.feed_forward.w1": 0.5782941579818726, "blocks.6.feed_forward.w2": 0.49462127685546875, "blocks.6.feed_forward.w3": 0.5232048034667969, "blocks.7.attention.w_q": 0.6489372253417969, "blocks.7.attention.w_k": 0.6044120788574219, "blocks.7.attention.w_v": 0.4251213073730469, "blocks.7.attention.w_out": 0.25113677978515625, "blocks.7.feed_forward.w1": 0.582977294921875, "blocks.7.feed_forward.w2": 0.41045889258384705, "blocks.7.feed_forward.w3": 0.5185534358024597, "blocks.8.attention.w_q": 0.6504859924316406, "blocks.8.attention.w_k": 0.6239547729492188, "blocks.8.attention.w_v": 0.35308074951171875, "blocks.8.attention.w_out": 0.19090652465820312, "blocks.8.feed_forward.w1": 0.598486602306366, "blocks.8.feed_forward.w2": 0.3210131525993347, "blocks.8.feed_forward.w3": 0.5176200866699219}} +{"step": 4700, "changed_frac": 0.5122330168373564, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5741310119628906, "blocks.0.attention.w_k": 0.5613632202148438, "blocks.0.attention.w_v": 0.2816162109375, "blocks.0.attention.w_out": 0.30499267578125, "blocks.0.feed_forward.w1": 0.5410995483398438, "blocks.0.feed_forward.w2": 0.6596946716308594, "blocks.0.feed_forward.w3": 0.4942614436149597, "blocks.1.attention.w_q": 0.6373443603515625, "blocks.1.attention.w_k": 0.5806808471679688, "blocks.1.attention.w_v": 0.3635520935058594, "blocks.1.attention.w_out": 0.505615234375, "blocks.1.feed_forward.w1": 0.5269063711166382, "blocks.1.feed_forward.w2": 0.6506284475326538, "blocks.1.feed_forward.w3": 0.5000966787338257, "blocks.2.attention.w_q": 0.6261634826660156, "blocks.2.attention.w_k": 0.5845451354980469, "blocks.2.attention.w_v": 0.3489952087402344, "blocks.2.attention.w_out": 0.47589111328125, "blocks.2.feed_forward.w1": 0.5323740839958191, "blocks.2.feed_forward.w2": 0.6290105581283569, "blocks.2.feed_forward.w3": 0.5035502314567566, "blocks.3.attention.w_q": 0.6376953125, "blocks.3.attention.w_k": 0.576995849609375, "blocks.3.attention.w_v": 0.38034820556640625, "blocks.3.attention.w_out": 0.4000244140625, "blocks.3.feed_forward.w1": 0.5418752431869507, "blocks.3.feed_forward.w2": 0.584564208984375, "blocks.3.feed_forward.w3": 0.5070317983627319, "blocks.4.attention.w_q": 0.6172409057617188, "blocks.4.attention.w_k": 0.5678863525390625, "blocks.4.attention.w_v": 0.2605743408203125, "blocks.4.attention.w_out": 0.46958160400390625, "blocks.4.feed_forward.w1": 0.5502764582633972, "blocks.4.feed_forward.w2": 0.5429954528808594, "blocks.4.feed_forward.w3": 0.5001806020736694, "blocks.5.attention.w_q": 0.6472854614257812, "blocks.5.attention.w_k": 0.5733299255371094, "blocks.5.attention.w_v": 0.41452789306640625, "blocks.5.attention.w_out": 0.3418121337890625, "blocks.5.feed_forward.w1": 0.5587705373764038, "blocks.5.feed_forward.w2": 0.5097987055778503, "blocks.5.feed_forward.w3": 0.5025991201400757, "blocks.6.attention.w_q": 0.6254348754882812, "blocks.6.attention.w_k": 0.5697517395019531, "blocks.6.attention.w_v": 0.4158782958984375, "blocks.6.attention.w_out": 0.25704193115234375, "blocks.6.feed_forward.w1": 0.5634142756462097, "blocks.6.feed_forward.w2": 0.4776954650878906, "blocks.6.feed_forward.w3": 0.5063718557357788, "blocks.7.attention.w_q": 0.6343460083007812, "blocks.7.attention.w_k": 0.5860939025878906, "blocks.7.attention.w_v": 0.4075355529785156, "blocks.7.attention.w_out": 0.240753173828125, "blocks.7.feed_forward.w1": 0.5672264099121094, "blocks.7.feed_forward.w2": 0.39445242285728455, "blocks.7.feed_forward.w3": 0.5005607604980469, "blocks.8.attention.w_q": 0.6345481872558594, "blocks.8.attention.w_k": 0.6099853515625, "blocks.8.attention.w_v": 0.337677001953125, "blocks.8.attention.w_out": 0.18169403076171875, "blocks.8.feed_forward.w1": 0.5824458003044128, "blocks.8.feed_forward.w2": 0.30328625440597534, "blocks.8.feed_forward.w3": 0.5016098022460938}} +{"step": 4800, "changed_frac": 0.49677935217180824, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5581016540527344, "blocks.0.attention.w_k": 0.5484580993652344, "blocks.0.attention.w_v": 0.27219390869140625, "blocks.0.attention.w_out": 0.2935638427734375, "blocks.0.feed_forward.w1": 0.5281804800033569, "blocks.0.feed_forward.w2": 0.6467399597167969, "blocks.0.feed_forward.w3": 0.47937649488449097, "blocks.1.attention.w_q": 0.6212348937988281, "blocks.1.attention.w_k": 0.5650444030761719, "blocks.1.attention.w_v": 0.3505706787109375, "blocks.1.attention.w_out": 0.4893836975097656, "blocks.1.feed_forward.w1": 0.5097516775131226, "blocks.1.feed_forward.w2": 0.6366793513298035, "blocks.1.feed_forward.w3": 0.4827715754508972, "blocks.2.attention.w_q": 0.6079940795898438, "blocks.2.attention.w_k": 0.5678138732910156, "blocks.2.attention.w_v": 0.3358154296875, "blocks.2.attention.w_out": 0.460723876953125, "blocks.2.feed_forward.w1": 0.5155309438705444, "blocks.2.feed_forward.w2": 0.6132024526596069, "blocks.2.feed_forward.w3": 0.4873237609863281, "blocks.3.attention.w_q": 0.6220817565917969, "blocks.3.attention.w_k": 0.560333251953125, "blocks.3.attention.w_v": 0.36761474609375, "blocks.3.attention.w_out": 0.3847084045410156, "blocks.3.feed_forward.w1": 0.5248667597770691, "blocks.3.feed_forward.w2": 0.5680376887321472, "blocks.3.feed_forward.w3": 0.4895477294921875, "blocks.4.attention.w_q": 0.5984992980957031, "blocks.4.attention.w_k": 0.5504188537597656, "blocks.4.attention.w_v": 0.2470855712890625, "blocks.4.attention.w_out": 0.45563507080078125, "blocks.4.feed_forward.w1": 0.5320765376091003, "blocks.4.feed_forward.w2": 0.5259298086166382, "blocks.4.feed_forward.w3": 0.48403677344322205, "blocks.5.attention.w_q": 0.6354255676269531, "blocks.5.attention.w_k": 0.5580101013183594, "blocks.5.attention.w_v": 0.4024467468261719, "blocks.5.attention.w_out": 0.330352783203125, "blocks.5.feed_forward.w1": 0.5418828725814819, "blocks.5.feed_forward.w2": 0.4914194941520691, "blocks.5.feed_forward.w3": 0.4873186945915222, "blocks.6.attention.w_q": 0.6105155944824219, "blocks.6.attention.w_k": 0.5559501647949219, "blocks.6.attention.w_v": 0.40235137939453125, "blocks.6.attention.w_out": 0.24607086181640625, "blocks.6.feed_forward.w1": 0.545599639415741, "blocks.6.feed_forward.w2": 0.4614105224609375, "blocks.6.feed_forward.w3": 0.4905204772949219, "blocks.7.attention.w_q": 0.6215019226074219, "blocks.7.attention.w_k": 0.569244384765625, "blocks.7.attention.w_v": 0.3890342712402344, "blocks.7.attention.w_out": 0.231048583984375, "blocks.7.feed_forward.w1": 0.5516294240951538, "blocks.7.feed_forward.w2": 0.37654241919517517, "blocks.7.feed_forward.w3": 0.4856414794921875, "blocks.8.attention.w_q": 0.6202964782714844, "blocks.8.attention.w_k": 0.5948829650878906, "blocks.8.attention.w_v": 0.3240242004394531, "blocks.8.attention.w_out": 0.17316818237304688, "blocks.8.feed_forward.w1": 0.5674718618392944, "blocks.8.feed_forward.w2": 0.2905718684196472, "blocks.8.feed_forward.w3": 0.48646801710128784}} +{"step": 4900, "changed_frac": 0.4800527849767962, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5408477783203125, "blocks.0.attention.w_k": 0.5282058715820312, "blocks.0.attention.w_v": 0.2589378356933594, "blocks.0.attention.w_out": 0.2824974060058594, "blocks.0.feed_forward.w1": 0.5098165273666382, "blocks.0.feed_forward.w2": 0.6341361999511719, "blocks.0.feed_forward.w3": 0.4618619382381439, "blocks.1.attention.w_q": 0.6081161499023438, "blocks.1.attention.w_k": 0.5538978576660156, "blocks.1.attention.w_v": 0.33510589599609375, "blocks.1.attention.w_out": 0.47394561767578125, "blocks.1.feed_forward.w1": 0.4930572509765625, "blocks.1.feed_forward.w2": 0.6226882934570312, "blocks.1.feed_forward.w3": 0.46457037329673767, "blocks.2.attention.w_q": 0.5915794372558594, "blocks.2.attention.w_k": 0.5514297485351562, "blocks.2.attention.w_v": 0.31902313232421875, "blocks.2.attention.w_out": 0.44470977783203125, "blocks.2.feed_forward.w1": 0.4978141784667969, "blocks.2.feed_forward.w2": 0.5984789729118347, "blocks.2.feed_forward.w3": 0.4700431823730469, "blocks.3.attention.w_q": 0.6024665832519531, "blocks.3.attention.w_k": 0.5450363159179688, "blocks.3.attention.w_v": 0.3518104553222656, "blocks.3.attention.w_out": 0.367156982421875, "blocks.3.feed_forward.w1": 0.5069770812988281, "blocks.3.feed_forward.w2": 0.5532493591308594, "blocks.3.feed_forward.w3": 0.47050732374191284, "blocks.4.attention.w_q": 0.5848960876464844, "blocks.4.attention.w_k": 0.5360069274902344, "blocks.4.attention.w_v": 0.23342132568359375, "blocks.4.attention.w_out": 0.438751220703125, "blocks.4.feed_forward.w1": 0.515692412853241, "blocks.4.feed_forward.w2": 0.5085678100585938, "blocks.4.feed_forward.w3": 0.4660441279411316, "blocks.5.attention.w_q": 0.6211051940917969, "blocks.5.attention.w_k": 0.5397262573242188, "blocks.5.attention.w_v": 0.3860969543457031, "blocks.5.attention.w_out": 0.31314849853515625, "blocks.5.feed_forward.w1": 0.5244789123535156, "blocks.5.feed_forward.w2": 0.47464752197265625, "blocks.5.feed_forward.w3": 0.4681396484375, "blocks.6.attention.w_q": 0.5939598083496094, "blocks.6.attention.w_k": 0.5398635864257812, "blocks.6.attention.w_v": 0.3858795166015625, "blocks.6.attention.w_out": 0.23286056518554688, "blocks.6.feed_forward.w1": 0.5283699035644531, "blocks.6.feed_forward.w2": 0.43854522705078125, "blocks.6.feed_forward.w3": 0.47097015380859375, "blocks.7.attention.w_q": 0.6027717590332031, "blocks.7.attention.w_k": 0.5555648803710938, "blocks.7.attention.w_v": 0.37320709228515625, "blocks.7.attention.w_out": 0.21826171875, "blocks.7.feed_forward.w1": 0.5348803400993347, "blocks.7.feed_forward.w2": 0.3582878112792969, "blocks.7.feed_forward.w3": 0.4666087031364441, "blocks.8.attention.w_q": 0.6063919067382812, "blocks.8.attention.w_k": 0.580902099609375, "blocks.8.attention.w_v": 0.3076515197753906, "blocks.8.attention.w_out": 0.16352081298828125, "blocks.8.feed_forward.w1": 0.5503260493278503, "blocks.8.feed_forward.w2": 0.2758077085018158, "blocks.8.feed_forward.w3": 0.4679069519042969}} +{"step": 5000, "changed_frac": 0.4625599200908954, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.5215187072753906, "blocks.0.attention.w_k": 0.5135993957519531, "blocks.0.attention.w_v": 0.24881362915039062, "blocks.0.attention.w_out": 0.269195556640625, "blocks.0.feed_forward.w1": 0.49078625440597534, "blocks.0.feed_forward.w2": 0.6185659170150757, "blocks.0.feed_forward.w3": 0.4433797299861908, "blocks.1.attention.w_q": 0.5908966064453125, "blocks.1.attention.w_k": 0.5359725952148438, "blocks.1.attention.w_v": 0.3203010559082031, "blocks.1.attention.w_out": 0.452728271484375, "blocks.1.feed_forward.w1": 0.47356289625167847, "blocks.1.feed_forward.w2": 0.6073850393295288, "blocks.1.feed_forward.w3": 0.4460703730583191, "blocks.2.attention.w_q": 0.5786170959472656, "blocks.2.attention.w_k": 0.534881591796875, "blocks.2.attention.w_v": 0.302032470703125, "blocks.2.attention.w_out": 0.4290962219238281, "blocks.2.feed_forward.w1": 0.4789835810661316, "blocks.2.feed_forward.w2": 0.5804392695426941, "blocks.2.feed_forward.w3": 0.4503720700740814, "blocks.3.attention.w_q": 0.5907249450683594, "blocks.3.attention.w_k": 0.5254135131835938, "blocks.3.attention.w_v": 0.3295745849609375, "blocks.3.attention.w_out": 0.3532867431640625, "blocks.3.feed_forward.w1": 0.49040478467941284, "blocks.3.feed_forward.w2": 0.5331052541732788, "blocks.3.feed_forward.w3": 0.4523506164550781, "blocks.4.attention.w_q": 0.5667915344238281, "blocks.4.attention.w_k": 0.5174560546875, "blocks.4.attention.w_v": 0.22078704833984375, "blocks.4.attention.w_out": 0.4190788269042969, "blocks.4.feed_forward.w1": 0.49625271558761597, "blocks.4.feed_forward.w2": 0.4902191162109375, "blocks.4.feed_forward.w3": 0.4467735290527344, "blocks.5.attention.w_q": 0.60589599609375, "blocks.5.attention.w_k": 0.5189208984375, "blocks.5.attention.w_v": 0.3674583435058594, "blocks.5.attention.w_out": 0.2958564758300781, "blocks.5.feed_forward.w1": 0.5054537653923035, "blocks.5.feed_forward.w2": 0.45505523681640625, "blocks.5.feed_forward.w3": 0.4483019709587097, "blocks.6.attention.w_q": 0.5817794799804688, "blocks.6.attention.w_k": 0.5201568603515625, "blocks.6.attention.w_v": 0.3681831359863281, "blocks.6.attention.w_out": 0.22037124633789062, "blocks.6.feed_forward.w1": 0.5107256770133972, "blocks.6.feed_forward.w2": 0.4195963740348816, "blocks.6.feed_forward.w3": 0.45208868384361267, "blocks.7.attention.w_q": 0.5890617370605469, "blocks.7.attention.w_k": 0.5384941101074219, "blocks.7.attention.w_v": 0.3579826354980469, "blocks.7.attention.w_out": 0.20701980590820312, "blocks.7.feed_forward.w1": 0.5143407583236694, "blocks.7.feed_forward.w2": 0.34291839599609375, "blocks.7.feed_forward.w3": 0.44976553320884705, "blocks.8.attention.w_q": 0.5907516479492188, "blocks.8.attention.w_k": 0.56317138671875, "blocks.8.attention.w_v": 0.2941741943359375, "blocks.8.attention.w_out": 0.15542984008789062, "blocks.8.feed_forward.w1": 0.5341288447380066, "blocks.8.feed_forward.w2": 0.2614898681640625, "blocks.8.feed_forward.w3": 0.44882965087890625}} +{"step": 5100, "changed_frac": 0.4451993632520366, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.503143310546875, "blocks.0.attention.w_k": 0.4896202087402344, "blocks.0.attention.w_v": 0.23581314086914062, "blocks.0.attention.w_out": 0.25811004638671875, "blocks.0.feed_forward.w1": 0.4738299250602722, "blocks.0.feed_forward.w2": 0.6028366088867188, "blocks.0.feed_forward.w3": 0.4287312924861908, "blocks.1.attention.w_q": 0.5773200988769531, "blocks.1.attention.w_k": 0.5135993957519531, "blocks.1.attention.w_v": 0.3052177429199219, "blocks.1.attention.w_out": 0.4376335144042969, "blocks.1.feed_forward.w1": 0.45398586988449097, "blocks.1.feed_forward.w2": 0.590301513671875, "blocks.1.feed_forward.w3": 0.4279124140739441, "blocks.2.attention.w_q": 0.5615386962890625, "blocks.2.attention.w_k": 0.5182991027832031, "blocks.2.attention.w_v": 0.2861480712890625, "blocks.2.attention.w_out": 0.41040802001953125, "blocks.2.feed_forward.w1": 0.4600486755371094, "blocks.2.feed_forward.w2": 0.563605010509491, "blocks.2.feed_forward.w3": 0.4336484372615814, "blocks.3.attention.w_q": 0.5723342895507812, "blocks.3.attention.w_k": 0.5096817016601562, "blocks.3.attention.w_v": 0.3176383972167969, "blocks.3.attention.w_out": 0.33896636962890625, "blocks.3.feed_forward.w1": 0.47078579664230347, "blocks.3.feed_forward.w2": 0.5150591731071472, "blocks.3.feed_forward.w3": 0.43326061964035034, "blocks.4.attention.w_q": 0.5493011474609375, "blocks.4.attention.w_k": 0.4983329772949219, "blocks.4.attention.w_v": 0.20844650268554688, "blocks.4.attention.w_out": 0.40409088134765625, "blocks.4.feed_forward.w1": 0.4764493405818939, "blocks.4.feed_forward.w2": 0.47092437744140625, "blocks.4.feed_forward.w3": 0.4280637204647064, "blocks.5.attention.w_q": 0.5861778259277344, "blocks.5.attention.w_k": 0.5049896240234375, "blocks.5.attention.w_v": 0.3532295227050781, "blocks.5.attention.w_out": 0.2837791442871094, "blocks.5.feed_forward.w1": 0.488555908203125, "blocks.5.feed_forward.w2": 0.4359245300292969, "blocks.5.feed_forward.w3": 0.4304656982421875, "blocks.6.attention.w_q": 0.5615577697753906, "blocks.6.attention.w_k": 0.5035362243652344, "blocks.6.attention.w_v": 0.3548011779785156, "blocks.6.attention.w_out": 0.20951461791992188, "blocks.6.feed_forward.w1": 0.49158352613449097, "blocks.6.feed_forward.w2": 0.40486782789230347, "blocks.6.feed_forward.w3": 0.4327049255371094, "blocks.7.attention.w_q": 0.5707473754882812, "blocks.7.attention.w_k": 0.521453857421875, "blocks.7.attention.w_v": 0.3378028869628906, "blocks.7.attention.w_out": 0.19573593139648438, "blocks.7.feed_forward.w1": 0.49626415967941284, "blocks.7.feed_forward.w2": 0.3248189389705658, "blocks.7.feed_forward.w3": 0.4295806884765625, "blocks.8.attention.w_q": 0.5744400024414062, "blocks.8.attention.w_k": 0.54510498046875, "blocks.8.attention.w_v": 0.27701568603515625, "blocks.8.attention.w_out": 0.1464385986328125, "blocks.8.feed_forward.w1": 0.5147132873535156, "blocks.8.feed_forward.w2": 0.24758784472942352, "blocks.8.feed_forward.w3": 0.42894238233566284}} +{"step": 5200, "changed_frac": 0.4271888732910156, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4827461242675781, "blocks.0.attention.w_k": 0.4747276306152344, "blocks.0.attention.w_v": 0.22081756591796875, "blocks.0.attention.w_out": 0.24634170532226562, "blocks.0.feed_forward.w1": 0.45442962646484375, "blocks.0.feed_forward.w2": 0.5880076289176941, "blocks.0.feed_forward.w3": 0.408966064453125, "blocks.1.attention.w_q": 0.5550651550292969, "blocks.1.attention.w_k": 0.4959983825683594, "blocks.1.attention.w_v": 0.2863426208496094, "blocks.1.attention.w_out": 0.4211273193359375, "blocks.1.feed_forward.w1": 0.4341278076171875, "blocks.1.feed_forward.w2": 0.5725555419921875, "blocks.1.feed_forward.w3": 0.40873846411705017, "blocks.2.attention.w_q": 0.5403900146484375, "blocks.2.attention.w_k": 0.49898529052734375, "blocks.2.attention.w_v": 0.2707481384277344, "blocks.2.attention.w_out": 0.3948173522949219, "blocks.2.feed_forward.w1": 0.4403025507926941, "blocks.2.feed_forward.w2": 0.5443929433822632, "blocks.2.feed_forward.w3": 0.4125010371208191, "blocks.3.attention.w_q": 0.5532302856445312, "blocks.3.attention.w_k": 0.4894447326660156, "blocks.3.attention.w_v": 0.30112457275390625, "blocks.3.attention.w_out": 0.31915283203125, "blocks.3.feed_forward.w1": 0.45152920484542847, "blocks.3.feed_forward.w2": 0.49884796142578125, "blocks.3.feed_forward.w3": 0.41474661231040955, "blocks.4.attention.w_q": 0.5277137756347656, "blocks.4.attention.w_k": 0.4816436767578125, "blocks.4.attention.w_v": 0.1958465576171875, "blocks.4.attention.w_out": 0.3845939636230469, "blocks.4.feed_forward.w1": 0.4590695798397064, "blocks.4.feed_forward.w2": 0.45095953345298767, "blocks.4.feed_forward.w3": 0.41015753149986267, "blocks.5.attention.w_q": 0.5688819885253906, "blocks.5.attention.w_k": 0.48709869384765625, "blocks.5.attention.w_v": 0.33626556396484375, "blocks.5.attention.w_out": 0.268035888671875, "blocks.5.feed_forward.w1": 0.4677785336971283, "blocks.5.feed_forward.w2": 0.41679638624191284, "blocks.5.feed_forward.w3": 0.4110247492790222, "blocks.6.attention.w_q": 0.5438346862792969, "blocks.6.attention.w_k": 0.48220062255859375, "blocks.6.attention.w_v": 0.3354759216308594, "blocks.6.attention.w_out": 0.20019149780273438, "blocks.6.feed_forward.w1": 0.4712066650390625, "blocks.6.feed_forward.w2": 0.38321560621261597, "blocks.6.feed_forward.w3": 0.4146626889705658, "blocks.7.attention.w_q": 0.5548515319824219, "blocks.7.attention.w_k": 0.5015525817871094, "blocks.7.attention.w_v": 0.3223304748535156, "blocks.7.attention.w_out": 0.185333251953125, "blocks.7.feed_forward.w1": 0.47836431860923767, "blocks.7.feed_forward.w2": 0.3075981140136719, "blocks.7.feed_forward.w3": 0.4113655090332031, "blocks.8.attention.w_q": 0.5546073913574219, "blocks.8.attention.w_k": 0.5297393798828125, "blocks.8.attention.w_v": 0.2646598815917969, "blocks.8.attention.w_out": 0.140380859375, "blocks.8.feed_forward.w1": 0.49639639258384705, "blocks.8.feed_forward.w2": 0.23432159423828125, "blocks.8.feed_forward.w3": 0.4128710627555847}} +{"step": 5300, "changed_frac": 0.40923769046098757, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4663047790527344, "blocks.0.attention.w_k": 0.4526824951171875, "blocks.0.attention.w_v": 0.21033096313476562, "blocks.0.attention.w_out": 0.23603439331054688, "blocks.0.feed_forward.w1": 0.43644842505455017, "blocks.0.feed_forward.w2": 0.5725008845329285, "blocks.0.feed_forward.w3": 0.39065808057785034, "blocks.1.attention.w_q": 0.5389366149902344, "blocks.1.attention.w_k": 0.4762153625488281, "blocks.1.attention.w_v": 0.2767295837402344, "blocks.1.attention.w_out": 0.40236663818359375, "blocks.1.feed_forward.w1": 0.4156430661678314, "blocks.1.feed_forward.w2": 0.5561612844467163, "blocks.1.feed_forward.w3": 0.38969677686691284, "blocks.2.attention.w_q": 0.5215911865234375, "blocks.2.attention.w_k": 0.4789581298828125, "blocks.2.attention.w_v": 0.25524139404296875, "blocks.2.attention.w_out": 0.3760795593261719, "blocks.2.feed_forward.w1": 0.42082977294921875, "blocks.2.feed_forward.w2": 0.52593994140625, "blocks.2.feed_forward.w3": 0.3934733271598816, "blocks.3.attention.w_q": 0.5334510803222656, "blocks.3.attention.w_k": 0.47121429443359375, "blocks.3.attention.w_v": 0.28588104248046875, "blocks.3.attention.w_out": 0.3018455505371094, "blocks.3.feed_forward.w1": 0.43037670850753784, "blocks.3.feed_forward.w2": 0.47569021582603455, "blocks.3.feed_forward.w3": 0.3947906494140625, "blocks.4.attention.w_q": 0.5121307373046875, "blocks.4.attention.w_k": 0.4617156982421875, "blocks.4.attention.w_v": 0.18421554565429688, "blocks.4.attention.w_out": 0.3665771484375, "blocks.4.feed_forward.w1": 0.4386495053768158, "blocks.4.feed_forward.w2": 0.4321340024471283, "blocks.4.feed_forward.w3": 0.3898404538631439, "blocks.5.attention.w_q": 0.5526161193847656, "blocks.5.attention.w_k": 0.4679412841796875, "blocks.5.attention.w_v": 0.3220252990722656, "blocks.5.attention.w_out": 0.2550697326660156, "blocks.5.feed_forward.w1": 0.4481010437011719, "blocks.5.feed_forward.w2": 0.39833325147628784, "blocks.5.feed_forward.w3": 0.3919881284236908, "blocks.6.attention.w_q": 0.5270881652832031, "blocks.6.attention.w_k": 0.4639701843261719, "blocks.6.attention.w_v": 0.3224754333496094, "blocks.6.attention.w_out": 0.1884918212890625, "blocks.6.feed_forward.w1": 0.45132192969322205, "blocks.6.feed_forward.w2": 0.36417388916015625, "blocks.6.feed_forward.w3": 0.39543280005455017, "blocks.7.attention.w_q": 0.5345726013183594, "blocks.7.attention.w_k": 0.48319244384765625, "blocks.7.attention.w_v": 0.3091926574707031, "blocks.7.attention.w_out": 0.17454910278320312, "blocks.7.feed_forward.w1": 0.4576962888240814, "blocks.7.feed_forward.w2": 0.2918701171875, "blocks.7.feed_forward.w3": 0.39262646436691284, "blocks.8.attention.w_q": 0.5373115539550781, "blocks.8.attention.w_k": 0.5094642639160156, "blocks.8.attention.w_v": 0.25434112548828125, "blocks.8.attention.w_out": 0.13042449951171875, "blocks.8.feed_forward.w1": 0.47790655493736267, "blocks.8.feed_forward.w2": 0.22206497192382812, "blocks.8.feed_forward.w3": 0.39217886328697205}} +{"step": 5400, "changed_frac": 0.39020130776951456, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4405479431152344, "blocks.0.attention.w_k": 0.4298095703125, "blocks.0.attention.w_v": 0.19952392578125, "blocks.0.attention.w_out": 0.2220611572265625, "blocks.0.feed_forward.w1": 0.41883978247642517, "blocks.0.feed_forward.w2": 0.5530586242675781, "blocks.0.feed_forward.w3": 0.3721555173397064, "blocks.1.attention.w_q": 0.5193328857421875, "blocks.1.attention.w_k": 0.46231842041015625, "blocks.1.attention.w_v": 0.2635955810546875, "blocks.1.attention.w_out": 0.3860969543457031, "blocks.1.feed_forward.w1": 0.39638519287109375, "blocks.1.feed_forward.w2": 0.5374400019645691, "blocks.1.feed_forward.w3": 0.3705940246582031, "blocks.2.attention.w_q": 0.5001754760742188, "blocks.2.attention.w_k": 0.4579811096191406, "blocks.2.attention.w_v": 0.2408599853515625, "blocks.2.attention.w_out": 0.3577766418457031, "blocks.2.feed_forward.w1": 0.4023119807243347, "blocks.2.feed_forward.w2": 0.5057615041732788, "blocks.2.feed_forward.w3": 0.3724810481071472, "blocks.3.attention.w_q": 0.5126075744628906, "blocks.3.attention.w_k": 0.4531364440917969, "blocks.3.attention.w_v": 0.2692756652832031, "blocks.3.attention.w_out": 0.2890739440917969, "blocks.3.feed_forward.w1": 0.4110158383846283, "blocks.3.feed_forward.w2": 0.4568290710449219, "blocks.3.feed_forward.w3": 0.3752937316894531, "blocks.4.attention.w_q": 0.49031829833984375, "blocks.4.attention.w_k": 0.4395256042480469, "blocks.4.attention.w_v": 0.173004150390625, "blocks.4.attention.w_out": 0.3475532531738281, "blocks.4.feed_forward.w1": 0.4189084470272064, "blocks.4.feed_forward.w2": 0.41155752539634705, "blocks.4.feed_forward.w3": 0.36919912695884705, "blocks.5.attention.w_q": 0.5343666076660156, "blocks.5.attention.w_k": 0.4421424865722656, "blocks.5.attention.w_v": 0.3068351745605469, "blocks.5.attention.w_out": 0.24136734008789062, "blocks.5.feed_forward.w1": 0.4263750910758972, "blocks.5.feed_forward.w2": 0.3760821223258972, "blocks.5.feed_forward.w3": 0.37178677320480347, "blocks.6.attention.w_q": 0.5027389526367188, "blocks.6.attention.w_k": 0.44451141357421875, "blocks.6.attention.w_v": 0.3030967712402344, "blocks.6.attention.w_out": 0.1779632568359375, "blocks.6.feed_forward.w1": 0.43080902099609375, "blocks.6.feed_forward.w2": 0.3438352048397064, "blocks.6.feed_forward.w3": 0.3742205500602722, "blocks.7.attention.w_q": 0.5114288330078125, "blocks.7.attention.w_k": 0.4610443115234375, "blocks.7.attention.w_v": 0.2936820983886719, "blocks.7.attention.w_out": 0.16585922241210938, "blocks.7.feed_forward.w1": 0.4381396174430847, "blocks.7.feed_forward.w2": 0.27377447485923767, "blocks.7.feed_forward.w3": 0.37181347608566284, "blocks.8.attention.w_q": 0.5172920227050781, "blocks.8.attention.w_k": 0.4899024963378906, "blocks.8.attention.w_v": 0.23869705200195312, "blocks.8.attention.w_out": 0.12443161010742188, "blocks.8.feed_forward.w1": 0.4566993713378906, "blocks.8.feed_forward.w2": 0.20788702368736267, "blocks.8.feed_forward.w3": 0.3712857663631439}} +{"step": 5500, "changed_frac": 0.3715015313564203, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.42414093017578125, "blocks.0.attention.w_k": 0.4140472412109375, "blocks.0.attention.w_v": 0.18914794921875, "blocks.0.attention.w_out": 0.21121597290039062, "blocks.0.feed_forward.w1": 0.39707183837890625, "blocks.0.feed_forward.w2": 0.5355275869369507, "blocks.0.feed_forward.w3": 0.3531786799430847, "blocks.1.attention.w_q": 0.49935150146484375, "blocks.1.attention.w_k": 0.4416351318359375, "blocks.1.attention.w_v": 0.24942779541015625, "blocks.1.attention.w_out": 0.3697624206542969, "blocks.1.feed_forward.w1": 0.3741391599178314, "blocks.1.feed_forward.w2": 0.5162811279296875, "blocks.1.feed_forward.w3": 0.3506253659725189, "blocks.2.attention.w_q": 0.4772605895996094, "blocks.2.attention.w_k": 0.43767547607421875, "blocks.2.attention.w_v": 0.22833633422851562, "blocks.2.attention.w_out": 0.33826446533203125, "blocks.2.feed_forward.w1": 0.38030242919921875, "blocks.2.feed_forward.w2": 0.48495739698410034, "blocks.2.feed_forward.w3": 0.3543230891227722, "blocks.3.attention.w_q": 0.4928398132324219, "blocks.3.attention.w_k": 0.4315910339355469, "blocks.3.attention.w_v": 0.25614166259765625, "blocks.3.attention.w_out": 0.270538330078125, "blocks.3.feed_forward.w1": 0.3911527097225189, "blocks.3.feed_forward.w2": 0.4366251826286316, "blocks.3.feed_forward.w3": 0.3562609553337097, "blocks.4.attention.w_q": 0.46976470947265625, "blocks.4.attention.w_k": 0.42034912109375, "blocks.4.attention.w_v": 0.1650390625, "blocks.4.attention.w_out": 0.3297462463378906, "blocks.4.feed_forward.w1": 0.39688748121261597, "blocks.4.feed_forward.w2": 0.3910319209098816, "blocks.4.feed_forward.w3": 0.3500468134880066, "blocks.5.attention.w_q": 0.5102195739746094, "blocks.5.attention.w_k": 0.4263954162597656, "blocks.5.attention.w_v": 0.29058837890625, "blocks.5.attention.w_out": 0.22765350341796875, "blocks.5.feed_forward.w1": 0.40662893652915955, "blocks.5.feed_forward.w2": 0.3563677668571472, "blocks.5.feed_forward.w3": 0.3512675166130066, "blocks.6.attention.w_q": 0.4850425720214844, "blocks.6.attention.w_k": 0.4243812561035156, "blocks.6.attention.w_v": 0.2891197204589844, "blocks.6.attention.w_out": 0.16835403442382812, "blocks.6.feed_forward.w1": 0.41034191846847534, "blocks.6.feed_forward.w2": 0.3247184753417969, "blocks.6.feed_forward.w3": 0.35525640845298767, "blocks.7.attention.w_q": 0.4947662353515625, "blocks.7.attention.w_k": 0.4408760070800781, "blocks.7.attention.w_v": 0.27628326416015625, "blocks.7.attention.w_out": 0.15399169921875, "blocks.7.feed_forward.w1": 0.41618603467941284, "blocks.7.feed_forward.w2": 0.2578977048397064, "blocks.7.feed_forward.w3": 0.35254669189453125, "blocks.8.attention.w_q": 0.49922943115234375, "blocks.8.attention.w_k": 0.4723472595214844, "blocks.8.attention.w_v": 0.22362136840820312, "blocks.8.attention.w_out": 0.11676788330078125, "blocks.8.feed_forward.w1": 0.4358482360839844, "blocks.8.feed_forward.w2": 0.19596481323242188, "blocks.8.feed_forward.w3": 0.35181936621665955}} +{"step": 5600, "changed_frac": 0.35274244781233305, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.4054718017578125, "blocks.0.attention.w_k": 0.39688873291015625, "blocks.0.attention.w_v": 0.17998504638671875, "blocks.0.attention.w_out": 0.20059585571289062, "blocks.0.feed_forward.w1": 0.3775444030761719, "blocks.0.feed_forward.w2": 0.5157508850097656, "blocks.0.feed_forward.w3": 0.3334490656852722, "blocks.1.attention.w_q": 0.4826240539550781, "blocks.1.attention.w_k": 0.4208412170410156, "blocks.1.attention.w_v": 0.23712539672851562, "blocks.1.attention.w_out": 0.3497505187988281, "blocks.1.feed_forward.w1": 0.35455450415611267, "blocks.1.feed_forward.w2": 0.4947242736816406, "blocks.1.feed_forward.w3": 0.33125561475753784, "blocks.2.attention.w_q": 0.4578819274902344, "blocks.2.attention.w_k": 0.4185028076171875, "blocks.2.attention.w_v": 0.21475601196289062, "blocks.2.attention.w_out": 0.3187751770019531, "blocks.2.feed_forward.w1": 0.35980224609375, "blocks.2.feed_forward.w2": 0.46440762281417847, "blocks.2.feed_forward.w3": 0.3345184326171875, "blocks.3.attention.w_q": 0.4706687927246094, "blocks.3.attention.w_k": 0.4127349853515625, "blocks.3.attention.w_v": 0.23859786987304688, "blocks.3.attention.w_out": 0.2578277587890625, "blocks.3.feed_forward.w1": 0.37009939551353455, "blocks.3.feed_forward.w2": 0.4139251708984375, "blocks.3.feed_forward.w3": 0.3357785642147064, "blocks.4.attention.w_q": 0.4473457336425781, "blocks.4.attention.w_k": 0.3975830078125, "blocks.4.attention.w_v": 0.15337753295898438, "blocks.4.attention.w_out": 0.3143348693847656, "blocks.4.feed_forward.w1": 0.3762779235839844, "blocks.4.feed_forward.w2": 0.3706321716308594, "blocks.4.feed_forward.w3": 0.32916259765625, "blocks.5.attention.w_q": 0.4928131103515625, "blocks.5.attention.w_k": 0.4056053161621094, "blocks.5.attention.w_v": 0.2739219665527344, "blocks.5.attention.w_out": 0.2151031494140625, "blocks.5.feed_forward.w1": 0.38559088110923767, "blocks.5.feed_forward.w2": 0.3374888300895691, "blocks.5.feed_forward.w3": 0.33258184790611267, "blocks.6.attention.w_q": 0.46451568603515625, "blocks.6.attention.w_k": 0.40058135986328125, "blocks.6.attention.w_v": 0.2717857360839844, "blocks.6.attention.w_out": 0.15779495239257812, "blocks.6.feed_forward.w1": 0.3886528015136719, "blocks.6.feed_forward.w2": 0.30731838941574097, "blocks.6.feed_forward.w3": 0.3348388671875, "blocks.7.attention.w_q": 0.47284698486328125, "blocks.7.attention.w_k": 0.4234580993652344, "blocks.7.attention.w_v": 0.26226806640625, "blocks.7.attention.w_out": 0.14765167236328125, "blocks.7.feed_forward.w1": 0.3950691223144531, "blocks.7.feed_forward.w2": 0.24322891235351562, "blocks.7.feed_forward.w3": 0.3330078125, "blocks.8.attention.w_q": 0.4796333312988281, "blocks.8.attention.w_k": 0.45169830322265625, "blocks.8.attention.w_v": 0.21175384521484375, "blocks.8.attention.w_out": 0.11013412475585938, "blocks.8.feed_forward.w1": 0.41446051001548767, "blocks.8.feed_forward.w2": 0.1839701384305954, "blocks.8.feed_forward.w3": 0.3331197202205658}} +{"step": 5700, "changed_frac": 0.3338951371673845, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3770294189453125, "blocks.0.attention.w_k": 0.37464141845703125, "blocks.0.attention.w_v": 0.169921875, "blocks.0.attention.w_out": 0.189666748046875, "blocks.0.feed_forward.w1": 0.35687255859375, "blocks.0.feed_forward.w2": 0.49504852294921875, "blocks.0.feed_forward.w3": 0.3154284358024597, "blocks.1.attention.w_q": 0.4609184265136719, "blocks.1.attention.w_k": 0.4010887145996094, "blocks.1.attention.w_v": 0.221588134765625, "blocks.1.attention.w_out": 0.330902099609375, "blocks.1.feed_forward.w1": 0.33431243896484375, "blocks.1.feed_forward.w2": 0.4747861325740814, "blocks.1.feed_forward.w3": 0.31336086988449097, "blocks.2.attention.w_q": 0.43740081787109375, "blocks.2.attention.w_k": 0.39592742919921875, "blocks.2.attention.w_v": 0.2005615234375, "blocks.2.attention.w_out": 0.3046760559082031, "blocks.2.feed_forward.w1": 0.3398272395133972, "blocks.2.feed_forward.w2": 0.4425099790096283, "blocks.2.feed_forward.w3": 0.3151499629020691, "blocks.3.attention.w_q": 0.45212554931640625, "blocks.3.attention.w_k": 0.3921623229980469, "blocks.3.attention.w_v": 0.2248077392578125, "blocks.3.attention.w_out": 0.24076461791992188, "blocks.3.feed_forward.w1": 0.3492279052734375, "blocks.3.feed_forward.w2": 0.39134472608566284, "blocks.3.feed_forward.w3": 0.3158683776855469, "blocks.4.attention.w_q": 0.4267311096191406, "blocks.4.attention.w_k": 0.378265380859375, "blocks.4.attention.w_v": 0.14115524291992188, "blocks.4.attention.w_out": 0.2969627380371094, "blocks.4.feed_forward.w1": 0.3555399775505066, "blocks.4.feed_forward.w2": 0.3489875793457031, "blocks.4.feed_forward.w3": 0.3121172785758972, "blocks.5.attention.w_q": 0.47191619873046875, "blocks.5.attention.w_k": 0.3844451904296875, "blocks.5.attention.w_v": 0.2571563720703125, "blocks.5.attention.w_out": 0.20306777954101562, "blocks.5.feed_forward.w1": 0.36355718970298767, "blocks.5.feed_forward.w2": 0.3185819089412689, "blocks.5.feed_forward.w3": 0.3125445246696472, "blocks.6.attention.w_q": 0.44120025634765625, "blocks.6.attention.w_k": 0.3817253112792969, "blocks.6.attention.w_v": 0.2569770812988281, "blocks.6.attention.w_out": 0.15035247802734375, "blocks.6.feed_forward.w1": 0.36839041113853455, "blocks.6.feed_forward.w2": 0.28924560546875, "blocks.6.feed_forward.w3": 0.3162117004394531, "blocks.7.attention.w_q": 0.4533195495605469, "blocks.7.attention.w_k": 0.4027519226074219, "blocks.7.attention.w_v": 0.24685287475585938, "blocks.7.attention.w_out": 0.13759231567382812, "blocks.7.feed_forward.w1": 0.37372082471847534, "blocks.7.feed_forward.w2": 0.22659048438072205, "blocks.7.feed_forward.w3": 0.3136405944824219, "blocks.8.attention.w_q": 0.4602165222167969, "blocks.8.attention.w_k": 0.4292144775390625, "blocks.8.attention.w_v": 0.19806289672851562, "blocks.8.attention.w_out": 0.10277938842773438, "blocks.8.feed_forward.w1": 0.39523953199386597, "blocks.8.feed_forward.w2": 0.17220687866210938, "blocks.8.feed_forward.w3": 0.3132896423339844}} +{"step": 5800, "changed_frac": 0.3164222912910657, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.35672760009765625, "blocks.0.attention.w_k": 0.35501861572265625, "blocks.0.attention.w_v": 0.16165542602539062, "blocks.0.attention.w_out": 0.18039703369140625, "blocks.0.feed_forward.w1": 0.33967846632003784, "blocks.0.feed_forward.w2": 0.4764747619628906, "blocks.0.feed_forward.w3": 0.2980842590332031, "blocks.1.attention.w_q": 0.444000244140625, "blocks.1.attention.w_k": 0.3801536560058594, "blocks.1.attention.w_v": 0.21077346801757812, "blocks.1.attention.w_out": 0.3162345886230469, "blocks.1.feed_forward.w1": 0.31658172607421875, "blocks.1.feed_forward.w2": 0.4525248408317566, "blocks.1.feed_forward.w3": 0.2957242429256439, "blocks.2.attention.w_q": 0.4142112731933594, "blocks.2.attention.w_k": 0.3762664794921875, "blocks.2.attention.w_v": 0.18970489501953125, "blocks.2.attention.w_out": 0.28749847412109375, "blocks.2.feed_forward.w1": 0.32087963819503784, "blocks.2.feed_forward.w2": 0.4214922785758972, "blocks.2.feed_forward.w3": 0.2971789240837097, "blocks.3.attention.w_q": 0.4299888610839844, "blocks.3.attention.w_k": 0.3710174560546875, "blocks.3.attention.w_v": 0.21269989013671875, "blocks.3.attention.w_out": 0.2276611328125, "blocks.3.feed_forward.w1": 0.33037567138671875, "blocks.3.feed_forward.w2": 0.3713315427303314, "blocks.3.feed_forward.w3": 0.2982978820800781, "blocks.4.attention.w_q": 0.40177154541015625, "blocks.4.attention.w_k": 0.356536865234375, "blocks.4.attention.w_v": 0.13417816162109375, "blocks.4.attention.w_out": 0.2812385559082031, "blocks.4.feed_forward.w1": 0.33786138892173767, "blocks.4.feed_forward.w2": 0.3282063901424408, "blocks.4.feed_forward.w3": 0.2939643859863281, "blocks.5.attention.w_q": 0.4511680603027344, "blocks.5.attention.w_k": 0.3667755126953125, "blocks.5.attention.w_v": 0.2461395263671875, "blocks.5.attention.w_out": 0.19091796875, "blocks.5.feed_forward.w1": 0.3447049558162689, "blocks.5.feed_forward.w2": 0.3003031611442566, "blocks.5.feed_forward.w3": 0.2945645749568939, "blocks.6.attention.w_q": 0.41965484619140625, "blocks.6.attention.w_k": 0.3641319274902344, "blocks.6.attention.w_v": 0.24380874633789062, "blocks.6.attention.w_out": 0.14189529418945312, "blocks.6.feed_forward.w1": 0.34939703345298767, "blocks.6.feed_forward.w2": 0.27212652564048767, "blocks.6.feed_forward.w3": 0.2970466613769531, "blocks.7.attention.w_q": 0.4297637939453125, "blocks.7.attention.w_k": 0.3825187683105469, "blocks.7.attention.w_v": 0.23148345947265625, "blocks.7.attention.w_out": 0.12923049926757812, "blocks.7.feed_forward.w1": 0.35399121046066284, "blocks.7.feed_forward.w2": 0.21399816870689392, "blocks.7.feed_forward.w3": 0.2960955500602722, "blocks.8.attention.w_q": 0.4358978271484375, "blocks.8.attention.w_k": 0.40769195556640625, "blocks.8.attention.w_v": 0.19069671630859375, "blocks.8.attention.w_out": 0.09743118286132812, "blocks.8.feed_forward.w1": 0.3754565119743347, "blocks.8.feed_forward.w2": 0.16199493408203125, "blocks.8.feed_forward.w3": 0.2964871823787689}} +{"step": 5900, "changed_frac": 0.2977917337010049, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3348884582519531, "blocks.0.attention.w_k": 0.3358802795410156, "blocks.0.attention.w_v": 0.14846038818359375, "blocks.0.attention.w_out": 0.16880416870117188, "blocks.0.feed_forward.w1": 0.3181495666503906, "blocks.0.feed_forward.w2": 0.45350393652915955, "blocks.0.feed_forward.w3": 0.2789573669433594, "blocks.1.attention.w_q": 0.4193000793457031, "blocks.1.attention.w_k": 0.3594093322753906, "blocks.1.attention.w_v": 0.19344711303710938, "blocks.1.attention.w_out": 0.2943000793457031, "blocks.1.feed_forward.w1": 0.2979685664176941, "blocks.1.feed_forward.w2": 0.43139904737472534, "blocks.1.feed_forward.w3": 0.27752685546875, "blocks.2.attention.w_q": 0.3948402404785156, "blocks.2.attention.w_k": 0.3556632995605469, "blocks.2.attention.w_v": 0.1754913330078125, "blocks.2.attention.w_out": 0.26894378662109375, "blocks.2.feed_forward.w1": 0.3006172180175781, "blocks.2.feed_forward.w2": 0.4000701904296875, "blocks.2.feed_forward.w3": 0.27798590064048767, "blocks.3.attention.w_q": 0.4060630798339844, "blocks.3.attention.w_k": 0.35024261474609375, "blocks.3.attention.w_v": 0.1961822509765625, "blocks.3.attention.w_out": 0.2133026123046875, "blocks.3.feed_forward.w1": 0.3102963864803314, "blocks.3.feed_forward.w2": 0.35108184814453125, "blocks.3.feed_forward.w3": 0.2797800898551941, "blocks.4.attention.w_q": 0.3817558288574219, "blocks.4.attention.w_k": 0.33582305908203125, "blocks.4.attention.w_v": 0.12427520751953125, "blocks.4.attention.w_out": 0.26316070556640625, "blocks.4.feed_forward.w1": 0.3171526789665222, "blocks.4.feed_forward.w2": 0.30982083082199097, "blocks.4.feed_forward.w3": 0.2752227783203125, "blocks.5.attention.w_q": 0.427764892578125, "blocks.5.attention.w_k": 0.34554290771484375, "blocks.5.attention.w_v": 0.22893524169921875, "blocks.5.attention.w_out": 0.17965316772460938, "blocks.5.feed_forward.w1": 0.3233540952205658, "blocks.5.feed_forward.w2": 0.2803128659725189, "blocks.5.feed_forward.w3": 0.27760061621665955, "blocks.6.attention.w_q": 0.40056610107421875, "blocks.6.attention.w_k": 0.34232330322265625, "blocks.6.attention.w_v": 0.22691726684570312, "blocks.6.attention.w_out": 0.13245010375976562, "blocks.6.feed_forward.w1": 0.3279813230037689, "blocks.6.feed_forward.w2": 0.2539774775505066, "blocks.6.feed_forward.w3": 0.27950286865234375, "blocks.7.attention.w_q": 0.40868377685546875, "blocks.7.attention.w_k": 0.3598060607910156, "blocks.7.attention.w_v": 0.21857070922851562, "blocks.7.attention.w_out": 0.12210845947265625, "blocks.7.feed_forward.w1": 0.3328501582145691, "blocks.7.feed_forward.w2": 0.20039114356040955, "blocks.7.feed_forward.w3": 0.27713775634765625, "blocks.8.attention.w_q": 0.4159126281738281, "blocks.8.attention.w_k": 0.390045166015625, "blocks.8.attention.w_v": 0.17810821533203125, "blocks.8.attention.w_out": 0.09072494506835938, "blocks.8.feed_forward.w1": 0.35439300537109375, "blocks.8.feed_forward.w2": 0.15177027881145477, "blocks.8.feed_forward.w3": 0.2789573669433594}} +{"step": 6000, "changed_frac": 0.2818190745818309, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.32196807861328125, "blocks.0.attention.w_k": 0.31780242919921875, "blocks.0.attention.w_v": 0.14158248901367188, "blocks.0.attention.w_out": 0.16086196899414062, "blocks.0.feed_forward.w1": 0.30157217383384705, "blocks.0.feed_forward.w2": 0.43437832593917847, "blocks.0.feed_forward.w3": 0.2633476257324219, "blocks.1.attention.w_q": 0.400360107421875, "blocks.1.attention.w_k": 0.3419075012207031, "blocks.1.attention.w_v": 0.1866607666015625, "blocks.1.attention.w_out": 0.28260040283203125, "blocks.1.feed_forward.w1": 0.2818107604980469, "blocks.1.feed_forward.w2": 0.4102274775505066, "blocks.1.feed_forward.w3": 0.2615407407283783, "blocks.2.attention.w_q": 0.3726463317871094, "blocks.2.attention.w_k": 0.33592987060546875, "blocks.2.attention.w_v": 0.166900634765625, "blocks.2.attention.w_out": 0.2560844421386719, "blocks.2.feed_forward.w1": 0.28364309668540955, "blocks.2.feed_forward.w2": 0.3782857358455658, "blocks.2.feed_forward.w3": 0.26181158423423767, "blocks.3.attention.w_q": 0.39025115966796875, "blocks.3.attention.w_k": 0.3338775634765625, "blocks.3.attention.w_v": 0.18686294555664062, "blocks.3.attention.w_out": 0.20249557495117188, "blocks.3.feed_forward.w1": 0.2922554016113281, "blocks.3.feed_forward.w2": 0.33136749267578125, "blocks.3.feed_forward.w3": 0.26416271924972534, "blocks.4.attention.w_q": 0.3621826171875, "blocks.4.attention.w_k": 0.3161773681640625, "blocks.4.attention.w_v": 0.11883926391601562, "blocks.4.attention.w_out": 0.24950790405273438, "blocks.4.feed_forward.w1": 0.2982330322265625, "blocks.4.feed_forward.w2": 0.2931200861930847, "blocks.4.feed_forward.w3": 0.2600046992301941, "blocks.5.attention.w_q": 0.4081382751464844, "blocks.5.attention.w_k": 0.3261070251464844, "blocks.5.attention.w_v": 0.2176055908203125, "blocks.5.attention.w_out": 0.16939163208007812, "blocks.5.feed_forward.w1": 0.30608367919921875, "blocks.5.feed_forward.w2": 0.2645530700683594, "blocks.5.feed_forward.w3": 0.26124319434165955, "blocks.6.attention.w_q": 0.3804779052734375, "blocks.6.attention.w_k": 0.3238029479980469, "blocks.6.attention.w_v": 0.21417617797851562, "blocks.6.attention.w_out": 0.1251220703125, "blocks.6.feed_forward.w1": 0.3094126582145691, "blocks.6.feed_forward.w2": 0.23925399780273438, "blocks.6.feed_forward.w3": 0.2618243098258972, "blocks.7.attention.w_q": 0.39011383056640625, "blocks.7.attention.w_k": 0.34189605712890625, "blocks.7.attention.w_v": 0.20600128173828125, "blocks.7.attention.w_out": 0.1147308349609375, "blocks.7.feed_forward.w1": 0.31637829542160034, "blocks.7.feed_forward.w2": 0.18802261352539062, "blocks.7.feed_forward.w3": 0.26145681738853455, "blocks.8.attention.w_q": 0.3971748352050781, "blocks.8.attention.w_k": 0.36916351318359375, "blocks.8.attention.w_v": 0.1665496826171875, "blocks.8.attention.w_out": 0.08424758911132812, "blocks.8.feed_forward.w1": 0.3345082700252533, "blocks.8.feed_forward.w2": 0.14340083301067352, "blocks.8.feed_forward.w3": 0.2623125910758972}} +{"step": 6100, "changed_frac": 0.2657825763408954, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.3001670837402344, "blocks.0.attention.w_k": 0.2960052490234375, "blocks.0.attention.w_v": 0.13284683227539062, "blocks.0.attention.w_out": 0.151763916015625, "blocks.0.feed_forward.w1": 0.2858479917049408, "blocks.0.feed_forward.w2": 0.4146728515625, "blocks.0.feed_forward.w3": 0.2486063688993454, "blocks.1.attention.w_q": 0.3791542053222656, "blocks.1.attention.w_k": 0.32172393798828125, "blocks.1.attention.w_v": 0.17686080932617188, "blocks.1.attention.w_out": 0.2657737731933594, "blocks.1.feed_forward.w1": 0.2652829587459564, "blocks.1.feed_forward.w2": 0.3892326354980469, "blocks.1.feed_forward.w3": 0.24754588305950165, "blocks.2.attention.w_q": 0.35253143310546875, "blocks.2.attention.w_k": 0.319244384765625, "blocks.2.attention.w_v": 0.155609130859375, "blocks.2.attention.w_out": 0.24096298217773438, "blocks.2.feed_forward.w1": 0.26654052734375, "blocks.2.feed_forward.w2": 0.35927581787109375, "blocks.2.feed_forward.w3": 0.24849829077720642, "blocks.3.attention.w_q": 0.3705101013183594, "blocks.3.attention.w_k": 0.3152008056640625, "blocks.3.attention.w_v": 0.17544174194335938, "blocks.3.attention.w_out": 0.18943405151367188, "blocks.3.feed_forward.w1": 0.27484387159347534, "blocks.3.feed_forward.w2": 0.31257757544517517, "blocks.3.feed_forward.w3": 0.24766921997070312, "blocks.4.attention.w_q": 0.34101104736328125, "blocks.4.attention.w_k": 0.30034637451171875, "blocks.4.attention.w_v": 0.11182022094726562, "blocks.4.attention.w_out": 0.23324966430664062, "blocks.4.feed_forward.w1": 0.28015899658203125, "blocks.4.feed_forward.w2": 0.2752940058708191, "blocks.4.feed_forward.w3": 0.24447378516197205, "blocks.5.attention.w_q": 0.3905181884765625, "blocks.5.attention.w_k": 0.3071174621582031, "blocks.5.attention.w_v": 0.20627975463867188, "blocks.5.attention.w_out": 0.15997314453125, "blocks.5.feed_forward.w1": 0.28919094800949097, "blocks.5.feed_forward.w2": 0.24847540259361267, "blocks.5.feed_forward.w3": 0.24546179175376892, "blocks.6.attention.w_q": 0.3597297668457031, "blocks.6.attention.w_k": 0.3036384582519531, "blocks.6.attention.w_v": 0.2016754150390625, "blocks.6.attention.w_out": 0.1167755126953125, "blocks.6.feed_forward.w1": 0.2909495234489441, "blocks.6.feed_forward.w2": 0.2244110107421875, "blocks.6.feed_forward.w3": 0.24794897437095642, "blocks.7.attention.w_q": 0.368560791015625, "blocks.7.attention.w_k": 0.3225212097167969, "blocks.7.attention.w_v": 0.190887451171875, "blocks.7.attention.w_out": 0.1079864501953125, "blocks.7.feed_forward.w1": 0.29678472876548767, "blocks.7.feed_forward.w2": 0.17597071826457977, "blocks.7.feed_forward.w3": 0.24537786841392517, "blocks.8.attention.w_q": 0.3759918212890625, "blocks.8.attention.w_k": 0.35324859619140625, "blocks.8.attention.w_v": 0.15808486938476562, "blocks.8.attention.w_out": 0.08046722412109375, "blocks.8.feed_forward.w1": 0.3151143491268158, "blocks.8.feed_forward.w2": 0.13431930541992188, "blocks.8.feed_forward.w3": 0.24662399291992188}} +{"step": 6200, "changed_frac": 0.2507086859809028, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.28375244140625, "blocks.0.attention.w_k": 0.27823638916015625, "blocks.0.attention.w_v": 0.12679672241210938, "blocks.0.attention.w_out": 0.14404678344726562, "blocks.0.feed_forward.w1": 0.26752346754074097, "blocks.0.feed_forward.w2": 0.39433926343917847, "blocks.0.feed_forward.w3": 0.23424021899700165, "blocks.1.attention.w_q": 0.36283111572265625, "blocks.1.attention.w_k": 0.30623626708984375, "blocks.1.attention.w_v": 0.16361236572265625, "blocks.1.attention.w_out": 0.25135040283203125, "blocks.1.feed_forward.w1": 0.2490997314453125, "blocks.1.feed_forward.w2": 0.3681831359863281, "blocks.1.feed_forward.w3": 0.23114013671875, "blocks.2.attention.w_q": 0.333587646484375, "blocks.2.attention.w_k": 0.2988471984863281, "blocks.2.attention.w_v": 0.14706802368164062, "blocks.2.attention.w_out": 0.22655868530273438, "blocks.2.feed_forward.w1": 0.2523002624511719, "blocks.2.feed_forward.w2": 0.33986029028892517, "blocks.2.feed_forward.w3": 0.23409271240234375, "blocks.3.attention.w_q": 0.3498573303222656, "blocks.3.attention.w_k": 0.2973823547363281, "blocks.3.attention.w_v": 0.1654815673828125, "blocks.3.attention.w_out": 0.17922592163085938, "blocks.3.feed_forward.w1": 0.2594541013240814, "blocks.3.feed_forward.w2": 0.2945048213005066, "blocks.3.feed_forward.w3": 0.23333105444908142, "blocks.4.attention.w_q": 0.3237876892089844, "blocks.4.attention.w_k": 0.2838172912597656, "blocks.4.attention.w_v": 0.10520172119140625, "blocks.4.attention.w_out": 0.21975326538085938, "blocks.4.feed_forward.w1": 0.2660980224609375, "blocks.4.feed_forward.w2": 0.25825247168540955, "blocks.4.feed_forward.w3": 0.22938792407512665, "blocks.5.attention.w_q": 0.3688812255859375, "blocks.5.attention.w_k": 0.2892417907714844, "blocks.5.attention.w_v": 0.19424819946289062, "blocks.5.attention.w_out": 0.15030288696289062, "blocks.5.feed_forward.w1": 0.27105969190597534, "blocks.5.feed_forward.w2": 0.23417282104492188, "blocks.5.feed_forward.w3": 0.23020172119140625, "blocks.6.attention.w_q": 0.33864593505859375, "blocks.6.attention.w_k": 0.2878570556640625, "blocks.6.attention.w_v": 0.19195556640625, "blocks.6.attention.w_out": 0.11148452758789062, "blocks.6.feed_forward.w1": 0.273834228515625, "blocks.6.feed_forward.w2": 0.21056239306926727, "blocks.6.feed_forward.w3": 0.23342005908489227, "blocks.7.attention.w_q": 0.3503379821777344, "blocks.7.attention.w_k": 0.30651092529296875, "blocks.7.attention.w_v": 0.18097686767578125, "blocks.7.attention.w_out": 0.10133743286132812, "blocks.7.feed_forward.w1": 0.28077825903892517, "blocks.7.feed_forward.w2": 0.16568629443645477, "blocks.7.feed_forward.w3": 0.23078028857707977, "blocks.8.attention.w_q": 0.3582496643066406, "blocks.8.attention.w_k": 0.33295440673828125, "blocks.8.attention.w_v": 0.14912796020507812, "blocks.8.attention.w_out": 0.07524871826171875, "blocks.8.feed_forward.w1": 0.2988777160644531, "blocks.8.feed_forward.w2": 0.12613296508789062, "blocks.8.feed_forward.w3": 0.2320607602596283}} +{"step": 6300, "changed_frac": 0.23688546205178285, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.26654815673828125, "blocks.0.attention.w_k": 0.2657051086425781, "blocks.0.attention.w_v": 0.11915969848632812, "blocks.0.attention.w_out": 0.13687896728515625, "blocks.0.feed_forward.w1": 0.2538350522518158, "blocks.0.feed_forward.w2": 0.3764228820800781, "blocks.0.feed_forward.w3": 0.22158178687095642, "blocks.1.attention.w_q": 0.34239959716796875, "blocks.1.attention.w_k": 0.2876396179199219, "blocks.1.attention.w_v": 0.15496063232421875, "blocks.1.attention.w_out": 0.23914718627929688, "blocks.1.feed_forward.w1": 0.23539988696575165, "blocks.1.feed_forward.w2": 0.3507080078125, "blocks.1.feed_forward.w3": 0.21844229102134705, "blocks.2.attention.w_q": 0.3172607421875, "blocks.2.attention.w_k": 0.2827186584472656, "blocks.2.attention.w_v": 0.13898468017578125, "blocks.2.attention.w_out": 0.21317291259765625, "blocks.2.feed_forward.w1": 0.23611578345298767, "blocks.2.feed_forward.w2": 0.32128143310546875, "blocks.2.feed_forward.w3": 0.21967315673828125, "blocks.3.attention.w_q": 0.33054351806640625, "blocks.3.attention.w_k": 0.2793846130371094, "blocks.3.attention.w_v": 0.15460586547851562, "blocks.3.attention.w_out": 0.167938232421875, "blocks.3.feed_forward.w1": 0.24424362182617188, "blocks.3.feed_forward.w2": 0.27971142530441284, "blocks.3.feed_forward.w3": 0.21927516162395477, "blocks.4.attention.w_q": 0.3054924011230469, "blocks.4.attention.w_k": 0.2689552307128906, "blocks.4.attention.w_v": 0.09719085693359375, "blocks.4.attention.w_out": 0.205963134765625, "blocks.4.feed_forward.w1": 0.24871826171875, "blocks.4.feed_forward.w2": 0.24541601538658142, "blocks.4.feed_forward.w3": 0.21613948047161102, "blocks.5.attention.w_q": 0.348785400390625, "blocks.5.attention.w_k": 0.27283477783203125, "blocks.5.attention.w_v": 0.18384933471679688, "blocks.5.attention.w_out": 0.14159011840820312, "blocks.5.feed_forward.w1": 0.25687408447265625, "blocks.5.feed_forward.w2": 0.21928024291992188, "blocks.5.feed_forward.w3": 0.21843592822551727, "blocks.6.attention.w_q": 0.3236198425292969, "blocks.6.attention.w_k": 0.2723808288574219, "blocks.6.attention.w_v": 0.18016815185546875, "blocks.6.attention.w_out": 0.10425949096679688, "blocks.6.feed_forward.w1": 0.25912603735923767, "blocks.6.feed_forward.w2": 0.19902420043945312, "blocks.6.feed_forward.w3": 0.21975581347942352, "blocks.7.attention.w_q": 0.3306083679199219, "blocks.7.attention.w_k": 0.2908172607421875, "blocks.7.attention.w_v": 0.17052459716796875, "blocks.7.attention.w_out": 0.09635543823242188, "blocks.7.feed_forward.w1": 0.2640113830566406, "blocks.7.feed_forward.w2": 0.15730921924114227, "blocks.7.feed_forward.w3": 0.21790823340415955, "blocks.8.attention.w_q": 0.3411064147949219, "blocks.8.attention.w_k": 0.3146514892578125, "blocks.8.attention.w_v": 0.1395416259765625, "blocks.8.attention.w_out": 0.071258544921875, "blocks.8.feed_forward.w1": 0.2827898859977722, "blocks.8.feed_forward.w2": 0.11926396936178207, "blocks.8.feed_forward.w3": 0.21878942847251892}} +{"step": 6400, "changed_frac": 0.2244331490280282, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2555580139160156, "blocks.0.attention.w_k": 0.2533149719238281, "blocks.0.attention.w_v": 0.1126708984375, "blocks.0.attention.w_out": 0.12891769409179688, "blocks.0.feed_forward.w1": 0.24007543921470642, "blocks.0.feed_forward.w2": 0.3584251403808594, "blocks.0.feed_forward.w3": 0.20895513892173767, "blocks.1.attention.w_q": 0.32735443115234375, "blocks.1.attention.w_k": 0.2747955322265625, "blocks.1.attention.w_v": 0.146697998046875, "blocks.1.attention.w_out": 0.22518539428710938, "blocks.1.feed_forward.w1": 0.22178778052330017, "blocks.1.feed_forward.w2": 0.3338114619255066, "blocks.1.feed_forward.w3": 0.20721690356731415, "blocks.2.attention.w_q": 0.2997894287109375, "blocks.2.attention.w_k": 0.27062225341796875, "blocks.2.attention.w_v": 0.1319580078125, "blocks.2.attention.w_out": 0.20191574096679688, "blocks.2.feed_forward.w1": 0.2245534360408783, "blocks.2.feed_forward.w2": 0.3041496276855469, "blocks.2.feed_forward.w3": 0.20750045776367188, "blocks.3.attention.w_q": 0.3140220642089844, "blocks.3.attention.w_k": 0.2653541564941406, "blocks.3.attention.w_v": 0.146728515625, "blocks.3.attention.w_out": 0.160003662109375, "blocks.3.feed_forward.w1": 0.23171743750572205, "blocks.3.feed_forward.w2": 0.26515454053878784, "blocks.3.feed_forward.w3": 0.20804087817668915, "blocks.4.attention.w_q": 0.2892913818359375, "blocks.4.attention.w_k": 0.2547607421875, "blocks.4.attention.w_v": 0.09329986572265625, "blocks.4.attention.w_out": 0.19644546508789062, "blocks.4.feed_forward.w1": 0.23619715869426727, "blocks.4.feed_forward.w2": 0.22976303100585938, "blocks.4.feed_forward.w3": 0.20514297485351562, "blocks.5.attention.w_q": 0.332916259765625, "blocks.5.attention.w_k": 0.25792694091796875, "blocks.5.attention.w_v": 0.17587661743164062, "blocks.5.attention.w_out": 0.13454437255859375, "blocks.5.feed_forward.w1": 0.24208323657512665, "blocks.5.feed_forward.w2": 0.20779673755168915, "blocks.5.feed_forward.w3": 0.2063700407743454, "blocks.6.attention.w_q": 0.30629730224609375, "blocks.6.attention.w_k": 0.2579841613769531, "blocks.6.attention.w_v": 0.17115402221679688, "blocks.6.attention.w_out": 0.09957122802734375, "blocks.6.feed_forward.w1": 0.2451578825712204, "blocks.6.feed_forward.w2": 0.18816503882408142, "blocks.6.feed_forward.w3": 0.20646414160728455, "blocks.7.attention.w_q": 0.3135833740234375, "blocks.7.attention.w_k": 0.2754096984863281, "blocks.7.attention.w_v": 0.16249847412109375, "blocks.7.attention.w_out": 0.09076309204101562, "blocks.7.feed_forward.w1": 0.24967066943645477, "blocks.7.feed_forward.w2": 0.1487986296415329, "blocks.7.feed_forward.w3": 0.20605342090129852, "blocks.8.attention.w_q": 0.3214836120605469, "blocks.8.attention.w_k": 0.3006401062011719, "blocks.8.attention.w_v": 0.13320541381835938, "blocks.8.attention.w_out": 0.06669998168945312, "blocks.8.feed_forward.w1": 0.2669944763183594, "blocks.8.feed_forward.w2": 0.11254119873046875, "blocks.8.feed_forward.w3": 0.20722579956054688}} +{"step": 6500, "changed_frac": 0.21384143014239448, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.24689865112304688, "blocks.0.attention.w_k": 0.24333953857421875, "blocks.0.attention.w_v": 0.10812759399414062, "blocks.0.attention.w_out": 0.123504638671875, "blocks.0.feed_forward.w1": 0.2281951904296875, "blocks.0.feed_forward.w2": 0.3430417478084564, "blocks.0.feed_forward.w3": 0.2001444548368454, "blocks.1.attention.w_q": 0.3113136291503906, "blocks.1.attention.w_k": 0.260528564453125, "blocks.1.attention.w_v": 0.13972091674804688, "blocks.1.attention.w_out": 0.21445083618164062, "blocks.1.feed_forward.w1": 0.2106679379940033, "blocks.1.feed_forward.w2": 0.3178952634334564, "blocks.1.feed_forward.w3": 0.19655101001262665, "blocks.2.attention.w_q": 0.2857818603515625, "blocks.2.attention.w_k": 0.2569770812988281, "blocks.2.attention.w_v": 0.12552261352539062, "blocks.2.attention.w_out": 0.19423294067382812, "blocks.2.feed_forward.w1": 0.2138621062040329, "blocks.2.feed_forward.w2": 0.2900632321834564, "blocks.2.feed_forward.w3": 0.19755682349205017, "blocks.3.attention.w_q": 0.2996406555175781, "blocks.3.attention.w_k": 0.2542991638183594, "blocks.3.attention.w_v": 0.1404266357421875, "blocks.3.attention.w_out": 0.15216064453125, "blocks.3.feed_forward.w1": 0.21987661719322205, "blocks.3.feed_forward.w2": 0.2508634030818939, "blocks.3.feed_forward.w3": 0.1982371062040329, "blocks.4.attention.w_q": 0.2746925354003906, "blocks.4.attention.w_k": 0.24193572998046875, "blocks.4.attention.w_v": 0.08927154541015625, "blocks.4.attention.w_out": 0.18865966796875, "blocks.4.feed_forward.w1": 0.22467422485351562, "blocks.4.feed_forward.w2": 0.2191670835018158, "blocks.4.feed_forward.w3": 0.1950734555721283, "blocks.5.attention.w_q": 0.31573486328125, "blocks.5.attention.w_k": 0.246429443359375, "blocks.5.attention.w_v": 0.16709518432617188, "blocks.5.attention.w_out": 0.12909317016601562, "blocks.5.feed_forward.w1": 0.23180770874023438, "blocks.5.feed_forward.w2": 0.19817988574504852, "blocks.5.feed_forward.w3": 0.19557063281536102, "blocks.6.attention.w_q": 0.2892723083496094, "blocks.6.attention.w_k": 0.24488449096679688, "blocks.6.attention.w_v": 0.163055419921875, "blocks.6.attention.w_out": 0.09500503540039062, "blocks.6.feed_forward.w1": 0.2330068051815033, "blocks.6.feed_forward.w2": 0.1787465512752533, "blocks.6.feed_forward.w3": 0.1974131315946579, "blocks.7.attention.w_q": 0.3018836975097656, "blocks.7.attention.w_k": 0.26270294189453125, "blocks.7.attention.w_v": 0.15654754638671875, "blocks.7.attention.w_out": 0.086578369140625, "blocks.7.feed_forward.w1": 0.23732249438762665, "blocks.7.feed_forward.w2": 0.1417287290096283, "blocks.7.feed_forward.w3": 0.1968180388212204, "blocks.8.attention.w_q": 0.3091239929199219, "blocks.8.attention.w_k": 0.2865409851074219, "blocks.8.attention.w_v": 0.127685546875, "blocks.8.attention.w_out": 0.06343841552734375, "blocks.8.feed_forward.w1": 0.2545280456542969, "blocks.8.feed_forward.w2": 0.10721080005168915, "blocks.8.feed_forward.w3": 0.19609451293945312}} +{"step": 6600, "changed_frac": 0.20438681708441842, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2325286865234375, "blocks.0.attention.w_k": 0.23137283325195312, "blocks.0.attention.w_v": 0.10396957397460938, "blocks.0.attention.w_out": 0.11872482299804688, "blocks.0.feed_forward.w1": 0.21864700317382812, "blocks.0.feed_forward.w2": 0.3306388854980469, "blocks.0.feed_forward.w3": 0.1914876401424408, "blocks.1.attention.w_q": 0.2996177673339844, "blocks.1.attention.w_k": 0.2506065368652344, "blocks.1.attention.w_v": 0.13474273681640625, "blocks.1.attention.w_out": 0.2046051025390625, "blocks.1.feed_forward.w1": 0.2008311003446579, "blocks.1.feed_forward.w2": 0.3046061396598816, "blocks.1.feed_forward.w3": 0.18725332617759705, "blocks.2.attention.w_q": 0.2713737487792969, "blocks.2.attention.w_k": 0.24387359619140625, "blocks.2.attention.w_v": 0.11900711059570312, "blocks.2.attention.w_out": 0.18498611450195312, "blocks.2.feed_forward.w1": 0.20282237231731415, "blocks.2.feed_forward.w2": 0.2787412106990814, "blocks.2.feed_forward.w3": 0.18834304809570312, "blocks.3.attention.w_q": 0.2859001159667969, "blocks.3.attention.w_k": 0.24385452270507812, "blocks.3.attention.w_v": 0.1333160400390625, "blocks.3.attention.w_out": 0.1459808349609375, "blocks.3.feed_forward.w1": 0.20961634814739227, "blocks.3.feed_forward.w2": 0.23881658911705017, "blocks.3.feed_forward.w3": 0.1886698454618454, "blocks.4.attention.w_q": 0.2658538818359375, "blocks.4.attention.w_k": 0.22902297973632812, "blocks.4.attention.w_v": 0.08546066284179688, "blocks.4.attention.w_out": 0.17968368530273438, "blocks.4.feed_forward.w1": 0.21427282691001892, "blocks.4.feed_forward.w2": 0.20932769775390625, "blocks.4.feed_forward.w3": 0.18663153052330017, "blocks.5.attention.w_q": 0.3018341064453125, "blocks.5.attention.w_k": 0.23542022705078125, "blocks.5.attention.w_v": 0.15995407104492188, "blocks.5.attention.w_out": 0.12443161010742188, "blocks.5.feed_forward.w1": 0.2199605405330658, "blocks.5.feed_forward.w2": 0.18956756591796875, "blocks.5.feed_forward.w3": 0.18705496191978455, "blocks.6.attention.w_q": 0.28089141845703125, "blocks.6.attention.w_k": 0.23531341552734375, "blocks.6.attention.w_v": 0.15653610229492188, "blocks.6.attention.w_out": 0.09088516235351562, "blocks.6.feed_forward.w1": 0.22191493213176727, "blocks.6.feed_forward.w2": 0.17130915820598602, "blocks.6.feed_forward.w3": 0.18760935962200165, "blocks.7.attention.w_q": 0.2902336120605469, "blocks.7.attention.w_k": 0.2515373229980469, "blocks.7.attention.w_v": 0.1488189697265625, "blocks.7.attention.w_out": 0.08366775512695312, "blocks.7.feed_forward.w1": 0.2265726774930954, "blocks.7.feed_forward.w2": 0.13470587134361267, "blocks.7.feed_forward.w3": 0.18755850195884705, "blocks.8.attention.w_q": 0.2954559326171875, "blocks.8.attention.w_k": 0.2777976989746094, "blocks.8.attention.w_v": 0.12146377563476562, "blocks.8.attention.w_out": 0.06142425537109375, "blocks.8.feed_forward.w1": 0.24404780566692352, "blocks.8.feed_forward.w2": 0.1028493270277977, "blocks.8.feed_forward.w3": 0.18718084692955017}} +{"step": 6700, "changed_frac": 0.19670202793219152, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.22359085083007812, "blocks.0.attention.w_k": 0.22006607055664062, "blocks.0.attention.w_v": 0.0997467041015625, "blocks.0.attention.w_out": 0.11489486694335938, "blocks.0.feed_forward.w1": 0.2106831967830658, "blocks.0.feed_forward.w2": 0.31782659888267517, "blocks.0.feed_forward.w3": 0.18517303466796875, "blocks.1.attention.w_q": 0.28536224365234375, "blocks.1.attention.w_k": 0.24002456665039062, "blocks.1.attention.w_v": 0.12882232666015625, "blocks.1.attention.w_out": 0.19701004028320312, "blocks.1.feed_forward.w1": 0.19387690722942352, "blocks.1.feed_forward.w2": 0.293701171875, "blocks.1.feed_forward.w3": 0.1811269223690033, "blocks.2.attention.w_q": 0.2642631530761719, "blocks.2.attention.w_k": 0.23479461669921875, "blocks.2.attention.w_v": 0.113372802734375, "blocks.2.attention.w_out": 0.177825927734375, "blocks.2.feed_forward.w1": 0.19670741260051727, "blocks.2.feed_forward.w2": 0.26861828565597534, "blocks.2.feed_forward.w3": 0.18219122290611267, "blocks.3.attention.w_q": 0.2771644592285156, "blocks.3.attention.w_k": 0.2328338623046875, "blocks.3.attention.w_v": 0.12703704833984375, "blocks.3.attention.w_out": 0.13887786865234375, "blocks.3.feed_forward.w1": 0.20231756567955017, "blocks.3.feed_forward.w2": 0.23137792944908142, "blocks.3.feed_forward.w3": 0.18167369067668915, "blocks.4.attention.w_q": 0.2537841796875, "blocks.4.attention.w_k": 0.22016143798828125, "blocks.4.attention.w_v": 0.07975387573242188, "blocks.4.attention.w_out": 0.17234420776367188, "blocks.4.feed_forward.w1": 0.20710501074790955, "blocks.4.feed_forward.w2": 0.20094554126262665, "blocks.4.feed_forward.w3": 0.17993927001953125, "blocks.5.attention.w_q": 0.2940483093261719, "blocks.5.attention.w_k": 0.22814178466796875, "blocks.5.attention.w_v": 0.15507125854492188, "blocks.5.attention.w_out": 0.11748504638671875, "blocks.5.feed_forward.w1": 0.21174240112304688, "blocks.5.feed_forward.w2": 0.18180084228515625, "blocks.5.feed_forward.w3": 0.17951202392578125, "blocks.6.attention.w_q": 0.2677955627441406, "blocks.6.attention.w_k": 0.2250213623046875, "blocks.6.attention.w_v": 0.15100860595703125, "blocks.6.attention.w_out": 0.0866546630859375, "blocks.6.feed_forward.w1": 0.21432876586914062, "blocks.6.feed_forward.w2": 0.16303889453411102, "blocks.6.feed_forward.w3": 0.18016943335533142, "blocks.7.attention.w_q": 0.2781105041503906, "blocks.7.attention.w_k": 0.24337005615234375, "blocks.7.attention.w_v": 0.14392471313476562, "blocks.7.attention.w_out": 0.07935714721679688, "blocks.7.feed_forward.w1": 0.21844482421875, "blocks.7.feed_forward.w2": 0.12925975024700165, "blocks.7.feed_forward.w3": 0.1809844970703125, "blocks.8.attention.w_q": 0.2855186462402344, "blocks.8.attention.w_k": 0.26554107666015625, "blocks.8.attention.w_v": 0.1168670654296875, "blocks.8.attention.w_out": 0.05808258056640625, "blocks.8.feed_forward.w1": 0.23409780859947205, "blocks.8.feed_forward.w2": 0.0981597900390625, "blocks.8.feed_forward.w3": 0.18066661059856415}} +{"step": 6800, "changed_frac": 0.19107887072440904, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.21731185913085938, "blocks.0.attention.w_k": 0.21461105346679688, "blocks.0.attention.w_v": 0.09717559814453125, "blocks.0.attention.w_out": 0.11240005493164062, "blocks.0.feed_forward.w1": 0.20471446216106415, "blocks.0.feed_forward.w2": 0.31071600317955017, "blocks.0.feed_forward.w3": 0.18008551001548767, "blocks.1.attention.w_q": 0.27886962890625, "blocks.1.attention.w_k": 0.23329925537109375, "blocks.1.attention.w_v": 0.12452316284179688, "blocks.1.attention.w_out": 0.19191360473632812, "blocks.1.feed_forward.w1": 0.18785350024700165, "blocks.1.feed_forward.w2": 0.28571319580078125, "blocks.1.feed_forward.w3": 0.17527899146080017, "blocks.2.attention.w_q": 0.2542915344238281, "blocks.2.attention.w_k": 0.22952651977539062, "blocks.2.attention.w_v": 0.11006927490234375, "blocks.2.attention.w_out": 0.17350387573242188, "blocks.2.feed_forward.w1": 0.19065603613853455, "blocks.2.feed_forward.w2": 0.26107025146484375, "blocks.2.feed_forward.w3": 0.17633438110351562, "blocks.3.attention.w_q": 0.2666473388671875, "blocks.3.attention.w_k": 0.22797012329101562, "blocks.3.attention.w_v": 0.12358856201171875, "blocks.3.attention.w_out": 0.13656997680664062, "blocks.3.feed_forward.w1": 0.19661585986614227, "blocks.3.feed_forward.w2": 0.22452545166015625, "blocks.3.feed_forward.w3": 0.1768595427274704, "blocks.4.attention.w_q": 0.24376678466796875, "blocks.4.attention.w_k": 0.213043212890625, "blocks.4.attention.w_v": 0.0785980224609375, "blocks.4.attention.w_out": 0.16803359985351562, "blocks.4.feed_forward.w1": 0.20110957324504852, "blocks.4.feed_forward.w2": 0.19477209448814392, "blocks.4.feed_forward.w3": 0.1750640869140625, "blocks.5.attention.w_q": 0.2846488952636719, "blocks.5.attention.w_k": 0.22254562377929688, "blocks.5.attention.w_v": 0.14813613891601562, "blocks.5.attention.w_out": 0.11449432373046875, "blocks.5.feed_forward.w1": 0.20511119067668915, "blocks.5.feed_forward.w2": 0.1767425537109375, "blocks.5.feed_forward.w3": 0.174530029296875, "blocks.6.attention.w_q": 0.26326751708984375, "blocks.6.attention.w_k": 0.21816635131835938, "blocks.6.attention.w_v": 0.14605331420898438, "blocks.6.attention.w_out": 0.08391189575195312, "blocks.6.feed_forward.w1": 0.2073822021484375, "blocks.6.feed_forward.w2": 0.15877532958984375, "blocks.6.feed_forward.w3": 0.17554092407226562, "blocks.7.attention.w_q": 0.27001953125, "blocks.7.attention.w_k": 0.2360992431640625, "blocks.7.attention.w_v": 0.139862060546875, "blocks.7.attention.w_out": 0.07705307006835938, "blocks.7.feed_forward.w1": 0.21261215209960938, "blocks.7.feed_forward.w2": 0.12549591064453125, "blocks.7.feed_forward.w3": 0.17440669238567352, "blocks.8.attention.w_q": 0.27734375, "blocks.8.attention.w_k": 0.2590522766113281, "blocks.8.attention.w_v": 0.11309051513671875, "blocks.8.attention.w_out": 0.057056427001953125, "blocks.8.feed_forward.w1": 0.22710546851158142, "blocks.8.feed_forward.w2": 0.09497325122356415, "blocks.8.feed_forward.w3": 0.17586009204387665}} +{"step": 6900, "changed_frac": 0.18687240078917935, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.20995712280273438, "blocks.0.attention.w_k": 0.21038818359375, "blocks.0.attention.w_v": 0.0941619873046875, "blocks.0.attention.w_out": 0.10902023315429688, "blocks.0.feed_forward.w1": 0.19977697730064392, "blocks.0.feed_forward.w2": 0.3040008544921875, "blocks.0.feed_forward.w3": 0.17537307739257812, "blocks.1.attention.w_q": 0.2746009826660156, "blocks.1.attention.w_k": 0.22709274291992188, "blocks.1.attention.w_v": 0.12166213989257812, "blocks.1.attention.w_out": 0.18906402587890625, "blocks.1.feed_forward.w1": 0.18328475952148438, "blocks.1.feed_forward.w2": 0.27877044677734375, "blocks.1.feed_forward.w3": 0.1718190610408783, "blocks.2.attention.w_q": 0.2490692138671875, "blocks.2.attention.w_k": 0.22495269775390625, "blocks.2.attention.w_v": 0.10919189453125, "blocks.2.attention.w_out": 0.17061233520507812, "blocks.2.feed_forward.w1": 0.187225341796875, "blocks.2.feed_forward.w2": 0.25572460889816284, "blocks.2.feed_forward.w3": 0.1721140593290329, "blocks.3.attention.w_q": 0.2636756896972656, "blocks.3.attention.w_k": 0.22113800048828125, "blocks.3.attention.w_v": 0.12030792236328125, "blocks.3.attention.w_out": 0.13336563110351562, "blocks.3.feed_forward.w1": 0.1914723813533783, "blocks.3.feed_forward.w2": 0.22029495239257812, "blocks.3.feed_forward.w3": 0.17251333594322205, "blocks.4.attention.w_q": 0.24077987670898438, "blocks.4.attention.w_k": 0.21019363403320312, "blocks.4.attention.w_v": 0.07609176635742188, "blocks.4.attention.w_out": 0.16432571411132812, "blocks.4.feed_forward.w1": 0.19688162207603455, "blocks.4.feed_forward.w2": 0.1909637451171875, "blocks.4.feed_forward.w3": 0.17011897265911102, "blocks.5.attention.w_q": 0.2796478271484375, "blocks.5.attention.w_k": 0.21468734741210938, "blocks.5.attention.w_v": 0.147674560546875, "blocks.5.attention.w_out": 0.11274337768554688, "blocks.5.feed_forward.w1": 0.20109431445598602, "blocks.5.feed_forward.w2": 0.17311859130859375, "blocks.5.feed_forward.w3": 0.170867919921875, "blocks.6.attention.w_q": 0.25713348388671875, "blocks.6.attention.w_k": 0.21552276611328125, "blocks.6.attention.w_v": 0.14363479614257812, "blocks.6.attention.w_out": 0.08235931396484375, "blocks.6.feed_forward.w1": 0.20322291553020477, "blocks.6.feed_forward.w2": 0.1556345671415329, "blocks.6.feed_forward.w3": 0.17188262939453125, "blocks.7.attention.w_q": 0.2635383605957031, "blocks.7.attention.w_k": 0.2280120849609375, "blocks.7.attention.w_v": 0.1354827880859375, "blocks.7.attention.w_out": 0.07517242431640625, "blocks.7.feed_forward.w1": 0.2078348845243454, "blocks.7.feed_forward.w2": 0.12215550988912582, "blocks.7.feed_forward.w3": 0.17086157202720642, "blocks.8.attention.w_q": 0.2725830078125, "blocks.8.attention.w_k": 0.2512245178222656, "blocks.8.attention.w_v": 0.11130142211914062, "blocks.8.attention.w_out": 0.055568695068359375, "blocks.8.feed_forward.w1": 0.22158941626548767, "blocks.8.feed_forward.w2": 0.09297307580709457, "blocks.8.feed_forward.w3": 0.17114131152629852}} +{"step": 7000, "changed_frac": 0.18478321825337207, "changed_frac_per_layer": {"blocks.0.attention.w_q": 0.2052764892578125, "blocks.0.attention.w_k": 0.20940399169921875, "blocks.0.attention.w_v": 0.093841552734375, "blocks.0.attention.w_out": 0.10892486572265625, "blocks.0.feed_forward.w1": 0.1971333920955658, "blocks.0.feed_forward.w2": 0.3022397458553314, "blocks.0.feed_forward.w3": 0.172149658203125, "blocks.1.attention.w_q": 0.2709808349609375, "blocks.1.attention.w_k": 0.22665023803710938, "blocks.1.attention.w_v": 0.121185302734375, "blocks.1.attention.w_out": 0.18743133544921875, "blocks.1.feed_forward.w1": 0.1825408935546875, "blocks.1.feed_forward.w2": 0.27677154541015625, "blocks.1.feed_forward.w3": 0.16960272192955017, "blocks.2.attention.w_q": 0.24619293212890625, "blocks.2.attention.w_k": 0.22140884399414062, "blocks.2.attention.w_v": 0.10670089721679688, "blocks.2.attention.w_out": 0.168670654296875, "blocks.2.feed_forward.w1": 0.1834208220243454, "blocks.2.feed_forward.w2": 0.25220999121665955, "blocks.2.feed_forward.w3": 0.17100779712200165, "blocks.3.attention.w_q": 0.2601776123046875, "blocks.3.attention.w_k": 0.21932601928710938, "blocks.3.attention.w_v": 0.11997222900390625, "blocks.3.attention.w_out": 0.13051986694335938, "blocks.3.feed_forward.w1": 0.18997573852539062, "blocks.3.feed_forward.w2": 0.21735382080078125, "blocks.3.feed_forward.w3": 0.17117945849895477, "blocks.4.attention.w_q": 0.23566055297851562, "blocks.4.attention.w_k": 0.20665740966796875, "blocks.4.attention.w_v": 0.07410049438476562, "blocks.4.attention.w_out": 0.16276931762695312, "blocks.4.feed_forward.w1": 0.19335556030273438, "blocks.4.feed_forward.w2": 0.1886800229549408, "blocks.4.feed_forward.w3": 0.16833877563476562, "blocks.5.attention.w_q": 0.27733612060546875, "blocks.5.attention.w_k": 0.21066665649414062, "blocks.5.attention.w_v": 0.1441497802734375, "blocks.5.attention.w_out": 0.11198806762695312, "blocks.5.feed_forward.w1": 0.1992136687040329, "blocks.5.feed_forward.w2": 0.1711069792509079, "blocks.5.feed_forward.w3": 0.16861598193645477, "blocks.6.attention.w_q": 0.2537727355957031, "blocks.6.attention.w_k": 0.2130126953125, "blocks.6.attention.w_v": 0.14348220825195312, "blocks.6.attention.w_out": 0.08188629150390625, "blocks.6.feed_forward.w1": 0.20121130347251892, "blocks.6.feed_forward.w2": 0.1537017822265625, "blocks.6.feed_forward.w3": 0.1705576628446579, "blocks.7.attention.w_q": 0.25939178466796875, "blocks.7.attention.w_k": 0.228179931640625, "blocks.7.attention.w_v": 0.13404083251953125, "blocks.7.attention.w_out": 0.0736083984375, "blocks.7.feed_forward.w1": 0.20461909472942352, "blocks.7.feed_forward.w2": 0.1214599609375, "blocks.7.feed_forward.w3": 0.16870245337486267, "blocks.8.attention.w_q": 0.26824188232421875, "blocks.8.attention.w_k": 0.2516822814941406, "blocks.8.attention.w_v": 0.11111068725585938, "blocks.8.attention.w_out": 0.055171966552734375, "blocks.8.feed_forward.w1": 0.21925990283489227, "blocks.8.feed_forward.w2": 0.09184519946575165, "blocks.8.feed_forward.w3": 0.16910044848918915}} diff --git a/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/perturb_blocks.json b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..13b0db719ab40dff06a2ca6e096271b48b8e969a --- /dev/null +++ b/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D1.84B-f6D-adamw-lr1.2e-03-s101", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.03579538315534592, + "blocks.0.attention.w_out.weight": 0.04934689402580261, + "blocks.0.attention.w_q.weight": 0.03733170032501221, + "blocks.0.attention.w_v.weight": 0.05923828110098839, + "blocks.0.feed_forward.w1.weight": 0.038740143179893494, + "blocks.0.feed_forward.w2.weight": 0.02094249613583088, + "blocks.0.feed_forward.w3.weight": 0.04289073124527931, + "blocks.1.attention.w_k.weight": 0.03547549992799759, + "blocks.1.attention.w_out.weight": 0.03157811984419823, + "blocks.1.attention.w_q.weight": 0.03014812245965004, + "blocks.1.attention.w_v.weight": 0.04801168665289879, + "blocks.1.feed_forward.w1.weight": 0.04285428673028946, + "blocks.1.feed_forward.w2.weight": 0.024818051606416702, + "blocks.1.feed_forward.w3.weight": 0.04518647491931915, + "blocks.2.attention.w_k.weight": 0.035335466265678406, + "blocks.2.attention.w_out.weight": 0.035742562264204025, + "blocks.2.attention.w_q.weight": 0.03333597630262375, + "blocks.2.attention.w_v.weight": 0.058168333023786545, + "blocks.2.feed_forward.w1.weight": 0.04277409240603447, + "blocks.2.feed_forward.w2.weight": 0.02783612348139286, + "blocks.2.feed_forward.w3.weight": 0.04527174308896065, + "blocks.3.attention.w_k.weight": 0.03852938488125801, + "blocks.3.attention.w_out.weight": 0.04550563544034958, + "blocks.3.attention.w_q.weight": 0.03285974636673927, + "blocks.3.attention.w_v.weight": 0.05249840021133423, + "blocks.3.feed_forward.w1.weight": 0.04122811555862427, + "blocks.3.feed_forward.w2.weight": 0.03218059986829758, + "blocks.3.feed_forward.w3.weight": 0.04533703252673149, + "blocks.4.attention.w_k.weight": 0.03939693421125412, + "blocks.4.attention.w_out.weight": 0.04199798405170441, + "blocks.4.attention.w_q.weight": 0.034757569432258606, + "blocks.4.attention.w_v.weight": 0.07850440591573715, + "blocks.4.feed_forward.w1.weight": 0.04038531333208084, + "blocks.4.feed_forward.w2.weight": 0.036753926426172256, + "blocks.4.feed_forward.w3.weight": 0.04560360312461853, + "blocks.5.attention.w_k.weight": 0.039305415004491806, + "blocks.5.attention.w_out.weight": 0.05870950222015381, + "blocks.5.attention.w_q.weight": 0.030567850917577744, + "blocks.5.attention.w_v.weight": 0.0403321236371994, + "blocks.5.feed_forward.w1.weight": 0.03994705528020859, + "blocks.5.feed_forward.w2.weight": 0.04174012690782547, + "blocks.5.feed_forward.w3.weight": 0.04651913791894913, + "blocks.6.attention.w_k.weight": 0.041856907308101654, + "blocks.6.attention.w_out.weight": 0.06657076627016068, + "blocks.6.attention.w_q.weight": 0.03454572334885597, + "blocks.6.attention.w_v.weight": 0.044679056853055954, + "blocks.6.feed_forward.w1.weight": 0.039848748594522476, + "blocks.6.feed_forward.w2.weight": 0.04650549963116646, + "blocks.6.feed_forward.w3.weight": 0.046307120472192764, + "blocks.7.attention.w_k.weight": 0.0394807867705822, + "blocks.7.attention.w_out.weight": 0.08001749962568283, + "blocks.7.attention.w_q.weight": 0.03330857679247856, + "blocks.7.attention.w_v.weight": 0.04770408570766449, + "blocks.7.feed_forward.w1.weight": 0.039437271654605865, + "blocks.7.feed_forward.w2.weight": 0.056872304528951645, + "blocks.7.feed_forward.w3.weight": 0.046531688421964645, + "blocks.8.attention.w_k.weight": 0.03602339327335358, + "blocks.8.attention.w_out.weight": 0.11167500913143158, + "blocks.8.attention.w_q.weight": 0.03374782204627991, + "blocks.8.attention.w_v.weight": 0.055345550179481506, + "blocks.8.feed_forward.w1.weight": 0.036724284291267395, + "blocks.8.feed_forward.w2.weight": 0.06761439144611359, + "blocks.8.feed_forward.w3.weight": 0.04632730782032013 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.929202462064875 + }, + "base_loss": 3.929202462064875, + "loss": { + "0.005": [ + 3.9291316216518037, + 3.9289748236137485, + 3.929090405340338 + ], + "0.01": [ + 3.9294760155260966, + 3.929914582901413, + 3.92984290525145 + ], + "0.02": [ + 3.9298874626858384, + 3.9305363369261594, + 3.9301305782824945 + ], + "0.05": [ + 3.934522046184428, + 3.9349759303000758, + 3.934938254639951 + ], + "0.1": [ + 3.9484791803128454, + 3.9477565666289083, + 3.9466689411341402 + ], + "0.15": [ + 3.972232615957466, + 3.971786912506961, + 3.96971478641101 + ], + "0.2": [ + 4.00826460009685, + 4.008376238202941, + 4.005030545688158 + ], + "0.3": [ + 4.128361208783532, + 4.128253357805064, + 4.1238313972660565 + ], + "0.4": [ + 4.350334171759571, + 4.349026282540503, + 4.341715025313036 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.9291316216518037, + 3.9289748236137485, + 3.929090405340338 + ], + "0.01": [ + 3.9294760155260966, + 3.929914582901413, + 3.92984290525145 + ], + "0.02": [ + 3.9298874626858384, + 3.9305363369261594, + 3.9301305782824945 + ], + "0.05": [ + 3.934522046184428, + 3.9349759303000758, + 3.934938254639951 + ], + "0.1": [ + 3.9484791803128454, + 3.9477565666289083, + 3.9466689411341402 + ], + "0.15": [ + 3.972232615957466, + 3.971786912506961, + 3.96971478641101 + ], + "0.2": [ + 4.00826460009685, + 4.008376238202941, + 4.005030545688158 + ], + "0.3": [ + 4.128361208783532, + 4.128253357805064, + 4.1238313972660565 + ], + "0.4": [ + 4.350334171759571, + 4.349026282540503, + 4.341715025313036 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": -0.00013684519624490576, + "0.01": 0.0005420391614450182, + "0.02": 0.000982330566622558, + "0.05": 0.005609614976610278, + "0.1": 0.018432433960422934, + "0.15": 0.04204230956027111, + "0.2": 0.07802133259777522, + "0.3": 0.19761285922000882, + "0.4": 0.4178226978061619 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": -0.00013684519624490576, + "0.01": 0.0005420391614450182, + "0.02": 0.000982330566622558, + "0.05": 0.005609614976610278, + "0.1": 0.018432433960422934, + "0.15": 0.04204230956027111, + "0.2": 0.07802133259777522, + "0.3": 0.19761285922000882, + "0.4": 0.4178226978061619 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 75.5 +} \ No newline at end of file diff --git a/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/bf16_start.json b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/bf16_start.json new file mode 100644 index 0000000000000000000000000000000000000000..1adb5654f525e86310d8a9d34bf913b97a800975 --- /dev/null +++ b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/bf16_start.json @@ -0,0 +1,40 @@ +{ + "run": "pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103", + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103", + "weights": "final-unsharded (fp32 master)", + "quantization": "none (weights, block-Linear inputs, q/k/v); pre-norm kept; BF16 autocast; uncompiled", + "run_bits": { + "w": 6, + "a": 6, + "qkv": 6 + }, + "pre_norm": true, + "val_files": [ + "/data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy", + "/data/user_data/jgai/precision-perturb/dolma/heldout_cc_en_tail_4M.npy" + ], + "n_inst": null, + "batch": 8, + "primary": "val_dolma_cc", + "n_seqs": { + "val_dolma_cc": 488, + "heldout_cc_en_tail_4M": 4096 + }, + "losses_bf16_master": { + "val_dolma_cc": 3.7867076421355943, + "heldout_cc_en_tail_4M": 4.378495031321386 + }, + "loss_bf16_master": 3.7867076421355943, + "losses_eval_model": { + "val_dolma_cc": 3.812330659057858 + }, + "bf16_minus_quant": -0.025623016922263542, + "checks": { + "master_weights_fp32": true, + "loss_finite": true + }, + "ok": true, + "host": "babel-n5-28", + "gpu": "NVIDIA L40S", + "t_s": 18.6 +} \ No newline at end of file diff --git a/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/finalize.json b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/finalize.json new file mode 100644 index 0000000000000000000000000000000000000000..5362ef8c235ed6c12fe1944644c7895004f36f0f --- /dev/null +++ b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/finalize.json @@ -0,0 +1,102 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103", + "w_bits": 6, + "a_bits": 6, + "qkv_bits": 6, + "pre_norm": true, + "primary": "val_dolma_cc", + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "losses_master_plain": { + "val_dolma_cc": 3.809248105403336 + }, + "losses_fakequant_master": { + "val_dolma_cc": 3.812330659057858 + }, + "losses_eval_model": { + "val_dolma_cc": 3.812330659057858 + }, + "max_distinct_values_per_row": 61, + "rel_quant_noise_per_layer": { + "blocks.0.attention.w_q": 0.0317123718559742, + "blocks.0.attention.w_k": 0.03469786047935486, + "blocks.0.attention.w_v": 0.031686317175626755, + "blocks.0.attention.w_out": 0.032908711582422256, + "blocks.0.feed_forward.w1": 0.03040812909603119, + "blocks.0.feed_forward.w2": 0.038855548948049545, + "blocks.0.feed_forward.w3": 0.030431784689426422, + "blocks.1.attention.w_q": 0.032807957381010056, + "blocks.1.attention.w_k": 0.048468440771102905, + "blocks.1.attention.w_v": 0.032814256846904755, + "blocks.1.attention.w_out": 0.03926654905080795, + "blocks.1.feed_forward.w1": 0.0310849379748106, + "blocks.1.feed_forward.w2": 0.03742148354649544, + "blocks.1.feed_forward.w3": 0.03139351308345795, + "blocks.2.attention.w_q": 0.03342672064900398, + "blocks.2.attention.w_k": 0.041122451424598694, + "blocks.2.attention.w_v": 0.03246428444981575, + "blocks.2.attention.w_out": 0.03366396203637123, + "blocks.2.feed_forward.w1": 0.03130171447992325, + "blocks.2.feed_forward.w2": 0.042388685047626495, + "blocks.2.feed_forward.w3": 0.031368039548397064, + "blocks.3.attention.w_q": 0.03225043788552284, + "blocks.3.attention.w_k": 0.04886463284492493, + "blocks.3.attention.w_v": 0.03201742842793465, + "blocks.3.attention.w_out": 0.03306422010064125, + "blocks.3.feed_forward.w1": 0.031103475019335747, + "blocks.3.feed_forward.w2": 0.037032488733530045, + "blocks.3.feed_forward.w3": 0.0314888060092926, + "blocks.4.attention.w_q": 0.03383558616042137, + "blocks.4.attention.w_k": 0.05353178083896637, + "blocks.4.attention.w_v": 0.03258998319506645, + "blocks.4.attention.w_out": 0.03174182400107384, + "blocks.4.feed_forward.w1": 0.031217798590660095, + "blocks.4.feed_forward.w2": 0.03698069229722023, + "blocks.4.feed_forward.w3": 0.0313887894153595, + "blocks.5.attention.w_q": 0.033187106251716614, + "blocks.5.attention.w_k": 0.051447607576847076, + "blocks.5.attention.w_v": 0.032321106642484665, + "blocks.5.attention.w_out": 0.033096253871917725, + "blocks.5.feed_forward.w1": 0.03126804530620575, + "blocks.5.feed_forward.w2": 0.039384499192237854, + "blocks.5.feed_forward.w3": 0.031971149146556854, + "blocks.6.attention.w_q": 0.0327836237847805, + "blocks.6.attention.w_k": 0.045040614902973175, + "blocks.6.attention.w_v": 0.032615289092063904, + "blocks.6.attention.w_out": 0.03586641699075699, + "blocks.6.feed_forward.w1": 0.031602565199136734, + "blocks.6.feed_forward.w2": 0.038599833846092224, + "blocks.6.feed_forward.w3": 0.032520610839128494, + "blocks.7.attention.w_q": 0.0331927053630352, + "blocks.7.attention.w_k": 0.043572768568992615, + "blocks.7.attention.w_v": 0.032720185816287994, + "blocks.7.attention.w_out": 0.03944181650876999, + "blocks.7.feed_forward.w1": 0.031492020934820175, + "blocks.7.feed_forward.w2": 0.04241097345948219, + "blocks.7.feed_forward.w3": 0.03252849727869034, + "blocks.8.attention.w_q": 0.03506213426589966, + "blocks.8.attention.w_k": 0.04331529885530472, + "blocks.8.attention.w_v": 0.03315919265151024, + "blocks.8.attention.w_out": 0.035326357930898666, + "blocks.8.feed_forward.w1": 0.032574158161878586, + "blocks.8.feed_forward.w2": 0.049125030636787415, + "blocks.8.feed_forward.w3": 0.03471020609140396 + }, + "rel_quant_noise_mean": 0.03566885290164796, + "n_quantized_params": 30670848, + "n_params": 82191872, + "param_dtypes": [ + "torch.float32" + ], + "norm_gain_max_abs_dev_from_1": 2.0493109226226807, + "loss_eval_model": 3.812330659057858, + "checks": { + "snap_reproduces_fakequant": true, + "grid_levels_ok": true, + "master_weights_fp32": true, + "norm_gains_trained": true, + "loss_finite": true + }, + "ok": true +} \ No newline at end of file diff --git a/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/launch.yaml b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/launch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa9f9edcd1ca8184b498ec36ff7ad26fca676ea8 --- /dev/null +++ b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/launch.yaml @@ -0,0 +1,119 @@ +model: + d_model: 512 + vocab_size: 50304 + n_layers: 9 + block: + attention: + name: default + n_heads: 8 + bias: false + rope: + name: default + theta: 10000 + full_precision: true + _CLASS_: olmo_core.nn.rope.RoPEConfig + backend: flash_2 + dtype: float32 + _CLASS_: olmo_core.nn.attention.AttentionConfig + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + feed_forward: + hidden_size: 1536 + name: default + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.feed_forward.FeedForwardConfig + name: default + _CLASS_: olmo_core.nn.transformer.config.TransformerBlockConfig + lm_head: + name: default + layer_norm: + name: rms + eps: 1.0e-06 + bias: false + dtype: float32 + _CLASS_: olmo_core.nn.layer_norm.LayerNormConfig + bias: false + dtype: float32 + loss_implementation: default + _CLASS_: olmo_core.nn.lm_head.LMHeadConfig + name: default + dtype: float32 + init_method: normal + init_seed: 103 + init_std: 0.02 + _CLASS_: olmo_core.nn.transformer.config.TransformerConfig +dataset: + _CLASS_: olmo_core.data.NumpyFSLDatasetConfig + tokenizer: + _CLASS_: olmo_core.data.TokenizerConfig + identifier: allenai/gpt-neox-olmo-dolma-v1_5 + vocab_size: 50280 + eos_token_id: 50279 + pad_token_id: 1 + sequence_length: 1024 + work_dir: /data/user_data/jgai/precision-optimizer/work/dataset-cache + paths: + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-000-00000.npy + - /data/user_data/jgai/precision-perturb/dolma/cc_en_head/part-001-00000.npy +data_loader: + _CLASS_: olmo_core.data.NumpyDataLoaderConfig + global_batch_size: 262144 + seed: 0 + num_workers: 4 +train_module_type: normal +train_module: + _CLASS_: precq.train_module.FakeQuantTransformerTrainModuleConfig + fq_weight_bits: 6 + fq_act_bits: 6 + fq_qkv_bits: 6 + fq_pre_norm: true + rank_microbatch_size: 32768 + max_sequence_length: 1024 + optim: + _CLASS_: olmo_core.optim.AdamWConfig + lr: 0.0024 + betas: + - 0.9 + - 0.95 + eps: 1.0e-15 + weight_decay: 0.1 + fused: true + scheduler: + _CLASS_: olmo_core.optim.CosWithWarmup + warmup: 2105 + alpha_f: 0.1 + compile_model: true + dp_config: + _CLASS_: olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig + name: ddp + param_dtype: float32 + reduce_dtype: float32 + autocast_precision: bfloat16 + max_grad_norm: 1.0 +trainer: + _CLASS_: olmo_core.train.config.TrainerConfig + save_folder: /data/user_data/jgai/precision-optimizer/runs/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/ckpt + save_overwrite: true + metrics_collect_interval: 10 + cancel_check_interval: 5 + callbacks: + config_saver: + _CLASS_: olmo_core.train.callbacks.config_saver.ConfigSaverCallback + checkpointer: + _CLASS_: olmo_core.train.callbacks.checkpointer.CheckpointerCallback + save_interval: 1021057 + save_async: false +wandb: + enabled: false +init_seed: 103 +n_tokens: 5519966208 +load_path: null +validation_datasets: + dolma_cc_val: + - /data/user_data/jgai/precision-perturb/dolma/val_dolma_cc.npy +validation_eval_interval: 500 diff --git a/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/perturb_all.json b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/perturb_all.json new file mode 100644 index 0000000000000000000000000000000000000000..71a5857a18fdcfcbad2b9b950799f7091088718d --- /dev/null +++ b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/perturb_all.json @@ -0,0 +1,284 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103", + "scope": "all", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.06254545599222183, + "blocks.0.attention.w_out.weight": 0.05789056792855263, + "blocks.0.attention.w_q.weight": 0.06114041060209274, + "blocks.0.attention.w_v.weight": 0.5299122929573059, + "blocks.0.attention_norm.weight": 0.09001659601926804, + "blocks.0.feed_forward.w1.weight": 0.04941612109541893, + "blocks.0.feed_forward.w2.weight": 0.04582080990076065, + "blocks.0.feed_forward.w3.weight": 0.06795532256364822, + "blocks.0.feed_forward_norm.weight": 0.09247597306966782, + "blocks.1.attention.w_k.weight": 0.045679181814193726, + "blocks.1.attention.w_out.weight": 0.047478385269641876, + "blocks.1.attention.w_q.weight": 0.03706132993102074, + "blocks.1.attention.w_v.weight": 0.10529263317584991, + "blocks.1.attention_norm.weight": 0.08609475940465927, + "blocks.1.feed_forward.w1.weight": 0.060230206698179245, + "blocks.1.feed_forward.w2.weight": 0.04396585375070572, + "blocks.1.feed_forward.w3.weight": 0.06796219944953918, + "blocks.1.feed_forward_norm.weight": 0.09469867497682571, + "blocks.2.attention.w_k.weight": 0.05524877831339836, + "blocks.2.attention.w_out.weight": 0.04149368032813072, + "blocks.2.attention.w_q.weight": 0.029878919944167137, + "blocks.2.attention.w_v.weight": 0.13932104408740997, + "blocks.2.attention_norm.weight": 0.08466124534606934, + "blocks.2.feed_forward.w1.weight": 0.0565994456410408, + "blocks.2.feed_forward.w2.weight": 0.05023452639579773, + "blocks.2.feed_forward.w3.weight": 0.06780119240283966, + "blocks.2.feed_forward_norm.weight": 0.09335678815841675, + "blocks.3.attention.w_k.weight": 0.05697837471961975, + "blocks.3.attention.w_out.weight": 0.06385812163352966, + "blocks.3.attention.w_q.weight": 0.02807307243347168, + "blocks.3.attention.w_v.weight": 0.08693751692771912, + "blocks.3.attention_norm.weight": 0.08815661072731018, + "blocks.3.feed_forward.w1.weight": 0.05525568500161171, + "blocks.3.feed_forward.w2.weight": 0.0540318489074707, + "blocks.3.feed_forward.w3.weight": 0.0670810267329216, + "blocks.3.feed_forward_norm.weight": 0.09318064898252487, + "blocks.4.attention.w_k.weight": 0.048730283975601196, + "blocks.4.attention.w_out.weight": 0.06752345710992813, + "blocks.4.attention.w_q.weight": 0.02894987165927887, + "blocks.4.attention.w_v.weight": 0.13944830000400543, + "blocks.4.attention_norm.weight": 0.08969980478286743, + "blocks.4.feed_forward.w1.weight": 0.05493234843015671, + "blocks.4.feed_forward.w2.weight": 0.058218058198690414, + "blocks.4.feed_forward.w3.weight": 0.06742749363183975, + "blocks.4.feed_forward_norm.weight": 0.09288641065359116, + "blocks.5.attention.w_k.weight": 0.04924924299120903, + "blocks.5.attention.w_out.weight": 0.08078045397996902, + "blocks.5.attention.w_q.weight": 0.031226739287376404, + "blocks.5.attention.w_v.weight": 0.08463749289512634, + "blocks.5.attention_norm.weight": 0.08843649923801422, + "blocks.5.feed_forward.w1.weight": 0.053925175219774246, + "blocks.5.feed_forward.w2.weight": 0.06412149965763092, + "blocks.5.feed_forward.w3.weight": 0.06745769828557968, + "blocks.5.feed_forward_norm.weight": 0.0927981436252594, + "blocks.6.attention.w_k.weight": 0.05353730171918869, + "blocks.6.attention.w_out.weight": 0.09852109104394913, + "blocks.6.attention.w_q.weight": 0.027996035292744637, + "blocks.6.attention.w_v.weight": 0.10261815786361694, + "blocks.6.attention_norm.weight": 0.08759564906358719, + "blocks.6.feed_forward.w1.weight": 0.053143423050642014, + "blocks.6.feed_forward.w2.weight": 0.07586660236120224, + "blocks.6.feed_forward.w3.weight": 0.06787356734275818, + "blocks.6.feed_forward_norm.weight": 0.09368889778852463, + "blocks.7.attention.w_k.weight": 0.04684089496731758, + "blocks.7.attention.w_out.weight": 0.1212855651974678, + "blocks.7.attention.w_q.weight": 0.029368974268436432, + "blocks.7.attention.w_v.weight": 0.14844746887683868, + "blocks.7.attention_norm.weight": 0.09191121906042099, + "blocks.7.feed_forward.w1.weight": 0.0514235682785511, + "blocks.7.feed_forward.w2.weight": 0.09100648015737534, + "blocks.7.feed_forward.w3.weight": 0.06772653758525848, + "blocks.7.feed_forward_norm.weight": 0.09333803504705429, + "blocks.8.attention.w_k.weight": 0.040730416774749756, + "blocks.8.attention.w_out.weight": 0.16268809139728546, + "blocks.8.attention.w_q.weight": 0.031699661165475845, + "blocks.8.attention.w_v.weight": 0.13097651302814484, + "blocks.8.attention_norm.weight": 0.09232737869024277, + "blocks.8.feed_forward.w1.weight": 0.049838837236166, + "blocks.8.feed_forward.w2.weight": 0.11346770077943802, + "blocks.8.feed_forward.w3.weight": 0.06710774451494217, + "blocks.8.feed_forward_norm.weight": 0.08977222442626953, + "embeddings.weight": 0.06266719847917557, + "lm_head.norm.weight": 1.7597737312316895, + "lm_head.w_out.weight": 0.05861980840563774 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 84, + "n_perturbed_params": 82191872, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.812330659057858 + }, + "base_loss": 3.812330659057858, + "loss": { + "0.005": [ + 3.813957504234071, + 3.813170904525073, + 3.812876764425889 + ], + "0.01": [ + 3.8140404689327836, + 3.8138143875857833, + 3.813937070176664 + ], + "0.02": [ + 3.8183975084017896, + 3.8158668190531704, + 3.8160503147760325 + ], + "0.05": [ + 3.8423071708854644, + 3.8330551476829537, + 3.8338673965744836 + ], + "0.1": [ + 3.9261016065683942, + 3.899838565078001, + 3.9039151653991997 + ], + "0.15": [ + 4.075838408240839, + 4.025402944901738, + 4.038379854596834 + ], + "0.2": [ + 4.328043866267948, + 4.233821187357779, + 4.260587408220058 + ], + "0.3": [ + 5.337918551153891, + 5.149502105392569, + 5.246617875943464 + ], + "0.4": [ + 7.094560578330069, + 6.876203242432255, + 7.135410915553439 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.813957504234071, + 3.813170904525073, + 3.812876764425889 + ], + "0.01": [ + 3.8140404689327836, + 3.8138143875857833, + 3.813937070176664 + ], + "0.02": [ + 3.8183975084017896, + 3.8158668190531704, + 3.8160503147760325 + ], + "0.05": [ + 3.8423071708854644, + 3.8330551476829537, + 3.8338673965744836 + ], + "0.1": [ + 3.9261016065683942, + 3.899838565078001, + 3.9039151653991997 + ], + "0.15": [ + 4.075838408240839, + 4.025402944901738, + 4.038379854596834 + ], + "0.2": [ + 4.328043866267948, + 4.233821187357779, + 4.260587408220058 + ], + "0.3": [ + 5.337918551153891, + 5.149502105392569, + 5.246617875943464 + ], + "0.4": [ + 7.094560578330069, + 6.876203242432255, + 7.135410915553439 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0010043986704868502, + "0.01": 0.0015999831738859882, + "0.02": 0.004440888352473138, + "0.05": 0.02407924598977562, + "0.1": 0.09762111995734069, + "0.15": 0.23420974352194568, + "0.2": 0.4618201615574038, + "0.3": 1.4323488517721166, + "0.4": 3.223060919714063 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9965, + 1.0028 + ], + "0.01": [ + 0.9965, + 1.0028 + ], + "0.02": [ + 0.9965, + 1.0028 + ], + "0.05": [ + 0.9965, + 1.0028 + ], + "0.1": [ + 0.9965, + 1.0028 + ], + "0.15": [ + 0.9965, + 1.0028 + ], + "0.2": [ + 0.9965, + 1.0028 + ], + "0.3": [ + 0.9965, + 1.0028 + ], + "0.4": [ + 0.9965, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0010043986704868502, + "0.01": 0.0015999831738859882, + "0.02": 0.004440888352473138, + "0.05": 0.02407924598977562, + "0.1": 0.09762111995734069, + "0.15": 0.23420974352194568, + "0.2": 0.4618201615574038, + "0.3": 1.4323488517721166, + "0.4": 3.223060919714063 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 84.6 +} \ No newline at end of file diff --git a/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/perturb_blocks.json b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/perturb_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..ab0a58157bc16641c852627f66183b3a8cd10b83 --- /dev/null +++ b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/perturb_blocks.json @@ -0,0 +1,263 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103", + "scope": "blocks", + "noise_ref": null, + "noise_scale": "rms", + "noise_std_per_tensor_over_gamma": { + "blocks.0.attention.w_k.weight": 0.06254545599222183, + "blocks.0.attention.w_out.weight": 0.05789056792855263, + "blocks.0.attention.w_q.weight": 0.06114041060209274, + "blocks.0.attention.w_v.weight": 0.5299122929573059, + "blocks.0.feed_forward.w1.weight": 0.04941612109541893, + "blocks.0.feed_forward.w2.weight": 0.04582080990076065, + "blocks.0.feed_forward.w3.weight": 0.06795532256364822, + "blocks.1.attention.w_k.weight": 0.045679181814193726, + "blocks.1.attention.w_out.weight": 0.047478385269641876, + "blocks.1.attention.w_q.weight": 0.03706132993102074, + "blocks.1.attention.w_v.weight": 0.10529263317584991, + "blocks.1.feed_forward.w1.weight": 0.060230206698179245, + "blocks.1.feed_forward.w2.weight": 0.04396585375070572, + "blocks.1.feed_forward.w3.weight": 0.06796219944953918, + "blocks.2.attention.w_k.weight": 0.05524877831339836, + "blocks.2.attention.w_out.weight": 0.04149368032813072, + "blocks.2.attention.w_q.weight": 0.029878919944167137, + "blocks.2.attention.w_v.weight": 0.13932104408740997, + "blocks.2.feed_forward.w1.weight": 0.0565994456410408, + "blocks.2.feed_forward.w2.weight": 0.05023452639579773, + "blocks.2.feed_forward.w3.weight": 0.06780119240283966, + "blocks.3.attention.w_k.weight": 0.05697837471961975, + "blocks.3.attention.w_out.weight": 0.06385812163352966, + "blocks.3.attention.w_q.weight": 0.02807307243347168, + "blocks.3.attention.w_v.weight": 0.08693751692771912, + "blocks.3.feed_forward.w1.weight": 0.05525568500161171, + "blocks.3.feed_forward.w2.weight": 0.0540318489074707, + "blocks.3.feed_forward.w3.weight": 0.0670810267329216, + "blocks.4.attention.w_k.weight": 0.048730283975601196, + "blocks.4.attention.w_out.weight": 0.06752345710992813, + "blocks.4.attention.w_q.weight": 0.02894987165927887, + "blocks.4.attention.w_v.weight": 0.13944830000400543, + "blocks.4.feed_forward.w1.weight": 0.05493234843015671, + "blocks.4.feed_forward.w2.weight": 0.058218058198690414, + "blocks.4.feed_forward.w3.weight": 0.06742749363183975, + "blocks.5.attention.w_k.weight": 0.04924924299120903, + "blocks.5.attention.w_out.weight": 0.08078045397996902, + "blocks.5.attention.w_q.weight": 0.031226739287376404, + "blocks.5.attention.w_v.weight": 0.08463749289512634, + "blocks.5.feed_forward.w1.weight": 0.053925175219774246, + "blocks.5.feed_forward.w2.weight": 0.06412149965763092, + "blocks.5.feed_forward.w3.weight": 0.06745769828557968, + "blocks.6.attention.w_k.weight": 0.05353730171918869, + "blocks.6.attention.w_out.weight": 0.09852109104394913, + "blocks.6.attention.w_q.weight": 0.027996035292744637, + "blocks.6.attention.w_v.weight": 0.10261815786361694, + "blocks.6.feed_forward.w1.weight": 0.053143423050642014, + "blocks.6.feed_forward.w2.weight": 0.07586660236120224, + "blocks.6.feed_forward.w3.weight": 0.06787356734275818, + "blocks.7.attention.w_k.weight": 0.04684089496731758, + "blocks.7.attention.w_out.weight": 0.1212855651974678, + "blocks.7.attention.w_q.weight": 0.029368974268436432, + "blocks.7.attention.w_v.weight": 0.14844746887683868, + "blocks.7.feed_forward.w1.weight": 0.0514235682785511, + "blocks.7.feed_forward.w2.weight": 0.09100648015737534, + "blocks.7.feed_forward.w3.weight": 0.06772653758525848, + "blocks.8.attention.w_k.weight": 0.040730416774749756, + "blocks.8.attention.w_out.weight": 0.16268809139728546, + "blocks.8.attention.w_q.weight": 0.031699661165475845, + "blocks.8.attention.w_v.weight": 0.13097651302814484, + "blocks.8.feed_forward.w1.weight": 0.049838837236166, + "blocks.8.feed_forward.w2.weight": 0.11346770077943802, + "blocks.8.feed_forward.w3.weight": 0.06710774451494217 + }, + "gammas": [ + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4 + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_tensors": 63, + "n_perturbed_params": 30670848, + "eval_tokens": { + "val_dolma_cc": 499712 + }, + "primary": "val_dolma_cc", + "w_bits": 6, + "base_losses": { + "val_dolma_cc": 3.812330659057858 + }, + "base_loss": 3.812330659057858, + "loss": { + "0.005": [ + 3.8131173331009727, + 3.8118572446311676, + 3.812590355074826 + ], + "0.01": [ + 3.812846694319584, + 3.813055893997484, + 3.8133650261142793 + ], + "0.02": [ + 3.813101073491058, + 3.8133403393629415, + 3.8145441882050943 + ], + "0.05": [ + 3.8188814860976734, + 3.8197258355905865, + 3.8199919243415783 + ], + "0.1": [ + 3.837976363534956, + 3.838548775355602, + 3.8396364438859107 + ], + "0.15": [ + 3.8735010525397917, + 3.8731008220070047, + 3.8743091856686576 + ], + "0.2": [ + 3.9264286062594147, + 3.925910446606884, + 3.926886144483739 + ], + "0.3": [ + 4.127769168843796, + 4.126580155526377, + 4.129037312784441 + ], + "0.4": [ + 4.554659080893547, + 4.6041280285753485, + 4.579290191189226 + ] + }, + "loss_by_set": { + "val_dolma_cc": { + "0.005": [ + 3.8131173331009727, + 3.8118572446311676, + 3.812590355074826 + ], + "0.01": [ + 3.812846694319584, + 3.813055893997484, + 3.8133650261142793 + ], + "0.02": [ + 3.813101073491058, + 3.8133403393629415, + 3.8145441882050943 + ], + "0.05": [ + 3.8188814860976734, + 3.8197258355905865, + 3.8199919243415783 + ], + "0.1": [ + 3.837976363534956, + 3.838548775355602, + 3.8396364438859107 + ], + "0.15": [ + 3.8735010525397917, + 3.8731008220070047, + 3.8743091856686576 + ], + "0.2": [ + 3.9264286062594147, + 3.925910446606884, + 3.926886144483739 + ], + "0.3": [ + 4.127769168843796, + 4.126580155526377, + 4.129037312784441 + ], + "0.4": [ + 4.554659080893547, + 4.6041280285753485, + 4.579290191189226 + ] + } + }, + "mean_delta_by_set": { + "val_dolma_cc": { + "0.005": 0.0001909852111308119, + "0.01": 0.0007585457525913775, + "0.02": 0.001331207961840164, + "0.05": 0.00720242295208795, + "0.1": 0.0263898685342987, + "0.15": 0.06130636101396014, + "0.2": 0.11407774005882132, + "0.3": 0.3154648866603469, + "0.4": 0.7670284411615165 + } + }, + "rel_noise_over_gamma": { + "0.005": [ + 0.9966, + 1.0028 + ], + "0.01": [ + 0.9966, + 1.0028 + ], + "0.02": [ + 0.9966, + 1.0028 + ], + "0.05": [ + 0.9966, + 1.0028 + ], + "0.1": [ + 0.9966, + 1.0028 + ], + "0.15": [ + 0.9966, + 1.0028 + ], + "0.2": [ + 0.9966, + 1.0028 + ], + "0.3": [ + 0.9966, + 1.0028 + ], + "0.4": [ + 0.9966, + 1.0028 + ] + }, + "mean_delta": { + "0.005": 0.0001909852111308119, + "0.01": 0.0007585457525913775, + "0.02": 0.001331207961840164, + "0.05": 0.00720242295208795, + "0.1": 0.0263898685342987, + "0.15": 0.06130636101396014, + "0.2": 0.11407774005882132, + "0.3": 0.3154648866603469, + "0.4": 0.7670284411615165 + }, + "checks": { + "base_loss_finite": true, + "noise_scale_matches_gamma": true, + "base_restored": true + }, + "ok": true, + "t_total_s": 77.6 +} \ No newline at end of file diff --git a/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/ptq.json b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..ff92c2ff720c6477dd55879a66cbebff283b26a1 --- /dev/null +++ b/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103/ptq.json @@ -0,0 +1,52 @@ +{ + "run_dir": "/data/user_data/jgai/precision-optimizer/runs/pom-30M-D5.52B-f6-adamw-lr2.4e-03-s103", + "w_bits": 6, + "method": "RTN, symmetric absmax per output channel", + "scope": "transformer-block Linear weights", + "primary": "val_dolma_cc", + "base_losses": { + "val_dolma_cc": 3.812330659057858 + }, + "losses_by_bits": { + "3": { + "val_dolma_cc": 4.7350477863519185 + }, + "4": { + "val_dolma_cc": 3.8849751454882977 + }, + "5": { + "val_dolma_cc": 3.833963944854915 + }, + "6": { + "val_dolma_cc": 3.812343651878966 + }, + "8": { + "val_dolma_cc": 3.8122212157643665 + } + }, + "delta_by_bits": { + "3": { + "val_dolma_cc": 0.9227171272940606 + }, + "4": { + "val_dolma_cc": 0.07264448643043986 + }, + "5": { + "val_dolma_cc": 0.021633285797057145 + }, + "6": { + "val_dolma_cc": 1.2992821108337438e-05 + }, + "8": { + "val_dolma_cc": -0.00010944329349138471 + } + }, + "own_grid_max_rel_weight_change": 2.2822871414973633e-07, + "own_grid_delta_loss": 1.2992821108337438e-05, + "checks": { + "weights_restored": true, + "own_grid_weights_unchanged": true, + "own_grid_is_fixed_point": true + }, + "ok": true +} \ No newline at end of file