Upload weights and configs - David-fully_shared-weighted_sum - Run 20251012_135249

weights/David-fully_shared-weighted_sum/20251012_135249/MODEL_SUMMARY.txt

@@ -3,24 +3,24 @@
 ║                      DAVID MODEL SUMMARY                      ║
 ╠══════════════════════════════════════════════════════════════╣
 ║                                                               ║
-║  🎯 VALIDATION ACCURACY: 71.77%                          ║
+║  🎯 VALIDATION ACCURACY: 71.78%                          ║
 ║                                                               ║
 ╚══════════════════════════════════════════════════════════════╝
 
 MODEL: David-fully_shared-weighted_sum
 RUN ID: 20251012_135249
-BEST EPOCH: 7/10
+BEST EPOCH: 8/10
 
 ═══════════════════════════════════════════════════════════════
 
 📊 PERFORMANCE BREAKDOWN
 
-Final Training Accuracy:  72.29%
-Best Validation Accuracy: 71.77%
+Final Training Accuracy:  72.75%
+Best Validation Accuracy: 71.78%
 
 Per-Scale Accuracies:
-  • Scale  256: 71.44%
-  • Scale  512: 71.58%
+  • Scale  256: 71.49%
+  • Scale  512: 71.69%
 
 ═══════════════════════════════════════════════════════════════
 
@@ -45,17 +45,18 @@ Epoch | Train Acc | Val Acc  | Learning Rate
     4 |    70.91% |   70.62% | 6.55e-04
     5 |    71.36% |   70.97% | 5.01e-04
     6 |    71.84% |   71.46% | 3.46e-04
-    7 |    72.29% |   71.77% 👑 | 2.07e-04
+    7 |    72.29% |   71.77% | 2.07e-04
+    8 |    72.75% |   71.78% 👑 | 9.64e-05
 
 ═══════════════════════════════════════════════════════════════
 
 📁 FILES
 
-Best Model:    best_model_acc71.77.safetensors
+Best Model:    best_model_acc71.78.safetensors
 Config:        david_config.json
 Training Cfg:  train_config.json
 History:       training_history.json
 
 ═══════════════════════════════════════════════════════════════
 
-Generated: 2025-10-12 14:04:40
+Generated: 2025-10-12 14:06:21

weights/David-fully_shared-weighted_sum/20251012_135249/best_model_acc71.78.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ea094896d9059ac03b925e0e9e782899a7058cc515bb01201da368e1750a9ac
+size 2628344

weights/David-fully_shared-weighted_sum/20251012_135249/best_model_acc71.78_metadata.json

@@ -0,0 +1,286 @@
+{
+  "epoch": 7,
+  "optimizer_state_dict": {
+    "state": {
+      "0": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([[ 1.5062e-04,  1.2931e-03,  6.3652e-04,  ..., -6.9315e-04,\n          2.6788e-04, -1.9202e-04],\n        [ 1.3724e-05, -1.1480e-03, -1.8676e-03,  ...,  5.8825e-04,\n          8.9245e-04, -9.5413e-04],\n        [-3.6894e-05,  1.0142e-04,  1.5780e-03,  ...,  1.0864e-03,\n          6.7905e-04,  2.2829e-04],\n        ...,\n        [-6.2483e-04,  1.3959e-04,  4.2794e-05,  ..., -3.8634e-04,\n          2.0065e-04,  2.4963e-04],\n        [ 1.0356e-04, -1.9847e-03,  1.0875e-03,  ...,  6.0027e-04,\n         -5.4806e-04,  3.6143e-04],\n        [-2.6487e-04,  4.8516e-05,  1.0238e-03,  ...,  4.3398e-04,\n          1.9616e-04,  1.3210e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[6.2359e-06, 3.2165e-05, 1.4842e-05,  ..., 6.6419e-06, 3.0001e-06,\n         8.6127e-06],\n        [5.0889e-06, 2.2290e-05, 1.7448e-05,  ..., 7.6721e-06, 2.4160e-06,\n         8.9631e-06],\n        [9.8949e-06, 6.1075e-05, 2.0084e-05,  ..., 6.5694e-06, 3.1082e-06,\n         3.9574e-06],\n        ...,\n        [5.9008e-06, 2.5729e-05, 1.4348e-05,  ..., 3.3453e-06, 1.7197e-06,\n         2.5745e-06],\n        [7.1457e-06, 4.2581e-05, 1.4950e-05,  ..., 6.2363e-06, 3.6198e-06,\n         5.7770e-06],\n        [4.0019e-06, 4.6648e-05, 1.1256e-05,  ..., 2.9904e-06, 1.8174e-06,\n         2.0343e-06]], device='cuda:0')"
+      },
+      "1": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([ 0.0137, -0.0015, -0.0120,  0.0200,  0.0354, -0.0204,  0.0067,  0.0037,\n        -0.0484, -0.0283, -0.0105,  0.0067, -0.0483,  0.0038,  0.0092, -0.0191,\n        -0.0053, -0.0100,  0.0313, -0.0448,  0.0147, -0.0192,  0.0184,  0.0088,\n         0.0327,  0.0405,  0.0053,  0.0326, -0.0521, -0.0100,  0.0164, -0.0009,\n         0.0248,  0.0192,  0.0150, -0.0193, -0.0072,  0.0073,  0.0105, -0.0135,\n         0.0068,  0.0121,  0.0105,  0.0348, -0.0245,  0.0557, -0.0131, -0.0441,\n         0.0208,  0.0049, -0.0053,  0.0053, -0.0310,  0.0015,  0.0144, -0.0121,\n         0.0136,  0.0087, -0.0051,  0.0041, -0.0067, -0.0291,  0.0229, -0.0090,\n        -0.0125, -0.0052, -0.0070,  0.0209,  0.0080, -0.0162, -0.0014, -0.0150,\n        -0.0333,  0.0098,  0.0274,  0.0221,  0.0239,  0.0146,  0.0064,  0.0002,\n        -0.0216, -0.0174, -0.0013,  0.0102,  0.0336, -0.0254, -0.0058,  0.0396,\n         0.0027,  0.0157,  0.0174,  0.0283, -0.0100, -0.0055,  0.0065, -0.0057,\n        -0.0153,  0.0017, -0.0176,  0.0184, -0.0039,  0.0039, -0.0019,  0.0016,\n         0.0055, -0.0192,  0.0234, -0.0012,  0.0046,  0.0094, -0.0006, -0.0162,\n         0.0256, -0.0014, -0.0013, -0.0073, -0.0002,  0.0010, -0.0044, -0.0112,\n        -0.0112,  0.0102, -0.0083,  0.0006, -0.0174, -0.0145,  0.0008,  0.0094,\n        -0.0019, -0.0074,  0.0049, -0.0056, -0.0071,  0.0070,  0.0159,  0.0010,\n        -0.0150, -0.0076, -0.0123,  0.0560, -0.0321,  0.0044, -0.0126, -0.0069,\n        -0.0031, -0.0045,  0.0126,  0.0078, -0.0197, -0.0004,  0.0084, -0.0201,\n        -0.0002, -0.0442, -0.0052,  0.0097, -0.0271,  0.0359,  0.0343, -0.0124,\n        -0.0032, -0.0147, -0.0021,  0.0518, -0.0036, -0.0187,  0.0018, -0.0288,\n        -0.0197,  0.0124,  0.0076, -0.0194,  0.0058,  0.0070,  0.0158, -0.0258,\n        -0.0032, -0.0062,  0.0011,  0.0261,  0.0034, -0.0344, -0.0326, -0.0568,\n        -0.0108, -0.0083,  0.0015,  0.0015,  0.0142, -0.0037,  0.0275,  0.0027,\n         0.0162,  0.0064, -0.0071, -0.0131, -0.0040, -0.0299,  0.0144, -0.0187,\n        -0.0086,  0.0078, -0.0307,  0.0093,  0.0139, -0.0096,  0.0004, -0.0050,\n         0.0091,  0.0049, -0.0089, -0.0077, -0.0163,  0.0058, -0.0083,  0.0242,\n        -0.0146, -0.0118,  0.0105,  0.0239,  0.0139, -0.0338,  0.0139,  0.0507,\n        -0.0004, -0.0155,  0.0205,  0.0242,  0.0091, -0.0396,  0.0016, -0.0166,\n         0.0011,  0.0038, -0.0135,  0.0415,  0.0114,  0.0206, -0.0165, -0.0289,\n        -0.0041, -0.0294, -0.0182, -0.0127,  0.0246,  0.0175, -0.0277,  0.0006,\n        -0.0054,  0.0175, -0.0136,  0.0088,  0.0009,  0.0264,  0.0213, -0.0071,\n         0.0074,  0.0172,  0.0238, -0.0076,  0.0016,  0.0164,  0.0049, -0.0239,\n         0.0118, -0.0153, -0.0022,  0.0110,  0.0254,  0.0096,  0.0074,  0.0322,\n         0.0341,  0.0045,  0.0302, -0.0271,  0.0105,  0.0173, -0.0119,  0.0137,\n        -0.0137,  0.0496, -0.0643, -0.0068, -0.0187,  0.0087, -0.0038,  0.0149,\n        -0.0044, -0.0185,  0.0011, -0.0173, -0.0242,  0.0049, -0.0101, -0.0366,\n         0.0340, -0.0148,  0.0365,  0.0116,  0.0083, -0.0039,  0.0082,  0.0196,\n         0.0015,  0.0023, -0.0182,  0.0088, -0.0047,  0.0081, -0.0060,  0.0034,\n        -0.0321,  0.0072,  0.0017, -0.0001,  0.0012,  0.0301,  0.0135,  0.0152,\n        -0.0048,  0.0381, -0.0263,  0.0009,  0.0237, -0.0002,  0.0074,  0.0208,\n         0.0003,  0.0084,  0.0107,  0.0063, -0.0055,  0.0190, -0.0080,  0.0143,\n         0.0194,  0.0355, -0.0173, -0.0403,  0.0209,  0.0136, -0.0173, -0.0279,\n        -0.0231,  0.0110, -0.0181, -0.0043, -0.0165,  0.0085, -0.0132,  0.0011,\n         0.0160, -0.0090,  0.0058, -0.0296, -0.0077,  0.0106, -0.0039, -0.0036,\n        -0.0304,  0.0251, -0.0095, -0.0010,  0.0166,  0.0188, -0.0140,  0.0510,\n         0.0095,  0.0412,  0.0105, -0.0104,  0.0086, -0.0018,  0.0032, -0.0886,\n        -0.0017, -0.0060,  0.0457,  0.0046,  0.0004, -0.0099,  0.0203, -0.0061,\n         0.0179, -0.0080,  0.0110,  0.0356,  0.0535,  0.0178,  0.0166,  0.0143,\n         0.0137,  0.0228, -0.0192,  0.0062, -0.0231, -0.0242, -0.0062,  0.0151,\n         0.0016, -0.0068, -0.0072,  0.0087,  0.0004,  0.0127, -0.0182,  0.0045,\n        -0.0119,  0.0192,  0.0071, -0.0088, -0.0647, -0.0067, -0.0151,  0.0272,\n        -0.0195,  0.0131, -0.0177,  0.0370, -0.0130,  0.0234, -0.0111, -0.0293,\n        -0.0232, -0.0201,  0.0217, -0.0040, -0.0398,  0.0108,  0.0058, -0.0098,\n         0.0084, -0.0132,  0.0054, -0.0106,  0.0109,  0.0057,  0.0300,  0.0233,\n         0.0141, -0.0021, -0.0093, -0.0015, -0.0102, -0.0230, -0.0080, -0.0039,\n         0.0067, -0.0296,  0.0057, -0.0296,  0.0161, -0.0208,  0.0012, -0.0152,\n         0.0181, -0.0456, -0.0235,  0.0459,  0.0159, -0.0114, -0.0128, -0.0032,\n        -0.0082, -0.0160,  0.0315, -0.0134, -0.0025, -0.0280, -0.0289,  0.0315,\n        -0.0114,  0.0057,  0.0302, -0.0406,  0.0144, -0.0141, -0.0347, -0.0091,\n         0.0020, -0.0029,  0.0106, -0.0088, -0.0020, -0.0008,  0.0052, -0.0044,\n        -0.0088, -0.0071, -0.0037, -0.0288, -0.0065,  0.0040, -0.0372, -0.0143,\n        -0.0004, -0.0186, -0.0042,  0.0428, -0.0239,  0.0237,  0.0323,  0.0203,\n        -0.0428, -0.0600,  0.0217,  0.0026,  0.0297, -0.0040,  0.0113, -0.0015],\n       device='cuda:0')",
+        "exp_avg_sq": "tensor([0.0051, 0.0047, 0.0049, 0.0030, 0.0064, 0.0032, 0.0032, 0.0033, 0.0061,\n        0.0041, 0.0026, 0.0042, 0.0046, 0.0033, 0.0040, 0.0036, 0.0011, 0.0020,\n        0.0028, 0.0028, 0.0033, 0.0035, 0.0041, 0.0046, 0.0052, 0.0037, 0.0040,\n        0.0033, 0.0057, 0.0016, 0.0032, 0.0050, 0.0048, 0.0032, 0.0037, 0.0031,\n        0.0025, 0.0039, 0.0033, 0.0029, 0.0054, 0.0033, 0.0035, 0.0049, 0.0038,\n        0.0055, 0.0078, 0.0047, 0.0032, 0.0046, 0.0031, 0.0036, 0.0046, 0.0020,\n        0.0061, 0.0047, 0.0026, 0.0049, 0.0030, 0.0034, 0.0034, 0.0066, 0.0049,\n        0.0026, 0.0025, 0.0026, 0.0033, 0.0052, 0.0076, 0.0063, 0.0053, 0.0033,\n        0.0066, 0.0024, 0.0036, 0.0076, 0.0040, 0.0038, 0.0041, 0.0074, 0.0062,\n        0.0032, 0.0054, 0.0048, 0.0031, 0.0033, 0.0012, 0.0078, 0.0044, 0.0042,\n        0.0024, 0.0031, 0.0027, 0.0043, 0.0053, 0.0038, 0.0050, 0.0047, 0.0041,\n        0.0045, 0.0028, 0.0031, 0.0044, 0.0018, 0.0030, 0.0029, 0.0064, 0.0029,\n        0.0044, 0.0038, 0.0017, 0.0052, 0.0037, 0.0026, 0.0024, 0.0027, 0.0040,\n        0.0056, 0.0025, 0.0038, 0.0027, 0.0027, 0.0037, 0.0049, 0.0052, 0.0046,\n        0.0048, 0.0033, 0.0048, 0.0028, 0.0045, 0.0032, 0.0035, 0.0030, 0.0045,\n        0.0027, 0.0027, 0.0019, 0.0048, 0.0074, 0.0053, 0.0027, 0.0031, 0.0051,\n        0.0051, 0.0033, 0.0038, 0.0039, 0.0042, 0.0047, 0.0032, 0.0042, 0.0028,\n        0.0027, 0.0019, 0.0024, 0.0046, 0.0043, 0.0025, 0.0041, 0.0030, 0.0036,\n        0.0035, 0.0037, 0.0043, 0.0041, 0.0018, 0.0027, 0.0041, 0.0039, 0.0043,\n        0.0033, 0.0021, 0.0032, 0.0045, 0.0026, 0.0031, 0.0036, 0.0023, 0.0039,\n        0.0051, 0.0074, 0.0020, 0.0045, 0.0065, 0.0044, 0.0017, 0.0022, 0.0038,\n        0.0031, 0.0021, 0.0022, 0.0059, 0.0036, 0.0043, 0.0034, 0.0028, 0.0034,\n        0.0035, 0.0091, 0.0021, 0.0041, 0.0026, 0.0048, 0.0034, 0.0033, 0.0022,\n        0.0026, 0.0034, 0.0049, 0.0022, 0.0050, 0.0051, 0.0073, 0.0041, 0.0025,\n        0.0036, 0.0024, 0.0031, 0.0046, 0.0024, 0.0044, 0.0064, 0.0055, 0.0018,\n        0.0033, 0.0039, 0.0047, 0.0040, 0.0063, 0.0038, 0.0030, 0.0045, 0.0025,\n        0.0036, 0.0045, 0.0047, 0.0042, 0.0035, 0.0079, 0.0052, 0.0031, 0.0041,\n        0.0031, 0.0026, 0.0049, 0.0033, 0.0038, 0.0024, 0.0064, 0.0025, 0.0032,\n        0.0016, 0.0084, 0.0043, 0.0046, 0.0076, 0.0030, 0.0042, 0.0039, 0.0038,\n        0.0074, 0.0028, 0.0027, 0.0018, 0.0039, 0.0030, 0.0052, 0.0019, 0.0032,\n        0.0023, 0.0024, 0.0043, 0.0038, 0.0031, 0.0062, 0.0016, 0.0029, 0.0077,\n        0.0034, 0.0031, 0.0045, 0.0041, 0.0026, 0.0065, 0.0045, 0.0035, 0.0040,\n        0.0056, 0.0030, 0.0036, 0.0056, 0.0039, 0.0044, 0.0062, 0.0050, 0.0051,\n        0.0044, 0.0067, 0.0060, 0.0044, 0.0028, 0.0032, 0.0026, 0.0028, 0.0035,\n        0.0041, 0.0027, 0.0053, 0.0039, 0.0035, 0.0043, 0.0026, 0.0037, 0.0027,\n        0.0049, 0.0021, 0.0048, 0.0036, 0.0027, 0.0046, 0.0066, 0.0035, 0.0029,\n        0.0040, 0.0028, 0.0031, 0.0036, 0.0025, 0.0037, 0.0054, 0.0037, 0.0035,\n        0.0030, 0.0026, 0.0029, 0.0035, 0.0038, 0.0052, 0.0047, 0.0033, 0.0033,\n        0.0029, 0.0035, 0.0044, 0.0049, 0.0028, 0.0032, 0.0030, 0.0034, 0.0043,\n        0.0061, 0.0029, 0.0042, 0.0023, 0.0038, 0.0050, 0.0042, 0.0025, 0.0036,\n        0.0041, 0.0041, 0.0038, 0.0024, 0.0031, 0.0034, 0.0027, 0.0071, 0.0028,\n        0.0025, 0.0028, 0.0082, 0.0046, 0.0039, 0.0038, 0.0029, 0.0040, 0.0031,\n        0.0045, 0.0031, 0.0025, 0.0037, 0.0032, 0.0044, 0.0053, 0.0030, 0.0024,\n        0.0039, 0.0051, 0.0016, 0.0043, 0.0074, 0.0030, 0.0037, 0.0027, 0.0050,\n        0.0024, 0.0030, 0.0073, 0.0019, 0.0022, 0.0035, 0.0037, 0.0046, 0.0046,\n        0.0069, 0.0036, 0.0029, 0.0045, 0.0023, 0.0047, 0.0048, 0.0057, 0.0026,\n        0.0057, 0.0028, 0.0034, 0.0025, 0.0054, 0.0029, 0.0041, 0.0036, 0.0044,\n        0.0058, 0.0061, 0.0034, 0.0075, 0.0056, 0.0027, 0.0023, 0.0045, 0.0026,\n        0.0029, 0.0109, 0.0024, 0.0041, 0.0046, 0.0047, 0.0049, 0.0049, 0.0050,\n        0.0064, 0.0035, 0.0031, 0.0027, 0.0034, 0.0047, 0.0028, 0.0031, 0.0029,\n        0.0016, 0.0062, 0.0036, 0.0042, 0.0031, 0.0054, 0.0037, 0.0027, 0.0042,\n        0.0039, 0.0046, 0.0056, 0.0030, 0.0052, 0.0013, 0.0040, 0.0031, 0.0040,\n        0.0037, 0.0051, 0.0045, 0.0030, 0.0055, 0.0036, 0.0034, 0.0027, 0.0033,\n        0.0040, 0.0035, 0.0030, 0.0030, 0.0038, 0.0030, 0.0035, 0.0033, 0.0037,\n        0.0027, 0.0031, 0.0074, 0.0044, 0.0006, 0.0040, 0.0038, 0.0034, 0.0049,\n        0.0046, 0.0045, 0.0049, 0.0049, 0.0041, 0.0026, 0.0037, 0.0049, 0.0030,\n        0.0044, 0.0058, 0.0040, 0.0030, 0.0041, 0.0035, 0.0053, 0.0028],\n       device='cuda:0')"
+      },
+      "2": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([ 2.8375e-03,  1.3535e-03, -2.7553e-03,  5.6456e-03,  5.4468e-03,\n        -5.7977e-03,  2.3974e-03,  5.5486e-04, -7.3389e-03, -6.8782e-03,\n        -3.2585e-03,  7.0032e-04, -6.4685e-03,  3.4179e-04,  1.8669e-03,\n        -3.1449e-03, -7.5391e-03, -1.9407e-03,  6.5652e-03, -1.2108e-02,\n         6.1811e-03, -6.4812e-03,  3.6845e-03,  1.5596e-03,  6.1364e-03,\n         1.4628e-02,  1.8386e-03,  7.5160e-03, -5.4505e-03, -4.7077e-03,\n         5.4789e-03,  2.9645e-04,  6.0152e-03,  2.3659e-03,  3.4069e-03,\n        -7.5485e-03, -3.5640e-03,  1.4986e-03,  1.1039e-03, -3.4852e-03,\n         7.7407e-04,  1.7118e-03,  6.7085e-04,  4.8313e-03, -4.4359e-03,\n         7.7368e-03, -1.5355e-03, -1.0577e-02,  7.5887e-03,  8.7730e-04,\n        -1.6491e-03,  2.6842e-03, -4.4823e-03,  1.2027e-03,  1.6263e-03,\n        -1.8620e-03,  5.2005e-03,  1.4716e-03, -2.6815e-04,  1.3967e-03,\n        -2.2067e-03, -2.2742e-03,  4.1969e-03, -3.3412e-03, -4.3087e-03,\n        -2.2245e-03, -2.5782e-03,  4.0586e-03,  5.0051e-04, -2.2524e-03,\n         8.9200e-04, -2.4320e-03, -3.6181e-03,  2.5917e-03,  5.3974e-03,\n         3.0984e-03,  3.4244e-03,  1.4284e-03,  1.9389e-03,  4.1401e-04,\n        -1.9051e-03, -2.4263e-03,  3.4778e-04,  1.4245e-03,  5.4901e-03,\n        -7.4263e-03, -7.9353e-03,  4.0396e-03,  2.8205e-04,  3.3324e-03,\n         5.5535e-03,  9.8808e-03, -2.7190e-03,  1.3214e-03,  3.8150e-05,\n        -4.8397e-04, -2.3818e-03,  1.7999e-03, -2.7165e-03,  2.5373e-03,\n         6.4745e-04,  3.6487e-04,  3.9363e-05,  3.6247e-04,  1.0678e-03,\n        -3.6526e-03,  4.1558e-03,  2.4627e-04,  1.8589e-04,  1.7079e-03,\n        -1.7780e-03, -1.4744e-03,  6.7903e-03,  3.7442e-04, -1.4284e-03,\n        -7.9676e-04, -2.1426e-04,  8.5350e-05, -1.6671e-03,  8.5666e-06,\n        -4.3156e-03,  2.7628e-04, -1.8832e-03,  9.2824e-04, -2.0953e-03,\n        -8.9036e-04, -2.7263e-05,  2.4268e-03, -4.1519e-05, -2.7317e-03,\n         1.7068e-03, -7.0210e-04, -2.4485e-03,  1.9202e-03,  2.8979e-03,\n         2.1870e-04, -3.2559e-03, -2.8578e-03, -2.1180e-03,  6.4829e-03,\n        -7.7009e-03,  9.0784e-04, -1.3987e-03, -1.2616e-03, -1.0408e-03,\n         1.3535e-03,  2.1444e-03, -3.8278e-04, -3.6925e-03, -6.3666e-04,\n         1.7251e-03, -6.0147e-03, -7.8412e-04, -1.5654e-02, -7.1471e-06,\n         3.6217e-03, -5.4875e-03,  6.7620e-03,  1.7798e-02, -1.4701e-03,\n        -2.8241e-03, -1.6570e-03,  8.9415e-05,  1.1685e-02, -1.3950e-03,\n        -3.7693e-03,  1.8088e-03, -8.9242e-03, -2.8544e-03,  1.0115e-03,\n         4.1357e-04, -4.1563e-03,  1.2584e-03,  2.0942e-03,  2.5833e-03,\n        -7.2251e-03, -3.2244e-04, -1.8306e-03,  1.0626e-03,  3.4652e-03,\n         2.8465e-04, -3.6441e-03, -1.8611e-02, -1.2466e-02, -1.5082e-03,\n        -1.3764e-03, -5.3893e-04,  2.4184e-04,  2.6332e-03, -1.0894e-03,\n         1.4082e-02,  2.0549e-03,  3.2536e-03,  2.4388e-03, -1.4418e-03,\n        -2.6941e-03, -1.6621e-04, -5.6829e-03,  9.0940e-04, -2.7055e-03,\n        -4.0366e-03,  2.7160e-03, -5.7523e-03,  3.2785e-03,  3.8232e-03,\n        -7.2395e-04,  7.9369e-04, -8.3360e-04,  2.3598e-03, -5.2823e-04,\n        -2.1796e-03,  2.7209e-04, -2.7490e-03,  5.1666e-04, -3.4299e-03,\n         7.3466e-03, -1.8769e-03, -7.5408e-04,  9.2410e-04,  4.2547e-03,\n         2.7181e-03, -3.4684e-03,  1.0064e-03,  8.2306e-03,  7.5716e-04,\n        -1.6072e-03,  5.9790e-03,  2.5399e-03,  5.1409e-03, -6.2098e-03,\n         1.7577e-04, -3.4952e-03,  3.5963e-04,  9.0853e-04, -2.2026e-04,\n         8.2980e-03,  3.7933e-03,  3.3098e-03, -2.7057e-03, -4.2211e-03,\n        -4.5142e-04, -8.2566e-03, -2.7402e-03, -1.4193e-03,  8.8760e-03,\n         1.7131e-03, -6.3697e-03,  9.2674e-04, -2.8642e-03,  2.6942e-03,\n        -3.1983e-03,  2.3982e-03,  1.2146e-03,  2.7355e-03,  3.4800e-03,\n         4.3552e-04,  2.2231e-03,  1.3326e-03,  7.0464e-03, -2.2687e-03,\n        -1.7646e-05,  2.4891e-03,  8.8266e-04, -8.4387e-03,  4.2006e-03,\n        -1.8133e-03, -9.1806e-04,  2.2417e-03,  7.6526e-03,  1.5821e-03,\n         3.0004e-03,  9.6661e-03,  4.2268e-03, -6.3831e-04,  5.4751e-03,\n        -6.4145e-03,  9.9649e-03,  3.8888e-03, -2.0356e-03,  3.2374e-03,\n        -1.1937e-03,  1.0583e-02, -1.3244e-02, -1.2187e-03, -4.1651e-03,\n         1.2996e-03, -1.4837e-03,  2.9643e-03,  5.7382e-05, -5.4769e-03,\n        -3.2191e-05, -2.3993e-03, -5.4492e-03,  4.5744e-04, -9.3095e-04,\n        -5.6859e-03,  1.9633e-03, -2.5649e-03,  4.5103e-03,  1.1926e-03,\n         7.7801e-04, -4.2462e-04,  1.3188e-03,  6.3107e-03, -9.4734e-04,\n        -5.1496e-04, -3.2808e-03,  1.4573e-03, -1.2031e-03,  1.5710e-03,\n        -1.1633e-03,  3.2090e-04, -7.6562e-03,  8.0390e-04,  9.9254e-04,\n         2.0153e-04,  3.6737e-04,  5.4507e-03,  2.2796e-03,  2.0687e-03,\n        -1.9482e-03,  9.5068e-03, -3.7121e-03,  3.5014e-04,  2.8683e-03,\n         4.8721e-04,  1.6401e-03,  4.1363e-03,  1.0494e-03,  1.5750e-03,\n         1.5000e-03,  2.5234e-03, -1.6978e-03,  4.3632e-03, -4.6520e-03,\n         4.8418e-03,  4.0681e-03,  9.2853e-03, -2.4757e-03, -6.3419e-03,\n         5.0406e-03,  4.4078e-03, -3.0300e-03, -5.0548e-03, -4.8064e-03,\n         1.8752e-03, -3.5824e-03, -6.5158e-04, -4.2511e-03,  9.5766e-04,\n        -3.0832e-03,  2.5510e-04,  1.6071e-03, -1.6971e-03,  2.9262e-03,\n        -4.4548e-03, -2.2710e-05,  2.5480e-03, -1.5848e-03, -7.6709e-04,\n        -5.0292e-03,  2.6395e-03, -1.8570e-03, -3.2446e-05,  1.4183e-03,\n         4.7543e-03, -2.5099e-03,  5.6597e-03,  2.2245e-03,  1.1982e-02,\n         3.2650e-03, -1.9213e-03,  1.7624e-03, -1.9025e-04,  3.1113e-06,\n        -2.5662e-02, -2.1004e-04, -1.2045e-03,  8.0549e-03,  3.8763e-04,\n         1.7940e-04, -6.1034e-04,  3.9993e-03, -3.4045e-03,  3.5552e-03,\n        -3.2230e-03,  5.1449e-03,  5.8752e-03,  5.3260e-03,  1.3017e-02,\n         3.4083e-03,  1.1350e-03,  4.3996e-03,  4.9143e-03, -8.4623e-03,\n         1.3684e-03, -7.8345e-03, -7.4061e-03, -1.9650e-03,  5.9628e-03,\n         9.8297e-04, -2.2188e-03, -1.0886e-03,  9.3098e-04, -3.0553e-04,\n         1.1958e-03, -3.1205e-03,  4.6010e-05, -5.7613e-04,  7.1839e-03,\n         1.7963e-03, -8.9153e-04, -1.0528e-02, -2.1044e-03, -1.5039e-03,\n         8.3042e-03, -3.0155e-03,  4.3602e-03, -2.2865e-03,  1.1840e-02,\n        -3.3151e-03,  4.4994e-03, -1.1146e-03, -2.8625e-03, -2.0248e-03,\n        -4.7229e-03,  3.2686e-03, -1.5146e-03, -1.2495e-02,  4.7933e-03,\n         6.0934e-04, -4.1336e-03,  2.7984e-03,  4.1591e-04, -1.7445e-03,\n        -4.4389e-04,  1.2182e-03,  2.0789e-03,  3.5222e-03,  3.2377e-03,\n         1.5178e-03, -1.5717e-03, -1.3078e-03,  6.4786e-04, -1.8280e-03,\n        -4.6808e-03, -2.7392e-03, -1.3315e-03,  3.3610e-04, -8.6495e-03,\n         4.1521e-03, -4.6529e-03,  1.6042e-03, -5.1368e-03, -2.1217e-04,\n        -2.4820e-03,  3.1142e-03, -1.9866e-02, -4.5577e-03,  8.8114e-03,\n         1.5984e-03, -1.3593e-03, -2.8528e-03, -2.3805e-04, -4.3742e-03,\n        -4.9108e-03,  7.2395e-03, -1.5977e-03, -5.5164e-04, -3.0485e-03,\n        -4.1268e-03,  1.0960e-02, -2.3840e-03,  1.4461e-03,  6.0976e-03,\n        -1.1387e-02,  3.1417e-03, -3.3297e-03, -4.3982e-03, -2.0041e-03,\n         1.4531e-03, -1.3553e-04,  2.6576e-03, -2.9723e-03, -9.5282e-04,\n        -2.1869e-03,  1.0056e-03, -9.7822e-04, -1.2272e-03, -3.6942e-05,\n        -5.4432e-02, -3.1969e-03, -2.6840e-04,  3.7779e-05, -5.9333e-03,\n        -3.5353e-03,  3.5704e-05, -3.1499e-03, -1.0425e-03,  1.1524e-02,\n        -5.4188e-03,  4.7800e-03,  5.0317e-03,  4.5259e-03, -8.5412e-03,\n        -1.1938e-02,  5.2772e-03,  1.0852e-03,  6.7676e-03, -1.9216e-03,\n         2.7224e-03,  3.4042e-06], device='cuda:0')",
+        "exp_avg_sq": "tensor([2.1555e-04, 7.7813e-05, 1.8064e-04, 3.0132e-04, 7.6607e-05, 1.4804e-04,\n        1.1160e-04, 1.6945e-04, 7.4415e-05, 1.7897e-04, 1.4430e-04, 2.2701e-04,\n        7.2679e-05, 1.4158e-04, 5.9421e-05, 8.2895e-05, 2.4154e-03, 1.9134e-04,\n        1.5572e-04, 2.1034e-04, 3.1151e-04, 2.9329e-04, 1.9836e-04, 1.1860e-04,\n        1.5129e-04, 3.5619e-04, 2.2950e-04, 1.5712e-04, 6.3319e-05, 4.1161e-04,\n        1.9249e-04, 9.8454e-05, 2.8807e-04, 7.0494e-05, 4.0732e-04, 3.2331e-04,\n        2.6150e-04, 2.7725e-04, 6.8590e-05, 1.1971e-04, 1.1649e-04, 1.2766e-04,\n        5.8057e-05, 6.5707e-05, 1.2951e-04, 8.7643e-05, 2.8909e-04, 2.2015e-04,\n        3.6886e-04, 1.2738e-04, 2.6121e-04, 3.0625e-04, 9.4860e-05, 2.4026e-04,\n        1.3608e-04, 2.2361e-04, 4.3284e-04, 1.7529e-04, 1.6642e-04, 1.9031e-04,\n        2.7958e-04, 5.8701e-05, 1.5191e-04, 3.2066e-04, 6.0217e-04, 1.6972e-04,\n        2.0954e-04, 1.6809e-04, 5.3898e-05, 2.0483e-04, 5.9105e-05, 1.0472e-04,\n        7.0505e-05, 1.1347e-04, 1.3403e-04, 1.1247e-04, 1.5436e-04, 1.3489e-04,\n        2.4768e-04, 1.5511e-04, 1.0735e-04, 7.4151e-05, 1.2170e-04, 1.5923e-04,\n        1.4368e-04, 2.3768e-04, 2.4812e-03, 1.0645e-04, 7.1618e-05, 1.9742e-04,\n        1.8443e-04, 2.5653e-04, 2.4507e-04, 8.9304e-05, 9.4622e-05, 1.0056e-04,\n        1.3419e-04, 1.0513e-04, 1.2605e-04, 7.5211e-05, 7.3567e-05, 7.4875e-05,\n        1.0064e-04, 2.7057e-04, 2.8412e-04, 1.8315e-04, 6.4606e-05, 2.7980e-04,\n        1.0559e-04, 7.5359e-05, 4.6332e-04, 6.5244e-05, 1.5186e-04, 8.0921e-05,\n        3.9614e-04, 9.9605e-05, 7.5494e-05, 1.0997e-04, 2.1248e-04, 1.2664e-04,\n        1.0550e-04, 2.2367e-04, 9.9130e-05, 1.1113e-04, 1.1531e-04, 1.0632e-04,\n        7.8732e-05, 1.8702e-04, 8.4748e-05, 3.2080e-04, 8.2464e-05, 1.3077e-04,\n        3.5948e-04, 2.7194e-04, 1.4870e-04, 2.3365e-04, 8.1653e-05, 2.4939e-04,\n        1.1436e-04, 1.0761e-04, 2.4647e-04, 2.1444e-04, 1.2388e-04, 1.4816e-04,\n        1.0713e-04, 2.1566e-04, 1.4257e-04, 1.7037e-04, 1.1245e-04, 1.2059e-04,\n        6.7442e-05, 2.1445e-04, 1.4147e-04, 2.8212e-04, 1.1426e-04, 2.4104e-04,\n        1.4538e-04, 1.4252e-04, 6.1039e-04, 5.4014e-05, 2.6926e-04, 1.3527e-04,\n        1.8474e-04, 1.3757e-04, 1.5485e-04, 9.2733e-05, 3.9937e-04, 1.9585e-04,\n        6.9640e-05, 1.2060e-04, 3.6087e-04, 1.3531e-04, 4.2862e-04, 1.0752e-04,\n        1.2040e-04, 1.5481e-04, 1.2437e-04, 1.9867e-04, 1.1680e-04, 7.8240e-05,\n        1.2133e-04, 1.2943e-04, 6.0191e-04, 1.7152e-04, 1.2441e-04, 1.7956e-04,\n        7.3509e-04, 4.5497e-04, 8.4735e-05, 2.1298e-04, 4.0266e-04, 3.9716e-04,\n        1.7718e-04, 1.1842e-04, 1.3510e-04, 1.2169e-04, 6.6433e-05, 1.3985e-04,\n        6.2430e-05, 1.1427e-04, 3.1047e-04, 2.3342e-04, 8.2727e-05, 8.1699e-05,\n        1.2917e-04, 2.9346e-04, 4.0855e-04, 2.0626e-04, 1.8057e-04, 1.1494e-04,\n        1.6015e-04, 1.4048e-04, 1.3986e-04, 1.8875e-04, 3.2352e-04, 1.5949e-04,\n        2.0360e-04, 7.9722e-05, 1.3077e-04, 1.4089e-04, 1.6862e-04, 8.2735e-05,\n        7.0977e-05, 1.3698e-04, 2.6248e-04, 1.1638e-04, 2.2271e-04, 7.5025e-05,\n        4.2461e-04, 2.0224e-04, 8.9539e-05, 1.4861e-04, 7.3316e-05, 1.4185e-04,\n        1.3565e-04, 1.2873e-04, 1.7597e-04, 1.7017e-04, 1.8601e-04, 1.2940e-04,\n        7.2466e-05, 1.5809e-04, 1.5586e-04, 1.3730e-04, 2.9462e-04, 8.8861e-05,\n        1.3039e-04, 5.8148e-05, 2.0440e-04, 1.7340e-04, 1.5535e-04, 1.7691e-04,\n        3.8015e-04, 1.2268e-04, 1.1648e-04, 8.3980e-05, 3.2477e-04, 1.2133e-04,\n        1.8369e-04, 1.6452e-04, 1.0357e-04, 1.0449e-04, 2.4217e-04, 3.3317e-04,\n        6.7364e-04, 2.3208e-04, 1.4507e-04, 1.2203e-04, 2.0030e-04, 1.0926e-04,\n        2.5190e-04, 1.6264e-04, 6.9407e-05, 6.3747e-05, 8.7643e-05, 3.3947e-04,\n        7.8014e-04, 2.1751e-04, 1.3827e-04, 8.7738e-05, 3.7582e-05, 1.5715e-04,\n        1.3173e-04, 1.9397e-04, 1.8367e-04, 8.9917e-05, 7.9770e-04, 8.5582e-05,\n        1.1700e-04, 2.4905e-04, 2.4242e-04, 1.2327e-04, 1.6986e-04, 1.2040e-04,\n        1.2751e-04, 1.3888e-04, 5.5408e-05, 1.7745e-04, 7.4512e-05, 1.1206e-04,\n        3.0559e-04, 1.2102e-04, 1.0945e-04, 3.1121e-04, 1.0501e-04, 1.6946e-04,\n        1.5275e-04, 1.1991e-04, 1.2894e-04, 1.0661e-04, 1.1708e-04, 7.4243e-05,\n        1.3348e-04, 9.3691e-05, 3.8709e-04, 1.9613e-04, 8.5138e-05, 2.1727e-04,\n        8.8291e-05, 3.2449e-04, 6.7651e-05, 2.5943e-04, 1.0345e-04, 1.4483e-04,\n        9.2775e-05, 8.2599e-05, 1.7650e-04, 1.2433e-04, 7.3711e-05, 1.4014e-04,\n        1.2488e-04, 2.4296e-04, 1.1013e-04, 1.4872e-04, 5.2232e-04, 2.9588e-04,\n        1.3659e-04, 3.4872e-04, 1.4100e-04, 9.0251e-05, 1.8604e-04, 2.3255e-04,\n        1.1327e-04, 1.3999e-04, 1.0834e-04, 1.0410e-04, 1.2695e-04, 8.5889e-05,\n        3.2029e-04, 5.6676e-05, 2.1503e-04, 8.7341e-05, 5.7250e-05, 9.4624e-05,\n        1.7058e-04, 9.8517e-05, 4.3194e-05, 8.6369e-05, 4.8434e-04, 9.5384e-05,\n        1.0781e-04, 5.4793e-05, 1.1030e-04, 2.0897e-04, 7.3940e-05, 1.2313e-04,\n        1.2335e-04, 1.1244e-04, 1.6023e-04, 1.8729e-04, 2.8119e-04, 1.1970e-04,\n        9.6049e-05, 1.3218e-04, 1.0065e-04, 2.3341e-04, 8.0732e-05, 1.4550e-04,\n        1.1352e-04, 2.1741e-04, 6.3054e-05, 1.0441e-04, 1.8373e-04, 1.7753e-04,\n        9.0500e-05, 8.6497e-05, 2.1344e-04, 6.1957e-05, 5.5859e-05, 7.5331e-04,\n        1.3015e-04, 8.1969e-05, 2.2020e-04, 1.3554e-04, 2.7500e-04, 2.1112e-04,\n        2.9783e-04, 2.7221e-04, 1.6467e-04, 2.5260e-04, 2.4865e-04, 9.9374e-05,\n        6.9804e-05, 6.3802e-05, 7.9163e-05, 8.8425e-05, 1.4247e-04, 1.4628e-04,\n        9.6777e-05, 4.4151e-04, 1.2857e-04, 5.9857e-05, 1.4777e-04, 4.3918e-04,\n        6.7640e-05, 2.6557e-04, 9.6918e-05, 1.2541e-04, 9.3015e-05, 3.1786e-04,\n        2.1952e-04, 1.2709e-04, 9.9872e-05, 1.4445e-04, 6.4689e-05, 1.1371e-04,\n        1.2351e-04, 1.0004e-04, 2.3855e-04, 2.2650e-04, 6.5044e-05, 4.2228e-04,\n        2.1758e-04, 9.3007e-05, 1.7446e-04, 1.4474e-04, 6.4128e-05, 1.4374e-04,\n        1.3776e-04, 8.7228e-05, 1.3088e-04, 1.4899e-04, 1.1363e-04, 3.1232e-04,\n        1.6978e-04, 7.6281e-05, 8.5188e-05, 2.7622e-04, 2.5461e-04, 2.4154e-04,\n        4.9377e-04, 1.2306e-04, 6.6411e-05, 2.5915e-04, 2.3786e-04, 1.0386e-04,\n        5.4954e-05, 6.7607e-04, 1.4427e-04, 1.5479e-04, 1.9354e-04, 1.0021e-04,\n        2.3526e-04, 6.6357e-05, 9.5213e-04, 2.1014e-04, 1.2436e-04, 7.1788e-05,\n        1.7081e-04, 7.6414e-05, 1.0769e-04, 2.7099e-04, 9.1635e-05, 1.6768e-04,\n        2.7000e-04, 2.2953e-04, 1.4725e-04, 2.3701e-04, 3.8649e-05, 1.2715e-04,\n        8.7632e-05, 4.5906e-05, 2.0277e-04, 1.6722e-04, 1.0800e-04, 2.8050e-04,\n        8.6112e-05, 1.8254e-04, 1.1385e-04, 1.1971e-04, 2.1488e-01, 8.2514e-05,\n        1.7189e-04, 1.3103e-04, 1.2689e-04, 2.7143e-04, 2.5005e-04, 1.4445e-04,\n        1.0616e-04, 2.0308e-04, 9.9873e-05, 1.3417e-04, 9.9695e-05, 1.2627e-04,\n        1.5777e-04, 1.9388e-04, 2.4491e-04, 1.9967e-04, 1.9633e-04, 1.9357e-04,\n        2.1022e-04, 1.5135e-04], device='cuda:0')"
+      },
+      "3": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([ 2.4691e-03, -2.9514e-04, -2.6392e-03,  4.4644e-03,  5.3308e-03,\n        -4.3606e-03,  1.6087e-03,  1.0612e-03, -7.9424e-03, -6.2561e-03,\n        -2.0579e-03,  1.2298e-03, -7.5554e-03,  4.1900e-04,  1.7547e-03,\n        -2.8908e-03, -2.9117e-03, -2.3918e-03,  7.1253e-03, -9.5555e-03,\n         3.8632e-03, -5.1426e-03,  4.0989e-03,  2.0273e-03,  7.2691e-03,\n         1.2780e-02,  1.7304e-03,  6.5428e-03, -6.8895e-03, -2.6203e-03,\n         4.4531e-03,  3.2538e-04,  5.5383e-03,  3.4521e-03,  3.2456e-03,\n        -4.7717e-03, -2.1051e-03,  1.6186e-03,  1.8775e-03, -3.0894e-03,\n         1.3760e-03,  2.5592e-03,  1.3056e-03,  5.4349e-03, -4.9119e-03,\n         1.0809e-02, -2.0732e-03, -1.1279e-02,  5.3524e-03,  1.3967e-03,\n        -1.5785e-03,  2.1301e-03, -4.7680e-03,  1.0304e-03,  2.8816e-03,\n        -2.4907e-03,  3.7129e-03,  1.6576e-03, -1.3117e-03,  1.0712e-03,\n        -1.9386e-03, -4.4981e-03,  4.8922e-03, -1.9491e-03, -3.2388e-03,\n        -1.4366e-03, -1.4577e-03,  4.6205e-03,  1.2414e-03, -1.9656e-03,\n         2.5614e-04, -2.9641e-03, -5.5398e-03,  2.4683e-03,  5.2460e-03,\n         4.7459e-03,  4.5173e-03,  3.4278e-03,  2.3529e-03,  3.9431e-04,\n        -3.3936e-03, -4.3726e-03, -2.4702e-04,  1.6025e-03,  7.3523e-03,\n        -6.5353e-03, -2.6289e-03,  7.1785e-03,  9.4103e-04,  2.9865e-03,\n         5.3276e-03,  6.9689e-03, -2.5779e-03, -4.8816e-04,  1.4112e-03,\n        -1.1196e-03, -2.5012e-03,  5.9378e-04, -3.3199e-03,  2.9446e-03,\n        -5.5374e-04,  8.0771e-04, -8.8576e-05,  5.2237e-04,  1.3004e-03,\n        -3.9151e-03,  4.4798e-03,  1.5151e-04,  2.9105e-04,  1.9246e-03,\n        -6.6297e-04, -2.4437e-03,  5.2591e-03, -2.4867e-04, -3.0074e-04,\n        -1.5791e-03,  5.0484e-05,  3.7855e-04, -1.2120e-03, -1.8724e-03,\n        -2.2496e-03,  1.8015e-03, -1.0541e-03, -2.5969e-05, -3.4359e-03,\n        -2.1873e-03, -1.8676e-04,  1.9479e-03, -2.5667e-05, -2.3950e-03,\n         1.4069e-03, -6.6337e-04, -1.2886e-03,  1.7625e-03,  3.7846e-03,\n         2.0550e-04, -2.8236e-03, -2.0449e-03, -1.7523e-03,  7.5064e-03,\n        -8.5499e-03,  1.1664e-03, -2.3224e-03, -1.7618e-03, -3.2797e-04,\n        -3.4352e-04,  2.3878e-03,  8.9587e-04, -3.6537e-03, -6.3105e-05,\n         1.8949e-03, -4.6285e-03, -4.6641e-04, -1.3612e-02, -9.3565e-05,\n         2.3400e-03, -6.0487e-03,  6.8525e-03,  1.1451e-02, -2.2177e-03,\n        -7.7852e-04, -3.1366e-03, -4.0760e-04,  9.4585e-03, -6.7824e-04,\n        -3.4302e-03,  2.9195e-04, -6.5887e-03, -3.1354e-03,  2.3275e-03,\n         2.4501e-03, -4.7167e-03,  2.4752e-03,  1.8401e-03,  2.4988e-03,\n        -5.8736e-03, -3.5219e-04, -1.0542e-03,  3.9388e-04,  3.9392e-03,\n         8.4775e-05, -6.1759e-03, -9.9912e-03, -1.1083e-02, -2.0244e-03,\n        -9.2312e-04,  3.0346e-04,  5.0851e-04,  2.6659e-03, -1.3440e-03,\n         8.3344e-03,  1.1656e-03,  3.4425e-03,  1.2719e-03, -6.6295e-04,\n        -3.0274e-03, -5.8957e-04, -6.6025e-03,  2.0108e-03, -4.2595e-03,\n        -2.7857e-03,  7.5925e-04, -5.7584e-03,  1.9580e-03,  4.3144e-03,\n        -1.8123e-03,  9.2187e-04, -7.9940e-04,  2.9230e-03, -1.3135e-04,\n        -1.7749e-03, -1.7503e-03, -2.9588e-03,  1.5319e-03, -2.9962e-03,\n         5.6809e-03, -2.4631e-03, -1.7504e-03,  1.5274e-03,  3.7258e-03,\n         3.3931e-03, -6.0933e-03,  2.0080e-03,  9.1082e-03, -2.2627e-04,\n        -2.9904e-03,  4.4553e-03,  4.3895e-03,  3.8855e-03, -7.9389e-03,\n         2.0327e-04, -3.4505e-03,  1.5668e-04,  1.0694e-03, -1.5325e-03,\n         9.5212e-03,  6.9908e-04,  5.4245e-03, -3.3552e-03, -4.2376e-03,\n        -4.9208e-04, -7.1480e-03, -2.6718e-03, -2.3140e-03,  5.9987e-03,\n         3.1996e-03, -6.0167e-03,  4.2308e-04, -8.3146e-04,  3.4867e-03,\n        -3.0494e-03,  2.3664e-03,  1.0661e-03,  5.4049e-03,  4.1317e-03,\n        -1.2461e-03,  1.4739e-03,  3.2664e-03,  4.3852e-03, -2.2834e-03,\n         4.3761e-04,  2.9096e-03,  1.1896e-03, -6.6621e-03,  3.6056e-03,\n        -3.2671e-03, -5.2463e-04,  1.7795e-03,  6.2690e-03,  2.3607e-03,\n         2.1578e-03,  7.4030e-03,  5.7075e-03,  8.9907e-04,  6.6776e-03,\n        -6.4009e-03,  4.0743e-03,  4.7609e-03, -2.2378e-03,  3.5976e-03,\n        -1.9638e-03,  1.0550e-02, -1.3918e-02, -1.1644e-03, -4.3393e-03,\n         1.4365e-03, -1.0007e-03,  2.6189e-03, -9.3278e-04, -4.7678e-03,\n         2.0455e-04, -3.8723e-03, -5.9016e-03,  7.4967e-04, -1.8997e-03,\n        -7.1631e-03,  5.0112e-03, -3.4651e-03,  5.7883e-03,  2.0015e-03,\n         1.4629e-03, -5.7798e-04,  1.2023e-03,  5.1076e-03,  5.1328e-04,\n        -2.0635e-04, -3.6813e-03,  1.8478e-03, -8.4209e-04,  1.2050e-03,\n        -1.6814e-03,  5.2562e-04, -6.7022e-03,  1.3390e-03,  9.1385e-04,\n         2.3576e-04,  1.0638e-04,  6.4684e-03,  2.4054e-03,  3.1940e-03,\n        -8.6030e-04,  8.6705e-03, -4.6967e-03,  1.4521e-04,  4.3661e-03,\n         5.1538e-04,  1.7605e-03,  4.5133e-03,  1.6558e-04,  3.0574e-03,\n         1.4408e-03,  2.2100e-03, -2.0461e-03,  3.7392e-03, -1.8770e-03,\n         3.3404e-03,  4.0822e-03,  8.2738e-03, -3.2735e-03, -7.3730e-03,\n         4.9037e-03,  4.1965e-03, -3.6636e-03, -4.9123e-03, -4.0869e-03,\n         2.0677e-03, -4.5601e-03, -8.0863e-04, -4.4502e-03,  1.2276e-03,\n        -3.5251e-03,  2.3010e-04,  2.6693e-03, -1.6933e-03,  1.9335e-03,\n        -5.4525e-03, -8.5217e-04,  2.7826e-03, -8.5711e-04, -1.0246e-03,\n        -5.0364e-03,  4.0818e-03, -1.2126e-03, -1.4939e-04,  2.6339e-03,\n         4.6168e-03, -2.3347e-03,  9.6412e-03,  1.9839e-03,  9.0857e-03,\n         3.1109e-03, -2.5779e-03,  1.8361e-03, -2.9194e-04,  4.9699e-04,\n        -2.1183e-02, -5.2489e-04, -3.3468e-04,  8.3982e-03,  8.2383e-04,\n         6.2577e-04, -1.8959e-03,  4.7941e-03, -1.5009e-03,  3.7682e-03,\n        -1.9705e-03,  3.8849e-03,  6.4346e-03,  8.5642e-03,  7.7239e-03,\n         2.7950e-03,  2.0380e-03,  3.6404e-03,  4.6874e-03, -5.6297e-03,\n         1.3348e-03, -6.1639e-03, -4.9308e-03, -1.9406e-03,  4.3429e-03,\n         5.8927e-04, -9.9921e-04, -1.0535e-03,  1.3865e-03, -1.2872e-06,\n         2.1548e-03, -3.1502e-03,  6.2435e-04, -9.9891e-04,  4.6287e-03,\n         1.5641e-03, -1.5335e-03, -1.4563e-02, -8.7868e-04, -2.1042e-03,\n         6.8296e-03, -3.8179e-03,  2.9512e-03, -2.9602e-03,  9.1511e-03,\n        -2.4954e-03,  4.5986e-03, -1.8846e-03, -5.2058e-03, -3.5966e-03,\n        -4.2738e-03,  3.8633e-03, -3.3684e-04, -9.6070e-03,  3.0539e-03,\n         8.6687e-04, -2.7940e-03,  1.3193e-03, -1.4864e-03,  1.0872e-03,\n        -1.9697e-03,  1.5710e-03,  1.2525e-03,  6.5949e-03,  3.5485e-03,\n         2.4158e-03, -7.9095e-05, -1.7049e-03,  7.5840e-04, -2.2604e-03,\n        -4.1440e-03, -2.3710e-03, -1.0575e-03,  1.7944e-03, -7.0967e-03,\n         2.0217e-03, -4.4480e-03,  2.4646e-03, -5.0729e-03, -1.3922e-04,\n        -2.9222e-03,  3.7787e-03, -1.1944e-02, -5.4572e-03,  1.0333e-02,\n         3.8053e-03, -1.7916e-03, -2.8007e-03, -4.0451e-04, -2.4421e-03,\n        -3.5835e-03,  6.4496e-03, -1.9923e-03, -2.7364e-04, -4.4722e-03,\n        -5.2968e-03,  8.4558e-03, -2.1825e-03,  1.8021e-03,  6.2548e-03,\n        -9.2511e-03,  3.4799e-03, -2.8480e-03, -5.2631e-03, -2.2042e-03,\n         1.4023e-03,  7.6528e-06,  1.9616e-03, -2.1077e-03, -4.6826e-04,\n        -1.0020e-03,  1.3064e-03, -7.6680e-04, -2.3281e-03, -9.6983e-04,\n        -2.8214e-03, -4.9421e-03, -8.9221e-04,  6.9828e-04, -6.4953e-03,\n        -3.3464e-03, -4.0591e-04, -3.0841e-03, -4.4721e-04,  9.0263e-03,\n        -4.9061e-03,  4.6443e-03,  6.6478e-03,  4.5561e-03, -9.0008e-03,\n        -1.2495e-02,  4.8471e-03,  7.7134e-04,  6.4721e-03, -7.0910e-04,\n         2.5393e-03, -6.0683e-04], device='cuda:0')",
+        "exp_avg_sq": "tensor([2.2644e-04, 1.5893e-04, 1.5593e-04, 1.8242e-04, 1.4178e-04, 1.4900e-04,\n        1.2659e-04, 1.4832e-04, 1.5256e-04, 2.1102e-04, 1.1837e-04, 2.1697e-04,\n        1.0910e-04, 1.3473e-04, 1.0257e-04, 1.1631e-04, 2.8727e-04, 1.1734e-04,\n        1.3260e-04, 1.4359e-04, 1.8028e-04, 2.1168e-04, 1.7262e-04, 1.4135e-04,\n        2.1728e-04, 2.4181e-04, 1.8724e-04, 1.3317e-04, 1.1492e-04, 1.5439e-04,\n        1.5439e-04, 1.7000e-04, 2.6574e-04, 1.0371e-04, 2.2030e-04, 1.8686e-04,\n        1.6472e-04, 1.9180e-04, 9.6445e-05, 1.1994e-04, 1.5171e-04, 1.2693e-04,\n        1.0752e-04, 1.1899e-04, 1.5367e-04, 1.5031e-04, 3.3936e-04, 2.2621e-04,\n        1.9738e-04, 1.5614e-04, 1.5174e-04, 2.0573e-04, 1.3629e-04, 1.3655e-04,\n        2.1801e-04, 2.2135e-04, 2.0439e-04, 1.8901e-04, 1.4616e-04, 1.5484e-04,\n        2.2023e-04, 1.2406e-04, 2.0062e-04, 1.7812e-04, 2.0969e-04, 1.3117e-04,\n        1.7272e-04, 1.8485e-04, 1.5115e-04, 2.3841e-04, 1.4122e-04, 1.1269e-04,\n        1.7031e-04, 1.1304e-04, 1.4869e-04, 2.2781e-04, 1.6124e-04, 1.5375e-04,\n        2.0547e-04, 2.3182e-04, 2.0848e-04, 1.1177e-04, 1.9282e-04, 1.9538e-04,\n        1.4121e-04, 1.7740e-04, 2.5977e-04, 2.0495e-04, 1.0088e-04, 2.0294e-04,\n        1.5735e-04, 1.8131e-04, 1.6759e-04, 1.3585e-04, 1.5359e-04, 1.4157e-04,\n        1.9486e-04, 1.6404e-04, 1.5740e-04, 1.2361e-04, 9.3858e-05, 1.0615e-04,\n        1.4697e-04, 1.3717e-04, 1.9471e-04, 1.5904e-04, 1.5596e-04, 1.6710e-04,\n        1.6094e-04, 1.2088e-04, 1.6346e-04, 1.2491e-04, 1.6893e-04, 1.0631e-04,\n        1.9863e-04, 1.1071e-04, 9.5163e-05, 2.0617e-04, 1.4458e-04, 1.3158e-04,\n        9.7717e-05, 1.2708e-04, 1.2946e-04, 1.6363e-04, 1.6736e-04, 1.4068e-04,\n        1.2935e-04, 1.5347e-04, 1.2879e-04, 1.6953e-04, 1.2495e-04, 1.2861e-04,\n        2.1045e-04, 1.4993e-04, 1.6051e-04, 1.5485e-04, 1.0695e-04, 1.2881e-04,\n        1.6704e-04, 1.6643e-04, 2.5123e-04, 1.5185e-04, 1.0844e-04, 1.9759e-04,\n        1.5132e-04, 1.7199e-04, 1.6257e-04, 1.7469e-04, 1.5078e-04, 1.7919e-04,\n        9.5585e-05, 1.8443e-04, 1.2022e-04, 1.8988e-04, 9.3202e-05, 1.6953e-04,\n        1.8007e-04, 1.6402e-04, 2.4301e-04, 9.7702e-05, 1.7031e-04, 1.3454e-04,\n        1.8716e-04, 1.2885e-04, 1.8035e-04, 1.2102e-04, 1.5941e-04, 1.3541e-04,\n        1.1586e-04, 1.6645e-04, 2.8105e-04, 1.3643e-04, 1.8089e-04, 1.1214e-04,\n        1.4827e-04, 1.1802e-04, 1.3119e-04, 1.6992e-04, 9.9779e-05, 1.1620e-04,\n        1.6132e-04, 2.4842e-04, 1.9334e-04, 1.7952e-04, 2.0701e-04, 1.5782e-04,\n        1.8857e-04, 1.9636e-04, 1.1700e-04, 1.5471e-04, 1.6888e-04, 1.6278e-04,\n        2.4305e-04, 1.3201e-04, 1.4278e-04, 1.2890e-04, 9.3112e-05, 1.6284e-04,\n        1.0713e-04, 2.5009e-04, 1.6580e-04, 2.2349e-04, 9.6136e-05, 1.3731e-04,\n        1.4535e-04, 1.7905e-04, 1.7649e-04, 1.2632e-04, 1.7463e-04, 1.6114e-04,\n        1.1165e-04, 1.9928e-04, 1.9705e-04, 2.5655e-04, 2.3852e-04, 1.2802e-04,\n        1.7418e-04, 8.4380e-05, 1.2064e-04, 1.5477e-04, 1.4437e-04, 1.4924e-04,\n        1.5455e-04, 1.9258e-04, 1.3567e-04, 1.3687e-04, 1.5748e-04, 1.1592e-04,\n        2.6692e-04, 2.5307e-04, 1.3857e-04, 1.3104e-04, 1.3083e-04, 1.2052e-04,\n        1.5152e-04, 1.9718e-04, 1.9179e-04, 2.1796e-04, 1.7808e-04, 2.1928e-04,\n        1.2851e-04, 1.3912e-04, 1.8297e-04, 1.3899e-04, 1.7063e-04, 1.4872e-04,\n        1.3020e-04, 9.3617e-05, 1.4704e-04, 2.3976e-04, 1.2202e-04, 1.5341e-04,\n        1.3760e-04, 2.6831e-04, 1.5377e-04, 1.3616e-04, 3.7799e-04, 1.5253e-04,\n        1.7878e-04, 1.8341e-04, 1.3556e-04, 2.0660e-04, 1.4619e-04, 1.6913e-04,\n        1.9716e-04, 1.9436e-04, 1.1771e-04, 1.6986e-04, 1.1980e-04, 1.2481e-04,\n        1.4007e-04, 1.1186e-04, 1.1866e-04, 1.0937e-04, 1.2778e-04, 3.1341e-04,\n        1.7399e-04, 1.7813e-04, 2.4939e-04, 1.4017e-04, 7.7480e-05, 1.9466e-04,\n        1.6646e-04, 1.3449e-04, 2.4993e-04, 1.2927e-04, 2.6908e-04, 1.1622e-04,\n        1.6053e-04, 1.8373e-04, 2.1240e-04, 1.8959e-04, 1.9604e-04, 1.5954e-04,\n        2.2103e-04, 1.7868e-04, 1.2397e-04, 2.0949e-04, 1.5557e-04, 2.1824e-04,\n        2.4296e-04, 1.2168e-04, 1.3239e-04, 1.9246e-04, 1.1092e-04, 1.5051e-04,\n        1.6360e-04, 1.1507e-04, 1.7391e-04, 1.4079e-04, 1.3157e-04, 1.1346e-04,\n        1.1376e-04, 1.0858e-04, 1.7988e-04, 1.9075e-04, 8.9054e-05, 2.1189e-04,\n        1.3022e-04, 1.7928e-04, 1.1655e-04, 2.6711e-04, 1.5064e-04, 1.3512e-04,\n        1.1831e-04, 1.1465e-04, 1.4158e-04, 1.6390e-04, 7.9651e-05, 1.4716e-04,\n        1.7733e-04, 1.9769e-04, 1.2856e-04, 1.2284e-04, 2.0833e-04, 1.7079e-04,\n        1.3151e-04, 2.3318e-04, 1.8850e-04, 1.4651e-04, 1.6278e-04, 1.9415e-04,\n        1.2274e-04, 1.2868e-04, 1.4217e-04, 1.6911e-04, 1.1165e-04, 1.2021e-04,\n        1.9591e-04, 7.1375e-05, 1.9717e-04, 1.6114e-04, 8.3363e-05, 1.3764e-04,\n        1.3403e-04, 1.3228e-04, 1.0274e-04, 1.2946e-04, 2.3287e-04, 1.0977e-04,\n        1.2939e-04, 9.0019e-05, 1.2987e-04, 1.3441e-04, 9.8624e-05, 1.4456e-04,\n        1.1034e-04, 2.2856e-04, 1.2646e-04, 1.2640e-04, 1.6487e-04, 2.1449e-04,\n        1.5211e-04, 1.5327e-04, 1.2464e-04, 1.6685e-04, 1.2921e-04, 1.3880e-04,\n        1.5316e-04, 1.4125e-04, 8.6375e-05, 1.3670e-04, 1.6637e-04, 1.9203e-04,\n        1.4838e-04, 8.8215e-05, 1.3989e-04, 1.0954e-04, 1.3053e-04, 2.0737e-04,\n        1.5755e-04, 1.5909e-04, 1.6383e-04, 1.2731e-04, 1.7095e-04, 1.9492e-04,\n        1.6564e-04, 1.6392e-04, 2.4950e-04, 1.3285e-04, 1.2700e-04, 1.2951e-04,\n        1.1858e-04, 1.2144e-04, 1.2599e-04, 1.6057e-04, 1.4365e-04, 1.1855e-04,\n        1.4670e-04, 1.9766e-04, 1.5330e-04, 1.1267e-04, 2.0198e-04, 2.2647e-04,\n        1.3164e-04, 1.6071e-04, 1.1646e-04, 1.0286e-04, 1.7236e-04, 1.7584e-04,\n        1.8085e-04, 1.3126e-04, 1.5066e-04, 1.8640e-04, 1.5362e-04, 1.2609e-04,\n        2.1058e-04, 1.4082e-04, 1.5209e-04, 1.4212e-04, 1.2651e-04, 1.9262e-04,\n        1.6564e-04, 2.7624e-04, 1.2548e-04, 1.8109e-04, 1.1132e-04, 2.0313e-04,\n        1.8399e-04, 1.4063e-04, 1.6665e-04, 2.3458e-04, 1.4313e-04, 1.7556e-04,\n        1.3547e-04, 1.1028e-04, 1.4270e-04, 1.6627e-04, 1.7454e-04, 1.7525e-04,\n        1.7090e-04, 1.5597e-04, 9.6889e-05, 2.1626e-04, 1.8557e-04, 1.7563e-04,\n        1.1044e-04, 2.3563e-04, 1.6255e-04, 1.8569e-04, 1.9619e-04, 1.7752e-04,\n        1.6695e-04, 1.3142e-04, 1.6943e-04, 1.8515e-04, 1.3052e-04, 1.1851e-04,\n        1.4364e-04, 1.4317e-04, 1.3745e-04, 1.7792e-04, 1.6246e-04, 1.8092e-04,\n        1.6097e-04, 1.5560e-04, 1.4954e-04, 1.9613e-04, 7.8743e-05, 1.2656e-04,\n        1.0866e-04, 9.4973e-05, 1.5177e-04, 1.3930e-04, 1.2234e-04, 1.8931e-04,\n        9.4800e-05, 1.4709e-04, 1.9428e-04, 1.6073e-04, 5.7027e-04, 1.3043e-04,\n        1.7456e-04, 1.3455e-04, 1.9164e-04, 2.2191e-04, 2.3232e-04, 1.9201e-04,\n        1.4890e-04, 1.8182e-04, 1.2166e-04, 1.3647e-04, 1.5761e-04, 1.2079e-04,\n        1.8538e-04, 2.3328e-04, 1.8736e-04, 1.4366e-04, 1.6800e-04, 1.5039e-04,\n        2.2273e-04, 1.3143e-04], device='cuda:0')"
+      },
+      "4": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([[-3.0450e-04, -4.0247e-05,  4.3720e-05,  ...,  1.4158e-04,\n         -1.0961e-04,  1.0093e-04],\n        [ 6.7722e-05, -7.3806e-05, -9.9561e-05,  ..., -9.8573e-05,\n          6.4143e-05,  3.6865e-04],\n        [ 2.8943e-04, -3.4509e-04,  6.7326e-06,  ...,  1.1656e-04,\n         -3.4284e-05,  5.1277e-04],\n        ...,\n        [-2.0439e-05,  3.1380e-04,  8.8418e-05,  ..., -8.4914e-05,\n          6.8881e-05,  9.0541e-05],\n        [ 2.8314e-04, -1.2042e-04,  1.0266e-04,  ..., -2.9992e-04,\n          5.5598e-05, -2.2718e-04],\n        [ 1.3921e-04,  2.5525e-04,  6.7605e-05,  ..., -2.0436e-04,\n          2.4153e-04, -7.9088e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[3.0209e-07, 4.8766e-07, 9.8404e-08,  ..., 2.1160e-07, 2.3391e-07,\n         3.7056e-07],\n        [6.0518e-07, 1.1282e-06, 3.4573e-07,  ..., 6.9057e-07, 6.9243e-07,\n         9.8534e-07],\n        [7.2916e-07, 3.6830e-07, 3.0298e-07,  ..., 7.3898e-07, 3.7777e-07,\n         7.1135e-07],\n        ...,\n        [4.1800e-07, 1.0789e-06, 4.8219e-07,  ..., 5.6874e-07, 8.4207e-07,\n         7.6590e-07],\n        [4.2307e-07, 8.1034e-07, 2.3789e-07,  ..., 7.6288e-07, 4.8583e-07,\n         5.7039e-07],\n        [3.6727e-07, 5.6250e-07, 3.3988e-07,  ..., 1.2048e-06, 4.1510e-07,\n         7.7295e-07]], device='cuda:0')"
+      },
+      "5": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([[ 1.9237e-05, -1.6313e-04, -5.5503e-05,  ..., -6.2872e-05,\n         -8.4307e-05,  7.5269e-05],\n        [ 2.5058e-06, -4.6338e-05, -8.4680e-05,  ..., -1.1010e-04,\n          1.9623e-07,  1.1131e-04],\n        [ 1.7064e-04, -1.5162e-04,  9.7683e-05,  ...,  1.0028e-04,\n         -1.5760e-05,  2.1304e-04],\n        ...,\n        [ 1.3811e-04,  2.9728e-05,  2.0740e-06,  ..., -2.3390e-04,\n          1.5782e-04, -2.2252e-04],\n        [-8.9918e-05,  2.6170e-05,  1.8475e-05,  ...,  1.4851e-04,\n          4.2041e-05,  2.9234e-04],\n        [ 2.1163e-05,  6.5565e-05,  2.4031e-04,  ..., -1.1673e-04,\n          2.4081e-05, -1.1068e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[1.0397e-07, 8.8353e-08, 9.2843e-08,  ..., 1.1043e-07, 8.3492e-08,\n         1.3036e-07],\n        [2.0261e-07, 4.3267e-07, 1.7080e-07,  ..., 1.7205e-07, 2.3733e-07,\n         2.8634e-07],\n        [2.6645e-07, 1.5383e-07, 1.9638e-07,  ..., 3.5662e-07, 3.0237e-07,\n         2.4617e-07],\n        ...,\n        [2.0093e-07, 5.5840e-07, 8.7594e-08,  ..., 4.4692e-07, 2.9483e-07,\n         2.7520e-07],\n        [2.2474e-07, 3.0743e-07, 7.4642e-08,  ..., 3.5369e-07, 2.0328e-07,\n         2.4506e-07],\n        [2.3708e-07, 1.6579e-07, 4.2371e-07,  ..., 1.9914e-07, 1.6557e-07,\n         2.5361e-07]], device='cuda:0')"
+      },
+      "6": {
+        "step": "tensor(10016.)",
+        "exp_avg": "tensor([-0.0008,  0.0008], device='cuda:0')",
+        "exp_avg_sq": "tensor([7.8511e-06, 7.8511e-06], device='cuda:0')"
+      }
+    },
+    "param_groups": [
+      {
+        "lr": 9.639601130971382e-05,
+        "name": "shared",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.001,
+        "params": [
+          0,
+          1,
+          2,
+          3
+        ]
+      },
+      {
+        "lr": 9.639601130971382e-05,
+        "name": "scale_256",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.001,
+        "params": [
+          4
+        ]
+      },
+      {
+        "lr": 9.639601130971382e-05,
+        "name": "scale_512",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.001,
+        "params": [
+          5
+        ]
+      },
+      {
+        "lr": 4.865025990345063e-05,
+        "name": "fusion",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0005,
+        "params": [
+          6
+        ]
+      }
+    ]
+  },
+  "scheduler_state_dict": {
+    "T_0": 10,
+    "T_i": 10,
+    "T_mult": 2,
+    "eta_min": 1e-06,
+    "T_cur": 8,
+    "base_lrs": [
+      0.001,
+      0.001,
+      0.001,
+      0.0005
+    ],
+    "last_epoch": 8,
+    "_step_count": 0,
+    "_is_initial": false,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+      9.639601130971382e-05,
+      9.639601130971382e-05,
+      9.639601130971382e-05,
+      4.865025990345063e-05
+    ]
+  },
+  "metrics": {
+    "best_val_acc": 71.776,
+    "best_epoch": 7,
+    "scale_accuracies": {
+      "256": 71.486,
+      "512": 71.694
+    },
+    "training_history": {
+      "epochs": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8
+      ],
+      "train_loss": [
+        5.603825211905824,
+        4.145695094292918,
+        3.767625377201044,
+        3.572572173021091,
+        3.45402966711087,
+        3.3684399169854844,
+        3.3046513376906277,
+        3.255301171407913
+      ],
+      "train_acc": [
+        63.39829233815732,
+        69.48282308239285,
+        70.34266414917025,
+        70.91120829681064,
+        71.35642738222262,
+        71.84090754757186,
+        72.29198067074785,
+        72.7457856782137
+      ],
+      "val_acc": [
+        68.056,
+        69.624,
+        70.142,
+        70.622,
+        70.966,
+        71.462,
+        71.766,
+        71.776
+      ],
+      "scale_accs": {
+        "256": [
+          66.924,
+          68.828,
+          69.54,
+          69.996,
+          70.468,
+          71.106,
+          71.436,
+          71.486
+        ],
+        "512": [
+          67.882,
+          69.572,
+          70.028,
+          70.482,
+          70.854,
+          71.28,
+          71.578,
+          71.694
+        ]
+      },
+      "lr": [
+        0.0009755527298894294,
+        0.0009046039886902864,
+        0.0007940987335200904,
+        0.0006548539886902864,
+        0.0005005000000000001,
+        0.0003461460113097139,
+        0.00020690126647990973,
+        9.639601130971382e-05
+      ]
+    }
+  },
+  "train_config": {
+    "name": "david_training",
+    "run_id": "20251012_135249",
+    "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
+    "model_variant": "clip_vit_laion_b32",
+    "num_classes": 1000,
+    "preset": "small_fast",
+    "custom_config_path": null,
+    "num_classes_override": null,
+    "use_belly_override": null,
+    "belly_expand_override": null,
+    "progressive_training_override": false,
+    "scale_warmup_epochs_override": null,
+    "num_epochs": 10,
+    "batch_size": 1024,
+    "learning_rate": 0.001,
+    "weight_decay": 1e-05,
+    "warmup_epochs": 3,
+    "use_rose_loss": true,
+    "rose_initial_weight": 0.1,
+    "rose_max_weight": 0.5,
+    "rose_weight_schedule": "adaptive",
+    "use_cayley_loss": false,
+    "cayley_weight": 0.001,
+    "scale_loss_balance": null,
+    "use_mixed_precision": false,
+    "gradient_clip": 10.0,
+    "scheduler_type": "cosine_restarts",
+    "min_lr": 1e-06,
+    "freeze_strategy": "never",
+    "freeze_threshold": 90.0,
+    "unfreeze_on_plateau": true,
+    "patience": 10,
+    "track_gradients": true,
+    "gradient_scale_threshold": 1e-05,
+    "gradient_scale_multiplier": 10.0,
+    "log_interval": 50,
+    "val_interval": 1,
+    "save_interval": 5,
+    "log_fusion_weights": true,
+    "log_loss_components": true,
+    "save_format": "safetensors",
+    "hf_repo": "AbstractPhil/gated-david",
+    "upload_to_hub": true,
+    "base_dir": "./david_training",
+    "num_workers": 10,
+    "pin_memory": true,
+    "prefetch_factor": 4,
+    "persistent_workers": true
+  }
+}

weights/David-fully_shared-weighted_sum/20251012_135249/training_history.json

@@ -6,7 +6,8 @@
     4,
     5,
     6,
-    7
+    7,
+    8
   ],
   "train_loss": [
     5.603825211905824,
@@ -15,7 +16,8 @@
     3.572572173021091,
     3.45402966711087,
     3.3684399169854844,
-    3.3046513376906277
+    3.3046513376906277,
+    3.255301171407913
   ],
   "train_acc": [
     63.39829233815732,
@@ -24,7 +26,8 @@
     70.91120829681064,
     71.35642738222262,
     71.84090754757186,
-    72.29198067074785
+    72.29198067074785,
+    72.7457856782137
   ],
   "val_acc": [
     68.056,
@@ -33,7 +36,8 @@
     70.622,
     70.966,
     71.462,
-    71.766
+    71.766,
+    71.776
   ],
   "scale_accs": {
     "256": [
@@ -43,7 +47,8 @@
       69.996,
       70.468,
       71.106,
-      71.436
+      71.436,
+      71.486
     ],
     "512": [
       67.882,
@@ -52,7 +57,8 @@
       70.482,
       70.854,
       71.28,
-      71.578
+      71.578,
+      71.694
     ]
   },
   "lr": [
@@ -62,6 +68,7 @@
     0.0006548539886902864,
     0.0005005000000000001,
     0.0003461460113097139,
-    0.00020690126647990973
+    0.00020690126647990973,
+    9.639601130971382e-05
   ]
 }