Ship H16-seed1 (all arms retained)

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/benchmark_revisions.json

@@ -0,0 +1,10 @@
+{
+  "mteb/biosses-sts": "f4f1265a7a4b0c99e6da09859b887bdb1f36d8ed",
+  "mteb/sickr-sts": "b93b4e08b707e9c9bc478e543495c3ced0b9d81e",
+  "mteb/sts12-sts": "3a51344839ec620dd3b753e0945bdf6c7e37af3e",
+  "mteb/sts13-sts": "f475812ca75868066186343ebf60e01f95c75164",
+  "mteb/sts14-sts": "8ab5e540b20b58d42fa6b944d3accee9df3fbaca",
+  "mteb/sts15-sts": "ae8e1ecbd3b3bcf5c1c8fd798290033715d98b93",
+  "mteb/sts16-sts": "d8e3be175a943eecfea09c96795087771996197c",
+  "mteb/stsbenchmark-sts": "96943a16ea6a35129e253c659081cb59daf81b30"
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/config.json

@@ -0,0 +1,5871 @@
+{
+  "architectures": [
+    "CaptionBertModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_captionbert.CaptionBertConfig",
+    "AutoModel": "modeling_captionbert.CaptionBertModel"
+  },
+  "continuation_ids": [
+    2015,
+    2050,
+    2063,
+    2072,
+    2075,
+    2078,
+    2080,
+    2094,
+    2098,
+    2099,
+    2100,
+    2102,
+    2121,
+    2135,
+    2140,
+    2213,
+    2226,
+    2229,
+    2232,
+    2239,
+    2243,
+    2271,
+    2278,
+    2290,
+    2319,
+    2361,
+    2368,
+    2378,
+    2386,
+    2389,
+    2401,
+    2475,
+    2480,
+    2483,
+    2487,
+    2497,
+    2509,
+    2527,
+    2532,
+    2545,
+    2546,
+    2549,
+    2571,
+    2575,
+    2581,
+    2594,
+    2595,
+    2615,
+    2618,
+    2620,
+    2629,
+    2638,
+    2666,
+    2669,
+    2683,
+    2692,
+    2696,
+    2705,
+    2721,
+    2791,
+    2818,
+    2819,
+    2850,
+    2854,
+    2860,
+    2863,
+    2869,
+    2884,
+    2890,
+    2891,
+    2906,
+    2912,
+    2923,
+    2937,
+    2953,
+    2964,
+    2989,
+    3012,
+    3022,
+    3051,
+    3064,
+    3070,
+    3077,
+    3085,
+    3089,
+    3111,
+    3122,
+    3126,
+    3148,
+    3170,
+    3207,
+    3211,
+    3215,
+    3217,
+    3238,
+    3240,
+    3258,
+    3270,
+    3286,
+    3334,
+    3351,
+    3363,
+    3366,
+    3367,
+    3370,
+    3372,
+    3385,
+    3388,
+    3401,
+    3406,
+    3436,
+    3468,
+    3489,
+    3490,
+    3501,
+    3508,
+    3511,
+    3512,
+    3527,
+    3540,
+    3549,
+    3550,
+    3560,
+    3567,
+    3593,
+    3597,
+    3600,
+    3619,
+    3630,
+    3654,
+    3669,
+    3672,
+    3676,
+    3678,
+    3683,
+    3686,
+    3695,
+    3702,
+    3709,
+    3723,
+    3726,
+    3736,
+    3762,
+    3771,
+    3775,
+    3790,
+    3877,
+    3900,
+    3917,
+    3973,
+    3981,
+    3989,
+    3993,
+    3995,
+    4014,
+    4017,
+    4048,
+    4059,
+    4063,
+    4095,
+    4103,
+    4115,
+    4135,
+    4140,
+    4143,
+    4160,
+    4168,
+    4173,
+    4179,
+    4181,
+    4183,
+    4213,
+    4215,
+    4221,
+    4226,
+    4232,
+    4244,
+    4246,
+    4263,
+    4270,
+    4305,
+    4313,
+    4328,
+    4355,
+    4360,
+    4371,
+    4381,
+    4383,
+    4402,
+    4430,
+    4478,
+    4492,
+    4502,
+    4509,
+    4523,
+    4570,
+    4571,
+    4576,
+    4580,
+    4588,
+    4590,
+    4609,
+    4630,
+    4645,
+    4648,
+    4665,
+    4667,
+    4674,
+    4691,
+    4697,
+    4710,
+    4726,
+    4747,
+    4757,
+    4765,
+    4779,
+    4783,
+    4801,
+    4817,
+    4818,
+    4819,
+    4842,
+    4859,
+    4877,
+    4886,
+    4887,
+    4890,
+    4892,
+    4904,
+    4917,
+    4939,
+    4948,
+    4974,
+    5004,
+    5007,
+    5017,
+    5051,
+    5054,
+    5063,
+    5084,
+    5092,
+    5104,
+    5130,
+    5134,
+    5149,
+    5162,
+    5172,
+    5178,
+    5243,
+    5244,
+    5267,
+    5280,
+    5283,
+    5289,
+    5302,
+    5311,
+    5313,
+    5332,
+    5339,
+    5349,
+    5358,
+    5369,
+    5397,
+    5400,
+    5403,
+    5420,
+    5422,
+    5428,
+    5480,
+    5484,
+    5488,
+    5498,
+    5521,
+    5524,
+    5555,
+    5558,
+    5562,
+    5575,
+    5582,
+    5596,
+    5602,
+    5603,
+    5605,
+    5620,
+    5634,
+    5638,
+    5643,
+    5644,
+    5649,
+    5654,
+    5657,
+    5666,
+    5668,
+    5669,
+    5677,
+    5685,
+    5686,
+    5705,
+    5714,
+    5732,
+    5737,
+    5740,
+    5753,
+    5794,
+    5802,
+    5804,
+    5809,
+    5831,
+    5833,
+    5844,
+    5856,
+    5867,
+    5874,
+    5880,
+    5886,
+    5897,
+    5910,
+    5937,
+    5946,
+    5963,
+    5974,
+    5999,
+    6024,
+    6026,
+    6030,
+    6038,
+    6072,
+    6096,
+    6129,
+    6132,
+    6137,
+    6155,
+    6169,
+    6182,
+    6190,
+    6198,
+    6199,
+    6200,
+    6212,
+    6216,
+    6238,
+    6277,
+    6279,
+    6290,
+    6292,
+    6299,
+    6305,
+    6313,
+    6321,
+    6340,
+    6342,
+    6371,
+    6374,
+    6392,
+    6442,
+    6444,
+    6447,
+    6455,
+    6460,
+    6483,
+    6491,
+    6494,
+    6499,
+    6508,
+    6525,
+    6528,
+    6553,
+    6558,
+    6559,
+    6562,
+    6582,
+    6588,
+    6590,
+    6591,
+    6593,
+    6610,
+    6632,
+    6633,
+    6648,
+    6651,
+    6657,
+    6672,
+    6673,
+    6675,
+    6679,
+    6692,
+    6711,
+    6761,
+    6767,
+    6774,
+    6776,
+    6777,
+    6779,
+    6784,
+    6790,
+    6799,
+    6806,
+    6810,
+    6820,
+    6824,
+    6826,
+    6834,
+    6843,
+    6844,
+    6850,
+    6856,
+    6858,
+    6862,
+    6873,
+    6894,
+    6895,
+    6906,
+    6914,
+    6916,
+    6924,
+    6935,
+    6961,
+    6962,
+    6968,
+    6977,
+    6979,
+    7011,
+    7028,
+    7033,
+    7054,
+    7062,
+    7066,
+    7068,
+    7076,
+    7088,
+    7092,
+    7096,
+    7100,
+    7103,
+    7106,
+    7113,
+    7138,
+    7140,
+    7159,
+    7174,
+    7178,
+    7184,
+    7228,
+    7229,
+    7231,
+    7245,
+    7265,
+    7274,
+    7277,
+    7292,
+    7295,
+    7301,
+    7315,
+    7317,
+    7347,
+    7352,
+    7361,
+    7363,
+    7368,
+    7373,
+    7377,
+    7382,
+    7384,
+    7389,
+    7393,
+    7405,
+    7406,
+    7413,
+    7416,
+    7430,
+    7432,
+    7442,
+    7446,
+    7447,
+    7474,
+    7476,
+    7485,
+    7507,
+    7512,
+    7520,
+    7529,
+    7542,
+    7556,
+    7559,
+    7583,
+    7585,
+    7606,
+    7616,
+    7629,
+    7630,
+    7646,
+    7652,
+    7662,
+    7666,
+    7677,
+    7685,
+    7690,
+    7693,
+    7698,
+    7699,
+    7712,
+    7716,
+    7724,
+    7730,
+    7737,
+    7741,
+    7747,
+    7770,
+    7810,
+    7811,
+    7828,
+    7834,
+    7840,
+    7847,
+    7849,
+    7853,
+    7856,
+    7869,
+    7874,
+    7875,
+    7878,
+    7895,
+    7898,
+    7903,
+    7911,
+    7913,
+    7934,
+    7941,
+    7946,
+    7951,
+    7952,
+    7959,
+    7962,
+    7971,
+    7974,
+    7983,
+    7999,
+    8002,
+    8004,
+    8010,
+    8017,
+    8022,
+    8024,
+    8029,
+    8034,
+    8043,
+    8049,
+    8059,
+    8067,
+    8082,
+    8091,
+    8093,
+    8095,
+    8113,
+    8126,
+    8151,
+    8156,
+    8159,
+    8162,
+    8163,
+    8167,
+    8180,
+    8189,
+    8193,
+    8197,
+    8202,
+    8229,
+    8231,
+    8237,
+    8238,
+    8261,
+    8270,
+    8286,
+    8296,
+    8303,
+    8322,
+    8337,
+    8376,
+    8400,
+    8405,
+    8411,
+    8428,
+    8445,
+    8447,
+    8449,
+    8450,
+    8454,
+    8458,
+    8462,
+    8464,
+    8485,
+    8486,
+    8490,
+    8496,
+    8516,
+    8520,
+    8523,
+    8524,
+    8525,
+    8528,
+    8540,
+    8545,
+    8551,
+    8553,
+    8557,
+    8566,
+    8569,
+    8571,
+    8583,
+    8584,
+    8585,
+    8586,
+    8591,
+    8602,
+    8609,
+    8613,
+    8621,
+    8625,
+    8630,
+    8649,
+    8653,
+    8654,
+    8661,
+    8662,
+    8663,
+    8671,
+    8684,
+    8706,
+    8713,
+    8717,
+    8718,
+    8737,
+    8743,
+    8747,
+    8751,
+    8757,
+    8758,
+    8780,
+    8787,
+    8791,
+    8820,
+    8838,
+    8844,
+    8865,
+    8873,
+    8883,
+    8889,
+    8898,
+    8913,
+    8918,
+    8939,
+    8943,
+    8950,
+    8953,
+    8977,
+    8978,
+    8988,
+    9001,
+    9006,
+    9013,
+    9021,
+    9024,
+    9028,
+    9031,
+    9032,
+    9035,
+    9048,
+    9050,
+    9057,
+    9067,
+    9071,
+    9072,
+    9077,
+    9080,
+    9091,
+    9096,
+    9102,
+    9103,
+    9126,
+    9142,
+    9148,
+    9153,
+    9189,
+    9215,
+    9221,
+    9232,
+    9236,
+    9243,
+    9247,
+    9250,
+    9257,
+    9276,
+    9284,
+    9285,
+    9286,
+    9289,
+    9299,
+    9305,
+    9307,
+    9314,
+    9316,
+    9323,
+    9328,
+    9331,
+    9333,
+    9335,
+    9336,
+    9337,
+    9351,
+    9354,
+    9355,
+    9363,
+    9365,
+    9386,
+    9397,
+    9407,
+    9438,
+    9447,
+    9453,
+    9463,
+    9468,
+    9488,
+    9496,
+    9497,
+    9513,
+    9515,
+    9516,
+    9521,
+    9527,
+    9541,
+    9581,
+    9589,
+    9623,
+    9626,
+    9628,
+    9629,
+    9644,
+    9648,
+    9650,
+    9654,
+    9681,
+    9688,
+    9691,
+    9692,
+    9695,
+    9697,
+    9709,
+    9711,
+    9737,
+    9739,
+    9759,
+    9766,
+    9769,
+    9777,
+    9791,
+    9793,
+    9816,
+    9818,
+    9821,
+    9825,
+    9834,
+    9856,
+    9863,
+    9864,
+    9869,
+    9888,
+    9890,
+    9892,
+    9905,
+    9910,
+    9911,
+    9912,
+    9953,
+    9954,
+    9956,
+    9961,
+    9966,
+    10010,
+    10020,
+    10023,
+    10024,
+    10050,
+    10052,
+    10054,
+    10057,
+    10074,
+    10085,
+    10087,
+    10091,
+    10092,
+    10105,
+    10111,
+    10127,
+    10128,
+    10136,
+    10139,
+    10143,
+    10169,
+    10175,
+    10177,
+    10179,
+    10182,
+    10196,
+    10199,
+    10207,
+    10222,
+    10224,
+    10230,
+    10235,
+    10242,
+    10244,
+    10253,
+    10258,
+    10259,
+    10260,
+    10264,
+    10265,
+    10270,
+    10273,
+    10278,
+    10280,
+    10285,
+    10286,
+    10288,
+    10322,
+    10325,
+    10343,
+    10344,
+    10354,
+    10359,
+    10362,
+    10371,
+    10374,
+    10376,
+    10383,
+    10404,
+    10415,
+    10419,
+    10421,
+    10422,
+    10431,
+    10440,
+    10441,
+    10446,
+    10448,
+    10450,
+    10451,
+    10458,
+    10464,
+    10472,
+    10483,
+    10484,
+    10486,
+    10497,
+    10510,
+    10513,
+    10521,
+    10524,
+    10526,
+    10532,
+    10536,
+    10542,
+    10559,
+    10581,
+    10586,
+    10593,
+    10600,
+    10603,
+    10606,
+    10609,
+    10610,
+    10623,
+    10624,
+    10626,
+    10631,
+    10649,
+    10665,
+    10672,
+    10686,
+    10695,
+    10696,
+    10698,
+    10701,
+    10708,
+    10711,
+    10719,
+    10727,
+    10732,
+    10735,
+    10736,
+    10755,
+    10760,
+    10762,
+    10772,
+    10790,
+    10803,
+    10814,
+    10820,
+    10830,
+    10841,
+    10867,
+    10875,
+    10880,
+    10893,
+    10895,
+    10924,
+    10929,
+    10932,
+    10936,
+    10976,
+    10980,
+    10985,
+    10992,
+    10993,
+    10994,
+    10997,
+    11001,
+    11008,
+    11012,
+    11020,
+    11022,
+    11029,
+    11031,
+    11039,
+    11045,
+    11057,
+    11066,
+    11069,
+    11077,
+    11097,
+    11101,
+    11106,
+    11108,
+    11115,
+    11124,
+    11129,
+    11137,
+    11140,
+    11144,
+    11148,
+    11149,
+    11151,
+    11179,
+    11187,
+    11191,
+    11198,
+    11201,
+    11215,
+    11219,
+    11227,
+    11231,
+    11233,
+    11236,
+    11239,
+    11244,
+    11246,
+    11253,
+    11254,
+    11261,
+    11263,
+    11266,
+    11272,
+    11283,
+    11285,
+    11306,
+    11314,
+    11319,
+    11329,
+    11335,
+    11362,
+    11365,
+    11368,
+    11387,
+    11390,
+    11392,
+    11393,
+    11410,
+    11411,
+    11431,
+    11439,
+    11442,
+    11444,
+    11451,
+    11452,
+    11461,
+    11467,
+    11469,
+    11474,
+    11475,
+    11483,
+    11484,
+    11488,
+    11493,
+    11510,
+    11512,
+    11514,
+    11522,
+    11528,
+    11533,
+    11563,
+    11589,
+    11592,
+    11597,
+    11602,
+    11607,
+    11610,
+    11613,
+    11614,
+    11622,
+    11624,
+    11627,
+    11631,
+    11634,
+    11636,
+    11638,
+    11639,
+    11644,
+    11649,
+    11650,
+    11656,
+    11657,
+    11663,
+    11667,
+    11692,
+    11705,
+    11711,
+    11714,
+    11715,
+    11722,
+    11733,
+    11735,
+    11741,
+    11745,
+    11751,
+    11755,
+    11761,
+    11774,
+    11783,
+    11787,
+    11788,
+    11795,
+    11796,
+    11800,
+    11802,
+    11818,
+    11823,
+    11837,
+    11871,
+    11872,
+    11873,
+    11877,
+    11880,
+    11890,
+    11905,
+    11916,
+    11921,
+    11923,
+    11927,
+    11960,
+    11961,
+    11975,
+    11983,
+    11998,
+    12002,
+    12031,
+    12032,
+    12036,
+    12044,
+    12053,
+    12054,
+    12059,
+    12069,
+    12070,
+    12079,
+    12083,
+    12086,
+    12088,
+    12096,
+    12112,
+    12119,
+    12129,
+    12131,
+    12146,
+    12155,
+    12162,
+    12165,
+    12166,
+    12171,
+    12172,
+    12173,
+    12179,
+    12184,
+    12186,
+    12190,
+    12193,
+    12198,
+    12199,
+    12207,
+    12215,
+    12221,
+    12227,
+    12228,
+    12229,
+    12243,
+    12244,
+    12248,
+    12255,
+    12259,
+    12260,
+    12269,
+    12273,
+    12274,
+    12298,
+    12303,
+    12314,
+    12322,
+    12325,
+    12326,
+    12333,
+    12349,
+    12352,
+    12356,
+    12376,
+    12377,
+    12380,
+    12384,
+    12399,
+    12412,
+    12414,
+    12417,
+    12423,
+    12426,
+    12439,
+    12449,
+    12458,
+    12462,
+    12474,
+    12488,
+    12494,
+    12502,
+    12514,
+    12516,
+    12519,
+    12521,
+    12540,
+    12541,
+    12556,
+    12561,
+    12565,
+    12588,
+    12589,
+    12614,
+    12617,
+    12618,
+    12624,
+    12638,
+    12640,
+    12644,
+    12647,
+    12650,
+    12662,
+    12672,
+    12676,
+    12680,
+    12681,
+    12690,
+    12691,
+    12693,
+    12707,
+    12717,
+    12718,
+    12722,
+    12733,
+    12734,
+    12735,
+    12740,
+    12742,
+    12744,
+    12750,
+    12758,
+    12762,
+    12782,
+    12789,
+    12792,
+    12798,
+    12811,
+    12820,
+    12821,
+    12830,
+    12837,
+    12844,
+    12848,
+    12863,
+    12866,
+    12868,
+    12870,
+    12871,
+    12879,
+    12881,
+    12887,
+    12891,
+    12898,
+    12902,
+    12910,
+    12911,
+    12914,
+    12921,
+    12926,
+    12928,
+    12933,
+    12942,
+    12952,
+    12956,
+    12972,
+    12995,
+    13006,
+    13013,
+    13027,
+    13028,
+    13033,
+    13038,
+    13043,
+    13046,
+    13062,
+    13064,
+    13068,
+    13069,
+    13087,
+    13088,
+    13089,
+    13094,
+    13098,
+    13102,
+    13106,
+    13112,
+    13113,
+    13122,
+    13124,
+    13129,
+    13143,
+    13149,
+    13150,
+    13153,
+    13159,
+    13181,
+    13186,
+    13203,
+    13210,
+    13213,
+    13231,
+    13238,
+    13241,
+    13245,
+    13247,
+    13278,
+    13288,
+    13290,
+    13295,
+    13306,
+    13309,
+    13320,
+    13327,
+    13331,
+    13334,
+    13340,
+    13344,
+    13348,
+    13355,
+    13369,
+    13375,
+    13380,
+    13390,
+    13421,
+    13429,
+    13444,
+    13453,
+    13455,
+    13461,
+    13462,
+    13464,
+    13465,
+    13469,
+    13471,
+    13473,
+    13476,
+    13484,
+    13490,
+    13492,
+    13510,
+    13512,
+    13535,
+    13542,
+    13565,
+    13578,
+    13592,
+    13596,
+    13602,
+    13615,
+    13620,
+    13623,
+    13626,
+    13629,
+    13639,
+    13662,
+    13663,
+    13665,
+    13668,
+    13669,
+    13684,
+    13687,
+    13699,
+    13700,
+    13701,
+    13704,
+    13705,
+    13706,
+    13714,
+    13716,
+    13719,
+    13728,
+    13731,
+    13754,
+    13759,
+    13765,
+    13767,
+    13770,
+    13775,
+    13776,
+    13777,
+    13778,
+    13786,
+    13793,
+    13806,
+    13807,
+    13808,
+    13820,
+    13821,
+    13832,
+    13837,
+    13860,
+    13871,
+    13874,
+    13876,
+    13887,
+    13890,
+    13900,
+    13901,
+    13902,
+    13910,
+    13911,
+    13923,
+    13936,
+    13944,
+    13947,
+    13959,
+    13966,
+    13976,
+    14001,
+    14014,
+    14028,
+    14031,
+    14035,
+    14045,
+    14049,
+    14050,
+    14066,
+    14069,
+    14074,
+    14075,
+    14081,
+    14083,
+    14088,
+    14097,
+    14116,
+    14117,
+    14122,
+    14127,
+    14129,
+    14132,
+    14138,
+    14141,
+    14142,
+    14144,
+    14147,
+    14149,
+    14150,
+    14151,
+    14156,
+    14157,
+    14162,
+    14183,
+    14191,
+    14192,
+    14194,
+    14204,
+    14213,
+    14220,
+    14226,
+    14227,
+    14228,
+    14232,
+    14235,
+    14238,
+    14239,
+    14244,
+    14268,
+    14270,
+    14272,
+    14273,
+    14277,
+    14287,
+    14289,
+    14301,
+    14317,
+    14321,
+    14322,
+    14326,
+    14339,
+    14341,
+    14343,
+    14346,
+    14353,
+    14355,
+    14356,
+    14359,
+    14376,
+    14399,
+    14406,
+    14413,
+    14416,
+    14417,
+    14423,
+    14428,
+    14431,
+    14432,
+    14454,
+    14478,
+    14479,
+    14482,
+    14483,
+    14490,
+    14498,
+    14499,
+    14503,
+    14505,
+    14510,
+    14511,
+    14515,
+    14517,
+    14526,
+    14528,
+    14530,
+    14536,
+    14540,
+    14544,
+    14545,
+    14547,
+    14550,
+    14561,
+    14565,
+    14573,
+    14581,
+    14586,
+    14604,
+    14606,
+    14608,
+    14615,
+    14619,
+    14621,
+    14625,
+    14626,
+    14634,
+    14642,
+    14643,
+    14644,
+    14647,
+    14650,
+    14656,
+    14659,
+    14660,
+    14663,
+    14672,
+    14680,
+    14691,
+    14693,
+    14697,
+    14701,
+    14702,
+    14706,
+    14713,
+    14715,
+    14716,
+    14728,
+    14733,
+    14735,
+    14737,
+    14740,
+    14756,
+    14758,
+    14760,
+    14762,
+    14767,
+    14769,
+    14772,
+    14773,
+    14774,
+    14776,
+    14778,
+    14782,
+    14787,
+    14795,
+    14808,
+    14820,
+    14821,
+    14839,
+    14844,
+    14852,
+    14853,
+    14854,
+    14856,
+    14859,
+    14862,
+    14876,
+    14881,
+    14890,
+    14898,
+    14900,
+    14905,
+    14907,
+    14919,
+    14945,
+    14949,
+    14956,
+    14966,
+    14968,
+    14971,
+    14973,
+    14995,
+    15000,
+    15004,
+    15006,
+    15007,
+    15010,
+    15042,
+    15049,
+    15058,
+    15061,
+    15069,
+    15070,
+    15073,
+    15088,
+    15094,
+    15096,
+    15104,
+    15118,
+    15136,
+    15141,
+    15143,
+    15145,
+    15148,
+    15150,
+    15154,
+    15155,
+    15164,
+    15166,
+    15172,
+    15185,
+    15194,
+    15198,
+    15202,
+    15204,
+    15217,
+    15222,
+    15227,
+    15238,
+    15239,
+    15249,
+    15256,
+    15265,
+    15266,
+    15271,
+    15272,
+    15277,
+    15286,
+    15290,
+    15297,
+    15305,
+    15312,
+    15319,
+    15327,
+    15338,
+    15343,
+    15350,
+    15353,
+    15361,
+    15362,
+    15370,
+    15378,
+    15379,
+    15388,
+    15394,
+    15395,
+    15396,
+    15402,
+    15414,
+    15416,
+    15417,
+    15422,
+    15431,
+    15441,
+    15444,
+    15449,
+    15455,
+    15457,
+    15459,
+    15460,
+    15464,
+    15465,
+    15472,
+    15474,
+    15482,
+    15493,
+    15494,
+    15495,
+    15496,
+    15499,
+    15500,
+    15509,
+    15513,
+    15530,
+    15532,
+    15534,
+    15542,
+    15549,
+    15551,
+    15557,
+    15558,
+    15564,
+    15567,
+    15568,
+    15569,
+    15580,
+    15590,
+    15593,
+    15599,
+    15603,
+    15608,
+    15613,
+    15621,
+    15628,
+    15630,
+    15637,
+    15638,
+    15643,
+    15648,
+    15660,
+    15666,
+    15668,
+    15671,
+    15683,
+    15687,
+    15689,
+    15706,
+    15707,
+    15710,
+    15714,
+    15719,
+    15721,
+    15725,
+    15728,
+    15733,
+    15735,
+    15750,
+    15758,
+    15759,
+    15773,
+    15776,
+    15780,
+    15781,
+    15782,
+    15787,
+    15788,
+    15794,
+    15808,
+    15810,
+    15816,
+    15830,
+    15833,
+    15851,
+    15853,
+    15857,
+    15859,
+    15864,
+    15869,
+    15872,
+    15878,
+    15879,
+    15900,
+    15904,
+    15907,
+    15909,
+    15915,
+    15916,
+    15922,
+    15928,
+    15937,
+    15950,
+    15951,
+    15952,
+    15975,
+    15979,
+    15985,
+    15987,
+    15992,
+    15994,
+    16001,
+    16020,
+    16022,
+    16026,
+    16033,
+    16037,
+    16043,
+    16044,
+    16048,
+    16050,
+    16052,
+    16059,
+    16066,
+    16068,
+    16069,
+    16070,
+    16076,
+    16078,
+    16084,
+    16086,
+    16089,
+    16093,
+    16098,
+    16102,
+    16103,
+    16107,
+    16116,
+    16118,
+    16119,
+    16136,
+    16147,
+    16150,
+    16161,
+    16168,
+    16173,
+    16177,
+    16179,
+    16186,
+    16190,
+    16194,
+    16198,
+    16200,
+    16203,
+    16206,
+    16207,
+    16211,
+    16213,
+    16223,
+    16230,
+    16238,
+    16252,
+    16256,
+    16257,
+    16259,
+    16275,
+    16277,
+    16279,
+    16280,
+    16281,
+    16284,
+    16288,
+    16294,
+    16302,
+    16303,
+    16307,
+    16308,
+    16313,
+    16329,
+    16336,
+    16338,
+    16339,
+    16340,
+    16344,
+    16347,
+    16363,
+    16365,
+    16366,
+    16368,
+    16369,
+    16377,
+    16383,
+    16398,
+    16402,
+    16409,
+    16414,
+    16415,
+    16416,
+    16425,
+    16429,
+    16445,
+    16446,
+    16451,
+    16453,
+    16454,
+    16458,
+    16467,
+    16502,
+    16503,
+    16505,
+    16506,
+    16515,
+    16518,
+    16523,
+    16526,
+    16529,
+    16530,
+    16531,
+    16535,
+    16549,
+    16550,
+    16555,
+    16558,
+    16563,
+    16564,
+    16566,
+    16567,
+    16570,
+    16576,
+    16584,
+    16585,
+    16594,
+    16600,
+    16610,
+    16613,
+    16616,
+    16643,
+    16649,
+    16650,
+    16652,
+    16656,
+    16665,
+    16670,
+    16671,
+    16677,
+    16689,
+    16693,
+    16695,
+    16700,
+    16703,
+    16704,
+    16715,
+    16737,
+    16739,
+    16741,
+    16748,
+    16750,
+    16754,
+    16761,
+    16765,
+    16774,
+    16777,
+    16778,
+    16781,
+    16782,
+    16786,
+    16790,
+    16811,
+    16814,
+    16816,
+    16835,
+    16843,
+    16846,
+    16848,
+    16856,
+    16862,
+    16867,
+    16869,
+    16872,
+    16874,
+    16882,
+    16885,
+    16886,
+    16892,
+    16898,
+    16901,
+    16912,
+    16917,
+    16918,
+    16921,
+    16930,
+    16932,
+    16940,
+    16952,
+    16961,
+    16963,
+    16969,
+    16975,
+    16977,
+    16989,
+    16992,
+    16998,
+    17004,
+    17007,
+    17028,
+    17036,
+    17040,
+    17043,
+    17048,
+    17058,
+    17062,
+    17063,
+    17071,
+    17080,
+    17084,
+    17086,
+    17101,
+    17110,
+    17112,
+    17119,
+    17122,
+    17125,
+    17130,
+    17134,
+    17139,
+    17140,
+    17149,
+    17157,
+    17166,
+    17167,
+    17168,
+    17175,
+    17190,
+    17191,
+    17198,
+    17206,
+    17226,
+    17231,
+    17236,
+    17238,
+    17240,
+    17250,
+    17252,
+    17258,
+    17269,
+    17275,
+    17281,
+    17285,
+    17287,
+    17288,
+    17296,
+    17298,
+    17299,
+    17301,
+    17302,
+    17305,
+    17308,
+    17310,
+    17311,
+    17322,
+    17325,
+    17327,
+    17339,
+    17340,
+    17342,
+    17345,
+    17364,
+    17369,
+    17375,
+    17378,
+    17386,
+    17389,
+    17406,
+    17413,
+    17417,
+    17420,
+    17421,
+    17426,
+    17432,
+    17440,
+    17441,
+    17452,
+    17456,
+    17457,
+    17460,
+    17465,
+    17468,
+    17470,
+    17471,
+    17479,
+    17487,
+    17499,
+    17510,
+    17516,
+    17545,
+    17560,
+    17563,
+    17570,
+    17572,
+    17577,
+    17579,
+    17580,
+    17583,
+    17585,
+    17592,
+    17597,
+    17599,
+    17603,
+    17619,
+    17622,
+    17625,
+    17629,
+    17635,
+    17643,
+    17644,
+    17650,
+    17658,
+    17661,
+    17668,
+    17670,
+    17679,
+    17681,
+    17682,
+    17683,
+    17686,
+    17695,
+    17699,
+    17701,
+    17706,
+    17713,
+    17724,
+    17726,
+    17728,
+    17729,
+    17751,
+    17753,
+    17759,
+    17788,
+    17789,
+    17791,
+    17794,
+    17799,
+    17802,
+    17809,
+    17811,
+    17815,
+    17819,
+    17822,
+    17830,
+    17847,
+    17848,
+    17854,
+    17868,
+    17872,
+    17875,
+    17883,
+    17884,
+    17896,
+    17897,
+    17904,
+    17905,
+    17906,
+    17914,
+    17915,
+    17920,
+    17922,
+    17926,
+    17933,
+    17944,
+    17948,
+    17955,
+    17960,
+    17965,
+    17968,
+    17980,
+    17988,
+    17989,
+    17994,
+    17998,
+    18003,
+    18009,
+    18028,
+    18032,
+    18037,
+    18041,
+    18053,
+    18057,
+    18060,
+    18065,
+    18069,
+    18073,
+    18075,
+    18078,
+    18083,
+    18095,
+    18098,
+    18100,
+    18107,
+    18115,
+    18116,
+    18117,
+    18124,
+    18136,
+    18139,
+    18141,
+    18142,
+    18143,
+    18150,
+    18153,
+    18155,
+    18161,
+    18163,
+    18170,
+    18175,
+    18182,
+    18192,
+    18194,
+    18195,
+    18199,
+    18209,
+    18219,
+    18222,
+    18223,
+    18232,
+    18246,
+    18249,
+    18252,
+    18256,
+    18259,
+    18260,
+    18274,
+    18277,
+    18278,
+    18279,
+    18291,
+    18300,
+    18317,
+    18319,
+    18321,
+    18324,
+    18331,
+    18334,
+    18337,
+    18339,
+    18349,
+    18351,
+    18352,
+    18357,
+    18367,
+    18372,
+    18376,
+    18377,
+    18380,
+    18382,
+    18384,
+    18389,
+    18393,
+    18400,
+    18405,
+    18409,
+    18410,
+    18412,
+    18413,
+    18417,
+    18418,
+    18424,
+    18425,
+    18426,
+    18428,
+    18436,
+    18442,
+    18447,
+    18450,
+    18471,
+    18477,
+    18483,
+    18491,
+    18492,
+    18494,
+    18505,
+    18507,
+    18510,
+    18511,
+    18513,
+    18514,
+    18515,
+    18518,
+    18532,
+    18533,
+    18537,
+    18552,
+    18557,
+    18560,
+    18569,
+    18570,
+    18581,
+    18585,
+    18589,
+    18595,
+    18598,
+    18599,
+    18602,
+    18605,
+    18606,
+    18613,
+    18617,
+    18620,
+    18622,
+    18623,
+    18630,
+    18632,
+    18637,
+    18639,
+    18647,
+    18655,
+    18657,
+    18660,
+    18663,
+    18673,
+    18674,
+    18679,
+    18682,
+    18684,
+    18688,
+    18689,
+    18690,
+    18696,
+    18697,
+    18702,
+    18703,
+    18715,
+    18717,
+    18718,
+    18719,
+    18721,
+    18722,
+    18724,
+    18727,
+    18738,
+    18743,
+    18752,
+    18757,
+    18769,
+    18774,
+    18780,
+    18790,
+    18796,
+    18802,
+    18803,
+    18807,
+    18808,
+    18809,
+    18810,
+    18818,
+    18824,
+    18826,
+    18827,
+    18839,
+    18842,
+    18845,
+    18861,
+    18863,
+    18866,
+    18871,
+    18878,
+    18879,
+    18884,
+    18885,
+    18886,
+    18891,
+    18894,
+    18900,
+    18902,
+    18903,
+    18907,
+    18908,
+    18909,
+    18912,
+    18915,
+    18924,
+    18926,
+    18927,
+    18933,
+    18935,
+    18938,
+    18939,
+    18940,
+    18941,
+    18947,
+    18954,
+    18963,
+    18964,
+    18965,
+    18969,
+    18979,
+    18980,
+    18981,
+    18992,
+    18994,
+    19000,
+    19001,
+    19009,
+    19020,
+    19022,
+    19023,
+    19025,
+    19036,
+    19040,
+    19042,
+    19053,
+    19058,
+    19062,
+    19068,
+    19082,
+    19092,
+    19093,
+    19098,
+    19099,
+    19105,
+    19109,
+    19110,
+    19114,
+    19126,
+    19137,
+    19139,
+    19140,
+    19145,
+    19146,
+    19150,
+    19152,
+    19155,
+    19159,
+    19168,
+    19170,
+    19172,
+    19182,
+    19185,
+    19190,
+    19196,
+    19198,
+    19205,
+    19208,
+    19210,
+    19213,
+    19217,
+    19224,
+    19225,
+    19228,
+    19230,
+    19231,
+    19234,
+    19250,
+    19253,
+    19259,
+    19263,
+    19265,
+    19269,
+    19270,
+    19277,
+    19279,
+    19281,
+    19282,
+    19288,
+    19291,
+    19301,
+    19303,
+    19304,
+    19307,
+    19310,
+    19312,
+    19315,
+    19317,
+    19321,
+    19333,
+    19339,
+    19341,
+    19357,
+    19358,
+    19362,
+    19386,
+    19392,
+    19393,
+    19403,
+    19405,
+    19417,
+    19418,
+    19419,
+    19422,
+    19425,
+    19426,
+    19433,
+    19434,
+    19436,
+    19440,
+    19442,
+    19445,
+    19454,
+    19464,
+    19466,
+    19468,
+    19473,
+    19478,
+    19481,
+    19491,
+    19496,
+    19498,
+    19500,
+    19506,
+    19510,
+    19513,
+    19516,
+    19520,
+    19524,
+    19531,
+    19539,
+    19562,
+    19563,
+    19565,
+    19567,
+    19570,
+    19574,
+    19579,
+    19585,
+    19588,
+    19603,
+    19612,
+    19620,
+    19629,
+    19636,
+    19646,
+    19648,
+    19650,
+    19658,
+    19661,
+    19663,
+    19665,
+    19666,
+    19673,
+    19696,
+    19699,
+    19704,
+    19707,
+    19708,
+    19716,
+    19718,
+    19722,
+    19729,
+    19731,
+    19736,
+    19738,
+    19742,
+    19755,
+    19761,
+    19763,
+    19766,
+    19767,
+    19771,
+    19779,
+    19784,
+    19791,
+    19792,
+    19797,
+    19798,
+    19800,
+    19801,
+    19811,
+    19824,
+    19825,
+    19826,
+    19833,
+    19834,
+    19839,
+    19841,
+    19845,
+    19847,
+    19848,
+    19849,
+    19856,
+    19859,
+    19862,
+    19864,
+    19865,
+    19869,
+    19877,
+    19879,
+    19880,
+    19892,
+    19894,
+    19895,
+    19901,
+    19903,
+    19908,
+    19909,
+    19912,
+    19915,
+    19927,
+    19929,
+    19932,
+    19944,
+    19946,
+    19961,
+    19966,
+    19968,
+    19969,
+    19980,
+    19983,
+    19987,
+    19990,
+    19991,
+    19993,
+    20009,
+    20011,
+    20023,
+    20026,
+    20027,
+    20028,
+    20030,
+    20031,
+    20041,
+    20042,
+    20043,
+    20048,
+    20049,
+    20051,
+    20058,
+    20060,
+    20082,
+    20086,
+    20089,
+    20097,
+    20100,
+    20106,
+    20110,
+    20113,
+    20132,
+    20136,
+    20142,
+    20147,
+    20156,
+    20159,
+    20166,
+    20169,
+    20175,
+    20179,
+    20186,
+    20189,
+    20192,
+    20200,
+    20203,
+    20205,
+    20207,
+    20211,
+    20214,
+    20217,
+    20224,
+    20236,
+    20240,
+    20255,
+    20257,
+    20260,
+    20265,
+    20267,
+    20269,
+    20281,
+    20282,
+    20285,
+    20303,
+    20307,
+    20318,
+    20330,
+    20338,
+    20343,
+    20348,
+    20349,
+    20350,
+    20357,
+    20360,
+    20363,
+    20366,
+    20367,
+    20370,
+    20379,
+    20382,
+    20389,
+    20391,
+    20395,
+    20409,
+    20411,
+    20414,
+    20420,
+    20431,
+    20434,
+    20444,
+    20447,
+    20450,
+    20459,
+    20464,
+    20465,
+    20469,
+    20470,
+    20473,
+    20479,
+    20483,
+    20486,
+    20492,
+    20496,
+    20499,
+    20501,
+    20511,
+    20515,
+    20518,
+    20523,
+    20527,
+    20528,
+    20529,
+    20534,
+    20535,
+    20536,
+    20550,
+    20552,
+    20553,
+    20554,
+    20559,
+    20562,
+    20564,
+    20569,
+    20570,
+    20573,
+    20574,
+    20577,
+    20589,
+    20590,
+    20591,
+    20592,
+    20598,
+    20609,
+    20614,
+    20617,
+    20620,
+    20623,
+    20624,
+    20630,
+    20644,
+    20650,
+    20654,
+    20656,
+    20660,
+    20668,
+    20671,
+    20679,
+    20683,
+    20684,
+    20685,
+    20688,
+    20689,
+    20693,
+    20697,
+    20709,
+    20715,
+    20718,
+    20721,
+    20722,
+    20734,
+    20738,
+    20740,
+    20744,
+    20745,
+    20746,
+    20755,
+    20760,
+    20763,
+    20766,
+    20770,
+    20771,
+    20778,
+    20781,
+    20782,
+    20784,
+    20791,
+    20792,
+    20793,
+    20799,
+    20800,
+    20806,
+    20807,
+    20808,
+    20812,
+    20821,
+    20822,
+    20824,
+    20838,
+    20842,
+    20844,
+    20845,
+    20846,
+    20849,
+    20850,
+    20853,
+    20860,
+    20872,
+    20875,
+    20876,
+    20881,
+    20891,
+    20894,
+    20897,
+    20898,
+    20899,
+    20900,
+    20902,
+    20908,
+    20909,
+    20913,
+    20915,
+    20921,
+    20922,
+    20924,
+    20925,
+    20927,
+    20935,
+    20936,
+    20938,
+    20939,
+    20941,
+    20942,
+    20952,
+    20958,
+    20959,
+    20960,
+    20961,
+    20968,
+    20972,
+    20974,
+    20984,
+    20995,
+    21001,
+    21007,
+    21010,
+    21011,
+    21013,
+    21017,
+    21020,
+    21022,
+    21030,
+    21031,
+    21041,
+    21043,
+    21046,
+    21050,
+    21055,
+    21057,
+    21077,
+    21081,
+    21084,
+    21095,
+    21102,
+    21104,
+    21112,
+    21114,
+    21119,
+    21126,
+    21128,
+    21131,
+    21132,
+    21138,
+    21141,
+    21147,
+    21148,
+    21149,
+    21156,
+    21159,
+    21162,
+    21163,
+    21165,
+    21166,
+    21170,
+    21172,
+    21181,
+    21187,
+    21188,
+    21190,
+    21194,
+    21197,
+    21201,
+    21202,
+    21204,
+    21205,
+    21217,
+    21218,
+    21222,
+    21223,
+    21225,
+    21227,
+    21228,
+    21230,
+    21231,
+    21239,
+    21246,
+    21252,
+    21253,
+    21254,
+    21261,
+    21266,
+    21270,
+    21272,
+    21273,
+    21275,
+    21280,
+    21281,
+    21285,
+    21292,
+    21293,
+    21297,
+    21302,
+    21309,
+    21314,
+    21325,
+    21332,
+    21335,
+    21337,
+    21338,
+    21341,
+    21351,
+    21354,
+    21355,
+    21361,
+    21369,
+    21382,
+    21397,
+    21398,
+    21405,
+    21408,
+    21409,
+    21412,
+    21421,
+    21422,
+    21426,
+    21436,
+    21450,
+    21456,
+    21458,
+    21463,
+    21466,
+    21471,
+    21472,
+    21476,
+    21482,
+    21486,
+    21493,
+    21499,
+    21504,
+    21508,
+    21509,
+    21511,
+    21512,
+    21515,
+    21525,
+    21530,
+    21531,
+    21532,
+    21539,
+    21540,
+    21543,
+    21547,
+    21548,
+    21556,
+    21557,
+    21558,
+    21559,
+    21561,
+    21564,
+    21565,
+    21572,
+    21576,
+    21578,
+    21579,
+    21590,
+    21593,
+    21595,
+    21596,
+    21600,
+    21604,
+    21608,
+    21615,
+    21619,
+    21621,
+    21634,
+    21638,
+    21639,
+    21646,
+    21649,
+    21654,
+    21661,
+    21663,
+    21673,
+    21678,
+    21682,
+    21685,
+    21693,
+    21694,
+    21700,
+    21709,
+    21710,
+    21711,
+    21716,
+    21723,
+    21724,
+    21735,
+    21736,
+    21737,
+    21748,
+    21752,
+    21756,
+    21758,
+    21759,
+    21761,
+    21763,
+    21769,
+    21781,
+    21784,
+    21786,
+    21789,
+    21799,
+    21804,
+    21806,
+    21807,
+    21808,
+    21814,
+    21815,
+    21816,
+    21818,
+    21819,
+    21820,
+    21821,
+    21823,
+    21827,
+    21835,
+    21837,
+    21842,
+    21844,
+    21847,
+    21850,
+    21861,
+    21869,
+    21884,
+    21885,
+    21886,
+    21890,
+    21891,
+    21898,
+    21906,
+    21924,
+    21926,
+    21927,
+    21928,
+    21930,
+    21932,
+    21939,
+    21940,
+    21945,
+    21946,
+    21958,
+    21966,
+    21967,
+    21983,
+    21996,
+    21997,
+    21998,
+    22008,
+    22014,
+    22016,
+    22022,
+    22025,
+    22026,
+    22036,
+    22043,
+    22044,
+    22045,
+    22046,
+    22048,
+    22049,
+    22059,
+    22068,
+    22071,
+    22074,
+    22081,
+    22083,
+    22084,
+    22086,
+    22087,
+    22090,
+    22104,
+    22107,
+    22117,
+    22119,
+    22122,
+    22123,
+    22134,
+    22139,
+    22144,
+    22145,
+    22153,
+    22155,
+    22158,
+    22167,
+    22172,
+    22179,
+    22187,
+    22192,
+    22196,
+    22197,
+    22199,
+    22203,
+    22204,
+    22208,
+    22218,
+    22225,
+    22231,
+    22244,
+    22248,
+    22251,
+    22254,
+    22263,
+    22264,
+    22269,
+    22270,
+    22272,
+    22275,
+    22280,
+    22282,
+    22287,
+    22299,
+    22305,
+    22311,
+    22314,
+    22315,
+    22318,
+    22325,
+    22327,
+    22328,
+    22332,
+    22341,
+    22342,
+    22345,
+    22352,
+    22360,
+    22367,
+    22368,
+    22373,
+    22375,
+    22379,
+    22381,
+    22384,
+    22392,
+    22394,
+    22397,
+    22398,
+    22401,
+    22407,
+    22408,
+    22414,
+    22420,
+    22425,
+    22426,
+    22427,
+    22436,
+    22444,
+    22447,
+    22461,
+    22462,
+    22465,
+    22471,
+    22483,
+    22486,
+    22492,
+    22494,
+    22499,
+    22500,
+    22504,
+    22507,
+    22509,
+    22510,
+    22513,
+    22516,
+    22517,
+    22522,
+    22536,
+    22540,
+    22542,
+    22543,
+    22556,
+    22571,
+    22573,
+    22576,
+    22578,
+    22579,
+    22586,
+    22591,
+    22592,
+    22599,
+    22601,
+    22610,
+    22612,
+    22618,
+    22620,
+    22626,
+    22631,
+    22637,
+    22638,
+    22642,
+    22646,
+    22648,
+    22655,
+    22658,
+    22662,
+    22669,
+    22670,
+    22675,
+    22684,
+    22691,
+    22693,
+    22694,
+    22698,
+    22717,
+    22723,
+    22729,
+    22731,
+    22734,
+    22738,
+    22740,
+    22743,
+    22747,
+    22749,
+    22750,
+    22753,
+    22758,
+    22769,
+    22772,
+    22776,
+    22779,
+    22780,
+    22781,
+    22786,
+    22787,
+    22792,
+    22798,
+    22799,
+    22803,
+    22809,
+    22823,
+    22828,
+    22829,
+    22835,
+    22842,
+    22844,
+    22847,
+    22850,
+    22852,
+    22862,
+    22864,
+    22869,
+    22879,
+    22881,
+    22890,
+    22893,
+    22895,
+    22903,
+    22907,
+    22908,
+    22909,
+    22911,
+    22916,
+    22918,
+    22919,
+    22920,
+    22923,
+    22924,
+    22925,
+    22927,
+    22929,
+    22930,
+    22932,
+    22934,
+    22938,
+    22942,
+    22943,
+    22948,
+    22957,
+    22960,
+    22966,
+    22967,
+    22968,
+    22971,
+    22972,
+    22974,
+    22984,
+    22987,
+    22991,
+    22994,
+    23004,
+    23017,
+    23024,
+    23033,
+    23035,
+    23041,
+    23047,
+    23048,
+    23049,
+    23052,
+    23056,
+    23057,
+    23058,
+    23061,
+    23062,
+    23065,
+    23067,
+    23070,
+    23072,
+    23076,
+    23078,
+    23085,
+    23086,
+    23091,
+    23093,
+    23095,
+    23096,
+    23099,
+    23101,
+    23102,
+    23108,
+    23111,
+    23115,
+    23125,
+    23128,
+    23132,
+    23137,
+    23141,
+    23144,
+    23147,
+    23155,
+    23169,
+    23177,
+    23180,
+    23185,
+    23186,
+    23194,
+    23195,
+    23196,
+    23200,
+    23207,
+    23211,
+    23212,
+    23221,
+    23223,
+    23230,
+    23235,
+    23237,
+    23238,
+    23239,
+    23242,
+    23243,
+    23247,
+    23248,
+    23266,
+    23267,
+    23270,
+    23274,
+    23278,
+    23282,
+    23296,
+    23298,
+    23301,
+    23302,
+    23305,
+    23309,
+    23314,
+    23316,
+    23320,
+    23335,
+    23342,
+    23344,
+    23345,
+    23350,
+    23352,
+    23355,
+    23356,
+    23357,
+    23361,
+    23367,
+    23373,
+    23379,
+    23385,
+    23393,
+    23395,
+    23398,
+    23402,
+    23410,
+    23416,
+    23417,
+    23419,
+    23422,
+    23423,
+    23425,
+    23432,
+    23436,
+    23446,
+    23451,
+    23460,
+    23461,
+    23467,
+    23479,
+    23483,
+    23487,
+    23490,
+    23491,
+    23496,
+    23499,
+    23500,
+    23505,
+    23510,
+    23511,
+    23518,
+    23522,
+    23543,
+    23544,
+    23545,
+    23549,
+    23553,
+    23555,
+    23559,
+    23565,
+    23567,
+    23573,
+    23574,
+    23576,
+    23580,
+    23584,
+    23585,
+    23597,
+    23606,
+    23615,
+    23630,
+    23632,
+    23633,
+    23644,
+    23645,
+    23650,
+    23654,
+    23656,
+    23662,
+    23663,
+    23664,
+    23665,
+    23670,
+    23671,
+    23673,
+    23681,
+    23684,
+    23692,
+    23695,
+    23697,
+    23706,
+    23709,
+    23722,
+    23732,
+    23736,
+    23737,
+    23738,
+    23742,
+    23743,
+    23744,
+    23749,
+    23757,
+    23773,
+    23777,
+    23778,
+    23780,
+    23782,
+    23783,
+    23784,
+    23793,
+    23795,
+    23796,
+    23797,
+    23803,
+    23804,
+    23805,
+    23806,
+    23808,
+    23809,
+    23816,
+    23823,
+    23829,
+    23830,
+    23833,
+    23835,
+    23836,
+    23846,
+    23854,
+    23858,
+    23865,
+    23871,
+    23874,
+    23877,
+    23884,
+    23887,
+    23890,
+    23891,
+    23896,
+    23901,
+    23904,
+    23909,
+    23913,
+    23918,
+    23924,
+    23925,
+    23930,
+    23931,
+    23935,
+    23938,
+    23940,
+    23941,
+    23943,
+    23947,
+    23948,
+    23955,
+    23991,
+    23998,
+    24002,
+    24007,
+    24014,
+    24015,
+    24019,
+    24032,
+    24038,
+    24041,
+    24045,
+    24063,
+    24065,
+    24066,
+    24068,
+    24072,
+    24073,
+    24076,
+    24077,
+    24079,
+    24080,
+    24085,
+    24087,
+    24088,
+    24090,
+    24092,
+    24093,
+    24096,
+    24097,
+    24101,
+    24102,
+    24103,
+    24108,
+    24117,
+    24118,
+    24124,
+    24128,
+    24129,
+    24133,
+    24137,
+    24138,
+    24140,
+    24141,
+    24144,
+    24145,
+    24146,
+    24147,
+    24153,
+    24158,
+    24163,
+    24164,
+    24167,
+    24168,
+    24170,
+    24174,
+    24175,
+    24178,
+    24180,
+    24192,
+    24198,
+    24200,
+    24204,
+    24206,
+    24220,
+    24221,
+    24225,
+    24226,
+    24228,
+    24229,
+    24230,
+    24237,
+    24238,
+    24242,
+    24268,
+    24270,
+    24271,
+    24277,
+    24278,
+    24279,
+    24281,
+    24284,
+    24285,
+    24290,
+    24291,
+    24292,
+    24298,
+    24300,
+    24301,
+    24302,
+    24303,
+    24310,
+    24314,
+    24316,
+    24317,
+    24322,
+    24323,
+    24327,
+    24330,
+    24333,
+    24335,
+    24336,
+    24340,
+    24342,
+    24343,
+    24344,
+    24352,
+    24359,
+    24360,
+    24366,
+    24375,
+    24376,
+    24383,
+    24384,
+    24388,
+    24390,
+    24393,
+    24395,
+    24396,
+    24397,
+    24409,
+    24410,
+    24411,
+    24412,
+    24413,
+    24415,
+    24422,
+    24428,
+    24431,
+    24434,
+    24435,
+    24448,
+    24449,
+    24454,
+    24455,
+    24456,
+    24457,
+    24458,
+    24463,
+    24472,
+    24475,
+    24488,
+    24490,
+    24498,
+    24506,
+    24513,
+    24516,
+    24518,
+    24521,
+    24527,
+    24539,
+    24546,
+    24548,
+    24553,
+    24556,
+    24561,
+    24562,
+    24572,
+    24573,
+    24586,
+    24587,
+    24589,
+    24594,
+    24597,
+    24598,
+    24601,
+    24607,
+    24610,
+    24612,
+    24619,
+    24620,
+    24649,
+    24652,
+    24653,
+    24657,
+    24658,
+    24660,
+    24661,
+    24662,
+    24667,
+    24669,
+    24676,
+    24679,
+    24682,
+    24695,
+    24700,
+    24703,
+    24714,
+    24721,
+    24725,
+    24729,
+    24741,
+    24746,
+    24755,
+    24759,
+    24764,
+    24768,
+    24769,
+    24771,
+    24772,
+    24778,
+    24788,
+    24790,
+    24793,
+    24796,
+    24798,
+    24804,
+    24805,
+    24817,
+    24824,
+    24825,
+    24826,
+    24830,
+    24836,
+    24838,
+    24844,
+    24847,
+    24848,
+    24850,
+    24860,
+    24861,
+    24862,
+    24864,
+    24871,
+    24872,
+    24876,
+    24879,
+    24886,
+    24887,
+    24891,
+    24894,
+    24895,
+    24901,
+    24911,
+    24913,
+    24914,
+    24915,
+    24916,
+    24917,
+    24918,
+    24920,
+    24925,
+    24930,
+    24932,
+    24938,
+    24952,
+    24965,
+    24967,
+    24968,
+    24971,
+    24979,
+    24985,
+    24988,
+    24989,
+    24991,
+    24996,
+    24997,
+    24999,
+    25002,
+    25008,
+    25009,
+    25013,
+    25015,
+    25016,
+    25023,
+    25027,
+    25031,
+    25032,
+    25036,
+    25046,
+    25051,
+    25057,
+    25058,
+    25060,
+    25062,
+    25063,
+    25064,
+    25070,
+    25074,
+    25078,
+    25083,
+    25084,
+    25089,
+    25090,
+    25091,
+    25099,
+    25100,
+    25101,
+    25105,
+    25107,
+    25112,
+    25114,
+    25121,
+    25122,
+    25127,
+    25128,
+    25137,
+    25148,
+    25152,
+    25153,
+    25154,
+    25164,
+    25165,
+    25167,
+    25168,
+    25171,
+    25181,
+    25184,
+    25185,
+    25190,
+    25201,
+    25205,
+    25217,
+    25219,
+    25226,
+    25229,
+    25230,
+    25232,
+    25236,
+    25243,
+    25253,
+    25255,
+    25258,
+    25279,
+    25284,
+    25290,
+    25293,
+    25299,
+    25300,
+    25301,
+    25310,
+    25311,
+    25313,
+    25318,
+    25322,
+    25324,
+    25329,
+    25344,
+    25346,
+    25351,
+    25360,
+    25368,
+    25370,
+    25373,
+    25377,
+    25378,
+    25379,
+    25383,
+    25384,
+    25385,
+    25389,
+    25393,
+    25394,
+    25399,
+    25417,
+    25418,
+    25421,
+    25425,
+    25426,
+    25434,
+    25438,
+    25440,
+    25441,
+    25443,
+    25444,
+    25450,
+    25451,
+    25457,
+    25459,
+    25460,
+    25463,
+    25464,
+    25465,
+    25469,
+    25475,
+    25481,
+    25485,
+    25490,
+    25494,
+    25495,
+    25499,
+    25500,
+    25508,
+    25509,
+    25514,
+    25518,
+    25522,
+    25523,
+    25524,
+    25526,
+    25528,
+    25529,
+    25531,
+    25533,
+    25538,
+    25541,
+    25547,
+    25555,
+    25556,
+    25557,
+    25560,
+    25561,
+    25564,
+    25572,
+    25573,
+    25574,
+    25575,
+    25581,
+    25587,
+    25592,
+    25593,
+    25599,
+    25607,
+    25608,
+    25611,
+    25619,
+    25623,
+    25629,
+    25632,
+    25638,
+    25639,
+    25643,
+    25646,
+    25647,
+    25650,
+    25655,
+    25656,
+    25660,
+    25661,
+    25666,
+    25667,
+    25673,
+    25678,
+    25679,
+    25680,
+    25687,
+    25688,
+    25689,
+    25690,
+    25691,
+    25698,
+    25701,
+    25708,
+    25709,
+    25711,
+    25725,
+    25731,
+    25733,
+    25746,
+    25758,
+    25761,
+    25766,
+    25767,
+    25778,
+    25780,
+    25785,
+    25787,
+    25789,
+    25793,
+    25794,
+    25798,
+    25799,
+    25804,
+    25807,
+    25808,
+    25810,
+    25811,
+    25816,
+    25820,
+    25831,
+    25832,
+    25838,
+    25839,
+    25848,
+    25849,
+    25852,
+    25855,
+    25856,
+    25858,
+    25862,
+    25863,
+    25868,
+    25872,
+    25879,
+    25888,
+    25890,
+    25892,
+    25896,
+    25902,
+    25903,
+    25910,
+    25911,
+    25915,
+    25917,
+    25918,
+    25923,
+    25929,
+    25932,
+    25940,
+    25944,
+    25949,
+    25950,
+    25955,
+    25965,
+    25968,
+    25969,
+    25970,
+    25974,
+    25987,
+    25990,
+    25994,
+    25997,
+    26005,
+    26006,
+    26010,
+    26013,
+    26016,
+    26020,
+    26022,
+    26029,
+    26032,
+    26036,
+    26039,
+    26052,
+    26053,
+    26054,
+    26055,
+    26058,
+    26061,
+    26065,
+    26067,
+    26068,
+    26083,
+    26086,
+    26087,
+    26091,
+    26095,
+    26100,
+    26115,
+    26116,
+    26117,
+    26121,
+    26126,
+    26127,
+    26132,
+    26134,
+    26139,
+    26143,
+    26145,
+    26147,
+    26148,
+    26154,
+    26156,
+    26160,
+    26169,
+    26173,
+    26174,
+    26187,
+    26190,
+    26199,
+    26200,
+    26207,
+    26210,
+    26212,
+    26217,
+    26224,
+    26227,
+    26228,
+    26229,
+    26230,
+    26240,
+    26243,
+    26247,
+    26250,
+    26251,
+    26255,
+    26266,
+    26274,
+    26282,
+    26289,
+    26291,
+    26295,
+    26300,
+    26302,
+    26305,
+    26306,
+    26310,
+    26320,
+    26327,
+    26328,
+    26332,
+    26335,
+    26336,
+    26337,
+    26338,
+    26340,
+    26341,
+    26348,
+    26356,
+    26362,
+    26370,
+    26371,
+    26373,
+    26376,
+    26378,
+    26379,
+    26387,
+    26388,
+    26393,
+    26394,
+    26401,
+    26414,
+    26415,
+    26416,
+    26426,
+    26430,
+    26432,
+    26437,
+    26440,
+    26441,
+    26444,
+    26454,
+    26455,
+    26456,
+    26460,
+    26461,
+    26467,
+    26468,
+    26474,
+    26477,
+    26492,
+    26493,
+    26494,
+    26496,
+    26516,
+    26517,
+    26518,
+    26521,
+    26525,
+    26526,
+    26527,
+    26531,
+    26534,
+    26535,
+    26541,
+    26544,
+    26547,
+    26548,
+    26549,
+    26557,
+    26559,
+    26560,
+    26565,
+    26569,
+    26571,
+    26573,
+    26576,
+    26579,
+    26580,
+    26585,
+    26589,
+    26591,
+    26592,
+    26601,
+    26605,
+    26607,
+    26611,
+    26614,
+    26620,
+    26634,
+    26636,
+    26638,
+    26639,
+    26642,
+    26643,
+    26644,
+    26652,
+    26654,
+    26655,
+    26656,
+    26657,
+    26658,
+    26662,
+    26675,
+    26677,
+    26682,
+    26685,
+    26686,
+    26688,
+    26691,
+    26692,
+    26698,
+    26702,
+    26703,
+    26711,
+    26715,
+    26724,
+    26725,
+    26730,
+    26737,
+    26741,
+    26744,
+    26745,
+    26746,
+    26755,
+    26760,
+    26763,
+    26764,
+    26770,
+    26775,
+    26782,
+    26786,
+    26787,
+    26788,
+    26789,
+    26792,
+    26793,
+    26797,
+    26801,
+    26802,
+    26806,
+    26807,
+    26813,
+    26824,
+    26827,
+    26830,
+    26832,
+    26837,
+    26846,
+    26860,
+    26863,
+    26869,
+    26876,
+    26877,
+    26878,
+    26884,
+    26891,
+    26895,
+    26896,
+    26897,
+    26901,
+    26907,
+    26910,
+    26915,
+    26919,
+    26933,
+    26941,
+    26942,
+    26945,
+    26949,
+    26950,
+    26951,
+    26952,
+    26965,
+    26966,
+    26972,
+    26976,
+    26977,
+    26978,
+    26982,
+    26985,
+    26989,
+    26992,
+    26993,
+    26994,
+    27002,
+    27009,
+    27010,
+    27014,
+    27015,
+    27020,
+    27026,
+    27027,
+    27028,
+    27031,
+    27032,
+    27035,
+    27051,
+    27052,
+    27058,
+    27064,
+    27065,
+    27066,
+    27067,
+    27072,
+    27073,
+    27082,
+    27085,
+    27086,
+    27087,
+    27095,
+    27100,
+    27102,
+    27103,
+    27108,
+    27109,
+    27110,
+    27113,
+    27114,
+    27115,
+    27122,
+    27131,
+    27147,
+    27153,
+    27156,
+    27164,
+    27165,
+    27172,
+    27174,
+    27176,
+    27179,
+    27184,
+    27190,
+    27191,
+    27199,
+    27203,
+    27204,
+    27210,
+    27225,
+    27235,
+    27239,
+    27241,
+    27242,
+    27245,
+    27265,
+    27266,
+    27267,
+    27268,
+    27272,
+    27275,
+    27276,
+    27283,
+    27287,
+    27288,
+    27292,
+    27293,
+    27296,
+    27310,
+    27313,
+    27318,
+    27321,
+    27330,
+    27334,
+    27341,
+    27342,
+    27346,
+    27348,
+    27349,
+    27350,
+    27351,
+    27355,
+    27360,
+    27361,
+    27362,
+    27364,
+    27366,
+    27373,
+    27378,
+    27380,
+    27381,
+    27385,
+    27390,
+    27391,
+    27392,
+    27401,
+    27405,
+    27406,
+    27412,
+    27413,
+    27415,
+    27416,
+    27417,
+    27421,
+    27428,
+    27430,
+    27431,
+    27432,
+    27439,
+    27444,
+    27453,
+    27454,
+    27465,
+    27466,
+    27469,
+    27472,
+    27475,
+    27479,
+    27483,
+    27484,
+    27487,
+    27488,
+    27495,
+    27500,
+    27501,
+    27506,
+    27511,
+    27512,
+    27524,
+    27528,
+    27531,
+    27532,
+    27537,
+    27543,
+    27548,
+    27549,
+    27559,
+    27572,
+    27573,
+    27576,
+    27578,
+    27580,
+    27581,
+    27584,
+    27585,
+    27586,
+    27595,
+    27605,
+    27606,
+    27609,
+    27610,
+    27611,
+    27614,
+    27618,
+    27623,
+    27627,
+    27629,
+    27631,
+    27632,
+    27633,
+    27635,
+    27640,
+    27642,
+    27651,
+    27654,
+    27657,
+    27659,
+    27663,
+    27678,
+    27684,
+    27685,
+    27687,
+    27688,
+    27693,
+    27694,
+    27698,
+    27703,
+    27704,
+    27709,
+    27710,
+    27711,
+    27717,
+    27718,
+    27720,
+    27727,
+    27730,
+    27732,
+    27740,
+    27742,
+    27750,
+    27751,
+    27753,
+    27757,
+    27759,
+    27760,
+    27766,
+    27767,
+    27773,
+    27774,
+    27794,
+    27795,
+    27798,
+    27803,
+    27806,
+    27807,
+    27814,
+    27815,
+    27819,
+    27821,
+    27822,
+    27833,
+    27839,
+    27840,
+    27843,
+    27844,
+    27847,
+    27851,
+    27861,
+    27867,
+    27869,
+    27870,
+    27871,
+    27872,
+    27875,
+    27876,
+    27877,
+    27881,
+    27887,
+    27892,
+    27893,
+    27896,
+    27897,
+    27898,
+    27900,
+    27901,
+    27903,
+    27904,
+    27914,
+    27915,
+    27920,
+    27921,
+    27922,
+    27923,
+    27927,
+    27931,
+    27932,
+    27933,
+    27942,
+    27943,
+    27944,
+    27949,
+    27953,
+    27958,
+    27959,
+    27965,
+    27966,
+    27967,
+    27971,
+    27972,
+    27975,
+    27982,
+    27983,
+    27989,
+    28001,
+    28002,
+    28006,
+    28008,
+    28014,
+    28020,
+    28027,
+    28029,
+    28032,
+    28037,
+    28039,
+    28040,
+    28055,
+    28060,
+    28061,
+    28064,
+    28065,
+    28067,
+    28074,
+    28075,
+    28077,
+    28078,
+    28083,
+    28084,
+    28090,
+    28094,
+    28097,
+    28099,
+    28100,
+    28104,
+    28108,
+    28109,
+    28113,
+    28114,
+    28116,
+    28118,
+    28119,
+    28120,
+    28121,
+    28128,
+    28132,
+    28136,
+    28138,
+    28139,
+    28140,
+    28143,
+    28146,
+    28147,
+    28154,
+    28160,
+    28168,
+    28173,
+    28176,
+    28183,
+    28184,
+    28187,
+    28191,
+    28195,
+    28198,
+    28199,
+    28200,
+    28201,
+    28207,
+    28209,
+    28210,
+    28212,
+    28221,
+    28225,
+    28227,
+    28228,
+    28230,
+    28231,
+    28234,
+    28235,
+    28239,
+    28242,
+    28247,
+    28250,
+    28251,
+    28254,
+    28255,
+    28260,
+    28262,
+    28264,
+    28265,
+    28277,
+    28281,
+    28282,
+    28295,
+    28296,
+    28298,
+    28302,
+    28307,
+    28311,
+    28312,
+    28313,
+    28327,
+    28329,
+    28331,
+    28332,
+    28335,
+    28344,
+    28345,
+    28351,
+    28357,
+    28362,
+    28365,
+    28369,
+    28370,
+    28377,
+    28392,
+    28393,
+    28394,
+    28397,
+    28400,
+    28402,
+    28403,
+    28405,
+    28412,
+    28414,
+    28418,
+    28426,
+    28431,
+    28433,
+    28437,
+    28471,
+    28472,
+    28473,
+    28476,
+    28478,
+    28479,
+    28483,
+    28484,
+    28488,
+    28495,
+    28499,
+    28503,
+    28505,
+    28507,
+    28508,
+    28510,
+    28518,
+    28522,
+    28524,
+    28530,
+    28533,
+    28534,
+    28545,
+    28546,
+    28550,
+    28551,
+    28553,
+    28555,
+    28556,
+    28562,
+    28563,
+    28574,
+    28578,
+    28579,
+    28580,
+    28582,
+    28588,
+    28591,
+    28597,
+    28598,
+    28600,
+    28610,
+    28613,
+    28614,
+    28617,
+    28621,
+    28632,
+    28637,
+    28640,
+    28649,
+    28656,
+    28659,
+    28660,
+    28665,
+    28674,
+    28677,
+    28682,
+    28687,
+    28689,
+    28695,
+    28696,
+    28700,
+    28703,
+    28704,
+    28705,
+    28706,
+    28716,
+    28723,
+    28727,
+    28728,
+    28730,
+    28731,
+    28732,
+    28733,
+    28738,
+    28739,
+    28741,
+    28742,
+    28745,
+    28748,
+    28752,
+    28753,
+    28756,
+    28759,
+    28765,
+    28775,
+    28787,
+    28788,
+    28793,
+    28804,
+    28807,
+    28808,
+    28812,
+    28813,
+    28817,
+    28819,
+    28823,
+    28824,
+    28825,
+    28827,
+    28832,
+    28835,
+    28849,
+    28852,
+    28863,
+    28869,
+    28871,
+    28872,
+    28875,
+    28878,
+    28880,
+    28883,
+    28884,
+    28886,
+    28891,
+    28893,
+    28897,
+    28901,
+    28905,
+    28911,
+    28921,
+    28922,
+    28931,
+    28940,
+    28943,
+    28948,
+    28949,
+    28951,
+    28954,
+    28957,
+    28964,
+    28968,
+    28969,
+    28970,
+    28975,
+    28985,
+    28986,
+    28987,
+    28989,
+    28990,
+    28992,
+    28993,
+    28994,
+    28995,
+    29001,
+    29002,
+    29005,
+    29006,
+    29015,
+    29020,
+    29021,
+    29027,
+    29028,
+    29033,
+    29039,
+    29040,
+    29043,
+    29045,
+    29046,
+    29048,
+    29050,
+    29055,
+    29064,
+    29067,
+    29068,
+    29074,
+    29076,
+    29084,
+    29091,
+    29092,
+    29097,
+    29098,
+    29099,
+    29100,
+    29107,
+    29109,
+    29110,
+    29111,
+    29112,
+    29113,
+    29122,
+    29123,
+    29125,
+    29128,
+    29133,
+    29134,
+    29135,
+    29136,
+    29138,
+    29139,
+    29146,
+    29147,
+    29148,
+    29150,
+    29154,
+    29155,
+    29159,
+    29162,
+    29163,
+    29164,
+    29165,
+    29166,
+    29171,
+    29178,
+    29180,
+    29181,
+    29183,
+    29184,
+    29189,
+    29190,
+    29193,
+    29196,
+    29201,
+    29206,
+    29207,
+    29212,
+    29218,
+    29232,
+    29234,
+    29243,
+    29244,
+    29246,
+    29247,
+    29251,
+    29256,
+    29264,
+    29266,
+    29275,
+    29278,
+    29282,
+    29284,
+    29287,
+    29288,
+    29289,
+    29292,
+    29293,
+    29297,
+    29301,
+    29312,
+    29313,
+    29314,
+    29315,
+    29319,
+    29320,
+    29323,
+    29325,
+    29334,
+    29336,
+    29337,
+    29339,
+    29342,
+    29344,
+    29345,
+    29346,
+    29356,
+    29358,
+    29366,
+    29370,
+    29377,
+    29378,
+    29381,
+    29385,
+    29388,
+    29389,
+    29390,
+    29392,
+    29397,
+    29401,
+    29402,
+    29405,
+    29415,
+    29416,
+    29417,
+    29418,
+    29419,
+    29422,
+    29423,
+    29424,
+    29427,
+    29428,
+    29436,
+    29440,
+    29441,
+    29443,
+    29446,
+    29451,
+    29452,
+    29459,
+    29462,
+    29469,
+    29472,
+    29473,
+    29477,
+    29480,
+    29481,
+    29487,
+    29488,
+    29493,
+    29501,
+    29510,
+    29513,
+    29514,
+    29518,
+    29519,
+    29521,
+    29522,
+    29531,
+    29537,
+    29538,
+    29539,
+    29540,
+    29544,
+    29545,
+    29546,
+    29547,
+    29548,
+    29553,
+    29560,
+    29566,
+    29573,
+    29574,
+    29576,
+    29577,
+    29578,
+    29587,
+    29595,
+    29598,
+    29599,
+    29601,
+    29602,
+    29604,
+    29605,
+    29612,
+    29613,
+    29614,
+    29615,
+    29616,
+    29617,
+    29618,
+    29619,
+    29620,
+    29621,
+    29622,
+    29623,
+    29624,
+    29625,
+    29626,
+    29627,
+    29628,
+    29629,
+    29630,
+    29631,
+    29632,
+    29633,
+    29634,
+    29635,
+    29636,
+    29637,
+    29638,
+    29639,
+    29640,
+    29641,
+    29642,
+    29643,
+    29644,
+    29645,
+    29646,
+    29647,
+    29648,
+    29649,
+    29650,
+    29651,
+    29652,
+    29653,
+    29654,
+    29655,
+    29656,
+    29657,
+    29658,
+    29659,
+    29660,
+    29661,
+    29662,
+    29663,
+    29664,
+    29665,
+    29666,
+    29667,
+    29668,
+    29669,
+    29670,
+    29671,
+    29672,
+    29673,
+    29674,
+    29675,
+    29676,
+    29677,
+    29678,
+    29679,
+    29680,
+    29681,
+    29682,
+    29683,
+    29684,
+    29685,
+    29686,
+    29687,
+    29688,
+    29689,
+    29690,
+    29691,
+    29692,
+    29693,
+    29694,
+    29695,
+    29696,
+    29697,
+    29698,
+    29699,
+    29700,
+    29701,
+    29702,
+    29703,
+    29704,
+    29705,
+    29706,
+    29707,
+    29708,
+    29709,
+    29710,
+    29711,
+    29712,
+    29713,
+    29714,
+    29715,
+    29716,
+    29717,
+    29718,
+    29719,
+    29720,
+    29721,
+    29722,
+    29723,
+    29724,
+    29725,
+    29726,
+    29727,
+    29728,
+    29729,
+    29730,
+    29731,
+    29732,
+    29733,
+    29734,
+    29735,
+    29736,
+    29737,
+    29738,
+    29739,
+    29740,
+    29741,
+    29742,
+    29743,
+    29744,
+    29745,
+    29746,
+    29747,
+    29748,
+    29749,
+    29750,
+    29751,
+    29752,
+    29753,
+    29754,
+    29755,
+    29756,
+    29757,
+    29758,
+    29759,
+    29760,
+    29761,
+    29762,
+    29763,
+    29764,
+    29765,
+    29766,
+    29767,
+    29768,
+    29769,
+    29770,
+    29771,
+    29772,
+    29773,
+    29774,
+    29775,
+    29776,
+    29777,
+    29778,
+    29779,
+    29780,
+    29781,
+    29782,
+    29783,
+    29784,
+    29785,
+    29786,
+    29787,
+    29788,
+    29789,
+    29790,
+    29791,
+    29792,
+    29793,
+    29794,
+    29795,
+    29796,
+    29797,
+    29798,
+    29799,
+    29800,
+    29801,
+    29802,
+    29803,
+    29804,
+    29805,
+    29806,
+    29807,
+    29808,
+    29809,
+    29810,
+    29811,
+    29812,
+    29813,
+    29814,
+    29815,
+    29816,
+    29817,
+    29818,
+    29819,
+    29820,
+    29821,
+    29822,
+    29823,
+    29824,
+    29825,
+    29826,
+    29827,
+    29828,
+    29829,
+    29830,
+    29831,
+    29832,
+    29833,
+    29834,
+    29835,
+    29836,
+    29837,
+    29838,
+    29839,
+    29840,
+    29841,
+    29842,
+    29843,
+    29844,
+    29845,
+    29846,
+    29847,
+    29848,
+    29849,
+    29850,
+    29851,
+    29852,
+    29853,
+    29854,
+    29855,
+    29856,
+    29857,
+    29858,
+    29859,
+    29860,
+    29861,
+    29862,
+    29863,
+    29864,
+    29865,
+    29866,
+    29867,
+    29868,
+    29869,
+    29870,
+    29871,
+    29872,
+    29873,
+    29874,
+    29875,
+    29876,
+    29877,
+    29878,
+    29879,
+    29880,
+    29881,
+    29882,
+    29883,
+    29884,
+    29885,
+    29886,
+    29887,
+    29888,
+    29889,
+    29890,
+    29891,
+    29892,
+    29893,
+    29894,
+    29895,
+    29896,
+    29897,
+    29898,
+    29899,
+    29900,
+    29901,
+    29902,
+    29903,
+    29904,
+    29905,
+    29906,
+    29907,
+    29908,
+    29909,
+    29910,
+    29911,
+    29912,
+    29913,
+    29914,
+    29915,
+    29916,
+    29917,
+    29918,
+    29919,
+    29920,
+    29921,
+    29922,
+    29923,
+    29924,
+    29925,
+    29926,
+    29927,
+    29928,
+    29929,
+    29930,
+    29931,
+    29932,
+    29933,
+    29934,
+    29935,
+    29936,
+    29937,
+    29938,
+    29939,
+    29940,
+    29941,
+    29942,
+    29943,
+    29944,
+    29945,
+    29946,
+    29947,
+    29948,
+    29949,
+    29950,
+    29951,
+    29952,
+    29953,
+    29954,
+    29955,
+    29956,
+    29957,
+    29958,
+    29959,
+    29960,
+    29961,
+    29962,
+    29963,
+    29964,
+    29965,
+    29966,
+    29967,
+    29968,
+    29969,
+    29970,
+    29971,
+    29972,
+    29973,
+    29974,
+    29975,
+    29976,
+    29977,
+    29978,
+    29979,
+    29980,
+    29981,
+    29982,
+    29983,
+    29984,
+    29985,
+    29986,
+    29987,
+    29988,
+    29989,
+    29990,
+    29991,
+    29992,
+    29993,
+    29994,
+    29995,
+    29996,
+    29997,
+    29998,
+    29999,
+    30000,
+    30001,
+    30002,
+    30003,
+    30004,
+    30005,
+    30006,
+    30007,
+    30008,
+    30009,
+    30010,
+    30011,
+    30012,
+    30013,
+    30014,
+    30015,
+    30016,
+    30017,
+    30018,
+    30019,
+    30020,
+    30021,
+    30022,
+    30023,
+    30024,
+    30025,
+    30026,
+    30027,
+    30028,
+    30029,
+    30030,
+    30031,
+    30032,
+    30033,
+    30034,
+    30035,
+    30036,
+    30037,
+    30038,
+    30039,
+    30040,
+    30041,
+    30042,
+    30043,
+    30044,
+    30045,
+    30046,
+    30047,
+    30048,
+    30049,
+    30050,
+    30051,
+    30052,
+    30053,
+    30054,
+    30055,
+    30056,
+    30057,
+    30058,
+    30059,
+    30060,
+    30061,
+    30062,
+    30063,
+    30064,
+    30065,
+    30066,
+    30067,
+    30068,
+    30069,
+    30070,
+    30071,
+    30072,
+    30073,
+    30074,
+    30075,
+    30076,
+    30077,
+    30078,
+    30079,
+    30080,
+    30081,
+    30082,
+    30083,
+    30084,
+    30085,
+    30086,
+    30087,
+    30088,
+    30089,
+    30090,
+    30091,
+    30092,
+    30093,
+    30094,
+    30095,
+    30096,
+    30097,
+    30098,
+    30099,
+    30100,
+    30101,
+    30102,
+    30103,
+    30104,
+    30105,
+    30106,
+    30107,
+    30108,
+    30109,
+    30110,
+    30111,
+    30112,
+    30113,
+    30114,
+    30115,
+    30116,
+    30117,
+    30118,
+    30119,
+    30120,
+    30121,
+    30122,
+    30123,
+    30124,
+    30125,
+    30126,
+    30127,
+    30128,
+    30129,
+    30130,
+    30131,
+    30132,
+    30133,
+    30134,
+    30135,
+    30136,
+    30137,
+    30138,
+    30139,
+    30140,
+    30141,
+    30142,
+    30143,
+    30144,
+    30145,
+    30146,
+    30147,
+    30148,
+    30149,
+    30150,
+    30151,
+    30152,
+    30153,
+    30154,
+    30155,
+    30156,
+    30157,
+    30158,
+    30159,
+    30160,
+    30161,
+    30162,
+    30163,
+    30164,
+    30165,
+    30166,
+    30167,
+    30168,
+    30169,
+    30170,
+    30171,
+    30172,
+    30173,
+    30174,
+    30175,
+    30176,
+    30177,
+    30178,
+    30179,
+    30180,
+    30181,
+    30182,
+    30183,
+    30184,
+    30185,
+    30186,
+    30187,
+    30188,
+    30189,
+    30190,
+    30191,
+    30192,
+    30193,
+    30194,
+    30195,
+    30196,
+    30197,
+    30198,
+    30199,
+    30200,
+    30201,
+    30202,
+    30203,
+    30204,
+    30205,
+    30206,
+    30207,
+    30208,
+    30209,
+    30210,
+    30211,
+    30212,
+    30213,
+    30214,
+    30215,
+    30216,
+    30217,
+    30218,
+    30219,
+    30220,
+    30221,
+    30222,
+    30223,
+    30224,
+    30225,
+    30226,
+    30227,
+    30228,
+    30229,
+    30230,
+    30231,
+    30232,
+    30233,
+    30234,
+    30235,
+    30236,
+    30237,
+    30238,
+    30239,
+    30240,
+    30241,
+    30242,
+    30243,
+    30244,
+    30245,
+    30246,
+    30247,
+    30248,
+    30249,
+    30250,
+    30251,
+    30252,
+    30253,
+    30254,
+    30255,
+    30256,
+    30257,
+    30258,
+    30259,
+    30260,
+    30261,
+    30262,
+    30263,
+    30264,
+    30265,
+    30266,
+    30267,
+    30268,
+    30269,
+    30270,
+    30271,
+    30272,
+    30273,
+    30274,
+    30275,
+    30276,
+    30277,
+    30278,
+    30279,
+    30280,
+    30281,
+    30282,
+    30283,
+    30284,
+    30285,
+    30286,
+    30287,
+    30288,
+    30289,
+    30290,
+    30291,
+    30292,
+    30293,
+    30294,
+    30295,
+    30296,
+    30297,
+    30298,
+    30299,
+    30300,
+    30301,
+    30302,
+    30303,
+    30304,
+    30305,
+    30306,
+    30307,
+    30308,
+    30309,
+    30310,
+    30311,
+    30312,
+    30313,
+    30314,
+    30315,
+    30316,
+    30317,
+    30318,
+    30319,
+    30320,
+    30321,
+    30322,
+    30323,
+    30324,
+    30325,
+    30326,
+    30327,
+    30328,
+    30329,
+    30330,
+    30331,
+    30332,
+    30333,
+    30334,
+    30335,
+    30336,
+    30337,
+    30338,
+    30339,
+    30340,
+    30341,
+    30342,
+    30343,
+    30344,
+    30345,
+    30346,
+    30347,
+    30348,
+    30349,
+    30350,
+    30351,
+    30352,
+    30353,
+    30354,
+    30355,
+    30356,
+    30357,
+    30358,
+    30359,
+    30360,
+    30361,
+    30362,
+    30363,
+    30364,
+    30365,
+    30366,
+    30367,
+    30368,
+    30369,
+    30370,
+    30371,
+    30372,
+    30373,
+    30374,
+    30375,
+    30376,
+    30377,
+    30378,
+    30379,
+    30380,
+    30381,
+    30382,
+    30383,
+    30384,
+    30385,
+    30386,
+    30387,
+    30388,
+    30389,
+    30390,
+    30391,
+    30392,
+    30393,
+    30394,
+    30395,
+    30396,
+    30397,
+    30398,
+    30399,
+    30400,
+    30401,
+    30402,
+    30403,
+    30404,
+    30405,
+    30406,
+    30407,
+    30408,
+    30409,
+    30410,
+    30411,
+    30412,
+    30413,
+    30414,
+    30415,
+    30416,
+    30417,
+    30418,
+    30419,
+    30420,
+    30421,
+    30422,
+    30423,
+    30424,
+    30425,
+    30426,
+    30427,
+    30428,
+    30429,
+    30430,
+    30431,
+    30432,
+    30433,
+    30434,
+    30435,
+    30436,
+    30437,
+    30438,
+    30439,
+    30440,
+    30441,
+    30442,
+    30443,
+    30444,
+    30445,
+    30446,
+    30447,
+    30448,
+    30449,
+    30450,
+    30451,
+    30452,
+    30453,
+    30454,
+    30455,
+    30456,
+    30457,
+    30458,
+    30459,
+    30460,
+    30461,
+    30462,
+    30463,
+    30464,
+    30465,
+    30466,
+    30467,
+    30468,
+    30469,
+    30470,
+    30471,
+    30472,
+    30473,
+    30474,
+    30475,
+    30476,
+    30477,
+    30478,
+    30479,
+    30480,
+    30481,
+    30482,
+    30483,
+    30484,
+    30485,
+    30486,
+    30487,
+    30488,
+    30489,
+    30490,
+    30491,
+    30492,
+    30493,
+    30494,
+    30495,
+    30496,
+    30497,
+    30498,
+    30499,
+    30500,
+    30501,
+    30502,
+    30503,
+    30504,
+    30505,
+    30506,
+    30507,
+    30508,
+    30509,
+    30510,
+    30511,
+    30512,
+    30513,
+    30514,
+    30515,
+    30516,
+    30517,
+    30518,
+    30519,
+    30520,
+    30521
+  ],
+  "dtype": "float32",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "intermediate_size": 2048,
+  "key_rho": 2.18,
+  "lookup_head": 0,
+  "lookup_layer": 3,
+  "max_position_embeddings": 8192,
+  "model_type": "captionbert_structured_rotary",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 12,
+  "output_dim": 768,
+  "pad_token_id": 0,
+  "phase_seed": 1662197431,
+  "pointer_positions": "all",
+  "pointer_weight": 0.0,
+  "punctuation_ids": [
+    1012,
+    999,
+    1029
+  ],
+  "rotary_mode": "hierarchical",
+  "special_ids": [
+    100,
+    102,
+    0,
+    101,
+    103
+  ],
+  "transformers_version": "5.22.1",
+  "use_value_addresses": true,
+  "vocab_size": 30522
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/configuration_captionbert.py

@@ -0,0 +1,48 @@
+"""Self-contained configuration shipped beside the safetensors AutoModel."""
+from transformers import PretrainedConfig
+
+
+class CaptionBertConfig(PretrainedConfig):
+    model_type = "captionbert_structured_rotary"
+
+    def __init__(self, vocab_size=30522, hidden_size=512, num_hidden_layers=12,
+                 num_attention_heads=16, intermediate_size=2048, output_dim=768,
+                 max_position_embeddings=8192, hidden_dropout_prob=0.1,
+                 rotary_mode="key_all", lookup_layer=3, lookup_head=0,
+                 pointer_weight=0.1, pointer_positions="all", key_rho=2.18,
+                 use_value_addresses=True, gradient_checkpointing=True,
+                 continuation_ids=None, punctuation_ids=None, special_ids=None,
+                 phase_seed=1662197431, pad_token_id=0, **kwargs):
+        super().__init__(pad_token_id=pad_token_id, **kwargs)
+        self.vocab_size = int(vocab_size)
+        self.hidden_size = int(hidden_size)
+        self.num_hidden_layers = int(num_hidden_layers)
+        self.num_attention_heads = int(num_attention_heads)
+        self.intermediate_size = int(intermediate_size)
+        self.output_dim = int(output_dim)
+        self.max_position_embeddings = int(max_position_embeddings)
+        self.hidden_dropout_prob = float(hidden_dropout_prob)
+        self.rotary_mode = rotary_mode
+        self.lookup_layer, self.lookup_head = int(lookup_layer), int(lookup_head)
+        self.pointer_weight = float(pointer_weight)
+        self.pointer_positions = pointer_positions
+        self.key_rho = float(key_rho)
+        self.use_value_addresses = bool(use_value_addresses)
+        self.gradient_checkpointing = bool(gradient_checkpointing)
+        self.continuation_ids = list(continuation_ids or [])
+        self.punctuation_ids = list(punctuation_ids if punctuation_ids is not None else [1012, 999, 1029])
+        self.special_ids = list(special_ids if special_ids is not None else [0, 100, 101, 102, 103])
+        self.phase_seed = int(phase_seed)
+        if rotary_mode not in ("rope", "ntk", "hierarchical", "key_end", "key_all"):
+            raise ValueError("Unknown rotary_mode")
+        if hidden_size % num_attention_heads or (hidden_size // num_attention_heads) % 2:
+            raise ValueError("Attention heads must contain an even number of dimensions")
+        if rotary_mode in ("hierarchical", "key_end", "key_all") and hidden_size // num_attention_heads != 32:
+            raise ValueError("The source-faithful 8x3 band requires 32-dimensional heads")
+        if min(vocab_size, hidden_size, num_hidden_layers, intermediate_size, output_dim) < 1:
+            raise ValueError("Dimensions must be positive")
+        if not 0 <= lookup_layer < num_hidden_layers or not 0 <= lookup_head < num_attention_heads:
+            raise ValueError("Lookup layer/head is outside the model")
+        if pointer_positions not in ("end", "all") or pointer_weight < 0 or key_rho < 0:
+            raise ValueError("Invalid pointer/norm configuration")
+

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/dev-sts-predictions.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c88c26d64d5e5a61f8238eab13e7b08455390706f2fe7c6b5fc6bbb2d14529
+size 18514

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/environment.json

@@ -0,0 +1,10 @@
+{
+  "code_revision": "v0.3.0",
+  "cuda": "13.2",
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
+  "precision": "bf16 autocast, fp32 parameters and optimizer",
+  "python": "3.13.16",
+  "tf32": false,
+  "torch": "2.15.0+cu132"
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/eval-007808.json

@@ -0,0 +1,16 @@
+{
+  "above_128_tokens": {
+    "cos_to_target": 0.7362988591194153,
+    "erank": 73.03811752294244,
+    "n": 5000,
+    "self_cos": 0.011567466648996574
+  },
+  "above_128_total_rows": 29240,
+  "heldout": {
+    "cos_to_target": 0.8080115914344788,
+    "erank": 95.62741489359365,
+    "n": 5000,
+    "self_cos": 0.003444665011627598
+  },
+  "heldout_total_rows": 500000
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/experiment.json

@@ -0,0 +1,88 @@
+{
+  "arm_definitions": {
+    "H16": {
+      "pointer_weight": 0.0,
+      "rotary_mode": "hierarchical"
+    },
+    "KALL": {
+      "pointer_weight": 0.0,
+      "rotary_mode": "key_all"
+    },
+    "KALLP": {
+      "pointer_positions": "all",
+      "pointer_weight": 0.1,
+      "rotary_mode": "key_all"
+    },
+    "KALLP_L8": {
+      "lookup_layer": 7,
+      "pointer_positions": "all",
+      "pointer_weight": 0.1,
+      "rotary_mode": "key_all"
+    },
+    "KALLP_N0": {
+      "key_rho": 0.0,
+      "pointer_positions": "all",
+      "pointer_weight": 0.1,
+      "rotary_mode": "key_all"
+    },
+    "KEND": {
+      "pointer_positions": "end",
+      "pointer_weight": 0.0,
+      "rotary_mode": "key_end"
+    },
+    "N16": {
+      "pointer_weight": 0.0,
+      "rotary_mode": "ntk",
+      "use_value_addresses": false
+    },
+    "R8": {
+      "num_attention_heads": 8,
+      "pointer_weight": 0.0,
+      "rotary_mode": "rope",
+      "use_value_addresses": false
+    }
+  },
+  "config": {
+    "arms": [
+      "H16"
+    ],
+    "batch_size": 2048,
+    "bucket_window": 64,
+    "checkpoint_every": 100,
+    "code_revision": "v0.3.0",
+    "data_dir": "/content/captionbert-data",
+    "device": "cuda",
+    "epochs": 16,
+    "eval_batch": 64,
+    "eval_every": 500,
+    "eval_rows": 5000,
+    "final_test": true,
+    "grad_clip": 1.0,
+    "hub_repo": "AbstractPhil/captionbert-8192-rotary",
+    "log_every": 25,
+    "lr": 0.0006,
+    "max_steps": 7808,
+    "max_tokens": 256,
+    "memory_limit_gb": 92.0,
+    "min_lr": 1e-06,
+    "output_dir": "/content/captionbert-structured/exposure1m",
+    "push": true,
+    "resume_upload_minutes": 10.0,
+    "seeds": [
+      1
+    ],
+    "stage": "exposure1m",
+    "temperature": 0.07,
+    "train_chunks": [
+      0,
+      1
+    ],
+    "val_chunk": 61,
+    "warmup_steps": 400
+  },
+  "config_id": "fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e",
+  "dataset_id": "659b21c1c6eb1074eeec7acdbfa9bd010965595a4669338690cf3ae38a4448d5",
+  "selection": "paired-seed STS-B validation; teacher/rank viability; test splits unopened for selection",
+  "train_rows": 1000000,
+  "validation_rows": 500000
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1714edd317340b18c6da0a0fddc061e4647cdcf52365ef3308c0a883fa71fe5e
+size 216583564

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/modeling_captionbert.py

@@ -0,0 +1,288 @@
+"""CaptionBERT with the final key-at-every-position rotary adaptation.
+
+This file and configuration_captionbert.py are deliberately self-contained for
+AutoModel remote-code loading. No training package is needed for inference.
+"""
+import copy
+from dataclasses import dataclass
+import math
+from typing import Optional
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.utils.checkpoint import checkpoint
+from transformers import PreTrainedModel
+from transformers.utils import ModelOutput
+from .configuration_captionbert import CaptionBertConfig
+
+
+@dataclass
+class CaptionBertOutput(ModelOutput):
+    last_hidden_state: Optional[torch.Tensor] = None
+    pooler_output: Optional[torch.Tensor] = None
+    hidden_states: Optional[tuple] = None
+    attentions: Optional[tuple] = None
+    pointer_loss: Optional[torch.Tensor] = None
+
+
+def hierarchical_frequencies():
+    """v0.1.103's 8x3 form verbatim in value: 2 relative, 2 content, 12 anchor pairs."""
+    base = 10000.0 * 4.0 ** (4.0 / 2.0)
+    w = [base ** (-j / 2) for j in range(2)] + [0.0, 0.0]
+    kinds = [0, 0, 1, 1]
+    for level in range(3):
+        w += [2 * math.pi * k / 9 for k in range(1, 5)]
+        kinds += [2 + level] * 4
+    return torch.tensor(w, dtype=torch.float64), torch.tensor(kinds, dtype=torch.long)
+
+
+def rotate_half(t, angle):
+    pairs = t.shape[-1] // 2
+    a, b = t[..., :pairs].float(), t[..., pairs:].float()
+    c, s = angle.float().cos(), angle.float().sin()
+    return torch.cat((a * c - b * s, a * s + b * c), dim=-1).to(t.dtype)
+
+
+def hold_pairs(key, on, rho):
+    pairs = key.shape[-1] // 2
+    a, b = key[..., :pairs], key[..., pairs:]
+    # Clamp before sqrt: the same forward floor, with finite derivatives at
+    # zero/padded pairs (sqrt(0) followed by clamp can yield 0 * inf in backward).
+    length = (a.float().square() + b.float().square()).clamp_min(1e-12).sqrt()
+    factor = torch.where(on[..., None], rho / length, torch.ones_like(length)).to(key.dtype)
+    return torch.cat((a * factor, b * factor), dim=-1)
+
+
+def word_features(ids, valid, continuation, special, pairs=16, seed=1662197431):
+    """Stateless whole-WordPiece-key binding; all pieces and offsets contribute.
+
+    The source's fixed byte/offset phase table becomes a fixed 31-bit counter
+    hash over WordPiece IDs and reverse offsets. Integer accumulation is exact
+    and independent of scatter order. It has no key-length truncation. This is
+    a declared tokenizer adaptation, not the original byte table's draw.
+    """
+    batch, length = ids.shape
+    position = torch.arange(length, device=ids.device)[None].expand(batch, -1)
+    active = valid & ~special[ids]
+    starts = valid & ~continuation[ids]
+    group = starts.long().cumsum(1)
+    first = torch.where(starts, position, torch.zeros_like(position)).cummax(1).values
+    offset = position - first
+    counts = torch.zeros(batch, length + 1, device=ids.device, dtype=torch.long)
+    counts.scatter_add_(1, group, active.long())
+    reverse = (counts.gather(1, group) - 1 - offset).clamp_min(0)
+    pair = torch.arange(1, pairs + 1, device=ids.device, dtype=torch.long)
+    mask31 = 2147483647
+    u = (ids[..., None].long() * 1103515245 + reverse[..., None] * 1234567891
+         + pair * 1013904223 + int(seed)) & mask31
+    u = ((u ^ (u >> 16)) * 73244475) & mask31
+    u = ((u ^ (u >> 16)) * 73244475) & mask31
+    u = (u ^ (u >> 16)) & mask31
+    sums = torch.zeros(batch, length + 1, pairs, device=ids.device, dtype=torch.long)
+    sums.scatter_add_(1, group[..., None].expand(-1, -1, pairs), u * active[..., None])
+    keys = sums.gather(1, group[..., None].expand(-1, -1, pairs)) & mask31
+    phase = keys.float() * (2 * math.pi / 2147483648.0)
+    next_start = F.pad(starts[:, 1:], (0, 1), value=True)
+    next_active = F.pad(active[:, 1:], (0, 1), value=False)
+    ends = active & (next_start | ~next_active)
+
+    # First occurrence's ending is the record; later occurrences are queries.
+    # Two independent 31-bit components identify candidate groups. Training
+    # targets are computed from exact token tuples separately, never this hash.
+    identity = keys[..., 0] | (keys[..., 1] << 31)
+    identity = torch.where(active, identity, torch.full_like(identity, -1))
+    order = identity.argsort(dim=1, stable=True)
+    sorted_id = identity.gather(1, order)
+    changed = F.pad(sorted_id[:, 1:] != sorted_id[:, :-1], (1, 0), value=True)
+    sorted_group = changed.long().cumsum(1)
+    candidate = torch.where(ends, position, torch.full_like(position, length)).gather(1, order)
+    minimum = torch.full((batch, length + 1), length, device=ids.device, dtype=torch.long)
+    minimum.scatter_reduce_(1, sorted_group, candidate, reduce="amin", include_self=True)
+    sorted_record = minimum.gather(1, sorted_group)
+    record_at = torch.empty_like(sorted_record).scatter_(1, order, sorted_record)
+    records = ends & (record_at == position)
+    return phase, active, ends, records
+
+
+class CaptionBertLayer(nn.TransformerEncoderLayer):
+    def __init__(self, config):
+        super().__init__(config.hidden_size, config.num_attention_heads,
+                         config.intermediate_size, config.hidden_dropout_prob,
+                         activation="gelu", batch_first=True, norm_first=True)
+        self.lookup_head = config.lookup_head
+        self.rho = config.key_rho
+
+    def forward(self, x, valid, aq, ak, phase, binding_on, records,
+                pointer_targets=None, bind=False, output_attentions=False):
+        z = self.norm1(x)
+        batch, length, width = z.shape
+        heads = self.self_attn.num_heads
+        qkv = F.linear(z, self.self_attn.in_proj_weight, self.self_attn.in_proj_bias)
+        q, k, v = [a.reshape(batch, length, heads, width // heads).transpose(1, 2)
+                   for a in qkv.chunk(3, -1)]
+        qr, kr = rotate_half(q, aq), rotate_half(k, ak)
+        if bind:
+            h = self.lookup_head
+            qa = torch.where(binding_on[..., None], phase, aq[:, 0])
+            ka = torch.where(binding_on[..., None], phase, ak[:, 0])
+            kh = hold_pairs(k[:, h], records, self.rho) if self.rho else k[:, h]
+            qh, kh = rotate_half(q[:, h], qa), rotate_half(kh, ka)
+            qr = torch.cat((qr[:, :h], qh[:, None], qr[:, h + 1:]), 1)
+            kr = torch.cat((kr[:, :h], kh[:, None], kr[:, h + 1:]), 1)
+        pointer_loss = x.new_zeros((), dtype=torch.float32)
+        if pointer_targets is not None:
+            logits = qr[:, self.lookup_head].float() @ kr[:, self.lookup_head].float().transpose(1, 2)
+            logits = logits / math.sqrt(width // heads)
+            logits = logits.masked_fill(~valid[:, None, :], -1e9)
+            selected = pointer_targets >= 0
+            nll = -logits.log_softmax(-1).gather(-1, pointer_targets.clamp_min(0)[..., None]).squeeze(-1)
+            pointer_loss = (nll * selected).sum() / selected.sum().clamp_min(1)
+        y = F.scaled_dot_product_attention(qr, kr, v, attn_mask=valid[:, None, None, :],
+            dropout_p=self.self_attn.dropout if self.training else 0.0)
+        x = x + self.dropout1(self.self_attn.out_proj(y.transpose(1, 2).reshape(batch, length, width)))
+        x = x + self._ff_block(self.norm2(x))
+        if output_attentions:
+            logits = qr.float() @ kr.float().transpose(-1, -2) / math.sqrt(width // heads)
+            attention = logits.masked_fill(~valid[:, None, None, :], -1e9).softmax(-1)
+            return x, pointer_loss, attention
+        return x, pointer_loss
+
+
+class CaptionBertModel(PreTrainedModel):
+    config_class = CaptionBertConfig
+    base_model_prefix = "captionbert"
+    main_input_name = "input_ids"
+    supports_gradient_checkpointing = True
+    _supports_sdpa = True
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.gradient_checkpointing = config.gradient_checkpointing
+        self.token_emb = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
+        self.emb_norm = nn.LayerNorm(config.hidden_size)
+        self.emb_drop = nn.Dropout(config.hidden_dropout_prob)
+        template = CaptionBertLayer(config)
+        self.layers = nn.ModuleList([copy.deepcopy(template) for _ in range(config.num_hidden_layers)])
+        self.output_proj = nn.Sequential(nn.Linear(config.hidden_size, config.hidden_size), nn.GELU(),
+                                        nn.LayerNorm(config.hidden_size), nn.Linear(config.hidden_size, config.output_dim))
+        self.structured = config.rotary_mode in ("hierarchical", "key_end", "key_all")
+        self.value_address = nn.Parameter(torch.zeros(3, 8, config.hidden_size)) if self.structured and config.use_value_addresses else None
+        continuation = torch.zeros(config.vocab_size, dtype=torch.bool)
+        continuation[config.continuation_ids] = True
+        special = torch.zeros_like(continuation)
+        special[[i for i in config.special_ids if i < config.vocab_size]] = True
+        self.register_buffer("wordpiece_continuation", continuation)
+        self.register_buffer("special_token", special)
+        if self.structured:
+            frequencies, kinds = hierarchical_frequencies()
+        else:
+            pairs = config.hidden_size // config.num_attention_heads // 2
+            dim = pairs * 2
+            base = 10000.0 * (4.0 ** (dim / (dim - 2)) if config.rotary_mode == "ntk" else 1.0)
+            frequencies = base ** (-torch.arange(pairs, dtype=torch.float64) / pairs)
+            kinds = torch.zeros(pairs, dtype=torch.long)
+        # Save the small immutable rotary tables alongside the weights. Current
+        # Transformers materializes non-persistent buffers with empty_like when
+        # loading on meta; constructor-only initialization would then be lost.
+        self.register_buffer("frequencies", frequencies.float())
+        self.register_buffer("kinds", kinds)
+        self.post_init()
+
+    def _init_weights(self, module):
+        # Keep PyTorch's native CaptionBERT initialization; value-address tables
+        # are born zero. No HF reinitialization changes the paired controls.
+        pass
+
+    def get_input_embeddings(self):
+        return self.token_emb
+
+    def set_input_embeddings(self, value):
+        self.token_emb = value
+
+    def addresses(self, ids, valid):
+        punctuation = torch.zeros_like(valid)
+        for token in self.config.punctuation_ids:
+            punctuation = punctuation | (ids == token)
+        punctuation = punctuation & valid
+        sentence = punctuation.long().cumsum(1) - punctuation.long()
+        digits = torch.stack(((sentence // 64) % 8, (sentence // 8) % 8, sentence % 8), -1)
+        return torch.where((valid & (sentence < 512))[..., None], digits, torch.full_like(digits, 8))
+
+    def angles(self, ids, valid, query_addresses=None, key_addresses=None):
+        batch, length = ids.shape
+        position = torch.arange(length, device=ids.device, dtype=torch.float32)
+        base = position[:, None] * torch.where(self.kinds == 0, self.frequencies, 0)[None]
+        aq = base[None].expand(batch, -1, -1).clone()
+        ak = aq.clone()
+        addresses = self.addresses(ids, valid) if self.structured else None
+        if self.structured:
+            dq = addresses if query_addresses is None else query_addresses
+            dk = addresses if key_addresses is None else key_addresses
+            if dq.shape != addresses.shape or dk.shape != addresses.shape:
+                raise ValueError("Address streams must have shape (batch, length, 3)")
+            for level in range(3):
+                selected = self.kinds == level + 2
+                aq[..., selected] = (dq[..., level:level + 1].float() - 8) * self.frequencies[selected]
+                ak[..., selected] = (dk[..., level:level + 1].float() - 8) * self.frequencies[selected]
+            addresses = dk
+        return aq[:, None], ak[:, None], addresses
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
+                query_addresses=None, key_addresses=None, pointer_targets=None,
+                output_hidden_states=None, output_attentions=None, return_dict=None, **kwargs):
+        if input_ids.ndim != 2 or not 1 <= input_ids.shape[1] <= self.config.max_position_embeddings:
+            raise ValueError("input_ids must be (batch, length), with length in the configured range")
+        valid = input_ids.ne(self.config.pad_token_id) if attention_mask is None else attention_mask.bool()
+        if valid.shape != input_ids.shape:
+            raise ValueError("attention_mask must match input_ids")
+        output_hidden_states = self.config.output_hidden_states if output_hidden_states is None else output_hidden_states
+        output_attentions = self.config.output_attentions if output_attentions is None else output_attentions
+        return_dict = self.config.use_return_dict if return_dict is None else return_dict
+        aq, ak, addresses = self.angles(input_ids, valid, query_addresses, key_addresses)
+        binding = self.config.rotary_mode in ("key_end", "key_all")
+        if binding:
+            phase, active, ends, records = word_features(input_ids, valid, self.wordpiece_continuation,
+                                                       self.special_token, seed=self.config.phase_seed)
+            on = ends if self.config.rotary_mode == "key_end" else active
+        else:
+            phase = aq[:, 0]
+            on = records = torch.zeros_like(valid)
+        x = self.token_emb(input_ids)
+        if self.value_address is not None:
+            for level in range(3):
+                d = addresses[..., level].long()
+                d = torch.where((d >= 0) & (d < 8), d, torch.full_like(d, 8))
+                table = torch.cat((self.value_address[level], x.new_zeros(1, x.shape[-1])), 0)
+                x = x + F.embedding(d, table, padding_idx=8)
+        x = self.emb_drop(self.emb_norm(x))
+        states = (x,) if output_hidden_states else None
+        attentions = () if output_attentions else None
+        pointer_loss = x.new_zeros((), dtype=torch.float32)
+        for index, layer in enumerate(self.layers):
+            target = pointer_targets if index == self.config.lookup_layer else None
+            if target is not None and self.config.pointer_positions == "end":
+                if not binding:
+                    raise ValueError("End-position supervision requires the word-binding features")
+                target = torch.where(ends, target, torch.full_like(target, -1))
+            args = (x, valid, aq, ak, phase, on, records)
+            opts = dict(pointer_targets=target, bind=binding and index == self.config.lookup_layer,
+                        output_attentions=bool(output_attentions))
+            if self.training and self.gradient_checkpointing and not output_attentions:
+                result = checkpoint(layer, *args, use_reentrant=False, **opts)
+            else:
+                result = layer(*args, **opts)
+            x, term = result[:2]
+            pointer_loss = pointer_loss + term
+            if output_hidden_states:
+                states += (x,)
+            if output_attentions:
+                attentions += (result[2],)
+        mask = valid[..., None].float()
+        pooled = (x.float() * mask).sum(1) / mask.sum(1).clamp_min(1)
+        embedding = F.normalize(self.output_proj(pooled).float(), dim=-1)
+        embedding = torch.where(valid.any(1)[:, None], embedding, torch.zeros_like(embedding))
+        if not return_dict:
+            return tuple(v for v in (x, embedding, states, attentions, pointer_loss) if v is not None)
+        return CaptionBertOutput(last_hidden_state=x, pooler_output=embedding,
+                                 hidden_states=states, attentions=attentions, pointer_loss=pointer_loss)

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/pip_freeze.txt

@@ -0,0 +1,79 @@
+absl-py==2.5.1
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.5
+aiosignal==1.4.0
+annotated-doc==0.0.5
+anyio==4.16.0
+attrs==26.1.0
+captionbert-rotary @ git+https://github.com/AbstractEyes/captionbert-trainer.git@cb1388dd3d313313c2cbe59b28200e3738793067
+click==8.5.0
+cuda-bindings==13.5.0
+cuda-pathfinder==1.8.6
+cuda-toolkit==13.2.2
+datasets==5.1.0
+dill==0.4.1
+filelock==4.2.9
+frozenlist==1.8.0
+fsspec==2026.7.0
+grpcio==1.85.1
+h11==0.16.0
+hf-xet==1.8.0
+httpcore2==2.16.0
+httpx2==2.16.0
+huggingface_hub==2.4.0
+idna==3.20
+Jinja2==3.1.6
+Markdown==3.11.3
+markdown-it-py==4.2.0
+MarkupSafe==3.0.4
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==7.2.1
+multiprocess==0.70.19
+networkx==3.7
+numpy==2.5.4
+nvidia-cublas==13.4.1.3
+nvidia-cuda-cccl==13.3.5.0.6
+nvidia-cuda-cupti==13.2.86
+nvidia-cuda-nvrtc==13.2.86
+nvidia-cuda-runtime==13.2.86
+nvidia-cudnn-cu13==9.26.0.51
+nvidia-cufft==12.2.0.57
+nvidia-cufile==1.17.1.22
+nvidia-cusolver==12.2.0.11
+nvidia-cusparse==12.7.10.12
+nvidia-cusparselt-cu13==0.8.1
+nvidia-nccl-cu13==2.30.7
+nvidia-nvjitlink==13.5.37
+nvidia-nvshmem-cu13==3.7.2
+nvidia-nvtx==13.2.86
+packaging==26.3
+pandas==3.1.1
+pillow==13.0.0
+propcache==0.5.4
+protobuf==7.36.2
+pyarrow==26.0.1
+Pygments==2.21.0
+python-dateutil==4.0.0
+PyYAML==6.0.3
+regex==2026.9.29
+rich==15.0.0
+safetensors==0.8.0
+scipy==1.18.1
+setuptools==84.2.1
+shellingham==1.5.4
+sympy==1.14.0
+tensorboard==2.21.0
+tensorboard-data-server==0.7.2
+tokenizers==0.23.3
+torch==2.15.0
+tqdm==4.70.1
+transformers==5.22.1
+triton==3.9.0
+truststore==0.10.5
+typer==0.28.0
+typing_extensions==4.16.0
+tzdata==2026.6
+Werkzeug==3.1.9
+xxhash==4.0.2
+yarl==1.25.1

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/result.json

@@ -0,0 +1,6017 @@
+{
+  "arm": "H16",
+  "benchmark": {
+    "dataset_fingerprints": {
+      "BIOSSES": "f1568c59b08bf901",
+      "SICK-R": "ec39ecc7fe3d35c5",
+      "STS-B": "a5cf888e07361adf",
+      "STS12": "babcef74f2c87568",
+      "STS13": "0b95e9bf5b389b20",
+      "STS14": "60e6b4f337fc83b1",
+      "STS15": "2ce54f5616944e48",
+      "STS16": "5f0ab25e659613f2"
+    },
+    "dataset_revisions": {
+      "mteb/biosses-sts": "f4f1265a7a4b0c99e6da09859b887bdb1f36d8ed",
+      "mteb/sickr-sts": "b93b4e08b707e9c9bc478e543495c3ced0b9d81e",
+      "mteb/sts12-sts": "3a51344839ec620dd3b753e0945bdf6c7e37af3e",
+      "mteb/sts13-sts": "f475812ca75868066186343ebf60e01f95c75164",
+      "mteb/sts14-sts": "8ab5e540b20b58d42fa6b944d3accee9df3fbaca",
+      "mteb/sts15-sts": "ae8e1ecbd3b3bcf5c1c8fd798290033715d98b93",
+      "mteb/sts16-sts": "d8e3be175a943eecfea09c96795087771996197c",
+      "mteb/stsbenchmark-sts": "96943a16ea6a35129e253c659081cb59daf81b30"
+    },
+    "max_tokens": 64,
+    "mean": 0.5623457076684515,
+    "split": "test",
+    "tasks": {
+      "BIOSSES": {
+        "erank": 24.437389468807638,
+        "self_cos": 0.4656761249349465,
+        "spearman": 0.4656672370499183
+      },
+      "SICK-R": {
+        "erank": 38.20085056275256,
+        "self_cos": 0.3120356239061648,
+        "spearman": 0.655322839605705
+      },
+      "STS-B": {
+        "erank": 36.86486821200843,
+        "self_cos": 0.142679074805884,
+        "spearman": 0.5451143879567549
+      },
+      "STS12": {
+        "erank": 27.4129171820282,
+        "self_cos": 0.1497967960231351,
+        "spearman": 0.48368747768423376
+      },
+      "STS13": {
+        "erank": 23.151543247970757,
+        "self_cos": 0.15161106209072178,
+        "spearman": 0.5259553590367815
+      },
+      "STS14": {
+        "erank": 37.465508806407975,
+        "self_cos": 0.11057021291141438,
+        "spearman": 0.5173224547588797
+      },
+      "STS15": {
+        "erank": 29.032311371058142,
+        "self_cos": 0.15310973097335936,
+        "spearman": 0.6950294568387984
+      },
+      "STS16": {
+        "erank": 30.70747353412827,
+        "self_cos": 0.15432572507159223,
+        "spearman": 0.6106664484165398
+      }
+    }
+  },
+  "checkpoint_sha256": "1714edd317340b18c6da0a0fddc061e4647cdcf52365ef3308c0a883fa71fe5e",
+  "config_id": "fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e",
+  "dataset_id": "659b21c1c6eb1074eeec7acdbfa9bd010965595a4669338690cf3ae38a4448d5",
+  "development": {
+    "max_tokens": 64,
+    "n": 1500,
+    "revision": "96943a16ea6a35129e253c659081cb59daf81b30",
+    "spearman": 0.6833117099948678,
+    "split": "validation",
+    "used_for_selection": true
+  },
+  "epochs": 16,
+  "evaluation": {
+    "above_128_tokens": {
+      "cos_to_target": 0.7362988591194153,
+      "erank": 73.03811752294244,
+      "n": 5000,
+      "self_cos": 0.011567466648996574
+    },
+    "above_128_total_rows": 29240,
+    "heldout": {
+      "cos_to_target": 0.8080115914344788,
+      "erank": 95.62741489359365,
+      "n": 5000,
+      "self_cos": 0.003444665011627598
+    },
+    "heldout_total_rows": 500000
+  },
+  "memory": {
+    "max_observed_device_gb": 57.169412096,
+    "peak_allocated_gb": 33.179365376,
+    "peak_reserved_gb": 56.346279936,
+    "total_device_limit_gb": 92.0
+  },
+  "model_config": {
+    "_name_or_path": "",
+    "architectures": [
+      "CaptionBertModel"
+    ],
+    "auto_map": {
+      "AutoConfig": "configuration_captionbert.CaptionBertConfig",
+      "AutoModel": "modeling_captionbert.CaptionBertModel"
+    },
+    "chunk_size_feed_forward": 0,
+    "continuation_ids": [
+      2015,
+      2050,
+      2063,
+      2072,
+      2075,
+      2078,
+      2080,
+      2094,
+      2098,
+      2099,
+      2100,
+      2102,
+      2121,
+      2135,
+      2140,
+      2213,
+      2226,
+      2229,
+      2232,
+      2239,
+      2243,
+      2271,
+      2278,
+      2290,
+      2319,
+      2361,
+      2368,
+      2378,
+      2386,
+      2389,
+      2401,
+      2475,
+      2480,
+      2483,
+      2487,
+      2497,
+      2509,
+      2527,
+      2532,
+      2545,
+      2546,
+      2549,
+      2571,
+      2575,
+      2581,
+      2594,
+      2595,
+      2615,
+      2618,
+      2620,
+      2629,
+      2638,
+      2666,
+      2669,
+      2683,
+      2692,
+      2696,
+      2705,
+      2721,
+      2791,
+      2818,
+      2819,
+      2850,
+      2854,
+      2860,
+      2863,
+      2869,
+      2884,
+      2890,
+      2891,
+      2906,
+      2912,
+      2923,
+      2937,
+      2953,
+      2964,
+      2989,
+      3012,
+      3022,
+      3051,
+      3064,
+      3070,
+      3077,
+      3085,
+      3089,
+      3111,
+      3122,
+      3126,
+      3148,
+      3170,
+      3207,
+      3211,
+      3215,
+      3217,
+      3238,
+      3240,
+      3258,
+      3270,
+      3286,
+      3334,
+      3351,
+      3363,
+      3366,
+      3367,
+      3370,
+      3372,
+      3385,
+      3388,
+      3401,
+      3406,
+      3436,
+      3468,
+      3489,
+      3490,
+      3501,
+      3508,
+      3511,
+      3512,
+      3527,
+      3540,
+      3549,
+      3550,
+      3560,
+      3567,
+      3593,
+      3597,
+      3600,
+      3619,
+      3630,
+      3654,
+      3669,
+      3672,
+      3676,
+      3678,
+      3683,
+      3686,
+      3695,
+      3702,
+      3709,
+      3723,
+      3726,
+      3736,
+      3762,
+      3771,
+      3775,
+      3790,
+      3877,
+      3900,
+      3917,
+      3973,
+      3981,
+      3989,
+      3993,
+      3995,
+      4014,
+      4017,
+      4048,
+      4059,
+      4063,
+      4095,
+      4103,
+      4115,
+      4135,
+      4140,
+      4143,
+      4160,
+      4168,
+      4173,
+      4179,
+      4181,
+      4183,
+      4213,
+      4215,
+      4221,
+      4226,
+      4232,
+      4244,
+      4246,
+      4263,
+      4270,
+      4305,
+      4313,
+      4328,
+      4355,
+      4360,
+      4371,
+      4381,
+      4383,
+      4402,
+      4430,
+      4478,
+      4492,
+      4502,
+      4509,
+      4523,
+      4570,
+      4571,
+      4576,
+      4580,
+      4588,
+      4590,
+      4609,
+      4630,
+      4645,
+      4648,
+      4665,
+      4667,
+      4674,
+      4691,
+      4697,
+      4710,
+      4726,
+      4747,
+      4757,
+      4765,
+      4779,
+      4783,
+      4801,
+      4817,
+      4818,
+      4819,
+      4842,
+      4859,
+      4877,
+      4886,
+      4887,
+      4890,
+      4892,
+      4904,
+      4917,
+      4939,
+      4948,
+      4974,
+      5004,
+      5007,
+      5017,
+      5051,
+      5054,
+      5063,
+      5084,
+      5092,
+      5104,
+      5130,
+      5134,
+      5149,
+      5162,
+      5172,
+      5178,
+      5243,
+      5244,
+      5267,
+      5280,
+      5283,
+      5289,
+      5302,
+      5311,
+      5313,
+      5332,
+      5339,
+      5349,
+      5358,
+      5369,
+      5397,
+      5400,
+      5403,
+      5420,
+      5422,
+      5428,
+      5480,
+      5484,
+      5488,
+      5498,
+      5521,
+      5524,
+      5555,
+      5558,
+      5562,
+      5575,
+      5582,
+      5596,
+      5602,
+      5603,
+      5605,
+      5620,
+      5634,
+      5638,
+      5643,
+      5644,
+      5649,
+      5654,
+      5657,
+      5666,
+      5668,
+      5669,
+      5677,
+      5685,
+      5686,
+      5705,
+      5714,
+      5732,
+      5737,
+      5740,
+      5753,
+      5794,
+      5802,
+      5804,
+      5809,
+      5831,
+      5833,
+      5844,
+      5856,
+      5867,
+      5874,
+      5880,
+      5886,
+      5897,
+      5910,
+      5937,
+      5946,
+      5963,
+      5974,
+      5999,
+      6024,
+      6026,
+      6030,
+      6038,
+      6072,
+      6096,
+      6129,
+      6132,
+      6137,
+      6155,
+      6169,
+      6182,
+      6190,
+      6198,
+      6199,
+      6200,
+      6212,
+      6216,
+      6238,
+      6277,
+      6279,
+      6290,
+      6292,
+      6299,
+      6305,
+      6313,
+      6321,
+      6340,
+      6342,
+      6371,
+      6374,
+      6392,
+      6442,
+      6444,
+      6447,
+      6455,
+      6460,
+      6483,
+      6491,
+      6494,
+      6499,
+      6508,
+      6525,
+      6528,
+      6553,
+      6558,
+      6559,
+      6562,
+      6582,
+      6588,
+      6590,
+      6591,
+      6593,
+      6610,
+      6632,
+      6633,
+      6648,
+      6651,
+      6657,
+      6672,
+      6673,
+      6675,
+      6679,
+      6692,
+      6711,
+      6761,
+      6767,
+      6774,
+      6776,
+      6777,
+      6779,
+      6784,
+      6790,
+      6799,
+      6806,
+      6810,
+      6820,
+      6824,
+      6826,
+      6834,
+      6843,
+      6844,
+      6850,
+      6856,
+      6858,
+      6862,
+      6873,
+      6894,
+      6895,
+      6906,
+      6914,
+      6916,
+      6924,
+      6935,
+      6961,
+      6962,
+      6968,
+      6977,
+      6979,
+      7011,
+      7028,
+      7033,
+      7054,
+      7062,
+      7066,
+      7068,
+      7076,
+      7088,
+      7092,
+      7096,
+      7100,
+      7103,
+      7106,
+      7113,
+      7138,
+      7140,
+      7159,
+      7174,
+      7178,
+      7184,
+      7228,
+      7229,
+      7231,
+      7245,
+      7265,
+      7274,
+      7277,
+      7292,
+      7295,
+      7301,
+      7315,
+      7317,
+      7347,
+      7352,
+      7361,
+      7363,
+      7368,
+      7373,
+      7377,
+      7382,
+      7384,
+      7389,
+      7393,
+      7405,
+      7406,
+      7413,
+      7416,
+      7430,
+      7432,
+      7442,
+      7446,
+      7447,
+      7474,
+      7476,
+      7485,
+      7507,
+      7512,
+      7520,
+      7529,
+      7542,
+      7556,
+      7559,
+      7583,
+      7585,
+      7606,
+      7616,
+      7629,
+      7630,
+      7646,
+      7652,
+      7662,
+      7666,
+      7677,
+      7685,
+      7690,
+      7693,
+      7698,
+      7699,
+      7712,
+      7716,
+      7724,
+      7730,
+      7737,
+      7741,
+      7747,
+      7770,
+      7810,
+      7811,
+      7828,
+      7834,
+      7840,
+      7847,
+      7849,
+      7853,
+      7856,
+      7869,
+      7874,
+      7875,
+      7878,
+      7895,
+      7898,
+      7903,
+      7911,
+      7913,
+      7934,
+      7941,
+      7946,
+      7951,
+      7952,
+      7959,
+      7962,
+      7971,
+      7974,
+      7983,
+      7999,
+      8002,
+      8004,
+      8010,
+      8017,
+      8022,
+      8024,
+      8029,
+      8034,
+      8043,
+      8049,
+      8059,
+      8067,
+      8082,
+      8091,
+      8093,
+      8095,
+      8113,
+      8126,
+      8151,
+      8156,
+      8159,
+      8162,
+      8163,
+      8167,
+      8180,
+      8189,
+      8193,
+      8197,
+      8202,
+      8229,
+      8231,
+      8237,
+      8238,
+      8261,
+      8270,
+      8286,
+      8296,
+      8303,
+      8322,
+      8337,
+      8376,
+      8400,
+      8405,
+      8411,
+      8428,
+      8445,
+      8447,
+      8449,
+      8450,
+      8454,
+      8458,
+      8462,
+      8464,
+      8485,
+      8486,
+      8490,
+      8496,
+      8516,
+      8520,
+      8523,
+      8524,
+      8525,
+      8528,
+      8540,
+      8545,
+      8551,
+      8553,
+      8557,
+      8566,
+      8569,
+      8571,
+      8583,
+      8584,
+      8585,
+      8586,
+      8591,
+      8602,
+      8609,
+      8613,
+      8621,
+      8625,
+      8630,
+      8649,
+      8653,
+      8654,
+      8661,
+      8662,
+      8663,
+      8671,
+      8684,
+      8706,
+      8713,
+      8717,
+      8718,
+      8737,
+      8743,
+      8747,
+      8751,
+      8757,
+      8758,
+      8780,
+      8787,
+      8791,
+      8820,
+      8838,
+      8844,
+      8865,
+      8873,
+      8883,
+      8889,
+      8898,
+      8913,
+      8918,
+      8939,
+      8943,
+      8950,
+      8953,
+      8977,
+      8978,
+      8988,
+      9001,
+      9006,
+      9013,
+      9021,
+      9024,
+      9028,
+      9031,
+      9032,
+      9035,
+      9048,
+      9050,
+      9057,
+      9067,
+      9071,
+      9072,
+      9077,
+      9080,
+      9091,
+      9096,
+      9102,
+      9103,
+      9126,
+      9142,
+      9148,
+      9153,
+      9189,
+      9215,
+      9221,
+      9232,
+      9236,
+      9243,
+      9247,
+      9250,
+      9257,
+      9276,
+      9284,
+      9285,
+      9286,
+      9289,
+      9299,
+      9305,
+      9307,
+      9314,
+      9316,
+      9323,
+      9328,
+      9331,
+      9333,
+      9335,
+      9336,
+      9337,
+      9351,
+      9354,
+      9355,
+      9363,
+      9365,
+      9386,
+      9397,
+      9407,
+      9438,
+      9447,
+      9453,
+      9463,
+      9468,
+      9488,
+      9496,
+      9497,
+      9513,
+      9515,
+      9516,
+      9521,
+      9527,
+      9541,
+      9581,
+      9589,
+      9623,
+      9626,
+      9628,
+      9629,
+      9644,
+      9648,
+      9650,
+      9654,
+      9681,
+      9688,
+      9691,
+      9692,
+      9695,
+      9697,
+      9709,
+      9711,
+      9737,
+      9739,
+      9759,
+      9766,
+      9769,
+      9777,
+      9791,
+      9793,
+      9816,
+      9818,
+      9821,
+      9825,
+      9834,
+      9856,
+      9863,
+      9864,
+      9869,
+      9888,
+      9890,
+      9892,
+      9905,
+      9910,
+      9911,
+      9912,
+      9953,
+      9954,
+      9956,
+      9961,
+      9966,
+      10010,
+      10020,
+      10023,
+      10024,
+      10050,
+      10052,
+      10054,
+      10057,
+      10074,
+      10085,
+      10087,
+      10091,
+      10092,
+      10105,
+      10111,
+      10127,
+      10128,
+      10136,
+      10139,
+      10143,
+      10169,
+      10175,
+      10177,
+      10179,
+      10182,
+      10196,
+      10199,
+      10207,
+      10222,
+      10224,
+      10230,
+      10235,
+      10242,
+      10244,
+      10253,
+      10258,
+      10259,
+      10260,
+      10264,
+      10265,
+      10270,
+      10273,
+      10278,
+      10280,
+      10285,
+      10286,
+      10288,
+      10322,
+      10325,
+      10343,
+      10344,
+      10354,
+      10359,
+      10362,
+      10371,
+      10374,
+      10376,
+      10383,
+      10404,
+      10415,
+      10419,
+      10421,
+      10422,
+      10431,
+      10440,
+      10441,
+      10446,
+      10448,
+      10450,
+      10451,
+      10458,
+      10464,
+      10472,
+      10483,
+      10484,
+      10486,
+      10497,
+      10510,
+      10513,
+      10521,
+      10524,
+      10526,
+      10532,
+      10536,
+      10542,
+      10559,
+      10581,
+      10586,
+      10593,
+      10600,
+      10603,
+      10606,
+      10609,
+      10610,
+      10623,
+      10624,
+      10626,
+      10631,
+      10649,
+      10665,
+      10672,
+      10686,
+      10695,
+      10696,
+      10698,
+      10701,
+      10708,
+      10711,
+      10719,
+      10727,
+      10732,
+      10735,
+      10736,
+      10755,
+      10760,
+      10762,
+      10772,
+      10790,
+      10803,
+      10814,
+      10820,
+      10830,
+      10841,
+      10867,
+      10875,
+      10880,
+      10893,
+      10895,
+      10924,
+      10929,
+      10932,
+      10936,
+      10976,
+      10980,
+      10985,
+      10992,
+      10993,
+      10994,
+      10997,
+      11001,
+      11008,
+      11012,
+      11020,
+      11022,
+      11029,
+      11031,
+      11039,
+      11045,
+      11057,
+      11066,
+      11069,
+      11077,
+      11097,
+      11101,
+      11106,
+      11108,
+      11115,
+      11124,
+      11129,
+      11137,
+      11140,
+      11144,
+      11148,
+      11149,
+      11151,
+      11179,
+      11187,
+      11191,
+      11198,
+      11201,
+      11215,
+      11219,
+      11227,
+      11231,
+      11233,
+      11236,
+      11239,
+      11244,
+      11246,
+      11253,
+      11254,
+      11261,
+      11263,
+      11266,
+      11272,
+      11283,
+      11285,
+      11306,
+      11314,
+      11319,
+      11329,
+      11335,
+      11362,
+      11365,
+      11368,
+      11387,
+      11390,
+      11392,
+      11393,
+      11410,
+      11411,
+      11431,
+      11439,
+      11442,
+      11444,
+      11451,
+      11452,
+      11461,
+      11467,
+      11469,
+      11474,
+      11475,
+      11483,
+      11484,
+      11488,
+      11493,
+      11510,
+      11512,
+      11514,
+      11522,
+      11528,
+      11533,
+      11563,
+      11589,
+      11592,
+      11597,
+      11602,
+      11607,
+      11610,
+      11613,
+      11614,
+      11622,
+      11624,
+      11627,
+      11631,
+      11634,
+      11636,
+      11638,
+      11639,
+      11644,
+      11649,
+      11650,
+      11656,
+      11657,
+      11663,
+      11667,
+      11692,
+      11705,
+      11711,
+      11714,
+      11715,
+      11722,
+      11733,
+      11735,
+      11741,
+      11745,
+      11751,
+      11755,
+      11761,
+      11774,
+      11783,
+      11787,
+      11788,
+      11795,
+      11796,
+      11800,
+      11802,
+      11818,
+      11823,
+      11837,
+      11871,
+      11872,
+      11873,
+      11877,
+      11880,
+      11890,
+      11905,
+      11916,
+      11921,
+      11923,
+      11927,
+      11960,
+      11961,
+      11975,
+      11983,
+      11998,
+      12002,
+      12031,
+      12032,
+      12036,
+      12044,
+      12053,
+      12054,
+      12059,
+      12069,
+      12070,
+      12079,
+      12083,
+      12086,
+      12088,
+      12096,
+      12112,
+      12119,
+      12129,
+      12131,
+      12146,
+      12155,
+      12162,
+      12165,
+      12166,
+      12171,
+      12172,
+      12173,
+      12179,
+      12184,
+      12186,
+      12190,
+      12193,
+      12198,
+      12199,
+      12207,
+      12215,
+      12221,
+      12227,
+      12228,
+      12229,
+      12243,
+      12244,
+      12248,
+      12255,
+      12259,
+      12260,
+      12269,
+      12273,
+      12274,
+      12298,
+      12303,
+      12314,
+      12322,
+      12325,
+      12326,
+      12333,
+      12349,
+      12352,
+      12356,
+      12376,
+      12377,
+      12380,
+      12384,
+      12399,
+      12412,
+      12414,
+      12417,
+      12423,
+      12426,
+      12439,
+      12449,
+      12458,
+      12462,
+      12474,
+      12488,
+      12494,
+      12502,
+      12514,
+      12516,
+      12519,
+      12521,
+      12540,
+      12541,
+      12556,
+      12561,
+      12565,
+      12588,
+      12589,
+      12614,
+      12617,
+      12618,
+      12624,
+      12638,
+      12640,
+      12644,
+      12647,
+      12650,
+      12662,
+      12672,
+      12676,
+      12680,
+      12681,
+      12690,
+      12691,
+      12693,
+      12707,
+      12717,
+      12718,
+      12722,
+      12733,
+      12734,
+      12735,
+      12740,
+      12742,
+      12744,
+      12750,
+      12758,
+      12762,
+      12782,
+      12789,
+      12792,
+      12798,
+      12811,
+      12820,
+      12821,
+      12830,
+      12837,
+      12844,
+      12848,
+      12863,
+      12866,
+      12868,
+      12870,
+      12871,
+      12879,
+      12881,
+      12887,
+      12891,
+      12898,
+      12902,
+      12910,
+      12911,
+      12914,
+      12921,
+      12926,
+      12928,
+      12933,
+      12942,
+      12952,
+      12956,
+      12972,
+      12995,
+      13006,
+      13013,
+      13027,
+      13028,
+      13033,
+      13038,
+      13043,
+      13046,
+      13062,
+      13064,
+      13068,
+      13069,
+      13087,
+      13088,
+      13089,
+      13094,
+      13098,
+      13102,
+      13106,
+      13112,
+      13113,
+      13122,
+      13124,
+      13129,
+      13143,
+      13149,
+      13150,
+      13153,
+      13159,
+      13181,
+      13186,
+      13203,
+      13210,
+      13213,
+      13231,
+      13238,
+      13241,
+      13245,
+      13247,
+      13278,
+      13288,
+      13290,
+      13295,
+      13306,
+      13309,
+      13320,
+      13327,
+      13331,
+      13334,
+      13340,
+      13344,
+      13348,
+      13355,
+      13369,
+      13375,
+      13380,
+      13390,
+      13421,
+      13429,
+      13444,
+      13453,
+      13455,
+      13461,
+      13462,
+      13464,
+      13465,
+      13469,
+      13471,
+      13473,
+      13476,
+      13484,
+      13490,
+      13492,
+      13510,
+      13512,
+      13535,
+      13542,
+      13565,
+      13578,
+      13592,
+      13596,
+      13602,
+      13615,
+      13620,
+      13623,
+      13626,
+      13629,
+      13639,
+      13662,
+      13663,
+      13665,
+      13668,
+      13669,
+      13684,
+      13687,
+      13699,
+      13700,
+      13701,
+      13704,
+      13705,
+      13706,
+      13714,
+      13716,
+      13719,
+      13728,
+      13731,
+      13754,
+      13759,
+      13765,
+      13767,
+      13770,
+      13775,
+      13776,
+      13777,
+      13778,
+      13786,
+      13793,
+      13806,
+      13807,
+      13808,
+      13820,
+      13821,
+      13832,
+      13837,
+      13860,
+      13871,
+      13874,
+      13876,
+      13887,
+      13890,
+      13900,
+      13901,
+      13902,
+      13910,
+      13911,
+      13923,
+      13936,
+      13944,
+      13947,
+      13959,
+      13966,
+      13976,
+      14001,
+      14014,
+      14028,
+      14031,
+      14035,
+      14045,
+      14049,
+      14050,
+      14066,
+      14069,
+      14074,
+      14075,
+      14081,
+      14083,
+      14088,
+      14097,
+      14116,
+      14117,
+      14122,
+      14127,
+      14129,
+      14132,
+      14138,
+      14141,
+      14142,
+      14144,
+      14147,
+      14149,
+      14150,
+      14151,
+      14156,
+      14157,
+      14162,
+      14183,
+      14191,
+      14192,
+      14194,
+      14204,
+      14213,
+      14220,
+      14226,
+      14227,
+      14228,
+      14232,
+      14235,
+      14238,
+      14239,
+      14244,
+      14268,
+      14270,
+      14272,
+      14273,
+      14277,
+      14287,
+      14289,
+      14301,
+      14317,
+      14321,
+      14322,
+      14326,
+      14339,
+      14341,
+      14343,
+      14346,
+      14353,
+      14355,
+      14356,
+      14359,
+      14376,
+      14399,
+      14406,
+      14413,
+      14416,
+      14417,
+      14423,
+      14428,
+      14431,
+      14432,
+      14454,
+      14478,
+      14479,
+      14482,
+      14483,
+      14490,
+      14498,
+      14499,
+      14503,
+      14505,
+      14510,
+      14511,
+      14515,
+      14517,
+      14526,
+      14528,
+      14530,
+      14536,
+      14540,
+      14544,
+      14545,
+      14547,
+      14550,
+      14561,
+      14565,
+      14573,
+      14581,
+      14586,
+      14604,
+      14606,
+      14608,
+      14615,
+      14619,
+      14621,
+      14625,
+      14626,
+      14634,
+      14642,
+      14643,
+      14644,
+      14647,
+      14650,
+      14656,
+      14659,
+      14660,
+      14663,
+      14672,
+      14680,
+      14691,
+      14693,
+      14697,
+      14701,
+      14702,
+      14706,
+      14713,
+      14715,
+      14716,
+      14728,
+      14733,
+      14735,
+      14737,
+      14740,
+      14756,
+      14758,
+      14760,
+      14762,
+      14767,
+      14769,
+      14772,
+      14773,
+      14774,
+      14776,
+      14778,
+      14782,
+      14787,
+      14795,
+      14808,
+      14820,
+      14821,
+      14839,
+      14844,
+      14852,
+      14853,
+      14854,
+      14856,
+      14859,
+      14862,
+      14876,
+      14881,
+      14890,
+      14898,
+      14900,
+      14905,
+      14907,
+      14919,
+      14945,
+      14949,
+      14956,
+      14966,
+      14968,
+      14971,
+      14973,
+      14995,
+      15000,
+      15004,
+      15006,
+      15007,
+      15010,
+      15042,
+      15049,
+      15058,
+      15061,
+      15069,
+      15070,
+      15073,
+      15088,
+      15094,
+      15096,
+      15104,
+      15118,
+      15136,
+      15141,
+      15143,
+      15145,
+      15148,
+      15150,
+      15154,
+      15155,
+      15164,
+      15166,
+      15172,
+      15185,
+      15194,
+      15198,
+      15202,
+      15204,
+      15217,
+      15222,
+      15227,
+      15238,
+      15239,
+      15249,
+      15256,
+      15265,
+      15266,
+      15271,
+      15272,
+      15277,
+      15286,
+      15290,
+      15297,
+      15305,
+      15312,
+      15319,
+      15327,
+      15338,
+      15343,
+      15350,
+      15353,
+      15361,
+      15362,
+      15370,
+      15378,
+      15379,
+      15388,
+      15394,
+      15395,
+      15396,
+      15402,
+      15414,
+      15416,
+      15417,
+      15422,
+      15431,
+      15441,
+      15444,
+      15449,
+      15455,
+      15457,
+      15459,
+      15460,
+      15464,
+      15465,
+      15472,
+      15474,
+      15482,
+      15493,
+      15494,
+      15495,
+      15496,
+      15499,
+      15500,
+      15509,
+      15513,
+      15530,
+      15532,
+      15534,
+      15542,
+      15549,
+      15551,
+      15557,
+      15558,
+      15564,
+      15567,
+      15568,
+      15569,
+      15580,
+      15590,
+      15593,
+      15599,
+      15603,
+      15608,
+      15613,
+      15621,
+      15628,
+      15630,
+      15637,
+      15638,
+      15643,
+      15648,
+      15660,
+      15666,
+      15668,
+      15671,
+      15683,
+      15687,
+      15689,
+      15706,
+      15707,
+      15710,
+      15714,
+      15719,
+      15721,
+      15725,
+      15728,
+      15733,
+      15735,
+      15750,
+      15758,
+      15759,
+      15773,
+      15776,
+      15780,
+      15781,
+      15782,
+      15787,
+      15788,
+      15794,
+      15808,
+      15810,
+      15816,
+      15830,
+      15833,
+      15851,
+      15853,
+      15857,
+      15859,
+      15864,
+      15869,
+      15872,
+      15878,
+      15879,
+      15900,
+      15904,
+      15907,
+      15909,
+      15915,
+      15916,
+      15922,
+      15928,
+      15937,
+      15950,
+      15951,
+      15952,
+      15975,
+      15979,
+      15985,
+      15987,
+      15992,
+      15994,
+      16001,
+      16020,
+      16022,
+      16026,
+      16033,
+      16037,
+      16043,
+      16044,
+      16048,
+      16050,
+      16052,
+      16059,
+      16066,
+      16068,
+      16069,
+      16070,
+      16076,
+      16078,
+      16084,
+      16086,
+      16089,
+      16093,
+      16098,
+      16102,
+      16103,
+      16107,
+      16116,
+      16118,
+      16119,
+      16136,
+      16147,
+      16150,
+      16161,
+      16168,
+      16173,
+      16177,
+      16179,
+      16186,
+      16190,
+      16194,
+      16198,
+      16200,
+      16203,
+      16206,
+      16207,
+      16211,
+      16213,
+      16223,
+      16230,
+      16238,
+      16252,
+      16256,
+      16257,
+      16259,
+      16275,
+      16277,
+      16279,
+      16280,
+      16281,
+      16284,
+      16288,
+      16294,
+      16302,
+      16303,
+      16307,
+      16308,
+      16313,
+      16329,
+      16336,
+      16338,
+      16339,
+      16340,
+      16344,
+      16347,
+      16363,
+      16365,
+      16366,
+      16368,
+      16369,
+      16377,
+      16383,
+      16398,
+      16402,
+      16409,
+      16414,
+      16415,
+      16416,
+      16425,
+      16429,
+      16445,
+      16446,
+      16451,
+      16453,
+      16454,
+      16458,
+      16467,
+      16502,
+      16503,
+      16505,
+      16506,
+      16515,
+      16518,
+      16523,
+      16526,
+      16529,
+      16530,
+      16531,
+      16535,
+      16549,
+      16550,
+      16555,
+      16558,
+      16563,
+      16564,
+      16566,
+      16567,
+      16570,
+      16576,
+      16584,
+      16585,
+      16594,
+      16600,
+      16610,
+      16613,
+      16616,
+      16643,
+      16649,
+      16650,
+      16652,
+      16656,
+      16665,
+      16670,
+      16671,
+      16677,
+      16689,
+      16693,
+      16695,
+      16700,
+      16703,
+      16704,
+      16715,
+      16737,
+      16739,
+      16741,
+      16748,
+      16750,
+      16754,
+      16761,
+      16765,
+      16774,
+      16777,
+      16778,
+      16781,
+      16782,
+      16786,
+      16790,
+      16811,
+      16814,
+      16816,
+      16835,
+      16843,
+      16846,
+      16848,
+      16856,
+      16862,
+      16867,
+      16869,
+      16872,
+      16874,
+      16882,
+      16885,
+      16886,
+      16892,
+      16898,
+      16901,
+      16912,
+      16917,
+      16918,
+      16921,
+      16930,
+      16932,
+      16940,
+      16952,
+      16961,
+      16963,
+      16969,
+      16975,
+      16977,
+      16989,
+      16992,
+      16998,
+      17004,
+      17007,
+      17028,
+      17036,
+      17040,
+      17043,
+      17048,
+      17058,
+      17062,
+      17063,
+      17071,
+      17080,
+      17084,
+      17086,
+      17101,
+      17110,
+      17112,
+      17119,
+      17122,
+      17125,
+      17130,
+      17134,
+      17139,
+      17140,
+      17149,
+      17157,
+      17166,
+      17167,
+      17168,
+      17175,
+      17190,
+      17191,
+      17198,
+      17206,
+      17226,
+      17231,
+      17236,
+      17238,
+      17240,
+      17250,
+      17252,
+      17258,
+      17269,
+      17275,
+      17281,
+      17285,
+      17287,
+      17288,
+      17296,
+      17298,
+      17299,
+      17301,
+      17302,
+      17305,
+      17308,
+      17310,
+      17311,
+      17322,
+      17325,
+      17327,
+      17339,
+      17340,
+      17342,
+      17345,
+      17364,
+      17369,
+      17375,
+      17378,
+      17386,
+      17389,
+      17406,
+      17413,
+      17417,
+      17420,
+      17421,
+      17426,
+      17432,
+      17440,
+      17441,
+      17452,
+      17456,
+      17457,
+      17460,
+      17465,
+      17468,
+      17470,
+      17471,
+      17479,
+      17487,
+      17499,
+      17510,
+      17516,
+      17545,
+      17560,
+      17563,
+      17570,
+      17572,
+      17577,
+      17579,
+      17580,
+      17583,
+      17585,
+      17592,
+      17597,
+      17599,
+      17603,
+      17619,
+      17622,
+      17625,
+      17629,
+      17635,
+      17643,
+      17644,
+      17650,
+      17658,
+      17661,
+      17668,
+      17670,
+      17679,
+      17681,
+      17682,
+      17683,
+      17686,
+      17695,
+      17699,
+      17701,
+      17706,
+      17713,
+      17724,
+      17726,
+      17728,
+      17729,
+      17751,
+      17753,
+      17759,
+      17788,
+      17789,
+      17791,
+      17794,
+      17799,
+      17802,
+      17809,
+      17811,
+      17815,
+      17819,
+      17822,
+      17830,
+      17847,
+      17848,
+      17854,
+      17868,
+      17872,
+      17875,
+      17883,
+      17884,
+      17896,
+      17897,
+      17904,
+      17905,
+      17906,
+      17914,
+      17915,
+      17920,
+      17922,
+      17926,
+      17933,
+      17944,
+      17948,
+      17955,
+      17960,
+      17965,
+      17968,
+      17980,
+      17988,
+      17989,
+      17994,
+      17998,
+      18003,
+      18009,
+      18028,
+      18032,
+      18037,
+      18041,
+      18053,
+      18057,
+      18060,
+      18065,
+      18069,
+      18073,
+      18075,
+      18078,
+      18083,
+      18095,
+      18098,
+      18100,
+      18107,
+      18115,
+      18116,
+      18117,
+      18124,
+      18136,
+      18139,
+      18141,
+      18142,
+      18143,
+      18150,
+      18153,
+      18155,
+      18161,
+      18163,
+      18170,
+      18175,
+      18182,
+      18192,
+      18194,
+      18195,
+      18199,
+      18209,
+      18219,
+      18222,
+      18223,
+      18232,
+      18246,
+      18249,
+      18252,
+      18256,
+      18259,
+      18260,
+      18274,
+      18277,
+      18278,
+      18279,
+      18291,
+      18300,
+      18317,
+      18319,
+      18321,
+      18324,
+      18331,
+      18334,
+      18337,
+      18339,
+      18349,
+      18351,
+      18352,
+      18357,
+      18367,
+      18372,
+      18376,
+      18377,
+      18380,
+      18382,
+      18384,
+      18389,
+      18393,
+      18400,
+      18405,
+      18409,
+      18410,
+      18412,
+      18413,
+      18417,
+      18418,
+      18424,
+      18425,
+      18426,
+      18428,
+      18436,
+      18442,
+      18447,
+      18450,
+      18471,
+      18477,
+      18483,
+      18491,
+      18492,
+      18494,
+      18505,
+      18507,
+      18510,
+      18511,
+      18513,
+      18514,
+      18515,
+      18518,
+      18532,
+      18533,
+      18537,
+      18552,
+      18557,
+      18560,
+      18569,
+      18570,
+      18581,
+      18585,
+      18589,
+      18595,
+      18598,
+      18599,
+      18602,
+      18605,
+      18606,
+      18613,
+      18617,
+      18620,
+      18622,
+      18623,
+      18630,
+      18632,
+      18637,
+      18639,
+      18647,
+      18655,
+      18657,
+      18660,
+      18663,
+      18673,
+      18674,
+      18679,
+      18682,
+      18684,
+      18688,
+      18689,
+      18690,
+      18696,
+      18697,
+      18702,
+      18703,
+      18715,
+      18717,
+      18718,
+      18719,
+      18721,
+      18722,
+      18724,
+      18727,
+      18738,
+      18743,
+      18752,
+      18757,
+      18769,
+      18774,
+      18780,
+      18790,
+      18796,
+      18802,
+      18803,
+      18807,
+      18808,
+      18809,
+      18810,
+      18818,
+      18824,
+      18826,
+      18827,
+      18839,
+      18842,
+      18845,
+      18861,
+      18863,
+      18866,
+      18871,
+      18878,
+      18879,
+      18884,
+      18885,
+      18886,
+      18891,
+      18894,
+      18900,
+      18902,
+      18903,
+      18907,
+      18908,
+      18909,
+      18912,
+      18915,
+      18924,
+      18926,
+      18927,
+      18933,
+      18935,
+      18938,
+      18939,
+      18940,
+      18941,
+      18947,
+      18954,
+      18963,
+      18964,
+      18965,
+      18969,
+      18979,
+      18980,
+      18981,
+      18992,
+      18994,
+      19000,
+      19001,
+      19009,
+      19020,
+      19022,
+      19023,
+      19025,
+      19036,
+      19040,
+      19042,
+      19053,
+      19058,
+      19062,
+      19068,
+      19082,
+      19092,
+      19093,
+      19098,
+      19099,
+      19105,
+      19109,
+      19110,
+      19114,
+      19126,
+      19137,
+      19139,
+      19140,
+      19145,
+      19146,
+      19150,
+      19152,
+      19155,
+      19159,
+      19168,
+      19170,
+      19172,
+      19182,
+      19185,
+      19190,
+      19196,
+      19198,
+      19205,
+      19208,
+      19210,
+      19213,
+      19217,
+      19224,
+      19225,
+      19228,
+      19230,
+      19231,
+      19234,
+      19250,
+      19253,
+      19259,
+      19263,
+      19265,
+      19269,
+      19270,
+      19277,
+      19279,
+      19281,
+      19282,
+      19288,
+      19291,
+      19301,
+      19303,
+      19304,
+      19307,
+      19310,
+      19312,
+      19315,
+      19317,
+      19321,
+      19333,
+      19339,
+      19341,
+      19357,
+      19358,
+      19362,
+      19386,
+      19392,
+      19393,
+      19403,
+      19405,
+      19417,
+      19418,
+      19419,
+      19422,
+      19425,
+      19426,
+      19433,
+      19434,
+      19436,
+      19440,
+      19442,
+      19445,
+      19454,
+      19464,
+      19466,
+      19468,
+      19473,
+      19478,
+      19481,
+      19491,
+      19496,
+      19498,
+      19500,
+      19506,
+      19510,
+      19513,
+      19516,
+      19520,
+      19524,
+      19531,
+      19539,
+      19562,
+      19563,
+      19565,
+      19567,
+      19570,
+      19574,
+      19579,
+      19585,
+      19588,
+      19603,
+      19612,
+      19620,
+      19629,
+      19636,
+      19646,
+      19648,
+      19650,
+      19658,
+      19661,
+      19663,
+      19665,
+      19666,
+      19673,
+      19696,
+      19699,
+      19704,
+      19707,
+      19708,
+      19716,
+      19718,
+      19722,
+      19729,
+      19731,
+      19736,
+      19738,
+      19742,
+      19755,
+      19761,
+      19763,
+      19766,
+      19767,
+      19771,
+      19779,
+      19784,
+      19791,
+      19792,
+      19797,
+      19798,
+      19800,
+      19801,
+      19811,
+      19824,
+      19825,
+      19826,
+      19833,
+      19834,
+      19839,
+      19841,
+      19845,
+      19847,
+      19848,
+      19849,
+      19856,
+      19859,
+      19862,
+      19864,
+      19865,
+      19869,
+      19877,
+      19879,
+      19880,
+      19892,
+      19894,
+      19895,
+      19901,
+      19903,
+      19908,
+      19909,
+      19912,
+      19915,
+      19927,
+      19929,
+      19932,
+      19944,
+      19946,
+      19961,
+      19966,
+      19968,
+      19969,
+      19980,
+      19983,
+      19987,
+      19990,
+      19991,
+      19993,
+      20009,
+      20011,
+      20023,
+      20026,
+      20027,
+      20028,
+      20030,
+      20031,
+      20041,
+      20042,
+      20043,
+      20048,
+      20049,
+      20051,
+      20058,
+      20060,
+      20082,
+      20086,
+      20089,
+      20097,
+      20100,
+      20106,
+      20110,
+      20113,
+      20132,
+      20136,
+      20142,
+      20147,
+      20156,
+      20159,
+      20166,
+      20169,
+      20175,
+      20179,
+      20186,
+      20189,
+      20192,
+      20200,
+      20203,
+      20205,
+      20207,
+      20211,
+      20214,
+      20217,
+      20224,
+      20236,
+      20240,
+      20255,
+      20257,
+      20260,
+      20265,
+      20267,
+      20269,
+      20281,
+      20282,
+      20285,
+      20303,
+      20307,
+      20318,
+      20330,
+      20338,
+      20343,
+      20348,
+      20349,
+      20350,
+      20357,
+      20360,
+      20363,
+      20366,
+      20367,
+      20370,
+      20379,
+      20382,
+      20389,
+      20391,
+      20395,
+      20409,
+      20411,
+      20414,
+      20420,
+      20431,
+      20434,
+      20444,
+      20447,
+      20450,
+      20459,
+      20464,
+      20465,
+      20469,
+      20470,
+      20473,
+      20479,
+      20483,
+      20486,
+      20492,
+      20496,
+      20499,
+      20501,
+      20511,
+      20515,
+      20518,
+      20523,
+      20527,
+      20528,
+      20529,
+      20534,
+      20535,
+      20536,
+      20550,
+      20552,
+      20553,
+      20554,
+      20559,
+      20562,
+      20564,
+      20569,
+      20570,
+      20573,
+      20574,
+      20577,
+      20589,
+      20590,
+      20591,
+      20592,
+      20598,
+      20609,
+      20614,
+      20617,
+      20620,
+      20623,
+      20624,
+      20630,
+      20644,
+      20650,
+      20654,
+      20656,
+      20660,
+      20668,
+      20671,
+      20679,
+      20683,
+      20684,
+      20685,
+      20688,
+      20689,
+      20693,
+      20697,
+      20709,
+      20715,
+      20718,
+      20721,
+      20722,
+      20734,
+      20738,
+      20740,
+      20744,
+      20745,
+      20746,
+      20755,
+      20760,
+      20763,
+      20766,
+      20770,
+      20771,
+      20778,
+      20781,
+      20782,
+      20784,
+      20791,
+      20792,
+      20793,
+      20799,
+      20800,
+      20806,
+      20807,
+      20808,
+      20812,
+      20821,
+      20822,
+      20824,
+      20838,
+      20842,
+      20844,
+      20845,
+      20846,
+      20849,
+      20850,
+      20853,
+      20860,
+      20872,
+      20875,
+      20876,
+      20881,
+      20891,
+      20894,
+      20897,
+      20898,
+      20899,
+      20900,
+      20902,
+      20908,
+      20909,
+      20913,
+      20915,
+      20921,
+      20922,
+      20924,
+      20925,
+      20927,
+      20935,
+      20936,
+      20938,
+      20939,
+      20941,
+      20942,
+      20952,
+      20958,
+      20959,
+      20960,
+      20961,
+      20968,
+      20972,
+      20974,
+      20984,
+      20995,
+      21001,
+      21007,
+      21010,
+      21011,
+      21013,
+      21017,
+      21020,
+      21022,
+      21030,
+      21031,
+      21041,
+      21043,
+      21046,
+      21050,
+      21055,
+      21057,
+      21077,
+      21081,
+      21084,
+      21095,
+      21102,
+      21104,
+      21112,
+      21114,
+      21119,
+      21126,
+      21128,
+      21131,
+      21132,
+      21138,
+      21141,
+      21147,
+      21148,
+      21149,
+      21156,
+      21159,
+      21162,
+      21163,
+      21165,
+      21166,
+      21170,
+      21172,
+      21181,
+      21187,
+      21188,
+      21190,
+      21194,
+      21197,
+      21201,
+      21202,
+      21204,
+      21205,
+      21217,
+      21218,
+      21222,
+      21223,
+      21225,
+      21227,
+      21228,
+      21230,
+      21231,
+      21239,
+      21246,
+      21252,
+      21253,
+      21254,
+      21261,
+      21266,
+      21270,
+      21272,
+      21273,
+      21275,
+      21280,
+      21281,
+      21285,
+      21292,
+      21293,
+      21297,
+      21302,
+      21309,
+      21314,
+      21325,
+      21332,
+      21335,
+      21337,
+      21338,
+      21341,
+      21351,
+      21354,
+      21355,
+      21361,
+      21369,
+      21382,
+      21397,
+      21398,
+      21405,
+      21408,
+      21409,
+      21412,
+      21421,
+      21422,
+      21426,
+      21436,
+      21450,
+      21456,
+      21458,
+      21463,
+      21466,
+      21471,
+      21472,
+      21476,
+      21482,
+      21486,
+      21493,
+      21499,
+      21504,
+      21508,
+      21509,
+      21511,
+      21512,
+      21515,
+      21525,
+      21530,
+      21531,
+      21532,
+      21539,
+      21540,
+      21543,
+      21547,
+      21548,
+      21556,
+      21557,
+      21558,
+      21559,
+      21561,
+      21564,
+      21565,
+      21572,
+      21576,
+      21578,
+      21579,
+      21590,
+      21593,
+      21595,
+      21596,
+      21600,
+      21604,
+      21608,
+      21615,
+      21619,
+      21621,
+      21634,
+      21638,
+      21639,
+      21646,
+      21649,
+      21654,
+      21661,
+      21663,
+      21673,
+      21678,
+      21682,
+      21685,
+      21693,
+      21694,
+      21700,
+      21709,
+      21710,
+      21711,
+      21716,
+      21723,
+      21724,
+      21735,
+      21736,
+      21737,
+      21748,
+      21752,
+      21756,
+      21758,
+      21759,
+      21761,
+      21763,
+      21769,
+      21781,
+      21784,
+      21786,
+      21789,
+      21799,
+      21804,
+      21806,
+      21807,
+      21808,
+      21814,
+      21815,
+      21816,
+      21818,
+      21819,
+      21820,
+      21821,
+      21823,
+      21827,
+      21835,
+      21837,
+      21842,
+      21844,
+      21847,
+      21850,
+      21861,
+      21869,
+      21884,
+      21885,
+      21886,
+      21890,
+      21891,
+      21898,
+      21906,
+      21924,
+      21926,
+      21927,
+      21928,
+      21930,
+      21932,
+      21939,
+      21940,
+      21945,
+      21946,
+      21958,
+      21966,
+      21967,
+      21983,
+      21996,
+      21997,
+      21998,
+      22008,
+      22014,
+      22016,
+      22022,
+      22025,
+      22026,
+      22036,
+      22043,
+      22044,
+      22045,
+      22046,
+      22048,
+      22049,
+      22059,
+      22068,
+      22071,
+      22074,
+      22081,
+      22083,
+      22084,
+      22086,
+      22087,
+      22090,
+      22104,
+      22107,
+      22117,
+      22119,
+      22122,
+      22123,
+      22134,
+      22139,
+      22144,
+      22145,
+      22153,
+      22155,
+      22158,
+      22167,
+      22172,
+      22179,
+      22187,
+      22192,
+      22196,
+      22197,
+      22199,
+      22203,
+      22204,
+      22208,
+      22218,
+      22225,
+      22231,
+      22244,
+      22248,
+      22251,
+      22254,
+      22263,
+      22264,
+      22269,
+      22270,
+      22272,
+      22275,
+      22280,
+      22282,
+      22287,
+      22299,
+      22305,
+      22311,
+      22314,
+      22315,
+      22318,
+      22325,
+      22327,
+      22328,
+      22332,
+      22341,
+      22342,
+      22345,
+      22352,
+      22360,
+      22367,
+      22368,
+      22373,
+      22375,
+      22379,
+      22381,
+      22384,
+      22392,
+      22394,
+      22397,
+      22398,
+      22401,
+      22407,
+      22408,
+      22414,
+      22420,
+      22425,
+      22426,
+      22427,
+      22436,
+      22444,
+      22447,
+      22461,
+      22462,
+      22465,
+      22471,
+      22483,
+      22486,
+      22492,
+      22494,
+      22499,
+      22500,
+      22504,
+      22507,
+      22509,
+      22510,
+      22513,
+      22516,
+      22517,
+      22522,
+      22536,
+      22540,
+      22542,
+      22543,
+      22556,
+      22571,
+      22573,
+      22576,
+      22578,
+      22579,
+      22586,
+      22591,
+      22592,
+      22599,
+      22601,
+      22610,
+      22612,
+      22618,
+      22620,
+      22626,
+      22631,
+      22637,
+      22638,
+      22642,
+      22646,
+      22648,
+      22655,
+      22658,
+      22662,
+      22669,
+      22670,
+      22675,
+      22684,
+      22691,
+      22693,
+      22694,
+      22698,
+      22717,
+      22723,
+      22729,
+      22731,
+      22734,
+      22738,
+      22740,
+      22743,
+      22747,
+      22749,
+      22750,
+      22753,
+      22758,
+      22769,
+      22772,
+      22776,
+      22779,
+      22780,
+      22781,
+      22786,
+      22787,
+      22792,
+      22798,
+      22799,
+      22803,
+      22809,
+      22823,
+      22828,
+      22829,
+      22835,
+      22842,
+      22844,
+      22847,
+      22850,
+      22852,
+      22862,
+      22864,
+      22869,
+      22879,
+      22881,
+      22890,
+      22893,
+      22895,
+      22903,
+      22907,
+      22908,
+      22909,
+      22911,
+      22916,
+      22918,
+      22919,
+      22920,
+      22923,
+      22924,
+      22925,
+      22927,
+      22929,
+      22930,
+      22932,
+      22934,
+      22938,
+      22942,
+      22943,
+      22948,
+      22957,
+      22960,
+      22966,
+      22967,
+      22968,
+      22971,
+      22972,
+      22974,
+      22984,
+      22987,
+      22991,
+      22994,
+      23004,
+      23017,
+      23024,
+      23033,
+      23035,
+      23041,
+      23047,
+      23048,
+      23049,
+      23052,
+      23056,
+      23057,
+      23058,
+      23061,
+      23062,
+      23065,
+      23067,
+      23070,
+      23072,
+      23076,
+      23078,
+      23085,
+      23086,
+      23091,
+      23093,
+      23095,
+      23096,
+      23099,
+      23101,
+      23102,
+      23108,
+      23111,
+      23115,
+      23125,
+      23128,
+      23132,
+      23137,
+      23141,
+      23144,
+      23147,
+      23155,
+      23169,
+      23177,
+      23180,
+      23185,
+      23186,
+      23194,
+      23195,
+      23196,
+      23200,
+      23207,
+      23211,
+      23212,
+      23221,
+      23223,
+      23230,
+      23235,
+      23237,
+      23238,
+      23239,
+      23242,
+      23243,
+      23247,
+      23248,
+      23266,
+      23267,
+      23270,
+      23274,
+      23278,
+      23282,
+      23296,
+      23298,
+      23301,
+      23302,
+      23305,
+      23309,
+      23314,
+      23316,
+      23320,
+      23335,
+      23342,
+      23344,
+      23345,
+      23350,
+      23352,
+      23355,
+      23356,
+      23357,
+      23361,
+      23367,
+      23373,
+      23379,
+      23385,
+      23393,
+      23395,
+      23398,
+      23402,
+      23410,
+      23416,
+      23417,
+      23419,
+      23422,
+      23423,
+      23425,
+      23432,
+      23436,
+      23446,
+      23451,
+      23460,
+      23461,
+      23467,
+      23479,
+      23483,
+      23487,
+      23490,
+      23491,
+      23496,
+      23499,
+      23500,
+      23505,
+      23510,
+      23511,
+      23518,
+      23522,
+      23543,
+      23544,
+      23545,
+      23549,
+      23553,
+      23555,
+      23559,
+      23565,
+      23567,
+      23573,
+      23574,
+      23576,
+      23580,
+      23584,
+      23585,
+      23597,
+      23606,
+      23615,
+      23630,
+      23632,
+      23633,
+      23644,
+      23645,
+      23650,
+      23654,
+      23656,
+      23662,
+      23663,
+      23664,
+      23665,
+      23670,
+      23671,
+      23673,
+      23681,
+      23684,
+      23692,
+      23695,
+      23697,
+      23706,
+      23709,
+      23722,
+      23732,
+      23736,
+      23737,
+      23738,
+      23742,
+      23743,
+      23744,
+      23749,
+      23757,
+      23773,
+      23777,
+      23778,
+      23780,
+      23782,
+      23783,
+      23784,
+      23793,
+      23795,
+      23796,
+      23797,
+      23803,
+      23804,
+      23805,
+      23806,
+      23808,
+      23809,
+      23816,
+      23823,
+      23829,
+      23830,
+      23833,
+      23835,
+      23836,
+      23846,
+      23854,
+      23858,
+      23865,
+      23871,
+      23874,
+      23877,
+      23884,
+      23887,
+      23890,
+      23891,
+      23896,
+      23901,
+      23904,
+      23909,
+      23913,
+      23918,
+      23924,
+      23925,
+      23930,
+      23931,
+      23935,
+      23938,
+      23940,
+      23941,
+      23943,
+      23947,
+      23948,
+      23955,
+      23991,
+      23998,
+      24002,
+      24007,
+      24014,
+      24015,
+      24019,
+      24032,
+      24038,
+      24041,
+      24045,
+      24063,
+      24065,
+      24066,
+      24068,
+      24072,
+      24073,
+      24076,
+      24077,
+      24079,
+      24080,
+      24085,
+      24087,
+      24088,
+      24090,
+      24092,
+      24093,
+      24096,
+      24097,
+      24101,
+      24102,
+      24103,
+      24108,
+      24117,
+      24118,
+      24124,
+      24128,
+      24129,
+      24133,
+      24137,
+      24138,
+      24140,
+      24141,
+      24144,
+      24145,
+      24146,
+      24147,
+      24153,
+      24158,
+      24163,
+      24164,
+      24167,
+      24168,
+      24170,
+      24174,
+      24175,
+      24178,
+      24180,
+      24192,
+      24198,
+      24200,
+      24204,
+      24206,
+      24220,
+      24221,
+      24225,
+      24226,
+      24228,
+      24229,
+      24230,
+      24237,
+      24238,
+      24242,
+      24268,
+      24270,
+      24271,
+      24277,
+      24278,
+      24279,
+      24281,
+      24284,
+      24285,
+      24290,
+      24291,
+      24292,
+      24298,
+      24300,
+      24301,
+      24302,
+      24303,
+      24310,
+      24314,
+      24316,
+      24317,
+      24322,
+      24323,
+      24327,
+      24330,
+      24333,
+      24335,
+      24336,
+      24340,
+      24342,
+      24343,
+      24344,
+      24352,
+      24359,
+      24360,
+      24366,
+      24375,
+      24376,
+      24383,
+      24384,
+      24388,
+      24390,
+      24393,
+      24395,
+      24396,
+      24397,
+      24409,
+      24410,
+      24411,
+      24412,
+      24413,
+      24415,
+      24422,
+      24428,
+      24431,
+      24434,
+      24435,
+      24448,
+      24449,
+      24454,
+      24455,
+      24456,
+      24457,
+      24458,
+      24463,
+      24472,
+      24475,
+      24488,
+      24490,
+      24498,
+      24506,
+      24513,
+      24516,
+      24518,
+      24521,
+      24527,
+      24539,
+      24546,
+      24548,
+      24553,
+      24556,
+      24561,
+      24562,
+      24572,
+      24573,
+      24586,
+      24587,
+      24589,
+      24594,
+      24597,
+      24598,
+      24601,
+      24607,
+      24610,
+      24612,
+      24619,
+      24620,
+      24649,
+      24652,
+      24653,
+      24657,
+      24658,
+      24660,
+      24661,
+      24662,
+      24667,
+      24669,
+      24676,
+      24679,
+      24682,
+      24695,
+      24700,
+      24703,
+      24714,
+      24721,
+      24725,
+      24729,
+      24741,
+      24746,
+      24755,
+      24759,
+      24764,
+      24768,
+      24769,
+      24771,
+      24772,
+      24778,
+      24788,
+      24790,
+      24793,
+      24796,
+      24798,
+      24804,
+      24805,
+      24817,
+      24824,
+      24825,
+      24826,
+      24830,
+      24836,
+      24838,
+      24844,
+      24847,
+      24848,
+      24850,
+      24860,
+      24861,
+      24862,
+      24864,
+      24871,
+      24872,
+      24876,
+      24879,
+      24886,
+      24887,
+      24891,
+      24894,
+      24895,
+      24901,
+      24911,
+      24913,
+      24914,
+      24915,
+      24916,
+      24917,
+      24918,
+      24920,
+      24925,
+      24930,
+      24932,
+      24938,
+      24952,
+      24965,
+      24967,
+      24968,
+      24971,
+      24979,
+      24985,
+      24988,
+      24989,
+      24991,
+      24996,
+      24997,
+      24999,
+      25002,
+      25008,
+      25009,
+      25013,
+      25015,
+      25016,
+      25023,
+      25027,
+      25031,
+      25032,
+      25036,
+      25046,
+      25051,
+      25057,
+      25058,
+      25060,
+      25062,
+      25063,
+      25064,
+      25070,
+      25074,
+      25078,
+      25083,
+      25084,
+      25089,
+      25090,
+      25091,
+      25099,
+      25100,
+      25101,
+      25105,
+      25107,
+      25112,
+      25114,
+      25121,
+      25122,
+      25127,
+      25128,
+      25137,
+      25148,
+      25152,
+      25153,
+      25154,
+      25164,
+      25165,
+      25167,
+      25168,
+      25171,
+      25181,
+      25184,
+      25185,
+      25190,
+      25201,
+      25205,
+      25217,
+      25219,
+      25226,
+      25229,
+      25230,
+      25232,
+      25236,
+      25243,
+      25253,
+      25255,
+      25258,
+      25279,
+      25284,
+      25290,
+      25293,
+      25299,
+      25300,
+      25301,
+      25310,
+      25311,
+      25313,
+      25318,
+      25322,
+      25324,
+      25329,
+      25344,
+      25346,
+      25351,
+      25360,
+      25368,
+      25370,
+      25373,
+      25377,
+      25378,
+      25379,
+      25383,
+      25384,
+      25385,
+      25389,
+      25393,
+      25394,
+      25399,
+      25417,
+      25418,
+      25421,
+      25425,
+      25426,
+      25434,
+      25438,
+      25440,
+      25441,
+      25443,
+      25444,
+      25450,
+      25451,
+      25457,
+      25459,
+      25460,
+      25463,
+      25464,
+      25465,
+      25469,
+      25475,
+      25481,
+      25485,
+      25490,
+      25494,
+      25495,
+      25499,
+      25500,
+      25508,
+      25509,
+      25514,
+      25518,
+      25522,
+      25523,
+      25524,
+      25526,
+      25528,
+      25529,
+      25531,
+      25533,
+      25538,
+      25541,
+      25547,
+      25555,
+      25556,
+      25557,
+      25560,
+      25561,
+      25564,
+      25572,
+      25573,
+      25574,
+      25575,
+      25581,
+      25587,
+      25592,
+      25593,
+      25599,
+      25607,
+      25608,
+      25611,
+      25619,
+      25623,
+      25629,
+      25632,
+      25638,
+      25639,
+      25643,
+      25646,
+      25647,
+      25650,
+      25655,
+      25656,
+      25660,
+      25661,
+      25666,
+      25667,
+      25673,
+      25678,
+      25679,
+      25680,
+      25687,
+      25688,
+      25689,
+      25690,
+      25691,
+      25698,
+      25701,
+      25708,
+      25709,
+      25711,
+      25725,
+      25731,
+      25733,
+      25746,
+      25758,
+      25761,
+      25766,
+      25767,
+      25778,
+      25780,
+      25785,
+      25787,
+      25789,
+      25793,
+      25794,
+      25798,
+      25799,
+      25804,
+      25807,
+      25808,
+      25810,
+      25811,
+      25816,
+      25820,
+      25831,
+      25832,
+      25838,
+      25839,
+      25848,
+      25849,
+      25852,
+      25855,
+      25856,
+      25858,
+      25862,
+      25863,
+      25868,
+      25872,
+      25879,
+      25888,
+      25890,
+      25892,
+      25896,
+      25902,
+      25903,
+      25910,
+      25911,
+      25915,
+      25917,
+      25918,
+      25923,
+      25929,
+      25932,
+      25940,
+      25944,
+      25949,
+      25950,
+      25955,
+      25965,
+      25968,
+      25969,
+      25970,
+      25974,
+      25987,
+      25990,
+      25994,
+      25997,
+      26005,
+      26006,
+      26010,
+      26013,
+      26016,
+      26020,
+      26022,
+      26029,
+      26032,
+      26036,
+      26039,
+      26052,
+      26053,
+      26054,
+      26055,
+      26058,
+      26061,
+      26065,
+      26067,
+      26068,
+      26083,
+      26086,
+      26087,
+      26091,
+      26095,
+      26100,
+      26115,
+      26116,
+      26117,
+      26121,
+      26126,
+      26127,
+      26132,
+      26134,
+      26139,
+      26143,
+      26145,
+      26147,
+      26148,
+      26154,
+      26156,
+      26160,
+      26169,
+      26173,
+      26174,
+      26187,
+      26190,
+      26199,
+      26200,
+      26207,
+      26210,
+      26212,
+      26217,
+      26224,
+      26227,
+      26228,
+      26229,
+      26230,
+      26240,
+      26243,
+      26247,
+      26250,
+      26251,
+      26255,
+      26266,
+      26274,
+      26282,
+      26289,
+      26291,
+      26295,
+      26300,
+      26302,
+      26305,
+      26306,
+      26310,
+      26320,
+      26327,
+      26328,
+      26332,
+      26335,
+      26336,
+      26337,
+      26338,
+      26340,
+      26341,
+      26348,
+      26356,
+      26362,
+      26370,
+      26371,
+      26373,
+      26376,
+      26378,
+      26379,
+      26387,
+      26388,
+      26393,
+      26394,
+      26401,
+      26414,
+      26415,
+      26416,
+      26426,
+      26430,
+      26432,
+      26437,
+      26440,
+      26441,
+      26444,
+      26454,
+      26455,
+      26456,
+      26460,
+      26461,
+      26467,
+      26468,
+      26474,
+      26477,
+      26492,
+      26493,
+      26494,
+      26496,
+      26516,
+      26517,
+      26518,
+      26521,
+      26525,
+      26526,
+      26527,
+      26531,
+      26534,
+      26535,
+      26541,
+      26544,
+      26547,
+      26548,
+      26549,
+      26557,
+      26559,
+      26560,
+      26565,
+      26569,
+      26571,
+      26573,
+      26576,
+      26579,
+      26580,
+      26585,
+      26589,
+      26591,
+      26592,
+      26601,
+      26605,
+      26607,
+      26611,
+      26614,
+      26620,
+      26634,
+      26636,
+      26638,
+      26639,
+      26642,
+      26643,
+      26644,
+      26652,
+      26654,
+      26655,
+      26656,
+      26657,
+      26658,
+      26662,
+      26675,
+      26677,
+      26682,
+      26685,
+      26686,
+      26688,
+      26691,
+      26692,
+      26698,
+      26702,
+      26703,
+      26711,
+      26715,
+      26724,
+      26725,
+      26730,
+      26737,
+      26741,
+      26744,
+      26745,
+      26746,
+      26755,
+      26760,
+      26763,
+      26764,
+      26770,
+      26775,
+      26782,
+      26786,
+      26787,
+      26788,
+      26789,
+      26792,
+      26793,
+      26797,
+      26801,
+      26802,
+      26806,
+      26807,
+      26813,
+      26824,
+      26827,
+      26830,
+      26832,
+      26837,
+      26846,
+      26860,
+      26863,
+      26869,
+      26876,
+      26877,
+      26878,
+      26884,
+      26891,
+      26895,
+      26896,
+      26897,
+      26901,
+      26907,
+      26910,
+      26915,
+      26919,
+      26933,
+      26941,
+      26942,
+      26945,
+      26949,
+      26950,
+      26951,
+      26952,
+      26965,
+      26966,
+      26972,
+      26976,
+      26977,
+      26978,
+      26982,
+      26985,
+      26989,
+      26992,
+      26993,
+      26994,
+      27002,
+      27009,
+      27010,
+      27014,
+      27015,
+      27020,
+      27026,
+      27027,
+      27028,
+      27031,
+      27032,
+      27035,
+      27051,
+      27052,
+      27058,
+      27064,
+      27065,
+      27066,
+      27067,
+      27072,
+      27073,
+      27082,
+      27085,
+      27086,
+      27087,
+      27095,
+      27100,
+      27102,
+      27103,
+      27108,
+      27109,
+      27110,
+      27113,
+      27114,
+      27115,
+      27122,
+      27131,
+      27147,
+      27153,
+      27156,
+      27164,
+      27165,
+      27172,
+      27174,
+      27176,
+      27179,
+      27184,
+      27190,
+      27191,
+      27199,
+      27203,
+      27204,
+      27210,
+      27225,
+      27235,
+      27239,
+      27241,
+      27242,
+      27245,
+      27265,
+      27266,
+      27267,
+      27268,
+      27272,
+      27275,
+      27276,
+      27283,
+      27287,
+      27288,
+      27292,
+      27293,
+      27296,
+      27310,
+      27313,
+      27318,
+      27321,
+      27330,
+      27334,
+      27341,
+      27342,
+      27346,
+      27348,
+      27349,
+      27350,
+      27351,
+      27355,
+      27360,
+      27361,
+      27362,
+      27364,
+      27366,
+      27373,
+      27378,
+      27380,
+      27381,
+      27385,
+      27390,
+      27391,
+      27392,
+      27401,
+      27405,
+      27406,
+      27412,
+      27413,
+      27415,
+      27416,
+      27417,
+      27421,
+      27428,
+      27430,
+      27431,
+      27432,
+      27439,
+      27444,
+      27453,
+      27454,
+      27465,
+      27466,
+      27469,
+      27472,
+      27475,
+      27479,
+      27483,
+      27484,
+      27487,
+      27488,
+      27495,
+      27500,
+      27501,
+      27506,
+      27511,
+      27512,
+      27524,
+      27528,
+      27531,
+      27532,
+      27537,
+      27543,
+      27548,
+      27549,
+      27559,
+      27572,
+      27573,
+      27576,
+      27578,
+      27580,
+      27581,
+      27584,
+      27585,
+      27586,
+      27595,
+      27605,
+      27606,
+      27609,
+      27610,
+      27611,
+      27614,
+      27618,
+      27623,
+      27627,
+      27629,
+      27631,
+      27632,
+      27633,
+      27635,
+      27640,
+      27642,
+      27651,
+      27654,
+      27657,
+      27659,
+      27663,
+      27678,
+      27684,
+      27685,
+      27687,
+      27688,
+      27693,
+      27694,
+      27698,
+      27703,
+      27704,
+      27709,
+      27710,
+      27711,
+      27717,
+      27718,
+      27720,
+      27727,
+      27730,
+      27732,
+      27740,
+      27742,
+      27750,
+      27751,
+      27753,
+      27757,
+      27759,
+      27760,
+      27766,
+      27767,
+      27773,
+      27774,
+      27794,
+      27795,
+      27798,
+      27803,
+      27806,
+      27807,
+      27814,
+      27815,
+      27819,
+      27821,
+      27822,
+      27833,
+      27839,
+      27840,
+      27843,
+      27844,
+      27847,
+      27851,
+      27861,
+      27867,
+      27869,
+      27870,
+      27871,
+      27872,
+      27875,
+      27876,
+      27877,
+      27881,
+      27887,
+      27892,
+      27893,
+      27896,
+      27897,
+      27898,
+      27900,
+      27901,
+      27903,
+      27904,
+      27914,
+      27915,
+      27920,
+      27921,
+      27922,
+      27923,
+      27927,
+      27931,
+      27932,
+      27933,
+      27942,
+      27943,
+      27944,
+      27949,
+      27953,
+      27958,
+      27959,
+      27965,
+      27966,
+      27967,
+      27971,
+      27972,
+      27975,
+      27982,
+      27983,
+      27989,
+      28001,
+      28002,
+      28006,
+      28008,
+      28014,
+      28020,
+      28027,
+      28029,
+      28032,
+      28037,
+      28039,
+      28040,
+      28055,
+      28060,
+      28061,
+      28064,
+      28065,
+      28067,
+      28074,
+      28075,
+      28077,
+      28078,
+      28083,
+      28084,
+      28090,
+      28094,
+      28097,
+      28099,
+      28100,
+      28104,
+      28108,
+      28109,
+      28113,
+      28114,
+      28116,
+      28118,
+      28119,
+      28120,
+      28121,
+      28128,
+      28132,
+      28136,
+      28138,
+      28139,
+      28140,
+      28143,
+      28146,
+      28147,
+      28154,
+      28160,
+      28168,
+      28173,
+      28176,
+      28183,
+      28184,
+      28187,
+      28191,
+      28195,
+      28198,
+      28199,
+      28200,
+      28201,
+      28207,
+      28209,
+      28210,
+      28212,
+      28221,
+      28225,
+      28227,
+      28228,
+      28230,
+      28231,
+      28234,
+      28235,
+      28239,
+      28242,
+      28247,
+      28250,
+      28251,
+      28254,
+      28255,
+      28260,
+      28262,
+      28264,
+      28265,
+      28277,
+      28281,
+      28282,
+      28295,
+      28296,
+      28298,
+      28302,
+      28307,
+      28311,
+      28312,
+      28313,
+      28327,
+      28329,
+      28331,
+      28332,
+      28335,
+      28344,
+      28345,
+      28351,
+      28357,
+      28362,
+      28365,
+      28369,
+      28370,
+      28377,
+      28392,
+      28393,
+      28394,
+      28397,
+      28400,
+      28402,
+      28403,
+      28405,
+      28412,
+      28414,
+      28418,
+      28426,
+      28431,
+      28433,
+      28437,
+      28471,
+      28472,
+      28473,
+      28476,
+      28478,
+      28479,
+      28483,
+      28484,
+      28488,
+      28495,
+      28499,
+      28503,
+      28505,
+      28507,
+      28508,
+      28510,
+      28518,
+      28522,
+      28524,
+      28530,
+      28533,
+      28534,
+      28545,
+      28546,
+      28550,
+      28551,
+      28553,
+      28555,
+      28556,
+      28562,
+      28563,
+      28574,
+      28578,
+      28579,
+      28580,
+      28582,
+      28588,
+      28591,
+      28597,
+      28598,
+      28600,
+      28610,
+      28613,
+      28614,
+      28617,
+      28621,
+      28632,
+      28637,
+      28640,
+      28649,
+      28656,
+      28659,
+      28660,
+      28665,
+      28674,
+      28677,
+      28682,
+      28687,
+      28689,
+      28695,
+      28696,
+      28700,
+      28703,
+      28704,
+      28705,
+      28706,
+      28716,
+      28723,
+      28727,
+      28728,
+      28730,
+      28731,
+      28732,
+      28733,
+      28738,
+      28739,
+      28741,
+      28742,
+      28745,
+      28748,
+      28752,
+      28753,
+      28756,
+      28759,
+      28765,
+      28775,
+      28787,
+      28788,
+      28793,
+      28804,
+      28807,
+      28808,
+      28812,
+      28813,
+      28817,
+      28819,
+      28823,
+      28824,
+      28825,
+      28827,
+      28832,
+      28835,
+      28849,
+      28852,
+      28863,
+      28869,
+      28871,
+      28872,
+      28875,
+      28878,
+      28880,
+      28883,
+      28884,
+      28886,
+      28891,
+      28893,
+      28897,
+      28901,
+      28905,
+      28911,
+      28921,
+      28922,
+      28931,
+      28940,
+      28943,
+      28948,
+      28949,
+      28951,
+      28954,
+      28957,
+      28964,
+      28968,
+      28969,
+      28970,
+      28975,
+      28985,
+      28986,
+      28987,
+      28989,
+      28990,
+      28992,
+      28993,
+      28994,
+      28995,
+      29001,
+      29002,
+      29005,
+      29006,
+      29015,
+      29020,
+      29021,
+      29027,
+      29028,
+      29033,
+      29039,
+      29040,
+      29043,
+      29045,
+      29046,
+      29048,
+      29050,
+      29055,
+      29064,
+      29067,
+      29068,
+      29074,
+      29076,
+      29084,
+      29091,
+      29092,
+      29097,
+      29098,
+      29099,
+      29100,
+      29107,
+      29109,
+      29110,
+      29111,
+      29112,
+      29113,
+      29122,
+      29123,
+      29125,
+      29128,
+      29133,
+      29134,
+      29135,
+      29136,
+      29138,
+      29139,
+      29146,
+      29147,
+      29148,
+      29150,
+      29154,
+      29155,
+      29159,
+      29162,
+      29163,
+      29164,
+      29165,
+      29166,
+      29171,
+      29178,
+      29180,
+      29181,
+      29183,
+      29184,
+      29189,
+      29190,
+      29193,
+      29196,
+      29201,
+      29206,
+      29207,
+      29212,
+      29218,
+      29232,
+      29234,
+      29243,
+      29244,
+      29246,
+      29247,
+      29251,
+      29256,
+      29264,
+      29266,
+      29275,
+      29278,
+      29282,
+      29284,
+      29287,
+      29288,
+      29289,
+      29292,
+      29293,
+      29297,
+      29301,
+      29312,
+      29313,
+      29314,
+      29315,
+      29319,
+      29320,
+      29323,
+      29325,
+      29334,
+      29336,
+      29337,
+      29339,
+      29342,
+      29344,
+      29345,
+      29346,
+      29356,
+      29358,
+      29366,
+      29370,
+      29377,
+      29378,
+      29381,
+      29385,
+      29388,
+      29389,
+      29390,
+      29392,
+      29397,
+      29401,
+      29402,
+      29405,
+      29415,
+      29416,
+      29417,
+      29418,
+      29419,
+      29422,
+      29423,
+      29424,
+      29427,
+      29428,
+      29436,
+      29440,
+      29441,
+      29443,
+      29446,
+      29451,
+      29452,
+      29459,
+      29462,
+      29469,
+      29472,
+      29473,
+      29477,
+      29480,
+      29481,
+      29487,
+      29488,
+      29493,
+      29501,
+      29510,
+      29513,
+      29514,
+      29518,
+      29519,
+      29521,
+      29522,
+      29531,
+      29537,
+      29538,
+      29539,
+      29540,
+      29544,
+      29545,
+      29546,
+      29547,
+      29548,
+      29553,
+      29560,
+      29566,
+      29573,
+      29574,
+      29576,
+      29577,
+      29578,
+      29587,
+      29595,
+      29598,
+      29599,
+      29601,
+      29602,
+      29604,
+      29605,
+      29612,
+      29613,
+      29614,
+      29615,
+      29616,
+      29617,
+      29618,
+      29619,
+      29620,
+      29621,
+      29622,
+      29623,
+      29624,
+      29625,
+      29626,
+      29627,
+      29628,
+      29629,
+      29630,
+      29631,
+      29632,
+      29633,
+      29634,
+      29635,
+      29636,
+      29637,
+      29638,
+      29639,
+      29640,
+      29641,
+      29642,
+      29643,
+      29644,
+      29645,
+      29646,
+      29647,
+      29648,
+      29649,
+      29650,
+      29651,
+      29652,
+      29653,
+      29654,
+      29655,
+      29656,
+      29657,
+      29658,
+      29659,
+      29660,
+      29661,
+      29662,
+      29663,
+      29664,
+      29665,
+      29666,
+      29667,
+      29668,
+      29669,
+      29670,
+      29671,
+      29672,
+      29673,
+      29674,
+      29675,
+      29676,
+      29677,
+      29678,
+      29679,
+      29680,
+      29681,
+      29682,
+      29683,
+      29684,
+      29685,
+      29686,
+      29687,
+      29688,
+      29689,
+      29690,
+      29691,
+      29692,
+      29693,
+      29694,
+      29695,
+      29696,
+      29697,
+      29698,
+      29699,
+      29700,
+      29701,
+      29702,
+      29703,
+      29704,
+      29705,
+      29706,
+      29707,
+      29708,
+      29709,
+      29710,
+      29711,
+      29712,
+      29713,
+      29714,
+      29715,
+      29716,
+      29717,
+      29718,
+      29719,
+      29720,
+      29721,
+      29722,
+      29723,
+      29724,
+      29725,
+      29726,
+      29727,
+      29728,
+      29729,
+      29730,
+      29731,
+      29732,
+      29733,
+      29734,
+      29735,
+      29736,
+      29737,
+      29738,
+      29739,
+      29740,
+      29741,
+      29742,
+      29743,
+      29744,
+      29745,
+      29746,
+      29747,
+      29748,
+      29749,
+      29750,
+      29751,
+      29752,
+      29753,
+      29754,
+      29755,
+      29756,
+      29757,
+      29758,
+      29759,
+      29760,
+      29761,
+      29762,
+      29763,
+      29764,
+      29765,
+      29766,
+      29767,
+      29768,
+      29769,
+      29770,
+      29771,
+      29772,
+      29773,
+      29774,
+      29775,
+      29776,
+      29777,
+      29778,
+      29779,
+      29780,
+      29781,
+      29782,
+      29783,
+      29784,
+      29785,
+      29786,
+      29787,
+      29788,
+      29789,
+      29790,
+      29791,
+      29792,
+      29793,
+      29794,
+      29795,
+      29796,
+      29797,
+      29798,
+      29799,
+      29800,
+      29801,
+      29802,
+      29803,
+      29804,
+      29805,
+      29806,
+      29807,
+      29808,
+      29809,
+      29810,
+      29811,
+      29812,
+      29813,
+      29814,
+      29815,
+      29816,
+      29817,
+      29818,
+      29819,
+      29820,
+      29821,
+      29822,
+      29823,
+      29824,
+      29825,
+      29826,
+      29827,
+      29828,
+      29829,
+      29830,
+      29831,
+      29832,
+      29833,
+      29834,
+      29835,
+      29836,
+      29837,
+      29838,
+      29839,
+      29840,
+      29841,
+      29842,
+      29843,
+      29844,
+      29845,
+      29846,
+      29847,
+      29848,
+      29849,
+      29850,
+      29851,
+      29852,
+      29853,
+      29854,
+      29855,
+      29856,
+      29857,
+      29858,
+      29859,
+      29860,
+      29861,
+      29862,
+      29863,
+      29864,
+      29865,
+      29866,
+      29867,
+      29868,
+      29869,
+      29870,
+      29871,
+      29872,
+      29873,
+      29874,
+      29875,
+      29876,
+      29877,
+      29878,
+      29879,
+      29880,
+      29881,
+      29882,
+      29883,
+      29884,
+      29885,
+      29886,
+      29887,
+      29888,
+      29889,
+      29890,
+      29891,
+      29892,
+      29893,
+      29894,
+      29895,
+      29896,
+      29897,
+      29898,
+      29899,
+      29900,
+      29901,
+      29902,
+      29903,
+      29904,
+      29905,
+      29906,
+      29907,
+      29908,
+      29909,
+      29910,
+      29911,
+      29912,
+      29913,
+      29914,
+      29915,
+      29916,
+      29917,
+      29918,
+      29919,
+      29920,
+      29921,
+      29922,
+      29923,
+      29924,
+      29925,
+      29926,
+      29927,
+      29928,
+      29929,
+      29930,
+      29931,
+      29932,
+      29933,
+      29934,
+      29935,
+      29936,
+      29937,
+      29938,
+      29939,
+      29940,
+      29941,
+      29942,
+      29943,
+      29944,
+      29945,
+      29946,
+      29947,
+      29948,
+      29949,
+      29950,
+      29951,
+      29952,
+      29953,
+      29954,
+      29955,
+      29956,
+      29957,
+      29958,
+      29959,
+      29960,
+      29961,
+      29962,
+      29963,
+      29964,
+      29965,
+      29966,
+      29967,
+      29968,
+      29969,
+      29970,
+      29971,
+      29972,
+      29973,
+      29974,
+      29975,
+      29976,
+      29977,
+      29978,
+      29979,
+      29980,
+      29981,
+      29982,
+      29983,
+      29984,
+      29985,
+      29986,
+      29987,
+      29988,
+      29989,
+      29990,
+      29991,
+      29992,
+      29993,
+      29994,
+      29995,
+      29996,
+      29997,
+      29998,
+      29999,
+      30000,
+      30001,
+      30002,
+      30003,
+      30004,
+      30005,
+      30006,
+      30007,
+      30008,
+      30009,
+      30010,
+      30011,
+      30012,
+      30013,
+      30014,
+      30015,
+      30016,
+      30017,
+      30018,
+      30019,
+      30020,
+      30021,
+      30022,
+      30023,
+      30024,
+      30025,
+      30026,
+      30027,
+      30028,
+      30029,
+      30030,
+      30031,
+      30032,
+      30033,
+      30034,
+      30035,
+      30036,
+      30037,
+      30038,
+      30039,
+      30040,
+      30041,
+      30042,
+      30043,
+      30044,
+      30045,
+      30046,
+      30047,
+      30048,
+      30049,
+      30050,
+      30051,
+      30052,
+      30053,
+      30054,
+      30055,
+      30056,
+      30057,
+      30058,
+      30059,
+      30060,
+      30061,
+      30062,
+      30063,
+      30064,
+      30065,
+      30066,
+      30067,
+      30068,
+      30069,
+      30070,
+      30071,
+      30072,
+      30073,
+      30074,
+      30075,
+      30076,
+      30077,
+      30078,
+      30079,
+      30080,
+      30081,
+      30082,
+      30083,
+      30084,
+      30085,
+      30086,
+      30087,
+      30088,
+      30089,
+      30090,
+      30091,
+      30092,
+      30093,
+      30094,
+      30095,
+      30096,
+      30097,
+      30098,
+      30099,
+      30100,
+      30101,
+      30102,
+      30103,
+      30104,
+      30105,
+      30106,
+      30107,
+      30108,
+      30109,
+      30110,
+      30111,
+      30112,
+      30113,
+      30114,
+      30115,
+      30116,
+      30117,
+      30118,
+      30119,
+      30120,
+      30121,
+      30122,
+      30123,
+      30124,
+      30125,
+      30126,
+      30127,
+      30128,
+      30129,
+      30130,
+      30131,
+      30132,
+      30133,
+      30134,
+      30135,
+      30136,
+      30137,
+      30138,
+      30139,
+      30140,
+      30141,
+      30142,
+      30143,
+      30144,
+      30145,
+      30146,
+      30147,
+      30148,
+      30149,
+      30150,
+      30151,
+      30152,
+      30153,
+      30154,
+      30155,
+      30156,
+      30157,
+      30158,
+      30159,
+      30160,
+      30161,
+      30162,
+      30163,
+      30164,
+      30165,
+      30166,
+      30167,
+      30168,
+      30169,
+      30170,
+      30171,
+      30172,
+      30173,
+      30174,
+      30175,
+      30176,
+      30177,
+      30178,
+      30179,
+      30180,
+      30181,
+      30182,
+      30183,
+      30184,
+      30185,
+      30186,
+      30187,
+      30188,
+      30189,
+      30190,
+      30191,
+      30192,
+      30193,
+      30194,
+      30195,
+      30196,
+      30197,
+      30198,
+      30199,
+      30200,
+      30201,
+      30202,
+      30203,
+      30204,
+      30205,
+      30206,
+      30207,
+      30208,
+      30209,
+      30210,
+      30211,
+      30212,
+      30213,
+      30214,
+      30215,
+      30216,
+      30217,
+      30218,
+      30219,
+      30220,
+      30221,
+      30222,
+      30223,
+      30224,
+      30225,
+      30226,
+      30227,
+      30228,
+      30229,
+      30230,
+      30231,
+      30232,
+      30233,
+      30234,
+      30235,
+      30236,
+      30237,
+      30238,
+      30239,
+      30240,
+      30241,
+      30242,
+      30243,
+      30244,
+      30245,
+      30246,
+      30247,
+      30248,
+      30249,
+      30250,
+      30251,
+      30252,
+      30253,
+      30254,
+      30255,
+      30256,
+      30257,
+      30258,
+      30259,
+      30260,
+      30261,
+      30262,
+      30263,
+      30264,
+      30265,
+      30266,
+      30267,
+      30268,
+      30269,
+      30270,
+      30271,
+      30272,
+      30273,
+      30274,
+      30275,
+      30276,
+      30277,
+      30278,
+      30279,
+      30280,
+      30281,
+      30282,
+      30283,
+      30284,
+      30285,
+      30286,
+      30287,
+      30288,
+      30289,
+      30290,
+      30291,
+      30292,
+      30293,
+      30294,
+      30295,
+      30296,
+      30297,
+      30298,
+      30299,
+      30300,
+      30301,
+      30302,
+      30303,
+      30304,
+      30305,
+      30306,
+      30307,
+      30308,
+      30309,
+      30310,
+      30311,
+      30312,
+      30313,
+      30314,
+      30315,
+      30316,
+      30317,
+      30318,
+      30319,
+      30320,
+      30321,
+      30322,
+      30323,
+      30324,
+      30325,
+      30326,
+      30327,
+      30328,
+      30329,
+      30330,
+      30331,
+      30332,
+      30333,
+      30334,
+      30335,
+      30336,
+      30337,
+      30338,
+      30339,
+      30340,
+      30341,
+      30342,
+      30343,
+      30344,
+      30345,
+      30346,
+      30347,
+      30348,
+      30349,
+      30350,
+      30351,
+      30352,
+      30353,
+      30354,
+      30355,
+      30356,
+      30357,
+      30358,
+      30359,
+      30360,
+      30361,
+      30362,
+      30363,
+      30364,
+      30365,
+      30366,
+      30367,
+      30368,
+      30369,
+      30370,
+      30371,
+      30372,
+      30373,
+      30374,
+      30375,
+      30376,
+      30377,
+      30378,
+      30379,
+      30380,
+      30381,
+      30382,
+      30383,
+      30384,
+      30385,
+      30386,
+      30387,
+      30388,
+      30389,
+      30390,
+      30391,
+      30392,
+      30393,
+      30394,
+      30395,
+      30396,
+      30397,
+      30398,
+      30399,
+      30400,
+      30401,
+      30402,
+      30403,
+      30404,
+      30405,
+      30406,
+      30407,
+      30408,
+      30409,
+      30410,
+      30411,
+      30412,
+      30413,
+      30414,
+      30415,
+      30416,
+      30417,
+      30418,
+      30419,
+      30420,
+      30421,
+      30422,
+      30423,
+      30424,
+      30425,
+      30426,
+      30427,
+      30428,
+      30429,
+      30430,
+      30431,
+      30432,
+      30433,
+      30434,
+      30435,
+      30436,
+      30437,
+      30438,
+      30439,
+      30440,
+      30441,
+      30442,
+      30443,
+      30444,
+      30445,
+      30446,
+      30447,
+      30448,
+      30449,
+      30450,
+      30451,
+      30452,
+      30453,
+      30454,
+      30455,
+      30456,
+      30457,
+      30458,
+      30459,
+      30460,
+      30461,
+      30462,
+      30463,
+      30464,
+      30465,
+      30466,
+      30467,
+      30468,
+      30469,
+      30470,
+      30471,
+      30472,
+      30473,
+      30474,
+      30475,
+      30476,
+      30477,
+      30478,
+      30479,
+      30480,
+      30481,
+      30482,
+      30483,
+      30484,
+      30485,
+      30486,
+      30487,
+      30488,
+      30489,
+      30490,
+      30491,
+      30492,
+      30493,
+      30494,
+      30495,
+      30496,
+      30497,
+      30498,
+      30499,
+      30500,
+      30501,
+      30502,
+      30503,
+      30504,
+      30505,
+      30506,
+      30507,
+      30508,
+      30509,
+      30510,
+      30511,
+      30512,
+      30513,
+      30514,
+      30515,
+      30516,
+      30517,
+      30518,
+      30519,
+      30520,
+      30521
+    ],
+    "dtype": "float32",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 512,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "intermediate_size": 2048,
+    "is_encoder_decoder": false,
+    "key_rho": 2.18,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "lookup_head": 0,
+    "lookup_layer": 3,
+    "max_position_embeddings": 8192,
+    "model_type": "captionbert_structured_rotary",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 12,
+    "output_attentions": false,
+    "output_dim": 768,
+    "output_hidden_states": false,
+    "pad_token_id": 0,
+    "phase_seed": 1662197431,
+    "pointer_positions": "all",
+    "pointer_weight": 0.0,
+    "problem_type": null,
+    "punctuation_ids": [
+      1012,
+      999,
+      1029
+    ],
+    "return_dict": true,
+    "rotary_mode": "hierarchical",
+    "special_ids": [
+      100,
+      102,
+      0,
+      101,
+      103
+    ],
+    "transformers_version": "5.22.1",
+    "use_value_addresses": true,
+    "vocab_size": 30522
+  },
+  "parameters": 54126848,
+  "seconds_per_step": 0.49217270983149225,
+  "seed": 1,
+  "source_rotary_revision": "3f174e3f391f983428171cefc6901b34b22274d8",
+  "status": "complete",
+  "steps": 7808,
+  "tensorboard_event_files": [
+    "events.out.tfevents.1793991823.79fe39615b31.56708.2"
+  ],
+  "test_evaluation": {
+    "above_128_tokens": {
+      "cos_to_target": 0.7372538447380066,
+      "erank": 73.67732147778597,
+      "n": 5000,
+      "self_cos": 0.011612230501773386
+    },
+    "above_128_total_rows": 29220,
+    "heldout": {
+      "cos_to_target": 0.8076629042625427,
+      "erank": 94.97366337774051,
+      "n": 5000,
+      "self_cos": 0.0035747495065234897
+    },
+    "heldout_total_rows": 500000
+  },
+  "training_rows": 1000000,
+  "wall_seconds_this_session": 3943.1309813709995
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58a9c260610d787b6fa6769962a86c70c698ecac45b66fa6091f0c4302d4264b
+oid sha256:02186b607c5b44b0d36728f0c80ddda0ec8a9094a21476cc161369a30b172012
 size 649784662

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/tensorboard/events.out.tfevents.1793991823.79fe39615b31.56708.2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0094d687d045a39037ecf42cd475e7a8590ffa0d2d3233974adf32ff4ce52d3
-size 4885555
+oid sha256:66c178734b581bc0b28d71ef74a0db3f2418eb4c2b4edbd261d9de65e3e799f6
+size 5023221

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "is_local": true,
+  "local_files_only": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

experiments/fad07b2a05857a7abef278004986cacc1e3b8eb89b70943668505ea90498525e/H16-seed1/train.jsonl

@@ -7598,3 +7598,211 @@
 {"loss": 0.16233189404010773, "nce": 0.1618477702140808, "mse": 0.0004841249610763043, "pointer": 0.0, "grad_norm": 0.061685673892498016, "pointer_opportunities": 761, "pointer_positions": 0, "step": 7598, "epoch": 15.569672131147541, "lr": 2.198228767616643e-06, "seconds": 0.18161793800027226, "rows_per_second": 11276.419182762276}
 {"loss": 0.17912836372852325, "nce": 0.17865285277366638, "mse": 0.0004755061527248472, "pointer": 0.0, "grad_norm": 0.055240046232938766, "pointer_opportunities": 12301, "pointer_positions": 0, "step": 7599, "epoch": 15.57172131147541, "lr": 2.186905552570166e-06, "seconds": 0.6767780810005206, "rows_per_second": 3026.1027321870733}
 {"loss": 0.13184958696365356, "nce": 0.13145288825035095, "mse": 0.00039669842226430774, "pointer": 0.0, "grad_norm": 0.04396223649382591, "pointer_opportunities": 1006, "pointer_positions": 0, "step": 7600, "epoch": 15.573770491803279, "lr": 2.175635987570926e-06, "seconds": 0.2858951359994535, "rows_per_second": 7163.46569814995}
+{"loss": 0.13884711265563965, "nce": 0.13840752840042114, "mse": 0.00043958675814792514, "pointer": 0.0, "grad_norm": 0.050792526453733444, "pointer_opportunities": 1028, "pointer_positions": 0, "step": 7601, "epoch": 15.575819672131148, "lr": 2.164420074645682e-06, "seconds": 0.2230110170021362, "rows_per_second": 9183.402809110468}
+{"loss": 0.19682569801807404, "nce": 0.19637247920036316, "mse": 0.0004532245802693069, "pointer": 0.0, "grad_norm": 0.07086184620857239, "pointer_opportunities": 10683, "pointer_positions": 0, "step": 7602, "epoch": 15.577868852459016, "lr": 2.153257815811584e-06, "seconds": 0.5360277829968254, "rows_per_second": 3820.697480548558}
+{"loss": 0.2795000970363617, "nce": 0.2789137065410614, "mse": 0.0005863952683284879, "pointer": 0.0, "grad_norm": 0.06677594035863876, "pointer_opportunities": 13665, "pointer_positions": 0, "step": 7603, "epoch": 15.579918032786885, "lr": 2.1421492130760724e-06, "seconds": 0.9930616670026211, "rows_per_second": 2062.308986491767}
+{"loss": 0.12885768711566925, "nce": 0.1284586489200592, "mse": 0.0003990313853137195, "pointer": 0.0, "grad_norm": 0.04676111787557602, "pointer_opportunities": 1988, "pointer_positions": 0, "step": 7604, "epoch": 15.581967213114755, "lr": 2.1310942684369773e-06, "seconds": 0.35447663600280066, "rows_per_second": 5777.53169600667}
+{"loss": 0.3226403594017029, "nce": 0.3220248222351074, "mse": 0.00061552703846246, "pointer": 0.0, "grad_norm": 0.07727556675672531, "pointer_opportunities": 16287, "pointer_positions": 0, "step": 7605, "epoch": 15.584016393442623, "lr": 2.120092983882487e-06, "seconds": 1.1084838950009726, "rows_per_second": 1847.568565710378}
+{"loss": 0.1346786469221115, "nce": 0.13427940011024475, "mse": 0.0003992534475401044, "pointer": 0.0, "grad_norm": 0.04447690024971962, "pointer_opportunities": 1517, "pointer_positions": 0, "step": 7606, "epoch": 15.586065573770492, "lr": 2.1091453613911466e-06, "seconds": 0.31996025599801214, "rows_per_second": 6400.794978776126}
+{"loss": 0.2776247262954712, "nce": 0.2770453095436096, "mse": 0.0005794118042103946, "pointer": 0.0, "grad_norm": 0.0644955262541771, "pointer_opportunities": 12417, "pointer_positions": 0, "step": 7607, "epoch": 15.58811475409836, "lr": 2.0982514029317913e-06, "seconds": 0.9533351879981637, "rows_per_second": 2148.2475689378884}
+{"loss": 0.33836308121681213, "nce": 0.33773118257522583, "mse": 0.0006319013191387057, "pointer": 0.0, "grad_norm": 0.069765605032444, "pointer_opportunities": 17710, "pointer_positions": 0, "step": 7608, "epoch": 15.59016393442623, "lr": 2.0874111104636468e-06, "seconds": 1.2605696329992497, "rows_per_second": 1624.6623323197402}
+{"loss": 0.13504154980182648, "nce": 0.1346435844898224, "mse": 0.0003979614411946386, "pointer": 0.0, "grad_norm": 0.04525871202349663, "pointer_opportunities": 1579, "pointer_positions": 0, "step": 7609, "epoch": 15.592213114754099, "lr": 2.076624485936297e-06, "seconds": 0.332919905999006, "rows_per_second": 6151.629755674972}
+{"loss": 0.63038170337677, "nce": 0.6295324563980103, "mse": 0.0008492653141729534, "pointer": 0.0, "grad_norm": 0.15179237723350525, "pointer_opportunities": 19, "pointer_positions": 0, "step": 7610, "epoch": 15.594262295081966, "lr": 2.0658915312896483e-06, "seconds": 0.09958929899948998, "rows_per_second": 20564.45843654837}
+{"loss": 0.13080471754074097, "nce": 0.13040795922279358, "mse": 0.00039675840525887907, "pointer": 0.0, "grad_norm": 0.04258748143911362, "pointer_opportunities": 1592, "pointer_positions": 0, "step": 7611, "epoch": 15.596311475409836, "lr": 2.0552122484539654e-06, "seconds": 0.33138857499943697, "rows_per_second": 6180.05614708799}
+{"loss": 0.21901346743106842, "nce": 0.21847176551818848, "mse": 0.0005417092470452189, "pointer": 0.0, "grad_norm": 0.0823047086596489, "pointer_opportunities": 478, "pointer_positions": 0, "step": 7612, "epoch": 15.598360655737705, "lr": 2.0445866393498693e-06, "seconds": 0.15461257800052408, "rows_per_second": 13246.0115889993}
+{"loss": 0.1301943063735962, "nce": 0.12979795038700104, "mse": 0.0003963522322010249, "pointer": 0.0, "grad_norm": 0.04170355945825577, "pointer_opportunities": 1028, "pointer_positions": 0, "step": 7613, "epoch": 15.600409836065573, "lr": 2.0340147058883387e-06, "seconds": 0.2925609859994438, "rows_per_second": 7000.249855610938}
+{"loss": 0.19089388847351074, "nce": 0.1903999149799347, "mse": 0.0004939764039590955, "pointer": 0.0, "grad_norm": 0.055326979607343674, "pointer_opportunities": 11127, "pointer_positions": 0, "step": 7614, "epoch": 15.602459016393443, "lr": 2.023496449970643e-06, "seconds": 0.7234809710025729, "rows_per_second": 2830.7586268121995}
+{"loss": 0.12383011728525162, "nce": 0.12343110144138336, "mse": 0.0003990193363279104, "pointer": 0.0, "grad_norm": 0.03921229764819145, "pointer_opportunities": 1010, "pointer_positions": 0, "step": 7615, "epoch": 15.604508196721312, "lr": 2.0130318734884415e-06, "seconds": 0.26874786600092193, "rows_per_second": 7620.5256267708355}
+{"loss": 0.1863839328289032, "nce": 0.18589331209659576, "mse": 0.0004906269605271518, "pointer": 0.0, "grad_norm": 0.05285577476024628, "pointer_opportunities": 11118, "pointer_positions": 0, "step": 7616, "epoch": 15.60655737704918, "lr": 2.0026209783237845e-06, "seconds": 0.7277891909980099, "rows_per_second": 2814.001671543924}
+{"loss": 0.20137107372283936, "nce": 0.2008620798587799, "mse": 0.0005089928163215518, "pointer": 0.0, "grad_norm": 0.05775018036365509, "pointer_opportunities": 11428, "pointer_positions": 0, "step": 7617, "epoch": 15.60860655737705, "lr": 1.992263766348946e-06, "seconds": 0.7719776399972034, "rows_per_second": 2652.926579592926}
+{"loss": 0.30760419368743896, "nce": 0.30699729919433594, "mse": 0.000606884656008333, "pointer": 0.0, "grad_norm": 0.06284141540527344, "pointer_opportunities": 15490, "pointer_positions": 0, "step": 7618, "epoch": 15.610655737704917, "lr": 1.9819602394266565e-06, "seconds": 1.0595478960021865, "rows_per_second": 1932.8998790213952}
+{"loss": 0.1324012577533722, "nce": 0.13199496269226074, "mse": 0.0004062999505549669, "pointer": 0.0, "grad_norm": 0.04248014837503433, "pointer_opportunities": 995, "pointer_positions": 0, "step": 7619, "epoch": 15.612704918032787, "lr": 1.9717103994099388e-06, "seconds": 0.25258864699935657, "rows_per_second": 8108.044539330451}
+{"loss": 0.17817704379558563, "nce": 0.1776980459690094, "mse": 0.00047899092896841466, "pointer": 0.0, "grad_norm": 0.05366327241063118, "pointer_opportunities": 11450, "pointer_positions": 0, "step": 7620, "epoch": 15.614754098360656, "lr": 1.9615142481421713e-06, "seconds": 0.6766709209987312, "rows_per_second": 3026.5819565251277}
+{"loss": 0.2963044047355652, "nce": 0.29571107029914856, "mse": 0.0005933406646363437, "pointer": 0.0, "grad_norm": 0.09566429257392883, "pointer_opportunities": 288, "pointer_positions": 0, "step": 7621, "epoch": 15.616803278688524, "lr": 1.9513717874570902e-06, "seconds": 0.14092060800248873, "rows_per_second": 14533.005704629313}
+{"loss": 0.6241148710250854, "nce": 0.6232702136039734, "mse": 0.000844657130073756, "pointer": 0.0, "grad_norm": 0.14992111921310425, "pointer_opportunities": 24, "pointer_positions": 0, "step": 7622, "epoch": 15.618852459016393, "lr": 1.941283019178755e-06, "seconds": 0.09948731899930863, "rows_per_second": 20585.538142948975}
+{"loss": 0.18100230395793915, "nce": 0.18057505786418915, "mse": 0.00042725232196971774, "pointer": 0.0, "grad_norm": 0.07484837621450424, "pointer_opportunities": 7130, "pointer_positions": 0, "step": 7623, "epoch": 15.620901639344263, "lr": 1.9312479451215835e-06, "seconds": 0.44562357399991015, "rows_per_second": 4595.807132951213}
+{"loss": 0.13685792684555054, "nce": 0.1364559829235077, "mse": 0.00040193693712353706, "pointer": 0.0, "grad_norm": 0.044500455260276794, "pointer_opportunities": 1229, "pointer_positions": 0, "step": 7624, "epoch": 15.62295081967213, "lr": 1.9212665670903167e-06, "seconds": 0.30916900600277586, "rows_per_second": 6624.208637464818}
+{"loss": 0.2754371464252472, "nce": 0.2748696208000183, "mse": 0.0005675363354384899, "pointer": 0.0, "grad_norm": 0.06358511000871658, "pointer_opportunities": 12079, "pointer_positions": 0, "step": 7625, "epoch": 15.625, "lr": 1.911338886880052e-06, "seconds": 0.9231866379996063, "rows_per_second": 2218.4029920945122}
+{"loss": 0.25495532155036926, "nce": 0.2543811500072479, "mse": 0.0005741757340729237, "pointer": 0.0, "grad_norm": 0.08551927655935287, "pointer_opportunities": 327, "pointer_positions": 0, "step": 7626, "epoch": 15.62704918032787, "lr": 1.9014649062762787e-06, "seconds": 0.1492203480011085, "rows_per_second": 13724.66977482713}
+{"loss": 0.13668027520179749, "nce": 0.13628441095352173, "mse": 0.0003958697197958827, "pointer": 0.0, "grad_norm": 0.04553031548857689, "pointer_opportunities": 1346, "pointer_positions": 0, "step": 7627, "epoch": 15.629098360655737, "lr": 1.8916446270547092e-06, "seconds": 0.3192750859998341, "rows_per_second": 6414.531198344314}
+{"loss": 0.1395188719034195, "nce": 0.13909892737865448, "mse": 0.0004199441464152187, "pointer": 0.0, "grad_norm": 0.055881086736917496, "pointer_opportunities": 1134, "pointer_positions": 0, "step": 7628, "epoch": 15.631147540983607, "lr": 1.8818780509815464e-06, "seconds": 0.23702774599951226, "rows_per_second": 8640.33867159254}
+{"loss": 0.12544980645179749, "nce": 0.12505173683166504, "mse": 0.0003980649053119123, "pointer": 0.0, "grad_norm": 0.03945508226752281, "pointer_opportunities": 1070, "pointer_positions": 0, "step": 7629, "epoch": 15.633196721311476, "lr": 1.8721651798131843e-06, "seconds": 0.292470107000554, "rows_per_second": 7002.425037565021}
+{"loss": 0.17656637728214264, "nce": 0.17606431245803833, "mse": 0.000502062845043838, "pointer": 0.0, "grad_norm": 0.06869104504585266, "pointer_opportunities": 596, "pointer_positions": 0, "step": 7630, "epoch": 15.635245901639344, "lr": 1.8625060152964735e-06, "seconds": 0.17350397700283793, "rows_per_second": 11803.764013815671}
+{"loss": 0.6008394360542297, "nce": 0.5999913215637207, "mse": 0.0008481404511258006, "pointer": 0.0, "grad_norm": 0.1507216840982437, "pointer_opportunities": 23, "pointer_positions": 0, "step": 7631, "epoch": 15.637295081967213, "lr": 1.852900559168522e-06, "seconds": 0.09911431900036405, "rows_per_second": 20663.008338810032}
+{"loss": 0.12539157271385193, "nce": 0.12498976290225983, "mse": 0.0004018105682916939, "pointer": 0.0, "grad_norm": 0.04040437191724777, "pointer_opportunities": 1023, "pointer_positions": 0, "step": 7632, "epoch": 15.639344262295083, "lr": 1.8433488131568952e-06, "seconds": 0.2672772059995623, "rows_per_second": 7662.456633145716}
+{"loss": 0.1286119967699051, "nce": 0.12821292877197266, "mse": 0.0003990692785009742, "pointer": 0.0, "grad_norm": 0.04206107184290886, "pointer_opportunities": 989, "pointer_positions": 0, "step": 7633, "epoch": 15.64139344262295, "lr": 1.8338507789793822e-06, "seconds": 0.27841427699968335, "rows_per_second": 7355.944609127675}
+{"loss": 0.29933249950408936, "nce": 0.29869529604911804, "mse": 0.0006371890194714069, "pointer": 0.0, "grad_norm": 0.09037122130393982, "pointer_opportunities": 156, "pointer_positions": 0, "step": 7634, "epoch": 15.64344262295082, "lr": 1.824406458344129e-06, "seconds": 0.12653418800255167, "rows_per_second": 16185.349053322256}
+{"loss": 0.12728890776634216, "nce": 0.12688961625099182, "mse": 0.00039928799378685653, "pointer": 0.0, "grad_norm": 0.05304993316531181, "pointer_opportunities": 2681, "pointer_positions": 0, "step": 7635, "epoch": 15.645491803278688, "lr": 1.8150158529496725e-06, "seconds": 0.36753604499972425, "rows_per_second": 5572.242581000556}
+{"loss": 0.1400589793920517, "nce": 0.13966062664985657, "mse": 0.00039835035568103194, "pointer": 0.0, "grad_norm": 0.045810312032699585, "pointer_opportunities": 1727, "pointer_positions": 0, "step": 7636, "epoch": 15.647540983606557, "lr": 1.8056789644849067e-06, "seconds": 0.3364568559991312, "rows_per_second": 6086.961711385927}
+{"loss": 0.11886798590421677, "nce": 0.11846274137496948, "mse": 0.0004052422009408474, "pointer": 0.0, "grad_norm": 0.038495875895023346, "pointer_opportunities": 1008, "pointer_positions": 0, "step": 7637, "epoch": 15.649590163934427, "lr": 1.7963957946289165e-06, "seconds": 0.25120609700024943, "rows_per_second": 8152.668364565875}
+{"loss": 0.12860321998596191, "nce": 0.12820583581924438, "mse": 0.00039738480700179935, "pointer": 0.0, "grad_norm": 0.04364663362503052, "pointer_opportunities": 1176, "pointer_positions": 0, "step": 7638, "epoch": 15.651639344262295, "lr": 1.7871663450513435e-06, "seconds": 0.28547105599864153, "rows_per_second": 7174.107346314458}
+{"loss": 0.19516900181770325, "nce": 0.19472947716712952, "mse": 0.00043951888801530004, "pointer": 0.0, "grad_norm": 0.07668530941009521, "pointer_opportunities": 8754, "pointer_positions": 0, "step": 7639, "epoch": 15.653688524590164, "lr": 1.7779906174119866e-06, "seconds": 0.4856606440007454, "rows_per_second": 4216.936301712883}
+{"loss": 0.13303562998771667, "nce": 0.1326189786195755, "mse": 0.0004166557337157428, "pointer": 0.0, "grad_norm": 0.04720123112201691, "pointer_opportunities": 1044, "pointer_positions": 0, "step": 7640, "epoch": 15.655737704918034, "lr": 1.7688686133610684e-06, "seconds": 0.23579398700167076, "rows_per_second": 8685.548033018707}
+{"loss": 0.6540675759315491, "nce": 0.6532142758369446, "mse": 0.0008532858919352293, "pointer": 0.0, "grad_norm": 0.1597580760717392, "pointer_opportunities": 21, "pointer_positions": 0, "step": 7641, "epoch": 15.657786885245901, "lr": 1.7598003345391693e-06, "seconds": 0.09921381800086237, "rows_per_second": 20642.285936241246}
+{"loss": 0.3448385000228882, "nce": 0.3441719710826874, "mse": 0.0006665244582109153, "pointer": 0.0, "grad_norm": 0.09277510643005371, "pointer_opportunities": 123, "pointer_positions": 0, "step": 7642, "epoch": 15.65983606557377, "lr": 1.750785782577093e-06, "seconds": 0.11767531899749883, "rows_per_second": 17403.819402805526}
+{"loss": 0.13665267825126648, "nce": 0.13625192642211914, "mse": 0.0004007519455626607, "pointer": 0.0, "grad_norm": 0.04387049376964569, "pointer_opportunities": 1387, "pointer_positions": 0, "step": 7643, "epoch": 15.66188524590164, "lr": 1.741824959096167e-06, "seconds": 0.3176739059999818, "rows_per_second": 6446.862525750281}
+{"loss": 0.35040992498397827, "nce": 0.34977036714553833, "mse": 0.0006395551026798785, "pointer": 0.0, "grad_norm": 0.07900499552488327, "pointer_opportunities": 19569, "pointer_positions": 0, "step": 7644, "epoch": 15.663934426229508, "lr": 1.7329178657078437e-06, "seconds": 1.3102617430013197, "rows_per_second": 1563.046475972654}
+{"loss": 0.137155681848526, "nce": 0.13675743341445923, "mse": 0.0003982541966252029, "pointer": 0.0, "grad_norm": 0.04255659133195877, "pointer_opportunities": 1223, "pointer_positions": 0, "step": 7645, "epoch": 15.665983606557377, "lr": 1.7240645040140647e-06, "seconds": 0.30963659600092797, "rows_per_second": 6614.205253676998}
+{"loss": 0.11881590634584427, "nce": 0.11841324716806412, "mse": 0.00040265772258862853, "pointer": 0.0, "grad_norm": 0.03925376385450363, "pointer_opportunities": 1058, "pointer_positions": 0, "step": 7646, "epoch": 15.668032786885245, "lr": 1.715264875607063e-06, "seconds": 0.25955919599800836, "rows_per_second": 7890.300292098742}
+{"loss": 0.3836654722690582, "nce": 0.38300177454948425, "mse": 0.0006636888138018548, "pointer": 0.0, "grad_norm": 0.08642321079969406, "pointer_opportunities": 22109, "pointer_positions": 0, "step": 7647, "epoch": 15.670081967213115, "lr": 1.7065189820693952e-06, "seconds": 1.4040849619996152, "rows_per_second": 1458.601192539915}
+{"loss": 0.12432926893234253, "nce": 0.12392576038837433, "mse": 0.0004035082529298961, "pointer": 0.0, "grad_norm": 0.041356150060892105, "pointer_opportunities": 942, "pointer_positions": 0, "step": 7648, "epoch": 15.672131147540984, "lr": 1.697826824973975e-06, "seconds": 0.25247884599957615, "rows_per_second": 8111.57066205633}
+{"loss": 0.13128045201301575, "nce": 0.13088363409042358, "mse": 0.0003968170494772494, "pointer": 0.0, "grad_norm": 0.04235183075070381, "pointer_opportunities": 1374, "pointer_positions": 0, "step": 7649, "epoch": 15.674180327868852, "lr": 1.6891884058840403e-06, "seconds": 0.3314551260009466, "rows_per_second": 6178.815288541204}
+{"loss": 0.18474029004573822, "nce": 0.18430471420288086, "mse": 0.00043557010940276086, "pointer": 0.0, "grad_norm": 0.07305736094713211, "pointer_opportunities": 8442, "pointer_positions": 0, "step": 7650, "epoch": 15.676229508196721, "lr": 1.6806037263531524e-06, "seconds": 0.48597339400294004, "rows_per_second": 4214.222476524322}
+{"loss": 0.13507555425167084, "nce": 0.1346447467803955, "mse": 0.0004308143979869783, "pointer": 0.0, "grad_norm": 0.0483398400247097, "pointer_opportunities": 996, "pointer_positions": 0, "step": 7651, "epoch": 15.67827868852459, "lr": 1.6720727879252303e-06, "seconds": 0.21793563699975493, "rows_per_second": 9397.269892130138}
+{"loss": 0.18151141703128815, "nce": 0.18108642101287842, "mse": 0.0004249958728905767, "pointer": 0.0, "grad_norm": 0.07552935928106308, "pointer_opportunities": 6958, "pointer_positions": 0, "step": 7652, "epoch": 15.680327868852459, "lr": 1.6635955921345166e-06, "seconds": 0.43910390399832977, "rows_per_second": 4664.044162102895}
+{"loss": 0.12171829491853714, "nce": 0.12131113559007645, "mse": 0.00040716066723689437, "pointer": 0.0, "grad_norm": 0.0422336645424366, "pointer_opportunities": 1017, "pointer_positions": 0, "step": 7653, "epoch": 15.682377049180328, "lr": 1.6551721405056111e-06, "seconds": 0.25163801700182375, "rows_per_second": 8138.6748489007405}
+{"loss": 0.11920800060033798, "nce": 0.11880993843078613, "mse": 0.0003980593755841255, "pointer": 0.0, "grad_norm": 0.04382549226284027, "pointer_opportunities": 968, "pointer_positions": 0, "step": 7654, "epoch": 15.684426229508198, "lr": 1.6468024345534045e-06, "seconds": 0.2675172160015791, "rows_per_second": 7655.582061634162}
+{"loss": 0.14951956272125244, "nce": 0.1491067111492157, "mse": 0.0004128542204853147, "pointer": 0.0, "grad_norm": 0.059952057898044586, "pointer_opportunities": 4798, "pointer_positions": 0, "step": 7655, "epoch": 15.686475409836065, "lr": 1.6384864757831442e-06, "seconds": 0.40468013499776134, "rows_per_second": 5060.787083139946}
+{"loss": 0.28194957971572876, "nce": 0.28136152029037476, "mse": 0.0005880733951926231, "pointer": 0.0, "grad_norm": 0.06591229885816574, "pointer_opportunities": 13541, "pointer_positions": 0, "step": 7656, "epoch": 15.688524590163935, "lr": 1.6302242656904018e-06, "seconds": 0.9915856570005417, "rows_per_second": 2065.378805694929}
+{"loss": 0.274305522441864, "nce": 0.27371615171432495, "mse": 0.0005893739289604127, "pointer": 0.0, "grad_norm": 0.08818327635526657, "pointer_opportunities": 289, "pointer_positions": 0, "step": 7657, "epoch": 15.690573770491802, "lr": 1.622015805761106e-06, "seconds": 0.1415204580007412, "rows_per_second": 14471.405964424406}
+{"loss": 0.14262376725673676, "nce": 0.14221858978271484, "mse": 0.000405176542699337, "pointer": 0.0, "grad_norm": 0.041981909424066544, "pointer_opportunities": 1046, "pointer_positions": 0, "step": 7658, "epoch": 15.692622950819672, "lr": 1.6138610974715095e-06, "seconds": 0.3022185260015249, "rows_per_second": 6776.5534664465495}
+{"loss": 0.26686447858810425, "nce": 0.26629602909088135, "mse": 0.0005684506613761187, "pointer": 0.0, "grad_norm": 0.06220261752605438, "pointer_opportunities": 12142, "pointer_positions": 0, "step": 7659, "epoch": 15.694672131147541, "lr": 1.605760142288189e-06, "seconds": 0.9158364780014381, "rows_per_second": 2236.2070622795004}
+{"loss": 0.19646553695201874, "nce": 0.1960260570049286, "mse": 0.0004394750576466322, "pointer": 0.0, "grad_norm": 0.07443152368068695, "pointer_opportunities": 8970, "pointer_positions": 0, "step": 7660, "epoch": 15.69672131147541, "lr": 1.597712941668078e-06, "seconds": 0.4863616340007866, "rows_per_second": 4210.858457632141}
+{"loss": 0.2022269368171692, "nce": 0.20177164673805237, "mse": 0.0004552945902105421, "pointer": 0.0, "grad_norm": 0.08129077404737473, "pointer_opportunities": 10719, "pointer_positions": 0, "step": 7661, "epoch": 15.698770491803279, "lr": 1.5897194970584007e-06, "seconds": 0.5338964030015632, "rows_per_second": 3835.950174015321}
+{"loss": 0.623353898525238, "nce": 0.6225271224975586, "mse": 0.0008267610101029277, "pointer": 0.0, "grad_norm": 0.1320953220129013, "pointer_opportunities": 39, "pointer_positions": 0, "step": 7662, "epoch": 15.700819672131148, "lr": 1.5817798098967387e-06, "seconds": 0.10586034900188679, "rows_per_second": 19346.242661296135}
+{"loss": 0.28781914710998535, "nce": 0.2872059941291809, "mse": 0.0006131419213488698, "pointer": 0.0, "grad_norm": 0.08757726103067398, "pointer_opportunities": 202, "pointer_positions": 0, "step": 7663, "epoch": 15.702868852459016, "lr": 1.5738938816109975e-06, "seconds": 0.13316080800359487, "rows_per_second": 15379.900668256018}
+{"loss": 0.1838822364807129, "nce": 0.18339113891124725, "mse": 0.0004910958814434707, "pointer": 0.0, "grad_norm": 0.052637554705142975, "pointer_opportunities": 11693, "pointer_positions": 0, "step": 7664, "epoch": 15.704918032786885, "lr": 1.5660617136194395e-06, "seconds": 0.723816051002359, "rows_per_second": 2829.4481687217035}
+{"loss": 0.23554828763008118, "nce": 0.23497962951660156, "mse": 0.0005686579970642924, "pointer": 0.0, "grad_norm": 0.08297751843929291, "pointer_opportunities": 376, "pointer_positions": 0, "step": 7665, "epoch": 15.706967213114755, "lr": 1.5582833073306182e-06, "seconds": 0.1479427679987566, "rows_per_second": 13843.191037342309}
+{"loss": 0.3897095024585724, "nce": 0.3890458941459656, "mse": 0.00066360057098791, "pointer": 0.0, "grad_norm": 0.09067624807357788, "pointer_opportunities": 21710, "pointer_positions": 0, "step": 7666, "epoch": 15.709016393442623, "lr": 1.5505586641434438e-06, "seconds": 1.4044910209995578, "rows_per_second": 1458.17948949397}
+{"loss": 0.3699203431606293, "nce": 0.36924782395362854, "mse": 0.0006725098937749863, "pointer": 0.0, "grad_norm": 0.09483709931373596, "pointer_opportunities": 134, "pointer_positions": 0, "step": 7667, "epoch": 15.711065573770492, "lr": 1.5428877854471505e-06, "seconds": 0.11879466900063562, "rows_per_second": 17239.83085460714}
+{"loss": 0.2153284102678299, "nce": 0.21478211879730225, "mse": 0.0005462916451506317, "pointer": 0.0, "grad_norm": 0.07954410463571548, "pointer_opportunities": 421, "pointer_positions": 0, "step": 7668, "epoch": 15.71311475409836, "lr": 1.5352706726213299e-06, "seconds": 0.1547556479999912, "rows_per_second": 13233.765787986727}
+{"loss": 0.20304961502552032, "nce": 0.20253624022006989, "mse": 0.0005133775412105024, "pointer": 0.0, "grad_norm": 0.05432435870170593, "pointer_opportunities": 11081, "pointer_positions": 0, "step": 7669, "epoch": 15.71516393442623, "lr": 1.527707327035864e-06, "seconds": 0.7714941600024758, "rows_per_second": 2654.589115740588}
+{"loss": 0.24723733961582184, "nce": 0.2466931790113449, "mse": 0.00054416514467448, "pointer": 0.0, "grad_norm": 0.06897082924842834, "pointer_opportunities": 11031, "pointer_positions": 0, "step": 7670, "epoch": 15.717213114754099, "lr": 1.5201977500509587e-06, "seconds": 0.8380465189984534, "rows_per_second": 2443.7784222856244}
+{"loss": 0.28363919258117676, "nce": 0.28305891156196594, "mse": 0.0005802791565656662, "pointer": 0.0, "grad_norm": 0.07141465693712234, "pointer_opportunities": 12545, "pointer_positions": 0, "step": 7671, "epoch": 15.719262295081966, "lr": 1.5127419430172106e-06, "seconds": 0.9526837269986572, "rows_per_second": 2149.7165764046754}
+{"loss": 0.18006183207035065, "nce": 0.17958293855190277, "mse": 0.0004788862424902618, "pointer": 0.0, "grad_norm": 0.05881549417972565, "pointer_opportunities": 11990, "pointer_positions": 0, "step": 7672, "epoch": 15.721311475409836, "lr": 1.5053399072755066e-06, "seconds": 0.6825384110015875, "rows_per_second": 3000.5637294385715}
+{"loss": 0.24078798294067383, "nce": 0.24022209644317627, "mse": 0.0005658901063725352, "pointer": 0.0, "grad_norm": 0.08466272801160812, "pointer_opportunities": 323, "pointer_positions": 0, "step": 7673, "epoch": 15.723360655737705, "lr": 1.4979916441570247e-06, "seconds": 0.14772865800114232, "rows_per_second": 13863.254616339666}
+{"loss": 0.23822256922721863, "nce": 0.23765990138053894, "mse": 0.0005626749480143189, "pointer": 0.0, "grad_norm": 0.0813755914568901, "pointer_opportunities": 344, "pointer_positions": 0, "step": 7674, "epoch": 15.725409836065573, "lr": 1.4906971549833329e-06, "seconds": 0.14771911899879342, "rows_per_second": 13864.149839782947}
+{"loss": 0.30082520842552185, "nce": 0.3002270460128784, "mse": 0.0005981642752885818, "pointer": 0.0, "grad_norm": 0.07514165341854095, "pointer_opportunities": 14561, "pointer_positions": 0, "step": 7675, "epoch": 15.727459016393443, "lr": 1.483456441066323e-06, "seconds": 1.0255012360030378, "rows_per_second": 1997.0721907486159}
+{"loss": 0.12399915605783463, "nce": 0.12360251694917679, "mse": 0.0003966354706790298, "pointer": 0.0, "grad_norm": 0.042214229702949524, "pointer_opportunities": 1090, "pointer_positions": 0, "step": 7676, "epoch": 15.729508196721312, "lr": 1.4762695037081782e-06, "seconds": 0.28620008600046276, "rows_per_second": 7155.832930101526}
+{"loss": 0.17096981406211853, "nce": 0.17049598693847656, "mse": 0.00047382558113895357, "pointer": 0.0, "grad_norm": 0.049338243901729584, "pointer_opportunities": 11116, "pointer_positions": 0, "step": 7677, "epoch": 15.73155737704918, "lr": 1.4691363442014377e-06, "seconds": 0.6764469219997409, "rows_per_second": 3027.584180508378}
+{"loss": 0.1307135373353958, "nce": 0.13031426072120667, "mse": 0.0003992724814452231, "pointer": 0.0, "grad_norm": 0.04095999896526337, "pointer_opportunities": 1061, "pointer_positions": 0, "step": 7678, "epoch": 15.73360655737705, "lr": 1.462056963828966e-06, "seconds": 0.2851484659986454, "rows_per_second": 7182.223452710873}
+{"loss": 0.13058577477931976, "nce": 0.13018685579299927, "mse": 0.00039891264168545604, "pointer": 0.0, "grad_norm": 0.04325690492987633, "pointer_opportunities": 1199, "pointer_positions": 0, "step": 7679, "epoch": 15.735655737704917, "lr": 1.45503136386395e-06, "seconds": 0.30866257600064273, "rows_per_second": 6635.077133535409}
+{"loss": 0.12345430254936218, "nce": 0.1230543702840805, "mse": 0.00039993226528167725, "pointer": 0.0, "grad_norm": 0.04029344767332077, "pointer_opportunities": 907, "pointer_positions": 0, "step": 7680, "epoch": 15.737704918032787, "lr": 1.4480595455699018e-06, "seconds": 0.2668485959984537, "rows_per_second": 7674.764007421901}
+{"loss": 0.18338125944137573, "nce": 0.18287551403045654, "mse": 0.0005057445960119367, "pointer": 0.0, "grad_norm": 0.0715089961886406, "pointer_opportunities": 705, "pointer_positions": 0, "step": 7681, "epoch": 15.739754098360656, "lr": 1.4411415102006567e-06, "seconds": 0.17331573800038313, "rows_per_second": 11816.5841349934}
+{"loss": 0.14522936940193176, "nce": 0.1447688341140747, "mse": 0.0004605379363056272, "pointer": 0.0, "grad_norm": 0.05662015080451965, "pointer_opportunities": 980, "pointer_positions": 0, "step": 7682, "epoch": 15.741803278688524, "lr": 1.4342772590004074e-06, "seconds": 0.1984286869992502, "rows_per_second": 10321.088301147398}
+{"loss": 0.29233676195144653, "nce": 0.2917245328426361, "mse": 0.0006122277700342238, "pointer": 0.0, "grad_norm": 0.08704258501529694, "pointer_opportunities": 215, "pointer_positions": 0, "step": 7683, "epoch": 15.743852459016393, "lr": 1.4274667932036704e-06, "seconds": 0.28681089699966833, "rows_per_second": 7140.593406401739}
+{"loss": 0.12479986250400543, "nce": 0.12439845502376556, "mse": 0.0004014111473225057, "pointer": 0.0, "grad_norm": 0.05066711828112602, "pointer_opportunities": 3272, "pointer_positions": 0, "step": 7684, "epoch": 15.745901639344263, "lr": 1.4207101140352195e-06, "seconds": 0.37982708499839646, "rows_per_second": 5391.927223959414}
+{"loss": 0.37014177441596985, "nce": 0.36949604749679565, "mse": 0.0006457280251197517, "pointer": 0.0, "grad_norm": 0.0862058475613594, "pointer_opportunities": 19227, "pointer_positions": 0, "step": 7685, "epoch": 15.74795081967213, "lr": 1.414007222710286e-06, "seconds": 1.3108388529981312, "rows_per_second": 1562.3583290317072}
+{"loss": 0.1353497952222824, "nce": 0.13495177030563354, "mse": 0.00039801927050575614, "pointer": 0.0, "grad_norm": 0.04453054443001747, "pointer_opportunities": 1260, "pointer_positions": 0, "step": 7686, "epoch": 15.75, "lr": 1.4073581204342579e-06, "seconds": 0.31815745599669754, "rows_per_second": 6437.06429442049}
+{"loss": 0.13869912922382355, "nce": 0.1383005678653717, "mse": 0.000398555479478091, "pointer": 0.0, "grad_norm": 0.04237940162420273, "pointer_opportunities": 1316, "pointer_positions": 0, "step": 7687, "epoch": 15.75204918032787, "lr": 1.4007628084030142e-06, "seconds": 0.3179955859995971, "rows_per_second": 6440.340967508256}
+{"loss": 0.1300601363182068, "nce": 0.1296597421169281, "mse": 0.0004003961803391576, "pointer": 0.0, "grad_norm": 0.039765615016222, "pointer_opportunities": 1009, "pointer_positions": 0, "step": 7688, "epoch": 15.754098360655737, "lr": 1.3942212878026244e-06, "seconds": 0.28535565600031987, "rows_per_second": 7177.008609907155}
+{"loss": 0.17153538763523102, "nce": 0.1710493266582489, "mse": 0.00048606330528855324, "pointer": 0.0, "grad_norm": 0.06431035697460175, "pointer_opportunities": 788, "pointer_positions": 0, "step": 7689, "epoch": 15.756147540983607, "lr": 1.387733559809615e-06, "seconds": 0.18203509799786843, "rows_per_second": 11250.577622256018}
+{"loss": 0.2226525843143463, "nce": 0.22212469577789307, "mse": 0.000527887255884707, "pointer": 0.0, "grad_norm": 0.06013995781540871, "pointer_opportunities": 11172, "pointer_positions": 0, "step": 7690, "epoch": 15.758196721311476, "lr": 1.3812996255907032e-06, "seconds": 0.8096012189998874, "rows_per_second": 2529.6404599414086}
+{"loss": 0.13373373448848724, "nce": 0.13333550095558167, "mse": 0.0003982388589065522, "pointer": 0.0, "grad_norm": 0.04293748363852501, "pointer_opportunities": 1352, "pointer_positions": 0, "step": 7691, "epoch": 15.760245901639344, "lr": 1.3749194863030636e-06, "seconds": 0.3177005959987582, "rows_per_second": 6446.32092540363}
+{"loss": 0.13056015968322754, "nce": 0.1301632821559906, "mse": 0.0003968712408095598, "pointer": 0.0, "grad_norm": 0.04517826810479164, "pointer_opportunities": 1698, "pointer_positions": 0, "step": 7692, "epoch": 15.762295081967213, "lr": 1.3685931430940612e-06, "seconds": 0.3361081959992589, "rows_per_second": 6093.275987844449}
+{"loss": 0.24764898419380188, "nce": 0.24710491299629211, "mse": 0.000544064911082387, "pointer": 0.0, "grad_norm": 0.06499962508678436, "pointer_opportunities": 11111, "pointer_positions": 0, "step": 7693, "epoch": 15.764344262295083, "lr": 1.3623205971015183e-06, "seconds": 0.8503856389979774, "rows_per_second": 2408.319127323446}
+{"loss": 0.1602785736322403, "nce": 0.15979942679405212, "mse": 0.00047914739116095006, "pointer": 0.0, "grad_norm": 0.057673003524541855, "pointer_opportunities": 759, "pointer_positions": 0, "step": 7694, "epoch": 15.76639344262295, "lr": 1.3561018494535142e-06, "seconds": 0.18974941699707415, "rows_per_second": 10793.182041932594}
+{"loss": 0.13414528965950012, "nce": 0.13374321162700653, "mse": 0.0004020781198050827, "pointer": 0.0, "grad_norm": 0.041167180985212326, "pointer_opportunities": 1111, "pointer_positions": 0, "step": 7695, "epoch": 15.76844262295082, "lr": 1.349936901268386e-06, "seconds": 0.2926508370001102, "rows_per_second": 6998.100606830757}
+{"loss": 0.24547937512397766, "nce": 0.24493372440338135, "mse": 0.0005456500221043825, "pointer": 0.0, "grad_norm": 0.062247134745121, "pointer_opportunities": 11042, "pointer_positions": 0, "step": 7696, "epoch": 15.770491803278688, "lr": 1.3438257536549605e-06, "seconds": 0.8373149489998468, "rows_per_second": 2445.9135746307747}
+{"loss": 0.13132308423519135, "nce": 0.13092611730098724, "mse": 0.0003969737736042589, "pointer": 0.0, "grad_norm": 0.04389408975839615, "pointer_opportunities": 1434, "pointer_positions": 0, "step": 7697, "epoch": 15.772540983606557, "lr": 1.337768407712256e-06, "seconds": 0.31768671499958145, "rows_per_second": 6446.602590865968}
+{"loss": 0.13274824619293213, "nce": 0.13234704732894897, "mse": 0.0004011941491626203, "pointer": 0.0, "grad_norm": 0.04341912269592285, "pointer_opportunities": 1086, "pointer_positions": 0, "step": 7698, "epoch": 15.774590163934427, "lr": 1.3317648645296143e-06, "seconds": 0.26740733699989505, "rows_per_second": 7658.727778291303}
+{"loss": 0.28490057587623596, "nce": 0.2842608094215393, "mse": 0.0006397629040293396, "pointer": 0.0, "grad_norm": 0.09141883254051208, "pointer_opportunities": 174, "pointer_positions": 0, "step": 7699, "epoch": 15.776639344262295, "lr": 1.325815125186834e-06, "seconds": 0.1261839679973491, "rows_per_second": 16230.271028115274}
+{"loss": 0.11884533613920212, "nce": 0.11844981461763382, "mse": 0.00039552320959046483, "pointer": 0.0, "grad_norm": 0.04371274262666702, "pointer_opportunities": 2627, "pointer_positions": 0, "step": 7700, "epoch": 15.778688524590164, "lr": 1.3199191907538388e-06, "seconds": 0.3672485349998169, "rows_per_second": 5576.604955009612}
+{"loss": 0.23209264874458313, "nce": 0.23155169188976288, "mse": 0.0005409545265138149, "pointer": 0.0, "grad_norm": 0.057948049157857895, "pointer_opportunities": 11162, "pointer_positions": 0, "step": 7701, "epoch": 15.780737704918034, "lr": 1.3140770622910749e-06, "seconds": 0.8394668190012453, "rows_per_second": 2439.643775839295}
+{"loss": 0.12679514288902283, "nce": 0.12639766931533813, "mse": 0.00039746807306073606, "pointer": 0.0, "grad_norm": 0.040097955614328384, "pointer_opportunities": 1039, "pointer_positions": 0, "step": 7702, "epoch": 15.782786885245901, "lr": 1.3082887408491466e-06, "seconds": 0.29436976600118214, "rows_per_second": 6957.23622646687}
+{"loss": 0.15288016200065613, "nce": 0.15243329107761383, "mse": 0.0004468640254344791, "pointer": 0.0, "grad_norm": 0.05504298955202103, "pointer_opportunities": 947, "pointer_positions": 0, "step": 7703, "epoch": 15.78483606557377, "lr": 1.3025542274690819e-06, "seconds": 0.20867052700123168, "rows_per_second": 9814.514917039107}
+{"loss": 0.3367835283279419, "nce": 0.3361629545688629, "mse": 0.0006205693352967501, "pointer": 0.0, "grad_norm": 0.07167710363864899, "pointer_opportunities": 16307, "pointer_positions": 0, "step": 7704, "epoch": 15.78688524590164, "lr": 1.2968735231822328e-06, "seconds": 1.1068686559992784, "rows_per_second": 1850.2646984353084}
+{"loss": 0.22021904587745667, "nce": 0.21967247128486633, "mse": 0.000546575989574194, "pointer": 0.0, "grad_norm": 0.08143633604049683, "pointer_opportunities": 444, "pointer_positions": 0, "step": 7705, "epoch": 15.788934426229508, "lr": 1.291246629010175e-06, "seconds": 0.1554829169981531, "rows_per_second": 13171.865048199008}
+{"loss": 0.12610729038715363, "nce": 0.1257118582725525, "mse": 0.0003954287967644632, "pointer": 0.0, "grad_norm": 0.04029783979058266, "pointer_opportunities": 1632, "pointer_positions": 0, "step": 7706, "epoch": 15.790983606557377, "lr": 1.2856735459649415e-06, "seconds": 0.33209613600047305, "rows_per_second": 6166.888975778636}
+{"loss": 0.1246342882514, "nce": 0.12423606216907501, "mse": 0.00039822838152758777, "pointer": 0.0, "grad_norm": 0.04671815037727356, "pointer_opportunities": 2684, "pointer_positions": 0, "step": 7707, "epoch": 15.793032786885245, "lr": 1.2801542750488229e-06, "seconds": 0.36785552600122173, "rows_per_second": 5567.403111386719}
+{"loss": 0.22092539072036743, "nce": 0.22039850056171417, "mse": 0.000526895746588707, "pointer": 0.0, "grad_norm": 0.05699460208415985, "pointer_opportunities": 11075, "pointer_positions": 0, "step": 7708, "epoch": 15.795081967213115, "lr": 1.2746888172543664e-06, "seconds": 0.8101303799994639, "rows_per_second": 2527.988149267227}
+{"loss": 0.12763391435146332, "nce": 0.1272258460521698, "mse": 0.00040806340985000134, "pointer": 0.0, "grad_norm": 0.04338572919368744, "pointer_opportunities": 973, "pointer_positions": 0, "step": 7709, "epoch": 15.797131147540984, "lr": 1.2692771735645436e-06, "seconds": 0.2524879560005502, "rows_per_second": 8111.277989020345}
+{"loss": 0.17166514694690704, "nce": 0.17116236686706543, "mse": 0.0005027833394706249, "pointer": 0.0, "grad_norm": 0.06828565150499344, "pointer_opportunities": 644, "pointer_positions": 0, "step": 7710, "epoch": 15.799180327868852, "lr": 1.2639193449526166e-06, "seconds": 0.17359922800096683, "rows_per_second": 11797.287485567585}
+{"loss": 0.18441081047058105, "nce": 0.1839476078748703, "mse": 0.0004632007039617747, "pointer": 0.0, "grad_norm": 0.06408170610666275, "pointer_opportunities": 11968, "pointer_positions": 0, "step": 7711, "epoch": 15.801229508196721, "lr": 1.2586153323821714e-06, "seconds": 0.6285526419997041, "rows_per_second": 3258.2792007434823}
+{"loss": 0.1863166242837906, "nce": 0.18585540354251862, "mse": 0.0004612251650542021, "pointer": 0.0, "grad_norm": 0.06556370854377747, "pointer_opportunities": 11659, "pointer_positions": 0, "step": 7712, "epoch": 15.80327868852459, "lr": 1.2533651368070844e-06, "seconds": 0.6287460819985426, "rows_per_second": 3257.2767586721075}
+{"loss": 0.20006076991558075, "nce": 0.1995510458946228, "mse": 0.0005097297253087163, "pointer": 0.0, "grad_norm": 0.05727838724851608, "pointer_opportunities": 11134, "pointer_positions": 0, "step": 7713, "epoch": 15.805327868852459, "lr": 1.2481687591715894e-06, "seconds": 0.7610520700000052, "rows_per_second": 2691.0116675722147}
+{"loss": 0.1286018341779709, "nce": 0.1282048225402832, "mse": 0.00039700831985101104, "pointer": 0.0, "grad_norm": 0.04204298183321953, "pointer_opportunities": 937, "pointer_positions": 0, "step": 7714, "epoch": 15.807377049180328, "lr": 1.243026200410211e-06, "seconds": 0.2862303459987743, "rows_per_second": 7155.076422290913}
+{"loss": 0.13151878118515015, "nce": 0.1311139166355133, "mse": 0.0004048636765219271, "pointer": 0.0, "grad_norm": 0.04210983216762543, "pointer_opportunities": 882, "pointer_positions": 0, "step": 7715, "epoch": 15.809426229508198, "lr": 1.2379374614478304e-06, "seconds": 0.2594808470021235, "rows_per_second": 7892.6827303875725}
+{"loss": 0.367186039686203, "nce": 0.366527795791626, "mse": 0.0006582545465789735, "pointer": 0.0, "grad_norm": 0.07991835474967957, "pointer_opportunities": 22329, "pointer_positions": 0, "step": 7716, "epoch": 15.811475409836065, "lr": 1.2329025431996534e-06, "seconds": 1.4065431709968834, "rows_per_second": 1456.052001979069}
+{"loss": 0.2928682565689087, "nce": 0.2922303080558777, "mse": 0.0006379581755027175, "pointer": 0.0, "grad_norm": 0.09062029421329498, "pointer_opportunities": 171, "pointer_positions": 0, "step": 7717, "epoch": 15.813524590163935, "lr": 1.2279214465711094e-06, "seconds": 0.12772506900000735, "rows_per_second": 16034.440349371682}
+{"loss": 0.12978129088878632, "nce": 0.12936285138130188, "mse": 0.00041843854705803096, "pointer": 0.0, "grad_norm": 0.04807361215353012, "pointer_opportunities": 1156, "pointer_positions": 0, "step": 7718, "epoch": 15.815573770491802, "lr": 1.2229941724580852e-06, "seconds": 0.23633993700059364, "rows_per_second": 8665.48424270273}
+{"loss": 0.12739865481853485, "nce": 0.12700314819812775, "mse": 0.0003955130814574659, "pointer": 0.0, "grad_norm": 0.04321315512061119, "pointer_opportunities": 1955, "pointer_positions": 0, "step": 7719, "epoch": 15.817622950819672, "lr": 1.2181207217467243e-06, "seconds": 0.3441076349990908, "rows_per_second": 5951.626153268617}
+{"loss": 0.2002454251050949, "nce": 0.19973452389240265, "mse": 0.0005109035409986973, "pointer": 0.0, "grad_norm": 0.05781405046582222, "pointer_opportunities": 11217, "pointer_positions": 0, "step": 7720, "epoch": 15.819672131147541, "lr": 1.2133010953134284e-06, "seconds": 0.7719207999980426, "rows_per_second": 2653.1219265048862}
+{"loss": 0.13213521242141724, "nce": 0.1317378580570221, "mse": 0.0003973604179918766, "pointer": 0.0, "grad_norm": 0.040967196226119995, "pointer_opportunities": 1170, "pointer_positions": 0, "step": 7721, "epoch": 15.82172131147541, "lr": 1.2085352940250555e-06, "seconds": 0.30250061599872424, "rows_per_second": 6770.234147254223}
+{"loss": 0.1336226463317871, "nce": 0.13322478532791138, "mse": 0.0003978583845309913, "pointer": 0.0, "grad_norm": 0.040002282708883286, "pointer_opportunities": 1533, "pointer_positions": 0, "step": 7722, "epoch": 15.823770491803279, "lr": 1.2038233187386882e-06, "seconds": 0.33201885600283276, "rows_per_second": 6168.324367645332}
+{"loss": 0.13132590055465698, "nce": 0.13091859221458435, "mse": 0.00040731270564720035, "pointer": 0.0, "grad_norm": 0.04538511112332344, "pointer_opportunities": 960, "pointer_positions": 0, "step": 7723, "epoch": 15.825819672131148, "lr": 1.1991651703017328e-06, "seconds": 0.25180913700023666, "rows_per_second": 8133.144112233208}
+{"loss": 0.258675754070282, "nce": 0.2581058442592621, "mse": 0.0005699102184735239, "pointer": 0.0, "grad_norm": 0.08551893383264542, "pointer_opportunities": 343, "pointer_positions": 0, "step": 7724, "epoch": 15.827868852459016, "lr": 1.1945608495519526e-06, "seconds": 0.14788782899995567, "rows_per_second": 13848.333658347326}
+{"loss": 0.17343705892562866, "nce": 0.1730107218027115, "mse": 0.00042633022530935705, "pointer": 0.0, "grad_norm": 0.07471577078104019, "pointer_opportunities": 7057, "pointer_positions": 0, "step": 7725, "epoch": 15.829918032786885, "lr": 1.1900103573174021e-06, "seconds": 0.4475032439986535, "rows_per_second": 4576.5031370502475}
+{"loss": 0.12929223477840424, "nce": 0.12889516353607178, "mse": 0.0003970736579503864, "pointer": 0.0, "grad_norm": 0.04243413731455803, "pointer_opportunities": 1230, "pointer_positions": 0, "step": 7726, "epoch": 15.831967213114755, "lr": 1.185513694416459e-06, "seconds": 0.3112379160011187, "rows_per_second": 6580.1751480453895}
+{"loss": 0.334103524684906, "nce": 0.33347639441490173, "mse": 0.0006271329475566745, "pointer": 0.0, "grad_norm": 0.07789088785648346, "pointer_opportunities": 17803, "pointer_positions": 0, "step": 7727, "epoch": 15.834016393442623, "lr": 1.1810708616578253e-06, "seconds": 1.2598564930012799, "rows_per_second": 1625.5819701505634}
+{"loss": 0.13111422955989838, "nce": 0.13071784377098083, "mse": 0.0003963792696595192, "pointer": 0.0, "grad_norm": 0.04384293779730797, "pointer_opportunities": 1490, "pointer_positions": 0, "step": 7728, "epoch": 15.836065573770492, "lr": 1.1766818598405602e-06, "seconds": 0.33163775600041845, "rows_per_second": 6175.412669230025}
+{"loss": 0.3218655586242676, "nce": 0.32124418020248413, "mse": 0.0006213683518581092, "pointer": 0.0, "grad_norm": 0.06591250002384186, "pointer_opportunities": 16439, "pointer_positions": 0, "step": 7729, "epoch": 15.83811475409836, "lr": 1.1723466897539468e-06, "seconds": 1.1061483750017942, "rows_per_second": 1851.4695191742953}
+{"loss": 0.23532171547412872, "nce": 0.2347557544708252, "mse": 0.0005659550661221147, "pointer": 0.0, "grad_norm": 0.07953190058469772, "pointer_opportunities": 336, "pointer_positions": 0, "step": 7730, "epoch": 15.84016393442623, "lr": 1.1680653521776588e-06, "seconds": 0.1483649180008797, "rows_per_second": 13803.802324669885}
+{"loss": 0.179772287607193, "nce": 0.17927852272987366, "mse": 0.000493767554871738, "pointer": 0.0, "grad_norm": 0.04927651956677437, "pointer_opportunities": 11725, "pointer_positions": 0, "step": 7731, "epoch": 15.842213114754099, "lr": 1.1638378478816604e-06, "seconds": 0.7234693710015563, "rows_per_second": 2830.8040147778343}
+{"loss": 0.27833059430122375, "nce": 0.2777353525161743, "mse": 0.0005952424253337085, "pointer": 0.0, "grad_norm": 0.09023372083902359, "pointer_opportunities": 275, "pointer_positions": 0, "step": 7732, "epoch": 15.844262295081966, "lr": 1.1596641776262734e-06, "seconds": 0.1406148280002526, "rows_per_second": 14564.60907519882}
+{"loss": 0.12316426634788513, "nce": 0.12276404350996017, "mse": 0.00040022502071224153, "pointer": 0.0, "grad_norm": 0.04569743573665619, "pointer_opportunities": 2581, "pointer_positions": 0, "step": 7733, "epoch": 15.846311475409836, "lr": 1.1555443421621097e-06, "seconds": 0.36740648500199313, "rows_per_second": 5574.207542876903}
+{"loss": 0.4850490689277649, "nce": 0.48432469367980957, "mse": 0.0007243851432576776, "pointer": 0.0, "grad_norm": 0.1464843899011612, "pointer_opportunities": 43157, "pointer_positions": 0, "step": 7734, "epoch": 15.848360655737705, "lr": 1.1514783422300723e-06, "seconds": 2.383510749001289, "rows_per_second": 859.2367375973149}
+{"loss": 0.22143489122390747, "nce": 0.22090838849544525, "mse": 0.0005265056388452649, "pointer": 0.0, "grad_norm": 0.055163536220788956, "pointer_opportunities": 11010, "pointer_positions": 0, "step": 7735, "epoch": 15.850409836065573, "lr": 1.147466178561454e-06, "seconds": 0.8049125900033687, "rows_per_second": 2544.375656978392}
+{"loss": 0.18732039630413055, "nce": 0.18687087297439575, "mse": 0.000449526181910187, "pointer": 0.0, "grad_norm": 0.06708476692438126, "pointer_opportunities": 10415, "pointer_positions": 0, "step": 7736, "epoch": 15.852459016393443, "lr": 1.1435078518778056e-06, "seconds": 0.5339619430014864, "rows_per_second": 3835.4793386357483}
+{"loss": 0.12915721535682678, "nce": 0.1287548691034317, "mse": 0.00040234398329630494, "pointer": 0.0, "grad_norm": 0.04236217588186264, "pointer_opportunities": 1082, "pointer_positions": 0, "step": 7737, "epoch": 15.854508196721312, "lr": 1.1396033628909683e-06, "seconds": 0.2679774459975306, "rows_per_second": 7642.4342070148405}
+{"loss": 0.13284964859485626, "nce": 0.13245072960853577, "mse": 0.00039891432970762253, "pointer": 0.0, "grad_norm": 0.040976785123348236, "pointer_opportunities": 1105, "pointer_positions": 0, "step": 7738, "epoch": 15.85655737704918, "lr": 1.135752712303174e-06, "seconds": 0.2924234259990044, "rows_per_second": 7003.542869397108}
+{"loss": 0.4935918152332306, "nce": 0.4928687810897827, "mse": 0.0007230259361676872, "pointer": 0.0, "grad_norm": 0.15802878141403198, "pointer_opportunities": 43451, "pointer_positions": 0, "step": 7739, "epoch": 15.85860655737705, "lr": 1.131955900806978e-06, "seconds": 2.382894148999185, "rows_per_second": 859.4590745292483}
+{"loss": 0.12488467991352081, "nce": 0.12448600679636002, "mse": 0.000398670497816056, "pointer": 0.0, "grad_norm": 0.046492405235767365, "pointer_opportunities": 2530, "pointer_positions": 0, "step": 7740, "epoch": 15.860655737704917, "lr": 1.1282129290851601e-06, "seconds": 0.36862998500146205, "rows_per_second": 5555.706489779656}
+{"loss": 0.4300321936607361, "nce": 0.4293472170829773, "mse": 0.0006849708734080195, "pointer": 0.0, "grad_norm": 0.10295744240283966, "pointer_opportunities": 26713, "pointer_positions": 0, "step": 7741, "epoch": 15.862704918032787, "lr": 1.1245237978109237e-06, "seconds": 1.579228189002606, "rows_per_second": 1296.8360204445542}
+{"loss": 0.32196173071861267, "nce": 0.3213443160057068, "mse": 0.0006174194859340787, "pointer": 0.0, "grad_norm": 0.07177970558404922, "pointer_opportunities": 16486, "pointer_positions": 0, "step": 7742, "epoch": 15.864754098360656, "lr": 1.1208885076476965e-06, "seconds": 1.106786666001426, "rows_per_second": 1850.4017647763758}
+{"loss": 0.2160622775554657, "nce": 0.21551506221294403, "mse": 0.0005472154007293284, "pointer": 0.0, "grad_norm": 0.07949677109718323, "pointer_opportunities": 440, "pointer_positions": 0, "step": 7743, "epoch": 15.866803278688524, "lr": 1.117307059249263e-06, "seconds": 0.15537814799972693, "rows_per_second": 13180.746625990158}
+{"loss": 0.19074712693691254, "nce": 0.19022643566131592, "mse": 0.0005206938367336988, "pointer": 0.0, "grad_norm": 0.07283257693052292, "pointer_opportunities": 553, "pointer_positions": 0, "step": 7744, "epoch": 15.868852459016393, "lr": 1.1137794532597648e-06, "seconds": 0.163828836997709, "rows_per_second": 12500.85172751754}
+{"loss": 0.20590740442276, "nce": 0.20536699891090393, "mse": 0.0005404122057370842, "pointer": 0.0, "grad_norm": 0.07472312450408936, "pointer_opportunities": 479, "pointer_positions": 0, "step": 7745, "epoch": 15.870901639344263, "lr": 1.110305690313635e-06, "seconds": 0.16325521799808485, "rows_per_second": 12544.775138666779}
+{"loss": 0.14300037920475006, "nce": 0.14260077476501465, "mse": 0.00039959800778888166, "pointer": 0.0, "grad_norm": 0.04794936627149582, "pointer_opportunities": 1493, "pointer_positions": 0, "step": 7746, "epoch": 15.87295081967213, "lr": 1.106885771035563e-06, "seconds": 0.33146493599997484, "rows_per_second": 6178.632421017695}
+{"loss": 0.1264977902173996, "nce": 0.12609454989433289, "mse": 0.0004032388678751886, "pointer": 0.0, "grad_norm": 0.044592853635549545, "pointer_opportunities": 1051, "pointer_positions": 0, "step": 7747, "epoch": 15.875, "lr": 1.1035196960406294e-06, "seconds": 0.25149613699977635, "rows_per_second": 8143.26623236293}
+{"loss": 0.28228577971458435, "nce": 0.28169912099838257, "mse": 0.0005866453284397721, "pointer": 0.0, "grad_norm": 0.060342635959386826, "pointer_opportunities": 13605, "pointer_positions": 0, "step": 7748, "epoch": 15.87704918032787, "lr": 1.1002074659341718e-06, "seconds": 0.9911111670007813, "rows_per_second": 2066.3675964801087}
+{"loss": 0.2726795971393585, "nce": 0.27210164070129395, "mse": 0.0005779517814517021, "pointer": 0.0, "grad_norm": 0.06553326547145844, "pointer_opportunities": 12833, "pointer_positions": 0, "step": 7749, "epoch": 15.879098360655737, "lr": 1.0969490813119517e-06, "seconds": 0.9529363769979682, "rows_per_second": 2149.146626610904}
+{"loss": 0.7864700555801392, "nce": 0.7856677770614624, "mse": 0.0008023071568459272, "pointer": 0.0, "grad_norm": 0.11541283130645752, "pointer_opportunities": 38, "pointer_positions": 0, "step": 7750, "epoch": 15.881147540983607, "lr": 1.093744542759888e-06, "seconds": 0.10635058900152217, "rows_per_second": 19257.06307062096}
+{"loss": 0.1280253827571869, "nce": 0.12763197720050812, "mse": 0.00039341067895293236, "pointer": 0.0, "grad_norm": 0.03957828879356384, "pointer_opportunities": 1672, "pointer_positions": 0, "step": 7751, "epoch": 15.883196721311476, "lr": 1.0905938508543567e-06, "seconds": 0.3380104450006911, "rows_per_second": 6058.984360663211}
+{"loss": 0.14197051525115967, "nce": 0.14156818389892578, "mse": 0.0004023268702439964, "pointer": 0.0, "grad_norm": 0.04543597996234894, "pointer_opportunities": 1290, "pointer_positions": 0, "step": 7752, "epoch": 15.885245901639344, "lr": 1.0874970061619911e-06, "seconds": 0.30964187599965953, "rows_per_second": 6614.0924685595555}
+{"loss": 0.2406207174062729, "nce": 0.24005119502544403, "mse": 0.0005695234285667539, "pointer": 0.0, "grad_norm": 0.08229630440473557, "pointer_opportunities": 415, "pointer_positions": 0, "step": 7753, "epoch": 15.887295081967213, "lr": 1.0844540092397153e-06, "seconds": 0.14807577799729188, "rows_per_second": 13830.756303961174}
+{"loss": 0.12970778346061707, "nce": 0.12931233644485474, "mse": 0.00039545027539134026, "pointer": 0.0, "grad_norm": 0.043097663670778275, "pointer_opportunities": 2174, "pointer_positions": 0, "step": 7754, "epoch": 15.889344262295083, "lr": 1.0814648606348105e-06, "seconds": 0.3524870259971067, "rows_per_second": 5810.142924286838}
+{"loss": 0.7448334693908691, "nce": 0.7440341711044312, "mse": 0.0007992739556357265, "pointer": 0.0, "grad_norm": 0.1111343652009964, "pointer_opportunities": 48, "pointer_positions": 0, "step": 7755, "epoch": 15.89139344262295, "lr": 1.0785295608848483e-06, "seconds": 0.10583233800207381, "rows_per_second": 19351.36309621988}
+{"loss": 0.12967292964458466, "nce": 0.12927797436714172, "mse": 0.0003949585952796042, "pointer": 0.0, "grad_norm": 0.0397830605506897, "pointer_opportunities": 1656, "pointer_positions": 0, "step": 7756, "epoch": 15.89344262295082, "lr": 1.0756481105177247e-06, "seconds": 0.33645895600056974, "rows_per_second": 6086.923719743492}
+{"loss": 0.123875193297863, "nce": 0.12347391992807388, "mse": 0.00040127034299075603, "pointer": 0.0, "grad_norm": 0.04152970016002655, "pointer_opportunities": 966, "pointer_positions": 0, "step": 7757, "epoch": 15.895491803278688, "lr": 1.0728205100516925e-06, "seconds": 0.2591492869978538, "rows_per_second": 7902.780762877272}
+{"loss": 0.290649950504303, "nce": 0.29003357887268066, "mse": 0.000616370583884418, "pointer": 0.0, "grad_norm": 0.08904960006475449, "pointer_opportunities": 195, "pointer_positions": 0, "step": 7758, "epoch": 15.897540983606557, "lr": 1.0700467599952288e-06, "seconds": 0.13362916800178937, "rows_per_second": 15325.995294474753}
+{"loss": 0.14162760972976685, "nce": 0.1411939561367035, "mse": 0.00043364858720451593, "pointer": 0.0, "grad_norm": 0.05342375859618187, "pointer_opportunities": 1083, "pointer_positions": 0, "step": 7759, "epoch": 15.899590163934427, "lr": 1.0673268608472008e-06, "seconds": 0.21705277699948056, "rows_per_second": 9435.493193459124}
+{"loss": 0.2841365933418274, "nce": 0.28355658054351807, "mse": 0.0005800126236863434, "pointer": 0.0, "grad_norm": 0.07017670571804047, "pointer_opportunities": 12777, "pointer_positions": 0, "step": 7760, "epoch": 15.901639344262295, "lr": 1.0646608130968e-06, "seconds": 0.9527409280017309, "rows_per_second": 2149.587510946396}
+{"loss": 0.13363228738307953, "nce": 0.1332351565361023, "mse": 0.00039712549187242985, "pointer": 0.0, "grad_norm": 0.043517906218767166, "pointer_opportunities": 1329, "pointer_positions": 0, "step": 7761, "epoch": 15.903688524590164, "lr": 1.062048617223442e-06, "seconds": 0.3100436160020763, "rows_per_second": 6605.522237188347}
+{"loss": 0.12710854411125183, "nce": 0.12671279907226562, "mse": 0.00039573851972818375, "pointer": 0.0, "grad_norm": 0.04081220179796219, "pointer_opportunities": 1071, "pointer_positions": 0, "step": 7762, "epoch": 15.905737704918034, "lr": 1.0594902736969658e-06, "seconds": 0.2931736259997706, "rows_per_second": 6985.621551106383}
+{"loss": 0.2541964054107666, "nce": 0.2536401152610779, "mse": 0.0005562961450777948, "pointer": 0.0, "grad_norm": 0.06397799402475357, "pointer_opportunities": 11316, "pointer_positions": 0, "step": 7763, "epoch": 15.907786885245901, "lr": 1.0569857829774348e-06, "seconds": 0.8855935880019388, "rows_per_second": 2312.5732025913408}
+{"loss": 0.1401001214981079, "nce": 0.13966548442840576, "mse": 0.00043463087058626115, "pointer": 0.0, "grad_norm": 0.052040912210941315, "pointer_opportunities": 972, "pointer_positions": 0, "step": 7764, "epoch": 15.90983606557377, "lr": 1.0545351455153028e-06, "seconds": 0.2185427369986428, "rows_per_second": 9371.16478051942}
+{"loss": 0.12785856425762177, "nce": 0.12745678424835205, "mse": 0.00040177302435040474, "pointer": 0.0, "grad_norm": 0.04894884303212166, "pointer_opportunities": 3810, "pointer_positions": 0, "step": 7765, "epoch": 15.91188524590164, "lr": 1.0521383617512807e-06, "seconds": 0.38914853499954916, "rows_per_second": 5262.771964443789}
+{"loss": 0.15310287475585938, "nce": 0.1526368260383606, "mse": 0.00046604470117017627, "pointer": 0.0, "grad_norm": 0.05809956043958664, "pointer_opportunities": 912, "pointer_positions": 0, "step": 7766, "epoch": 15.913934426229508, "lr": 1.049795432116437e-06, "seconds": 0.19976126800247584, "rows_per_second": 10252.237685909247}
+{"loss": 0.24481673538684845, "nce": 0.244264155626297, "mse": 0.0005525752203539014, "pointer": 0.0, "grad_norm": 0.06209534406661987, "pointer_opportunities": 11569, "pointer_positions": 0, "step": 7767, "epoch": 15.915983606557377, "lr": 1.0475063570321307e-06, "seconds": 0.8839321679988643, "rows_per_second": 2316.9198657363845}
+{"loss": 0.1250455528497696, "nce": 0.12465029209852219, "mse": 0.0003952592087443918, "pointer": 0.0, "grad_norm": 0.04034718871116638, "pointer_opportunities": 1057, "pointer_positions": 0, "step": 7768, "epoch": 15.918032786885245, "lr": 1.0452711369100112e-06, "seconds": 0.2792423159989994, "rows_per_second": 7334.131980223723}
+{"loss": 0.3560555577278137, "nce": 0.35541224479675293, "mse": 0.0006433134549297392, "pointer": 0.0, "grad_norm": 0.07617843896150589, "pointer_opportunities": 19467, "pointer_positions": 0, "step": 7769, "epoch": 15.920081967213115, "lr": 1.0430897721521186e-06, "seconds": 1.3233936830001767, "rows_per_second": 1547.536478606364}
+{"loss": 0.12370948493480682, "nce": 0.123313307762146, "mse": 0.00039617656148038805, "pointer": 0.0, "grad_norm": 0.04525751248002052, "pointer_opportunities": 2555, "pointer_positions": 0, "step": 7770, "epoch": 15.922131147540984, "lr": 1.040962263150717e-06, "seconds": 0.36857070499900146, "rows_per_second": 5556.600055898497}
+{"loss": 0.14408886432647705, "nce": 0.14363929629325867, "mse": 0.0004495676257647574, "pointer": 0.0, "grad_norm": 0.05308176949620247, "pointer_opportunities": 896, "pointer_positions": 0, "step": 7771, "epoch": 15.924180327868852, "lr": 1.0388886102884614e-06, "seconds": 0.208003417999862, "rows_per_second": 9845.992050002556}
+{"loss": 0.3030906021595001, "nce": 0.3024837374687195, "mse": 0.0006068790680728853, "pointer": 0.0, "grad_norm": 0.0720989853143692, "pointer_opportunities": 15143, "pointer_positions": 0, "step": 7772, "epoch": 15.926229508196721, "lr": 1.0368688139382633e-06, "seconds": 1.0593269559976761, "rows_per_second": 1933.3030169813717}
+{"loss": 0.12319444119930267, "nce": 0.12279923260211945, "mse": 0.0003952086844947189, "pointer": 0.0, "grad_norm": 0.04200359806418419, "pointer_opportunities": 1193, "pointer_positions": 0, "step": 7773, "epoch": 15.92827868852459, "lr": 1.034902874463392e-06, "seconds": 0.2933002160025353, "rows_per_second": 6982.606518033716}
+{"loss": 0.3096016049385071, "nce": 0.30898526310920715, "mse": 0.0006163529469631612, "pointer": 0.0, "grad_norm": 0.08990844339132309, "pointer_opportunities": 201, "pointer_positions": 0, "step": 7774, "epoch": 15.930327868852459, "lr": 1.0329907922173738e-06, "seconds": 0.13353876800101716, "rows_per_second": 15336.370333927302}
+{"loss": 0.12851203978061676, "nce": 0.12810099124908447, "mse": 0.0004110555164515972, "pointer": 0.0, "grad_norm": 0.04451506957411766, "pointer_opportunities": 1060, "pointer_positions": 0, "step": 7775, "epoch": 15.932377049180328, "lr": 1.0311325675441592e-06, "seconds": 0.24343203700118465, "rows_per_second": 8413.025767804069}
+{"loss": 0.13134416937828064, "nce": 0.13094502687454224, "mse": 0.00039914852823130786, "pointer": 0.0, "grad_norm": 0.0418391227722168, "pointer_opportunities": 1040, "pointer_positions": 0, "step": 7776, "epoch": 15.934426229508198, "lr": 1.029328200777856e-06, "seconds": 0.29412558599869953, "rows_per_second": 6963.012051624285}
+{"loss": 0.13314469158649445, "nce": 0.1327454000711441, "mse": 0.0003992966085206717, "pointer": 0.0, "grad_norm": 0.041811756789684296, "pointer_opportunities": 1233, "pointer_positions": 0, "step": 7777, "epoch": 15.936475409836065, "lr": 1.027577692243029e-06, "seconds": 0.3094837259995984, "rows_per_second": 6617.472351365764}
+{"loss": 0.3121008276939392, "nce": 0.31149283051490784, "mse": 0.0006079879822209477, "pointer": 0.0, "grad_norm": 0.07033444195985794, "pointer_opportunities": 15287, "pointer_positions": 0, "step": 7778, "epoch": 15.938524590163935, "lr": 1.0258810422544672e-06, "seconds": 1.063105965997238, "rows_per_second": 1926.4307279838188}
+{"loss": 0.12741775810718536, "nce": 0.1270117461681366, "mse": 0.00040601304499432445, "pointer": 0.0, "grad_norm": 0.04689177870750427, "pointer_opportunities": 987, "pointer_positions": 0, "step": 7779, "epoch": 15.940573770491802, "lr": 1.0242382511173164e-06, "seconds": 0.25194501700025285, "rows_per_second": 8128.757712235066}
+{"loss": 0.2865171730518341, "nce": 0.2859228551387787, "mse": 0.0005943220457993448, "pointer": 0.0, "grad_norm": 0.08742804080247879, "pointer_opportunities": 306, "pointer_positions": 0, "step": 7780, "epoch": 15.942622950819672, "lr": 1.0226493191270467e-06, "seconds": 0.1409852380020311, "rows_per_second": 14526.343530877291}
+{"loss": 0.18204976618289948, "nce": 0.18158583343029022, "mse": 0.00046392943477258086, "pointer": 0.0, "grad_norm": 0.061641544103622437, "pointer_opportunities": 11786, "pointer_positions": 0, "step": 7781, "epoch": 15.944672131147541, "lr": 1.0211142465693856e-06, "seconds": 0.6337326120010403, "rows_per_second": 3231.646851080213}
+{"loss": 0.12899236381053925, "nce": 0.12859532237052917, "mse": 0.0003970428369939327, "pointer": 0.0, "grad_norm": 0.0417359359562397, "pointer_opportunities": 2030, "pointer_positions": 0, "step": 7782, "epoch": 15.94672131147541, "lr": 1.0196330337203842e-06, "seconds": 0.34453986500011524, "rows_per_second": 5944.159756373373}
+{"loss": 0.12710709869861603, "nce": 0.12670613825321198, "mse": 0.00040096748853102326, "pointer": 0.0, "grad_norm": 0.04077175632119179, "pointer_opportunities": 914, "pointer_positions": 0, "step": 7783, "epoch": 15.948770491803279, "lr": 1.0182056808465174e-06, "seconds": 0.2856928869987314, "rows_per_second": 7168.536891186563}
+{"loss": 0.3012637794017792, "nce": 0.3006647229194641, "mse": 0.0005990597419440746, "pointer": 0.0, "grad_norm": 0.06852923333644867, "pointer_opportunities": 14085, "pointer_positions": 0, "step": 7784, "epoch": 15.950819672131148, "lr": 1.0168321882043848e-06, "seconds": 1.026085855999554, "rows_per_second": 1995.9343441148555}
+{"loss": 0.12121880799531937, "nce": 0.12082341313362122, "mse": 0.00039539506542496383, "pointer": 0.0, "grad_norm": 0.03994075953960419, "pointer_opportunities": 901, "pointer_positions": 0, "step": 7785, "epoch": 15.952868852459016, "lr": 1.0155125560410759e-06, "seconds": 0.26074226700075087, "rows_per_second": 7854.499477808492}
+{"loss": 0.7343835830688477, "nce": 0.7335866689682007, "mse": 0.0007969042053446174, "pointer": 0.0, "grad_norm": 0.11129336059093475, "pointer_opportunities": 47, "pointer_positions": 0, "step": 7786, "epoch": 15.954918032786885, "lr": 1.0142467845939042e-06, "seconds": 0.1070475880005688, "rows_per_second": 19131.678146630617}
+{"loss": 0.13113684952259064, "nce": 0.13073623180389404, "mse": 0.0004006223753094673, "pointer": 0.0, "grad_norm": 0.04067955166101456, "pointer_opportunities": 1067, "pointer_positions": 0, "step": 7787, "epoch": 15.956967213114755, "lr": 1.013034874090474e-06, "seconds": 0.3032350160028727, "rows_per_second": 6753.837426151992}
+{"loss": 0.11921103298664093, "nce": 0.11881120502948761, "mse": 0.00039982981979846954, "pointer": 0.0, "grad_norm": 0.04202762618660927, "pointer_opportunities": 906, "pointer_positions": 0, "step": 7788, "epoch": 15.959016393442623, "lr": 1.0118768247487803e-06, "seconds": 0.25308146699899225, "rows_per_second": 8092.255921719289}
+{"loss": 0.180742084980011, "nce": 0.1802382618188858, "mse": 0.0005038277013227344, "pointer": 0.0, "grad_norm": 0.06897936761379242, "pointer_opportunities": 700, "pointer_positions": 0, "step": 7789, "epoch": 15.961065573770492, "lr": 1.0107726367771082e-06, "seconds": 0.17334830799882184, "rows_per_second": 11814.363945299767}
+{"loss": 0.21169528365135193, "nce": 0.2111753672361374, "mse": 0.0005199185106903315, "pointer": 0.0, "grad_norm": 0.05546734109520912, "pointer_opportunities": 10867, "pointer_positions": 0, "step": 7790, "epoch": 15.96311475409836, "lr": 1.0097223103739675e-06, "seconds": 0.7908925499978068, "rows_per_second": 2589.4794432008234}
+{"loss": 0.19554157555103302, "nce": 0.19503480195999146, "mse": 0.0005067784222774208, "pointer": 0.0, "grad_norm": 0.05267604440450668, "pointer_opportunities": 11299, "pointer_positions": 0, "step": 7791, "epoch": 15.96516393442623, "lr": 1.0087258457283245e-06, "seconds": 0.7610790899998392, "rows_per_second": 2690.916130674977}
+{"loss": 0.13520687818527222, "nce": 0.134806826710701, "mse": 0.00040004667243920267, "pointer": 0.0, "grad_norm": 0.04282468929886818, "pointer_opportunities": 1134, "pointer_positions": 0, "step": 7792, "epoch": 15.967213114754099, "lr": 1.0077832430193697e-06, "seconds": 0.2929227759996138, "rows_per_second": 6991.6038212156645}
+{"loss": 0.2608206570148468, "nce": 0.26025471091270447, "mse": 0.0005659571033902466, "pointer": 0.0, "grad_norm": 0.05795153230428696, "pointer_opportunities": 12266, "pointer_positions": 0, "step": 7793, "epoch": 15.969262295081966, "lr": 1.0068945024166178e-06, "seconds": 0.9156816180002352, "rows_per_second": 2236.5852494370747}
+{"loss": 0.12562231719493866, "nce": 0.12522205710411072, "mse": 0.0004002636414952576, "pointer": 0.0, "grad_norm": 0.04079992324113846, "pointer_opportunities": 948, "pointer_positions": 0, "step": 7794, "epoch": 15.971311475409836, "lr": 1.0060596240799072e-06, "seconds": 0.26792371600095066, "rows_per_second": 7643.96683716022}
+{"loss": 0.38848876953125, "nce": 0.38782691955566406, "mse": 0.0006618546321988106, "pointer": 0.0, "grad_norm": 0.08305447548627853, "pointer_opportunities": 21932, "pointer_positions": 0, "step": 7795, "epoch": 15.973360655737705, "lr": 1.005278608159367e-06, "seconds": 1.4044980819999182, "rows_per_second": 1458.172158614681}
+{"loss": 0.3079628646373749, "nce": 0.30734843015670776, "mse": 0.0006144472863525152, "pointer": 0.0, "grad_norm": 0.08913210034370422, "pointer_opportunities": 226, "pointer_positions": 0, "step": 7796, "epoch": 15.975409836065573, "lr": 1.0045514547955164e-06, "seconds": 0.1343645880006079, "rows_per_second": 15242.111262163318}
+{"loss": 0.13700149953365326, "nce": 0.13660302758216858, "mse": 0.0003984665381722152, "pointer": 0.0, "grad_norm": 0.04391881451010704, "pointer_opportunities": 1199, "pointer_positions": 0, "step": 7797, "epoch": 15.977459016393443, "lr": 1.0038781641190327e-06, "seconds": 0.30978880600014236, "rows_per_second": 6610.95546492748}
+{"loss": 0.14081943035125732, "nce": 0.14040955901145935, "mse": 0.0004098695353604853, "pointer": 0.0, "grad_norm": 0.05782271921634674, "pointer_opportunities": 5039, "pointer_positions": 0, "step": 7798, "epoch": 15.979508196721312, "lr": 1.0032587362511167e-06, "seconds": 0.41272134500104585, "rows_per_second": 4962.185806006254}
+{"loss": 0.24890382587909698, "nce": 0.24834901094436646, "mse": 0.0005548197077587247, "pointer": 0.0, "grad_norm": 0.05801800265908241, "pointer_opportunities": 11634, "pointer_positions": 0, "step": 7799, "epoch": 15.98155737704918, "lr": 1.00269317130306e-06, "seconds": 0.8837817880012153, "rows_per_second": 2317.3141015179913}
+{"loss": 0.41012218594551086, "nce": 0.40944114327430725, "mse": 0.0006810533232055604, "pointer": 0.0, "grad_norm": 0.10055399686098099, "pointer_opportunities": 26401, "pointer_positions": 0, "step": 7800, "epoch": 15.98360655737705, "lr": 1.0021814693766774e-06, "seconds": 1.58120061000227, "rows_per_second": 1295.218321473491}
+{"loss": 0.17442503571510315, "nce": 0.1739596724510193, "mse": 0.00046536995796486735, "pointer": 0.0, "grad_norm": 0.05889223515987396, "pointer_opportunities": 11915, "pointer_positions": 0, "step": 7801, "epoch": 15.985655737704917, "lr": 1.0017236305639419e-06, "seconds": 0.6656546109988994, "rows_per_second": 3076.6706429430656}
+{"loss": 0.12443163990974426, "nce": 0.12403476983308792, "mse": 0.00039686710806563497, "pointer": 0.0, "grad_norm": 0.03952471911907196, "pointer_opportunities": 1123, "pointer_positions": 0, "step": 7802, "epoch": 15.987704918032787, "lr": 1.0013196549471834e-06, "seconds": 0.2799782359979872, "rows_per_second": 7314.854287512274}
+{"loss": 0.4212205410003662, "nce": 0.420534610748291, "mse": 0.0006859289715066552, "pointer": 0.0, "grad_norm": 0.11406459659337997, "pointer_opportunities": 27234, "pointer_positions": 0, "step": 7803, "epoch": 15.989754098360656, "lr": 1.0009695425990559e-06, "seconds": 1.5799672100001771, "rows_per_second": 1296.2294325081407}
+{"loss": 0.3626468777656555, "nce": 0.3620036244392395, "mse": 0.0006432391237467527, "pointer": 0.0, "grad_norm": 0.08041119575500488, "pointer_opportunities": 19225, "pointer_positions": 0, "step": 7804, "epoch": 15.991803278688524, "lr": 1.00067329358257e-06, "seconds": 1.3242621120007243, "rows_per_second": 1546.5216300010552}
+{"loss": 0.1314559429883957, "nce": 0.13105084002017975, "mse": 0.00040509592508897185, "pointer": 0.0, "grad_norm": 0.05663928762078285, "pointer_opportunities": 4542, "pointer_positions": 0, "step": 7805, "epoch": 15.993852459016393, "lr": 1.0004309079509611e-06, "seconds": 0.40543038500254625, "rows_per_second": 5051.422083194721}
+{"loss": 0.531017005443573, "nce": 0.5302868485450745, "mse": 0.0007301438599824905, "pointer": 0.0, "grad_norm": 0.10042908787727356, "pointer_opportunities": 70, "pointer_positions": 0, "step": 7806, "epoch": 15.995901639344263, "lr": 1.0002423857478548e-06, "seconds": 0.11106528899836121, "rows_per_second": 18439.604474717737}
+{"loss": 0.300460547208786, "nce": 0.2998608648777008, "mse": 0.0005996746476739645, "pointer": 0.0, "grad_norm": 0.06856462359428406, "pointer_opportunities": 14292, "pointer_positions": 0, "step": 7807, "epoch": 15.99795081967213, "lr": 1.0001077270071008e-06, "seconds": 1.0256997459982813, "rows_per_second": 1996.685685055665}
+{"loss": 0.1683766394853592, "nce": 0.16789257526397705, "mse": 0.00048406116547994316, "pointer": 0.0, "grad_norm": 0.06499374657869339, "pointer_opportunities": 769, "pointer_positions": 0, "step": 7808, "epoch": 16.0, "lr": 1.0000269317530054e-06, "seconds": 0.18273594800120918, "rows_per_second": 11207.428108159913}