{ "sample_count": 208, "samples": [ { "step": 100.0, "elapsed_seconds": 8.903452000000016, "window_seconds": 8.903452000000016, "steps_per_second": 11.231598710253037, "samples_per_second": 5750.578539649555, "tokens_per_second": 736074.053075143, "process_rss_mb": 1816.14453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 48.0, "gpu_power_w": 131.206 }, { "step": 200.0, "elapsed_seconds": 16.140120600000046, "window_seconds": 7.23666860000003, "steps_per_second": 13.818513120802518, "samples_per_second": 7075.078717850889, "tokens_per_second": 905610.0758849138, "process_rss_mb": 1832.77734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 42.0, "gpu_memory_util_percent": 30.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 56.0, "gpu_power_w": 114.181 }, { "step": 300.0, "elapsed_seconds": 23.81037910000032, "window_seconds": 7.670258500000273, "steps_per_second": 13.037370252905616, "samples_per_second": 6675.133569487675, "tokens_per_second": 854417.0968944224, "process_rss_mb": 1832.890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 59.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 58.0, "gpu_power_w": 118.542 }, { "step": 400.0, "elapsed_seconds": 31.07618870000033, "window_seconds": 7.265809600000011, "steps_per_second": 13.76309117706578, "samples_per_second": 7046.702682657679, "tokens_per_second": 901977.943380183, "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 134.38 }, { "step": 500.0, "elapsed_seconds": 38.41567729999997, "window_seconds": 7.33948859999964, "steps_per_second": 13.62492749154279, "samples_per_second": 6975.9628756699085, "tokens_per_second": 892923.2480857483, "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 106.578 }, { "step": 600.0, "elapsed_seconds": 46.05576810000002, "window_seconds": 7.640090800000053, "steps_per_second": 13.088849677022074, "samples_per_second": 6701.491034635302, "tokens_per_second": 857790.8524333186, "process_rss_mb": 1838.4140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 188.069 }, { "step": 700.0, "elapsed_seconds": 53.297965400000066, "window_seconds": 7.2421973000000435, "steps_per_second": 13.807964055328815, "samples_per_second": 7069.6775963283535, "tokens_per_second": 904918.7323300292, "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 160.406 }, { "step": 800.0, "elapsed_seconds": 60.61575200000016, "window_seconds": 7.31778660000009, "steps_per_second": 13.665334269244578, "samples_per_second": 6996.651145853224, "tokens_per_second": 895571.3466692127, "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 203.595 }, { "step": 900.0, "elapsed_seconds": 68.03899850000016, "window_seconds": 7.423246500000005, "steps_per_second": 13.471194847160193, "samples_per_second": 6897.251761746019, "tokens_per_second": 882848.2255034904, "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 191.101 }, { "step": 1000.0, "elapsed_seconds": 75.25265880000006, "window_seconds": 7.213660299999901, "steps_per_second": 13.862587901457097, "samples_per_second": 7097.645005546034, "tokens_per_second": 908498.5607098923, "process_rss_mb": 1838.26953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 213.308 }, { "step": 1100.0, "elapsed_seconds": 82.82529980000027, "window_seconds": 7.572641000000203, "steps_per_second": 13.205432556488194, "samples_per_second": 6761.1814689219555, "tokens_per_second": 865431.2280220103, "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 64.0, "gpu_power_w": 214.504 }, { "step": 1200.0, "elapsed_seconds": 90.14202300000034, "window_seconds": 7.3167232000000695, "steps_per_second": 13.667320365488072, "samples_per_second": 6997.668027129893, "tokens_per_second": 895701.5074726263, "process_rss_mb": 1839.296875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 209.776 }, { "step": 1300.0, "elapsed_seconds": 97.50183790000028, "window_seconds": 7.359814899999947, "steps_per_second": 13.587298234905436, "samples_per_second": 6956.696696271583, "tokens_per_second": 890457.1771227627, "process_rss_mb": 1839.3125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 215.402 }, { "step": 1400.0, "elapsed_seconds": 104.96856790000038, "window_seconds": 7.466730000000098, "steps_per_second": 13.392743543692982, "samples_per_second": 6857.084694370807, "tokens_per_second": 877706.8408794632, "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 190.537 }, { "step": 1500.0, "elapsed_seconds": 112.27981500000033, "window_seconds": 7.311247099999946, "steps_per_second": 13.677557143431898, "samples_per_second": 7002.909257437132, "tokens_per_second": 896372.3849519528, "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.813 }, { "step": 1600.0, "elapsed_seconds": 119.72480010000027, "window_seconds": 7.44498509999994, "steps_per_second": 13.431860327027492, "samples_per_second": 6877.112487438076, "tokens_per_second": 880270.3983920737, "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.653 }, { "step": 1700.0, "elapsed_seconds": 127.37679610000032, "window_seconds": 7.651996000000054, "steps_per_second": 13.068485660473332, "samples_per_second": 6691.064658162346, "tokens_per_second": 856456.2762447803, "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 188.916 }, { "step": 1800.0, "elapsed_seconds": 134.6600384000003, "window_seconds": 7.2832422999999835, "steps_per_second": 13.730148727854383, "samples_per_second": 7029.836148661444, "tokens_per_second": 899819.0270286648, "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.208 }, { "step": 1900.0, "elapsed_seconds": 141.9570794000001, "window_seconds": 7.297040999999808, "steps_per_second": 13.70418502513589, "samples_per_second": 7016.542732869576, "tokens_per_second": 898117.4698073057, "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 219.203 }, { "step": 2000.0, "elapsed_seconds": 149.39294990000008, "window_seconds": 7.4358704999999645, "steps_per_second": 13.448324577465474, "samples_per_second": 6885.542183662323, "tokens_per_second": 881349.3995087773, "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.769 }, { "step": 2100.0, "elapsed_seconds": 156.71874380000008, "window_seconds": 7.325793900000008, "steps_per_second": 13.650397672257732, "samples_per_second": 6989.003608195959, "tokens_per_second": 894592.4618490827, "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 211.466 }, { "step": 2200.0, "elapsed_seconds": 162.8732742000002, "window_seconds": 6.154530400000112, "steps_per_second": 16.248193363379627, "samples_per_second": 8319.07500205037, "tokens_per_second": 1064841.6002624473, "process_rss_mb": 1840.45703125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.198 }, { "step": 2300.0, "elapsed_seconds": 169.2013237000001, "window_seconds": 6.328049499999906, "steps_per_second": 15.802657675165385, "samples_per_second": 8090.960729684677, "tokens_per_second": 1035642.9733996387, "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 189.925 }, { "step": 2400.0, "elapsed_seconds": 175.35101350000014, "window_seconds": 6.149689800000033, "steps_per_second": 16.26098278973347, "samples_per_second": 8325.623188343538, "tokens_per_second": 1065679.7681079728, "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.845 }, { "step": 2500.0, "elapsed_seconds": 181.50287780000008, "window_seconds": 6.151864299999943, "steps_per_second": 16.255235018756984, "samples_per_second": 8322.680329603576, "tokens_per_second": 1065303.0821892577, "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 218.645 }, { "step": 2600.0, "elapsed_seconds": 187.78224480000017, "window_seconds": 6.279367000000093, "steps_per_second": 15.925172075465333, "samples_per_second": 8153.688102638251, "tokens_per_second": 1043672.0771376961, "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 220.123 }, { "step": 2700.0, "elapsed_seconds": 194.08218190000025, "window_seconds": 6.299937100000079, "steps_per_second": 15.873174352804053, "samples_per_second": 8127.065268635675, "tokens_per_second": 1040264.3543853664, "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 219.522 }, { "step": 2800.0, "elapsed_seconds": 200.2462502000003, "window_seconds": 6.164068300000054, "steps_per_second": 16.223051908753042, "samples_per_second": 8306.202577281558, "tokens_per_second": 1063193.9298920394, "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 221.693 }, { "step": 2900.0, "elapsed_seconds": 206.6105974000002, "window_seconds": 6.364347199999884, "steps_per_second": 15.712530579727302, "samples_per_second": 8044.8156568203785, "tokens_per_second": 1029736.4040730085, "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.768 }, { "step": 3000.0, "elapsed_seconds": 212.82193240000015, "window_seconds": 6.211334999999963, "steps_per_second": 16.09959855651009, "samples_per_second": 8242.994460933167, "tokens_per_second": 1055103.2909994454, "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.446 }, { "step": 3100.0, "elapsed_seconds": 219.14003420000017, "window_seconds": 6.318101800000022, "steps_per_second": 15.827538581287127, "samples_per_second": 8103.699753619009, "tokens_per_second": 1037273.5684632332, "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.686 }, { "step": 3200.0, "elapsed_seconds": 225.44381470000008, "window_seconds": 6.3037804999999025, "steps_per_second": 15.863496516098799, "samples_per_second": 8122.110216242585, "tokens_per_second": 1039630.1076790509, "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.552 }, { "step": 3300.0, "elapsed_seconds": 231.62088630000017, "window_seconds": 6.17707160000009, "steps_per_second": 16.188900902492136, "samples_per_second": 8288.717262075974, "tokens_per_second": 1060955.8095457247, "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.091 }, { "step": 3400.0, "elapsed_seconds": 237.77530750000005, "window_seconds": 6.154421199999888, "steps_per_second": 16.24848166063152, "samples_per_second": 8319.222610243338, "tokens_per_second": 1064860.4941111472, "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.805 }, { "step": 3500.0, "elapsed_seconds": 244.10230320000028, "window_seconds": 6.326995700000225, "steps_per_second": 15.805289704874692, "samples_per_second": 8092.308328895842, "tokens_per_second": 1035815.4660986678, "process_rss_mb": 1839.23046875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.828 }, { "step": 3600.0, "elapsed_seconds": 250.4137068, "window_seconds": 6.311403599999721, "steps_per_second": 15.844336115662832, "samples_per_second": 8112.30009121937, "tokens_per_second": 1038374.4116760794, "process_rss_mb": 1840.3125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.241 }, { "step": 3700.0, "elapsed_seconds": 256.8099503000003, "window_seconds": 6.39624350000031, "steps_per_second": 15.634176528769606, "samples_per_second": 8004.698382730038, "tokens_per_second": 1024601.3929894449, "process_rss_mb": 1840.3125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 191.349 }, { "step": 3800.0, "elapsed_seconds": 262.9805487000003, "window_seconds": 6.170598400000017, "steps_per_second": 16.205883695169618, "samples_per_second": 8297.412451926844, "tokens_per_second": 1062068.793846636, "process_rss_mb": 1840.3125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.153 }, { "step": 3900.0, "elapsed_seconds": 269.1176267000001, "window_seconds": 6.137077999999747, "steps_per_second": 16.294399386809836, "samples_per_second": 8342.732486046636, "tokens_per_second": 1067869.7582139694, "process_rss_mb": 1840.3125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.788 }, { "step": 4000.0, "elapsed_seconds": 275.4397656000001, "window_seconds": 6.322138900000027, "steps_per_second": 15.817431660667811, "samples_per_second": 8098.525010261919, "tokens_per_second": 1036611.2013135257, "process_rss_mb": 1840.3125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.738 }, { "step": 4100.0, "elapsed_seconds": 281.7172369, "window_seconds": 6.277471299999888, "steps_per_second": 15.929981233048693, "samples_per_second": 8156.150391320931, "tokens_per_second": 1043987.2500890791, "process_rss_mb": 1840.359375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.967 }, { "step": 4200.0, "elapsed_seconds": 287.87845830000015, "window_seconds": 6.161221400000159, "steps_per_second": 16.23054805334498, "samples_per_second": 8310.04060331263, "tokens_per_second": 1063685.1972240165, "process_rss_mb": 1841.484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.908 }, { "step": 4300.0, "elapsed_seconds": 294.2290678000004, "window_seconds": 6.350609500000246, "steps_per_second": 15.746520078111578, "samples_per_second": 8062.218279993128, "tokens_per_second": 1031963.9398391204, "process_rss_mb": 1840.921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 199.104 }, { "step": 4400.0, "elapsed_seconds": 300.36520040000005, "window_seconds": 6.136132599999655, "steps_per_second": 16.296909880990125, "samples_per_second": 8344.017859066944, "tokens_per_second": 1068034.2859605688, "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.851 }, { "step": 4500.0, "elapsed_seconds": 306.51977000000034, "window_seconds": 6.154569600000286, "steps_per_second": 16.248089874553592, "samples_per_second": 8319.02201577144, "tokens_per_second": 1064834.8180187442, "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.071 }, { "step": 4600.0, "elapsed_seconds": 312.8989471, "window_seconds": 6.3791770999996515, "steps_per_second": 15.676003100776974, "samples_per_second": 8026.113587597811, "tokens_per_second": 1027342.5392125198, "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.973 }, { "step": 4700.0, "elapsed_seconds": 319.32361089999995, "window_seconds": 6.424663799999962, "steps_per_second": 15.565016802902681, "samples_per_second": 7969.288603086173, "tokens_per_second": 1020068.9411950301, "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.04 }, { "step": 4800.0, "elapsed_seconds": 325.52438640000037, "window_seconds": 6.200775500000418, "steps_per_second": 16.12701508061262, "samples_per_second": 8257.03172127366, "tokens_per_second": 1056900.0603230286, "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.563 }, { "step": 4900.0, "elapsed_seconds": 331.6376997000002, "window_seconds": 6.113313299999845, "steps_per_second": 16.357741717572782, "samples_per_second": 8375.163759397265, "tokens_per_second": 1072020.9612028499, "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.124 }, { "step": 5000.0, "elapsed_seconds": 337.8368824000004, "window_seconds": 6.199182700000165, "steps_per_second": 16.131158709033908, "samples_per_second": 8259.153259025361, "tokens_per_second": 1057171.6171552462, "process_rss_mb": 1838.9296875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.683 }, { "step": 5100.0, "elapsed_seconds": 344.1479210000002, "window_seconds": 6.311038599999847, "steps_per_second": 15.845252475559636, "samples_per_second": 8112.769267486534, "tokens_per_second": 1038434.4662382763, "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.004 }, { "step": 5200.0, "elapsed_seconds": 350.4551905000003, "window_seconds": 6.307269500000075, "steps_per_second": 15.854721286287008, "samples_per_second": 8117.617298578948, "tokens_per_second": 1039055.0142181054, "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.681 }, { "step": 5300.0, "elapsed_seconds": 356.6461669, "window_seconds": 6.190976399999727, "steps_per_second": 16.152540978835653, "samples_per_second": 8270.100981163854, "tokens_per_second": 1058572.9255889733, "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 217.354 }, { "step": 5400.0, "elapsed_seconds": 363.0472904000003, "window_seconds": 6.401123500000267, "steps_per_second": 15.62225756150398, "samples_per_second": 7998.595871490038, "tokens_per_second": 1023820.2715507249, "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.263 }, { "step": 5500.0, "elapsed_seconds": 369.2443648000003, "window_seconds": 6.1970744000000195, "steps_per_second": 16.13664667314623, "samples_per_second": 8261.96309665087, "tokens_per_second": 1057531.2763713114, "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.977 }, { "step": 5600.0, "elapsed_seconds": 375.68432789999997, "window_seconds": 6.439963099999659, "steps_per_second": 15.528039283331498, "samples_per_second": 7950.356113065727, "tokens_per_second": 1017645.5824724131, "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 191.987 }, { "step": 5700.0, "elapsed_seconds": 381.8370156000001, "window_seconds": 6.152687700000115, "steps_per_second": 16.253059618156488, "samples_per_second": 8321.566524496122, "tokens_per_second": 1065160.5151355036, "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 91.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.18 }, { "step": 5800.0, "elapsed_seconds": 388.1134645000002, "window_seconds": 6.276448900000105, "steps_per_second": 15.93257614190061, "samples_per_second": 8157.478984653112, "tokens_per_second": 1044157.3100355983, "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.717 }, { "step": 5900.0, "elapsed_seconds": 394.33245940000006, "window_seconds": 6.2189948999998705, "steps_per_second": 16.079768774211743, "samples_per_second": 8232.841612396413, "tokens_per_second": 1053803.7263867408, "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.805 }, { "step": 6000.0, "elapsed_seconds": 400.6645559000003, "window_seconds": 6.332096500000262, "steps_per_second": 15.792557804511645, "samples_per_second": 8085.789595909962, "tokens_per_second": 1034981.0682764752, "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.074 }, { "step": 6100.0, "elapsed_seconds": 406.9181136000002, "window_seconds": 6.253557699999874, "steps_per_second": 15.990897469451992, "samples_per_second": 8187.33950435942, "tokens_per_second": 1047979.4565580058, "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.852 }, { "step": 6200.0, "elapsed_seconds": 413.1112514000001, "window_seconds": 6.193137799999931, "steps_per_second": 16.14690375531465, "samples_per_second": 8267.2147227211, "tokens_per_second": 1058203.484508301, "process_rss_mb": 1839.7578125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.232 }, { "step": 6300.0, "elapsed_seconds": 419.28662150000036, "window_seconds": 6.175370100000237, "steps_per_second": 16.193361431081865, "samples_per_second": 8291.001052713915, "tokens_per_second": 1061248.134747381, "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.08 }, { "step": 6400.0, "elapsed_seconds": 425.62520029999996, "window_seconds": 6.3385787999995955, "steps_per_second": 15.776407165594657, "samples_per_second": 8077.520468784464, "tokens_per_second": 1033922.6200044114, "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.541 }, { "step": 6500.0, "elapsed_seconds": 431.8106788, "window_seconds": 6.185478500000045, "steps_per_second": 16.166898001504535, "samples_per_second": 8277.451776770322, "tokens_per_second": 1059513.8274266012, "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.375 }, { "step": 6600.0, "elapsed_seconds": 438.0415554000001, "window_seconds": 6.2308766000001015, "steps_per_second": 16.04910615626674, "samples_per_second": 8217.14235200857, "tokens_per_second": 1051794.221057097, "process_rss_mb": 1840.390625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.544 }, { "step": 6700.0, "elapsed_seconds": 444.26439770000025, "window_seconds": 6.2228423000001385, "steps_per_second": 16.06982712706664, "samples_per_second": 8227.75148905812, "tokens_per_second": 1053152.1905994394, "process_rss_mb": 1840.453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.634 }, { "step": 6800.0, "elapsed_seconds": 450.4596012000002, "window_seconds": 6.195203499999934, "steps_per_second": 16.141519806411697, "samples_per_second": 8264.458140882789, "tokens_per_second": 1057850.642032997, "process_rss_mb": 1840.453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.006 }, { "step": 6900.0, "elapsed_seconds": 456.83013170000004, "window_seconds": 6.370530499999859, "steps_per_second": 15.69727984192246, "samples_per_second": 8037.0072790643, "tokens_per_second": 1028736.9317202304, "process_rss_mb": 1840.453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.954 }, { "step": 7000.0, "elapsed_seconds": 463.0098954, "window_seconds": 6.179763699999967, "steps_per_second": 16.18184850660237, "samples_per_second": 8285.106435380414, "tokens_per_second": 1060493.623728693, "process_rss_mb": 1840.453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.566 }, { "step": 7100.0, "elapsed_seconds": 469.3084415000003, "window_seconds": 6.298546100000294, "steps_per_second": 15.87667985791123, "samples_per_second": 8128.860087250549, "tokens_per_second": 1040494.0911680703, "process_rss_mb": 1840.46875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.363 }, { "step": 7200.0, "elapsed_seconds": 475.6791295000003, "window_seconds": 6.370687999999973, "steps_per_second": 15.69689176428047, "samples_per_second": 8036.808583311601, "tokens_per_second": 1028711.4986638849, "process_rss_mb": 1840.46875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.912 }, { "step": 7300.0, "elapsed_seconds": 481.88366570000017, "window_seconds": 6.204536199999893, "steps_per_second": 16.117240157290357, "samples_per_second": 8252.026960532663, "tokens_per_second": 1056259.4509481809, "process_rss_mb": 1840.46875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.10546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.171 }, { "step": 7400.0, "elapsed_seconds": 488.0593595, "window_seconds": 6.175693799999863, "steps_per_second": 16.192512653396484, "samples_per_second": 8290.566478539, "tokens_per_second": 1061192.509252992, "process_rss_mb": 1840.46875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.429 }, { "step": 7500.0, "elapsed_seconds": 494.4528210000003, "window_seconds": 6.393461500000285, "steps_per_second": 15.640979460030461, "samples_per_second": 8008.181483535596, "tokens_per_second": 1025047.2298925563, "process_rss_mb": 1840.5, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 65.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 184.835 }, { "step": 7600.0, "elapsed_seconds": 500.7447495000001, "window_seconds": 6.291928499999813, "steps_per_second": 15.893378317951798, "samples_per_second": 8137.409698791321, "tokens_per_second": 1041588.4414452891, "process_rss_mb": 1839.875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.816 }, { "step": 7700.0, "elapsed_seconds": 506.94524920000003, "window_seconds": 6.200499699999909, "steps_per_second": 16.127732414857057, "samples_per_second": 8257.398996406813, "tokens_per_second": 1056947.071540072, "process_rss_mb": 1839.875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.423 }, { "step": 7800.0, "elapsed_seconds": 513.3290940000002, "window_seconds": 6.3838448000001335, "steps_per_second": 15.664541218169639, "samples_per_second": 8020.245103702855, "tokens_per_second": 1026591.3732739654, "process_rss_mb": 1839.875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.89 }, { "step": 7900.0, "elapsed_seconds": 519.5080165000004, "window_seconds": 6.178922500000226, "steps_per_second": 16.184051507361733, "samples_per_second": 8286.234371769207, "tokens_per_second": 1060637.9995864586, "process_rss_mb": 1839.875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.442 }, { "step": 8000.0, "elapsed_seconds": 525.8964941000004, "window_seconds": 6.388477599999987, "steps_per_second": 15.6531815968174, "samples_per_second": 8014.428977570509, "tokens_per_second": 1025846.9091290252, "process_rss_mb": 1839.9140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 211.876 }, { "step": 8100.0, "elapsed_seconds": 532.1567189000002, "window_seconds": 6.260224799999833, "steps_per_second": 15.973867264319754, "samples_per_second": 8178.620039331714, "tokens_per_second": 1046863.3650344594, "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.597 }, { "step": 8200.0, "elapsed_seconds": 538.3156355000001, "window_seconds": 6.158916599999884, "steps_per_second": 16.236621875997134, "samples_per_second": 8313.150400510533, "tokens_per_second": 1064083.2512653482, "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.291 }, { "step": 8300.0, "elapsed_seconds": 544.6330875000003, "window_seconds": 6.31745200000023, "steps_per_second": 15.829166569052896, "samples_per_second": 8104.533283355083, "tokens_per_second": 1037380.2602694506, "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.714 }, { "step": 8400.0, "elapsed_seconds": 550.7974985000001, "window_seconds": 6.164410999999745, "steps_per_second": 16.22215001563071, "samples_per_second": 8305.740808002924, "tokens_per_second": 1063134.8234243742, "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 93.0, "gpu_memory_util_percent": 65.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.152 }, { "step": 8500.0, "elapsed_seconds": 556.9742562000001, "window_seconds": 6.176757700000053, "steps_per_second": 16.1897236150285, "samples_per_second": 8289.138490894593, "tokens_per_second": 1061009.726834508, "process_rss_mb": 1841.30078125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.416 }, { "step": 8600.0, "elapsed_seconds": 563.4249459000002, "window_seconds": 6.450689700000112, "steps_per_second": 15.502218313182583, "samples_per_second": 7937.1357763494825, "tokens_per_second": 1015953.3793727338, "process_rss_mb": 1841.34375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.005 }, { "step": 8700.0, "elapsed_seconds": 569.5674870000003, "window_seconds": 6.142541100000017, "steps_per_second": 16.279907349744835, "samples_per_second": 8335.312563069356, "tokens_per_second": 1066920.0080728775, "process_rss_mb": 1841.34375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.413 }, { "step": 8800.0, "elapsed_seconds": 575.7692353000002, "window_seconds": 6.201748299999963, "steps_per_second": 16.124485413250422, "samples_per_second": 8255.736531584216, "tokens_per_second": 1056734.2760427797, "process_rss_mb": 1841.34375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.764 }, { "step": 8900.0, "elapsed_seconds": 582.1036005000001, "window_seconds": 6.334365199999866, "steps_per_second": 15.786901582498293, "samples_per_second": 8082.893610239126, "tokens_per_second": 1034610.3821106082, "process_rss_mb": 1841.421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.763 }, { "step": 9000.0, "elapsed_seconds": 588.2590813000002, "window_seconds": 6.155480800000078, "steps_per_second": 16.245684658783883, "samples_per_second": 8317.790545297348, "tokens_per_second": 1064677.1897980606, "process_rss_mb": 1841.421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.451 }, { "step": 9100.0, "elapsed_seconds": 594.5160964000002, "window_seconds": 6.25701509999999, "steps_per_second": 15.982061478483592, "samples_per_second": 8182.815476983599, "tokens_per_second": 1047400.3810539007, "process_rss_mb": 1846.34375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.324 }, { "step": 9200.0, "elapsed_seconds": 600.6848840000002, "window_seconds": 6.168787600000087, "steps_per_second": 16.210640807279308, "samples_per_second": 8299.848093327006, "tokens_per_second": 1062380.5559458567, "process_rss_mb": 1846.34375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.643 }, { "step": 9300.0, "elapsed_seconds": 606.8815420000001, "window_seconds": 6.196657999999843, "steps_per_second": 16.13773101565433, "samples_per_second": 8262.518280015018, "tokens_per_second": 1057602.3398419223, "process_rss_mb": 1840.109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 58.0, "gpu_memory_util_percent": 40.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.376 }, { "step": 9400.0, "elapsed_seconds": 613.2031249000001, "window_seconds": 6.321582900000067, "steps_per_second": 15.818822845777905, "samples_per_second": 8099.237297038288, "tokens_per_second": 1036702.3740209008, "process_rss_mb": 1840.109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.48 }, { "step": 9500.0, "elapsed_seconds": 619.3316034, "window_seconds": 6.128478499999801, "steps_per_second": 16.31726373846351, "samples_per_second": 8354.439034093317, "tokens_per_second": 1069368.1963639446, "process_rss_mb": 1840.109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.293 }, { "step": 9600.0, "elapsed_seconds": 625.6393263, "window_seconds": 6.307722900000044, "steps_per_second": 15.853581646714268, "samples_per_second": 8117.033803117705, "tokens_per_second": 1038980.3267990663, "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.511 }, { "step": 9700.0, "elapsed_seconds": 632.0479509000002, "window_seconds": 6.4086246000001665, "steps_per_second": 15.603972184608441, "samples_per_second": 7989.233758519522, "tokens_per_second": 1022621.9210904988, "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.678 }, { "step": 9800.0, "elapsed_seconds": 638.1941039000003, "window_seconds": 6.14615300000014, "steps_per_second": 16.27034016237437, "samples_per_second": 8330.414163135678, "tokens_per_second": 1066293.0128813668, "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.176 }, { "step": 9900.0, "elapsed_seconds": 644.5170161000001, "window_seconds": 6.3229121999997915, "steps_per_second": 15.815497169168868, "samples_per_second": 8097.53455061446, "tokens_per_second": 1036484.4224786509, "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 194.573 }, { "step": 10000.0, "elapsed_seconds": 650.7136834000003, "window_seconds": 6.196667300000172, "steps_per_second": 16.137706796038128, "samples_per_second": 8262.505879571521, "tokens_per_second": 1057600.7525851547, "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.693 }, { "step": 10100.0, "elapsed_seconds": 657.0395794000001, "window_seconds": 6.32589599999983, "steps_per_second": 15.808037312027054, "samples_per_second": 8093.7151037578515, "tokens_per_second": 1035995.533281005, "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.778 }, { "step": 10200.0, "elapsed_seconds": 663.2299209000003, "window_seconds": 6.190341500000159, "steps_per_second": 16.15419763190729, "samples_per_second": 8270.949187536533, "tokens_per_second": 1058681.4960046762, "process_rss_mb": 1840.6796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.314 }, { "step": 10300.0, "elapsed_seconds": 669.3964020000003, "window_seconds": 6.1664811000000554, "steps_per_second": 16.216704207525925, "samples_per_second": 8302.952554253274, "tokens_per_second": 1062777.926944419, "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.095 }, { "step": 10400.0, "elapsed_seconds": 675.7291157, "window_seconds": 6.332713699999658, "steps_per_second": 15.79101862760753, "samples_per_second": 8085.001537335055, "tokens_per_second": 1034880.1967788871, "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.979 }, { "step": 10500.0, "elapsed_seconds": 681.9226350000004, "window_seconds": 6.193519300000389, "steps_per_second": 16.145909160240077, "samples_per_second": 8266.70549004292, "tokens_per_second": 1058138.3027254937, "process_rss_mb": 1841.1875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.273 }, { "step": 10600.0, "elapsed_seconds": 688.2179854000001, "window_seconds": 6.295350399999734, "steps_per_second": 15.884739314908385, "samples_per_second": 8132.986529233093, "tokens_per_second": 1041022.2757418359, "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.615 }, { "step": 10700.0, "elapsed_seconds": 694.3896768, "window_seconds": 6.171691399999872, "steps_per_second": 16.20301365035881, "samples_per_second": 8295.94298898371, "tokens_per_second": 1061880.702589915, "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.811 }, { "step": 10800.0, "elapsed_seconds": 700.5805862000002, "window_seconds": 6.190909400000237, "steps_per_second": 16.152715786794776, "samples_per_second": 8270.190482838925, "tokens_per_second": 1058584.3818033824, "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.656 }, { "step": 10900.0, "elapsed_seconds": 706.9403584000002, "window_seconds": 6.359772199999952, "steps_per_second": 15.723833630393358, "samples_per_second": 8050.602818761399, "tokens_per_second": 1030477.1608014591, "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 191.385 }, { "step": 11000.0, "elapsed_seconds": 713.1120964000002, "window_seconds": 6.171738000000005, "steps_per_second": 16.202891308736685, "samples_per_second": 8295.880350073183, "tokens_per_second": 1061872.6848093674, "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.764 }, { "step": 11100.0, "elapsed_seconds": 719.3734212000004, "window_seconds": 6.261324800000239, "steps_per_second": 15.971060948634415, "samples_per_second": 8177.1832057008205, "tokens_per_second": 1046679.450329705, "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.593 }, { "step": 11200.0, "elapsed_seconds": 725.7768551000004, "window_seconds": 6.403433899999982, "steps_per_second": 15.616620950830816, "samples_per_second": 7995.709926825378, "tokens_per_second": 1023450.8706336484, "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 199.127 }, { "step": 11300.0, "elapsed_seconds": 732.0621894000001, "window_seconds": 6.285334299999704, "steps_per_second": 15.910052707937064, "samples_per_second": 8145.946986463777, "tokens_per_second": 1042681.2142673634, "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.92 }, { "step": 11400.0, "elapsed_seconds": 738.2808528999999, "window_seconds": 6.2186634999998205, "steps_per_second": 16.08062568428134, "samples_per_second": 8233.280350352046, "tokens_per_second": 1053859.884845062, "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.169 }, { "step": 11500.0, "elapsed_seconds": 744.6068601000002, "window_seconds": 6.3260072000002765, "steps_per_second": 15.807759434734066, "samples_per_second": 8093.572830583842, "tokens_per_second": 1035977.3223147318, "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.144 }, { "step": 11600.0, "elapsed_seconds": 750.9451591000002, "window_seconds": 6.338299000000006, "steps_per_second": 15.777103604610621, "samples_per_second": 8077.877045560638, "tokens_per_second": 1033968.2618317617, "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.422 }, { "step": 11700.0, "elapsed_seconds": 757.2669271999998, "window_seconds": 6.321768099999645, "steps_per_second": 15.818359423846253, "samples_per_second": 8099.000025009282, "tokens_per_second": 1036672.003201188, "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 92.0, "gpu_memory_util_percent": 64.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 198.153 }, { "step": 11800.0, "elapsed_seconds": 763.4741757000006, "window_seconds": 6.207248500000787, "steps_per_second": 16.11019761815357, "samples_per_second": 8248.421180494628, "tokens_per_second": 1055797.9111033124, "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.061 }, { "step": 11900.0, "elapsed_seconds": 769.6611085, "window_seconds": 6.18693279999934, "steps_per_second": 16.16309781157, "samples_per_second": 8275.50607952384, "tokens_per_second": 1059264.7781790516, "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.567 }, { "step": 12000.0, "elapsed_seconds": 775.7945552000006, "window_seconds": 6.133446700000604, "steps_per_second": 16.304046467052554, "samples_per_second": 8347.671791130908, "tokens_per_second": 1068501.9892647562, "process_rss_mb": 1840.9453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.66 }, { "step": 12100.0, "elapsed_seconds": 782.2494649000005, "window_seconds": 6.4549096999999165, "steps_per_second": 15.492083491113949, "samples_per_second": 7931.946747450342, "tokens_per_second": 1015289.1836736437, "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.631 }, { "step": 12200.0, "elapsed_seconds": 788.4529053000001, "window_seconds": 6.203440399999636, "steps_per_second": 16.120087169694717, "samples_per_second": 8253.484630883695, "tokens_per_second": 1056446.032753113, "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 211.31 }, { "step": 12300.0, "elapsed_seconds": 794.6300482000001, "window_seconds": 6.177142900000035, "steps_per_second": 16.1887140412438, "samples_per_second": 8288.621589116825, "tokens_per_second": 1060943.5634069536, "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.951 }, { "step": 12400.0, "elapsed_seconds": 800.8262078000002, "window_seconds": 6.196159600000101, "steps_per_second": 16.139029085047838, "samples_per_second": 8263.182891544493, "tokens_per_second": 1057687.410117695, "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.829 }, { "step": 12500.0, "elapsed_seconds": 806.9713022999999, "window_seconds": 6.145094499999686, "steps_per_second": 16.273142748253115, "samples_per_second": 8331.849087105595, "tokens_per_second": 1066476.6831495161, "process_rss_mb": 1836.77734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.15 }, { "step": 12600.0, "elapsed_seconds": 813.4641734000002, "window_seconds": 6.4928711000002295, "steps_per_second": 15.40150704670488, "samples_per_second": 7885.571607912899, "tokens_per_second": 1009353.165812851, "process_rss_mb": 1846.0, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.027 }, { "step": 12700.0, "elapsed_seconds": 819.7062971999999, "window_seconds": 6.2421237999997174, "steps_per_second": 16.02018851340381, "samples_per_second": 8202.33651886275, "tokens_per_second": 1049899.074414432, "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.977 }, { "step": 12800.0, "elapsed_seconds": 825.9462103999999, "window_seconds": 6.239913200000046, "steps_per_second": 16.02586394951764, "samples_per_second": 8205.242342153031, "tokens_per_second": 1050271.019795588, "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.837 }, { "step": 12900.0, "elapsed_seconds": 832.1121918000003, "window_seconds": 6.165981400000419, "steps_per_second": 16.21801843255531, "samples_per_second": 8303.62543746832, "tokens_per_second": 1062864.055995945, "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.192 }, { "step": 13000.0, "elapsed_seconds": 838.3269452, "window_seconds": 6.2147533999996085, "steps_per_second": 16.09074303736755, "samples_per_second": 8238.460435132185, "tokens_per_second": 1054522.9356969197, "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.426 }, { "step": 13100.0, "elapsed_seconds": 844.8318112000002, "window_seconds": 6.50486600000022, "steps_per_second": 15.37310684032486, "samples_per_second": 7871.0307022463285, "tokens_per_second": 1007491.92988753, "process_rss_mb": 1846.9375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.266 }, { "step": 13200.0, "elapsed_seconds": 851.0576606, "window_seconds": 6.225849399999788, "steps_per_second": 16.062065362519593, "samples_per_second": 8223.777465610032, "tokens_per_second": 1052643.515598084, "process_rss_mb": 1846.9375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.055 }, { "step": 13300.0, "elapsed_seconds": 857.3689303000006, "window_seconds": 6.311269700000594, "steps_per_second": 15.844672269351854, "samples_per_second": 8112.472201908149, "tokens_per_second": 1038396.4418442431, "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 203.104 }, { "step": 13400.0, "elapsed_seconds": 863.5463449000003, "window_seconds": 6.177414599999793, "steps_per_second": 16.18800201624857, "samples_per_second": 8288.257032319269, "tokens_per_second": 1060896.9001368664, "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.807 }, { "step": 13500.0, "elapsed_seconds": 869.7278009000006, "window_seconds": 6.181456000000253, "steps_per_second": 16.17741839462999, "samples_per_second": 8282.838218050554, "tokens_per_second": 1060203.291910471, "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.409 }, { "step": 13600.0, "elapsed_seconds": 876.2413363000001, "window_seconds": 6.513535399999455, "steps_per_second": 15.352645507999906, "samples_per_second": 7860.554500095952, "tokens_per_second": 1006150.9760122818, "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 208.525 }, { "step": 13700.0, "elapsed_seconds": 882.4132949, "window_seconds": 6.171958599999925, "steps_per_second": 16.20231218012402, "samples_per_second": 8295.583836223499, "tokens_per_second": 1061834.7310366079, "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.794 }, { "step": 13800.0, "elapsed_seconds": 888.6569678999999, "window_seconds": 6.243672999999944, "steps_per_second": 16.01621353328416, "samples_per_second": 8200.30132904149, "tokens_per_second": 1049638.5701173106, "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.079 }, { "step": 13900.0, "elapsed_seconds": 894.8567674999999, "window_seconds": 6.199799600000006, "steps_per_second": 16.12955360686173, "samples_per_second": 8258.331446713206, "tokens_per_second": 1057066.4251792904, "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.853 }, { "step": 14000.0, "elapsed_seconds": 901.1639231000004, "window_seconds": 6.307155600000442, "steps_per_second": 15.855007604377636, "samples_per_second": 8117.76389344135, "tokens_per_second": 1039073.7783604928, "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.711 }, { "step": 14100.0, "elapsed_seconds": 907.4035153999998, "window_seconds": 6.239592299999458, "steps_per_second": 16.0266881539694, "samples_per_second": 8205.664334832332, "tokens_per_second": 1050325.0348585385, "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.501 }, { "step": 14200.0, "elapsed_seconds": 913.5769095999999, "window_seconds": 6.173394200000075, "steps_per_second": 16.198544392321292, "samples_per_second": 8293.654728868501, "tokens_per_second": 1061587.8052951682, "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.445 }, { "step": 14300.0, "elapsed_seconds": 919.7791953000001, "window_seconds": 6.202285700000175, "steps_per_second": 16.123088299527573, "samples_per_second": 8255.021209358118, "tokens_per_second": 1056642.714797839, "process_rss_mb": 1841.09375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.363 }, { "step": 14400.0, "elapsed_seconds": 926.1523864000005, "window_seconds": 6.37319110000044, "steps_per_second": 15.690726738131717, "samples_per_second": 8033.652089923439, "tokens_per_second": 1028307.4675102002, "process_rss_mb": 1841.09375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.859 }, { "step": 14500.0, "elapsed_seconds": 932.3401135999998, "window_seconds": 6.1877271999992445, "steps_per_second": 16.161022741922462, "samples_per_second": 8274.4436438643, "tokens_per_second": 1059128.7864146305, "process_rss_mb": 1841.09375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.6 }, { "step": 14600.0, "elapsed_seconds": 938.5920563000004, "window_seconds": 6.251942700000654, "steps_per_second": 15.995028233382486, "samples_per_second": 8189.454455491833, "tokens_per_second": 1048250.1703029546, "process_rss_mb": 1842.29296875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.288 }, { "step": 14700.0, "elapsed_seconds": 945.0286621, "window_seconds": 6.436605799999597, "steps_per_second": 15.536138627598765, "samples_per_second": 7954.502977330568, "tokens_per_second": 1018176.3810983127, "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 205.245 }, { "step": 14800.0, "elapsed_seconds": 951.2371669999998, "window_seconds": 6.2085048999997525, "steps_per_second": 16.106937436741653, "samples_per_second": 8246.751967611726, "tokens_per_second": 1055584.251854301, "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.874 }, { "step": 14900.0, "elapsed_seconds": 957.4043817000006, "window_seconds": 6.167214700000841, "steps_per_second": 16.214775204759185, "samples_per_second": 8301.964904836703, "tokens_per_second": 1062651.507819098, "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.964 }, { "step": 15000.0, "elapsed_seconds": 963.5301908000006, "window_seconds": 6.1258090999999695, "steps_per_second": 16.324374195728772, "samples_per_second": 8358.079588213132, "tokens_per_second": 1069834.1872912808, "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.454 }, { "step": 15100.0, "elapsed_seconds": 969.9937376000003, "window_seconds": 6.463546799999676, "steps_per_second": 15.471381749723697, "samples_per_second": 7921.347455858533, "tokens_per_second": 1013932.4743498922, "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.736 }, { "step": 15200.0, "elapsed_seconds": 976.2274011000004, "window_seconds": 6.233663500000148, "steps_per_second": 16.041931041031912, "samples_per_second": 8213.468693008339, "tokens_per_second": 1051323.9927050674, "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.854 }, { "step": 15300.0, "elapsed_seconds": 982.5118895999999, "window_seconds": 6.284488499999497, "steps_per_second": 15.912193967736275, "samples_per_second": 8147.043311480973, "tokens_per_second": 1042821.5438695645, "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 221.726 }, { "step": 15400.0, "elapsed_seconds": 988.6915613000006, "window_seconds": 6.179671700000654, "steps_per_second": 16.18208941423044, "samples_per_second": 8285.229780085985, "tokens_per_second": 1060509.411851006, "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.836 }, { "step": 15500.0, "elapsed_seconds": 994.8421758, "window_seconds": 6.150614499999392, "steps_per_second": 16.25853806965302, "samples_per_second": 8324.371491662347, "tokens_per_second": 1065519.5509327804, "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.404 }, { "step": 15600.0, "elapsed_seconds": 1001.3160514000006, "window_seconds": 6.473875600000611, "steps_per_second": 15.446697801853123, "samples_per_second": 7908.709274548799, "tokens_per_second": 1012314.7871422463, "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.271 }, { "step": 15700.0, "elapsed_seconds": 1007.5345473000002, "window_seconds": 6.2184958999996525, "steps_per_second": 16.081059086974005, "samples_per_second": 8233.50225253069, "tokens_per_second": 1053888.2883239284, "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.467 }, { "step": 15800.0, "elapsed_seconds": 1013.6817511000004, "window_seconds": 6.14720380000017, "steps_per_second": 16.267558918413805, "samples_per_second": 8328.990166227868, "tokens_per_second": 1066110.741277167, "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.21 }, { "step": 15900.0, "elapsed_seconds": 1019.9108327999998, "window_seconds": 6.2290816999993694, "steps_per_second": 16.053730680721387, "samples_per_second": 8219.51010852935, "tokens_per_second": 1052097.2938917568, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.818 }, { "step": 16000.0, "elapsed_seconds": 1026.0182086999998, "window_seconds": 6.107375900000079, "steps_per_second": 16.37364420290533, "samples_per_second": 8383.305831887528, "tokens_per_second": 1073063.1464816036, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.931 }, { "step": 16100.0, "elapsed_seconds": 1032.5023281999997, "window_seconds": 6.484119499999906, "steps_per_second": 15.422294422550578, "samples_per_second": 7896.214744345896, "tokens_per_second": 1010715.4872762747, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.928 }, { "step": 16200.0, "elapsed_seconds": 1038.6612810000001, "window_seconds": 6.158952800000407, "steps_per_second": 16.236526443260516, "samples_per_second": 8313.101538949384, "tokens_per_second": 1064076.9969855212, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.127 }, { "step": 16300.0, "elapsed_seconds": 1044.9991601000006, "window_seconds": 6.337879100000464, "steps_per_second": 15.778148876331938, "samples_per_second": 8078.4122246819525, "tokens_per_second": 1034036.7647592899, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.471 }, { "step": 16400.0, "elapsed_seconds": 1051.1497615000003, "window_seconds": 6.150601399999687, "steps_per_second": 16.258572698273877, "samples_per_second": 8324.389221516225, "tokens_per_second": 1065521.8203540768, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.167 }, { "step": 16500.0, "elapsed_seconds": 1057.5669788, "window_seconds": 6.41721729999972, "steps_per_second": 15.583078353915857, "samples_per_second": 7978.536117204919, "tokens_per_second": 1021252.6230022296, "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 2.0, "gpu_memory_util_percent": 2.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 202.148 }, { "step": 16600.0, "elapsed_seconds": 1063.8088315999998, "window_seconds": 6.241852799999833, "steps_per_second": 16.02088405545268, "samples_per_second": 8202.692636391772, "tokens_per_second": 1049944.6574581468, "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.486 }, { "step": 16700.0, "elapsed_seconds": 1069.9894651, "window_seconds": 6.180633500000113, "steps_per_second": 16.179571236508067, "samples_per_second": 8283.94047309213, "tokens_per_second": 1060344.3805557927, "process_rss_mb": 1836.96875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.486 }, { "step": 16800.0, "elapsed_seconds": 1076.0881381000004, "window_seconds": 6.098673000000417, "steps_per_second": 16.397009644555983, "samples_per_second": 8395.268938012663, "tokens_per_second": 1074594.4240656209, "process_rss_mb": 1841.1640625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.79 }, { "step": 16900.0, "elapsed_seconds": 1082.2608765, "window_seconds": 6.172738399999616, "steps_per_second": 16.200265347387184, "samples_per_second": 8294.535857862238, "tokens_per_second": 1061700.5898063665, "process_rss_mb": 1841.1796875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.783 }, { "step": 17000.0, "elapsed_seconds": 1088.6661049, "window_seconds": 6.405228399999942, "steps_per_second": 15.61224577097062, "samples_per_second": 7993.469834736958, "tokens_per_second": 1023164.1388463306, "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 206.759 }, { "step": 17100.0, "elapsed_seconds": 1094.9959791000006, "window_seconds": 6.329874200000631, "steps_per_second": 15.798102275079973, "samples_per_second": 8088.628364840946, "tokens_per_second": 1035344.4306996411, "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.513 }, { "step": 17200.0, "elapsed_seconds": 1101.1489482000002, "window_seconds": 6.152969099999609, "steps_per_second": 16.252316300435567, "samples_per_second": 8321.18594582301, "tokens_per_second": 1065111.8010653453, "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.116 }, { "step": 17300.0, "elapsed_seconds": 1107.3340062000002, "window_seconds": 6.185058000000026, "steps_per_second": 16.16799713115052, "samples_per_second": 8278.014531149067, "tokens_per_second": 1059585.8599870806, "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.397 }, { "step": 17400.0, "elapsed_seconds": 1113.5613736, "window_seconds": 6.227367399999821, "steps_per_second": 16.05815002981884, "samples_per_second": 8221.772815267246, "tokens_per_second": 1052386.9203542075, "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 209.307 }, { "step": 17500.0, "elapsed_seconds": 1119.9404208, "window_seconds": 6.379047200000059, "steps_per_second": 15.676322319734375, "samples_per_second": 8026.277027704, "tokens_per_second": 1027363.459546112, "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 207.851 }, { "step": 17600.0, "elapsed_seconds": 1126.1975831000004, "window_seconds": 6.257162300000346, "steps_per_second": 15.981685499830247, "samples_per_second": 8182.6229759130865, "tokens_per_second": 1047375.7409168751, "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.792 }, { "step": 17700.0, "elapsed_seconds": 1132.4045486999999, "window_seconds": 6.206965599999421, "steps_per_second": 16.1109318859459, "samples_per_second": 8248.7971256043, "tokens_per_second": 1055846.0320773504, "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.769 }, { "step": 17800.0, "elapsed_seconds": 1138.7792679, "window_seconds": 6.374719200000072, "steps_per_second": 15.68696547449476, "samples_per_second": 8031.726322941317, "tokens_per_second": 1028060.9693364886, "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.011 }, { "step": 17900.0, "elapsed_seconds": 1144.9590196000004, "window_seconds": 6.179751700000452, "steps_per_second": 16.181879928928648, "samples_per_second": 8285.122523611468, "tokens_per_second": 1060495.683022268, "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 208.388 }, { "step": 18000.0, "elapsed_seconds": 1151.2880605000005, "window_seconds": 6.329040900000109, "steps_per_second": 15.800182299343062, "samples_per_second": 8089.693337263648, "tokens_per_second": 1035480.7471697469, "process_rss_mb": 1841.46875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.769 }, { "step": 18100.0, "elapsed_seconds": 1157.5272288, "window_seconds": 6.239168299999619, "steps_per_second": 16.02777729204806, "samples_per_second": 8206.221973528607, "tokens_per_second": 1050396.4126116617, "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.449 }, { "step": 18200.0, "elapsed_seconds": 1163.6821913999997, "window_seconds": 6.154962599999635, "steps_per_second": 16.247052419133453, "samples_per_second": 8318.490838596328, "tokens_per_second": 1064766.82734033, "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.699 }, { "step": 18300.0, "elapsed_seconds": 1170.0860607, "window_seconds": 6.403869300000224, "steps_per_second": 15.6155591745129, "samples_per_second": 7995.1662973506045, "tokens_per_second": 1023381.2860608774, "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.019 }, { "step": 18400.0, "elapsed_seconds": 1176.2011317000001, "window_seconds": 6.115071000000171, "steps_per_second": 16.353039891114463, "samples_per_second": 8372.756424250605, "tokens_per_second": 1071712.8223040774, "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.517 }, { "step": 18500.0, "elapsed_seconds": 1182.3467949, "window_seconds": 6.1456631999999445, "steps_per_second": 16.271636883713526, "samples_per_second": 8331.078084461325, "tokens_per_second": 1066377.9948110497, "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.739 }, { "step": 18600.0, "elapsed_seconds": 1188.5754350000002, "window_seconds": 6.228640100000121, "steps_per_second": 16.0548688629478, "samples_per_second": 8220.092857829273, "tokens_per_second": 1052171.885802147, "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.657 }, { "step": 18700.0, "elapsed_seconds": 1194.9182524000003, "window_seconds": 6.342817400000058, "steps_per_second": 15.765864550979993, "samples_per_second": 8072.122650101756, "tokens_per_second": 1033231.6992130248, "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.241 }, { "step": 18800.0, "elapsed_seconds": 1201.0507309999998, "window_seconds": 6.132478599999558, "steps_per_second": 16.306620295422995, "samples_per_second": 8348.989591256573, "tokens_per_second": 1068670.6676808414, "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.962 }, { "step": 18900.0, "elapsed_seconds": 1207.2244426000002, "window_seconds": 6.173711600000388, "steps_per_second": 16.197711600262267, "samples_per_second": 8293.22833933428, "tokens_per_second": 1061533.227434788, "process_rss_mb": 1841.09765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.588 }, { "step": 19000.0, "elapsed_seconds": 1213.4391042000002, "window_seconds": 6.2146615999999995, "steps_per_second": 16.090980722104, "samples_per_second": 8238.582129717248, "tokens_per_second": 1054538.5126038077, "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.522 }, { "step": 19100.0, "elapsed_seconds": 1219.8708360000005, "window_seconds": 6.431731800000307, "steps_per_second": 15.547911994712718, "samples_per_second": 7960.530941292912, "tokens_per_second": 1018947.9604854927, "process_rss_mb": 1841.09765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.887 }, { "step": 19200.0, "elapsed_seconds": 1225.9997607, "window_seconds": 6.128924699999516, "steps_per_second": 16.316075803641038, "samples_per_second": 8353.830811464211, "tokens_per_second": 1069290.343867419, "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.308 }, { "step": 19300.0, "elapsed_seconds": 1232.3276240999999, "window_seconds": 6.327863399999842, "steps_per_second": 15.803122425177904, "samples_per_second": 8091.198681691087, "tokens_per_second": 1035673.4312564591, "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.109 }, { "step": 19400.0, "elapsed_seconds": 1238.4775437000003, "window_seconds": 6.14991960000043, "steps_per_second": 16.26037517628572, "samples_per_second": 8325.312090258289, "tokens_per_second": 1065639.947553061, "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.032 }, { "step": 19500.0, "elapsed_seconds": 1244.8447456, "window_seconds": 6.3672018999996, "steps_per_second": 15.705485952943675, "samples_per_second": 8041.208807907162, "tokens_per_second": 1029274.7274121167, "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.009 }, { "step": 19600.0, "elapsed_seconds": 1251.1243773, "window_seconds": 6.279631700000209, "steps_per_second": 15.92450079516553, "samples_per_second": 8153.344407124751, "tokens_per_second": 1043628.0841119682, "process_rss_mb": 1842.21484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.599 }, { "step": 19700.0, "elapsed_seconds": 1257.3108266000004, "window_seconds": 6.186449300000277, "steps_per_second": 16.164361033395284, "samples_per_second": 8276.152849098386, "tokens_per_second": 1059347.5646845934, "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.729 }, { "step": 19800.0, "elapsed_seconds": 1263.4479332000005, "window_seconds": 6.137106600000152, "steps_per_second": 16.294323452031538, "samples_per_second": 8342.693607440147, "tokens_per_second": 1067864.7817523389, "process_rss_mb": 1840.96484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.654 }, { "step": 19900.0, "elapsed_seconds": 1269.7032304999998, "window_seconds": 6.255297299999256, "steps_per_second": 15.986450396212485, "samples_per_second": 8185.062602860792, "tokens_per_second": 1047688.0131661814, "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 55.0, "gpu_memory_util_percent": 38.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 208.063 }, { "step": 20000.0, "elapsed_seconds": 1276.3866174999998, "window_seconds": 6.683387000000039, "steps_per_second": 14.962473368667625, "samples_per_second": 7660.786364757824, "tokens_per_second": 980580.6546890015, "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 204.752 }, { "step": 20100.0, "elapsed_seconds": 1283.3304402999997, "window_seconds": 6.943822799999907, "steps_per_second": 14.40128915732143, "samples_per_second": 7373.460048548573, "tokens_per_second": 943802.8862142173, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.72 }, { "step": 20200.0, "elapsed_seconds": 1290.0269410000005, "window_seconds": 6.696500700000797, "steps_per_second": 14.933172485144084, "samples_per_second": 7645.784312393771, "tokens_per_second": 978660.3919864027, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 202.753 }, { "step": 20300.0, "elapsed_seconds": 1296.9948632, "window_seconds": 6.967922199999521, "steps_per_second": 14.35148056044697, "samples_per_second": 7347.958046948848, "tokens_per_second": 940538.6300094526, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 205.906 }, { "step": 20400.0, "elapsed_seconds": 1303.2297758999998, "window_seconds": 6.234912699999768, "steps_per_second": 16.038716949477692, "samples_per_second": 8211.823078132578, "tokens_per_second": 1051113.35400097, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.309 }, { "step": 20500.0, "elapsed_seconds": 1309.6146435000005, "window_seconds": 6.384867600000689, "steps_per_second": 15.662031895538322, "samples_per_second": 8018.960330515621, "tokens_per_second": 1026426.9223059994, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.101 }, { "step": 20600.0, "elapsed_seconds": 1315.9102124000005, "window_seconds": 6.295568900000035, "steps_per_second": 15.88418800404193, "samples_per_second": 8132.704258069468, "tokens_per_second": 1040986.1450328919, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.189 }, { "step": 20700.0, "elapsed_seconds": 1322.1155768000003, "window_seconds": 6.20536439999978, "steps_per_second": 16.11508906713094, "samples_per_second": 8250.92560237104, "tokens_per_second": 1056118.4771034932, "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.66796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.534 }, { "step": 20800.0, "elapsed_seconds": 1328.2462650000002, "window_seconds": 6.130688199999895, "steps_per_second": 16.311382464370265, "samples_per_second": 8351.427821757576, "tokens_per_second": 1068982.7611849697, "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.425 } ], "background_sample_count": 1317, "background_samples": [ { "process_rss_mb": 1477.0, "cuda_allocated_mb": 3087.12255859375, "cuda_reserved_mb": 3232.0, "cuda_max_allocated_mb": 3197.85400390625, "cuda_max_reserved_mb": 3232.0, "gpu_util_percent": 4.0, "gpu_memory_util_percent": 1.0, "gpu_memory_used_mb": 5246.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 43.0, "gpu_power_w": 39.826, "elapsed_seconds": 1.0588439000002836 }, { "process_rss_mb": 1815.6953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 55.0, "gpu_power_w": 128.324, "elapsed_seconds": 2.0653787000001103 }, { "process_rss_mb": 1815.953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 56.0, "gpu_power_w": 196.367, "elapsed_seconds": 3.0765872000001764 }, { "process_rss_mb": 1816.015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 57.0, "gpu_power_w": 205.422, "elapsed_seconds": 4.087784300000294 }, { "process_rss_mb": 1816.015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 46.0, "gpu_power_w": 117.748, "elapsed_seconds": 5.099099000000024 }, { "process_rss_mb": 1816.08203125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 56.0, "gpu_power_w": 195.371, "elapsed_seconds": 6.110950900000262 }, { "process_rss_mb": 1816.14453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 58.0, "gpu_power_w": 207.73, "elapsed_seconds": 7.123562900000252 }, { "process_rss_mb": 1822.25, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 204.654, "elapsed_seconds": 8.135382100000243 }, { "process_rss_mb": 1832.55859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 57.0, "gpu_power_w": 90.707, "elapsed_seconds": 9.144094100000075 }, { "process_rss_mb": 1832.6328125, "cuda_allocated_mb": 2667.0126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 203.928, "elapsed_seconds": 10.151817800000117 }, { "process_rss_mb": 1832.63671875, "cuda_allocated_mb": 260.5029296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 212.727, "elapsed_seconds": 11.162295399999948 }, { "process_rss_mb": 1847.98046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 51.0, "gpu_power_w": 169.268, "elapsed_seconds": 12.166819800000212 }, { "process_rss_mb": 1832.71484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 163.673, "elapsed_seconds": 13.174661400000332 }, { "process_rss_mb": 1832.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 210.983, "elapsed_seconds": 14.181690300000355 }, { "process_rss_mb": 1832.77734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 59.0, "gpu_power_w": 211.89, "elapsed_seconds": 15.193586900000355 }, { "process_rss_mb": 1832.77734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 42.0, "gpu_memory_util_percent": 30.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 56.0, "gpu_power_w": 114.181, "elapsed_seconds": 16.206443500000205 }, { "process_rss_mb": 1832.8125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 209.388, "elapsed_seconds": 17.21214420000024 }, { "process_rss_mb": 1832.81640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 204.15, "elapsed_seconds": 18.224111900000025 }, { "process_rss_mb": 1841.98828125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 59.0, "gpu_power_w": 192.774, "elapsed_seconds": 19.23556950000011 }, { "process_rss_mb": 1832.828125, "cuda_allocated_mb": 1576.42041015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 59.0, "gpu_power_w": 90.645, "elapsed_seconds": 20.249188600000252 }, { "process_rss_mb": 1832.859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 210.686, "elapsed_seconds": 21.260575900000276 }, { "process_rss_mb": 1832.859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 209.231, "elapsed_seconds": 22.27378090000002 }, { "process_rss_mb": 1832.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 53.0, "gpu_power_w": 167.274, "elapsed_seconds": 23.29759690000037 }, { "process_rss_mb": 1832.890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 57.0, "gpu_memory_util_percent": 40.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 156.484, "elapsed_seconds": 24.299879200000305 }, { "process_rss_mb": 1832.96875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 211.169, "elapsed_seconds": 25.310374200000297 }, { "process_rss_mb": 1839.07421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 64.0, "gpu_power_w": 214.297, "elapsed_seconds": 26.32383430000027 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 1271.6103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 7.0, "gpu_memory_util_percent": 5.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 52.0, "gpu_power_w": 118.872, "elapsed_seconds": 27.333900900000117 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 209.904, "elapsed_seconds": 28.349133100000017 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 64.0, "gpu_power_w": 213.404, "elapsed_seconds": 29.358234900000298 }, { "process_rss_mb": 1848.23828125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 50.0, "gpu_memory_util_percent": 35.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 55.0, "gpu_power_w": 194.204, "elapsed_seconds": 30.36370150000039 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 134.38, "elapsed_seconds": 31.380809500000396 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 214.495, "elapsed_seconds": 32.38991560000022 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 209.292, "elapsed_seconds": 33.402320000000145 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 53.0, "gpu_power_w": 136.657, "elapsed_seconds": 34.41180370000029 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 2408.890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 206.899, "elapsed_seconds": 35.42008699999997 }, { "process_rss_mb": 1832.97265625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 215.576, "elapsed_seconds": 36.43388600000026 }, { "process_rss_mb": 1848.23828125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 54.0, "gpu_power_w": 140.902, "elapsed_seconds": 37.44490089999999 }, { "process_rss_mb": 1842.34375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 64.0, "gpu_power_w": 167.942, "elapsed_seconds": 38.4988556000003 }, { "process_rss_mb": 1838.47265625, "cuda_allocated_mb": 400.66357421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 170.064, "elapsed_seconds": 39.49958749999996 }, { "process_rss_mb": 1838.47265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 213.591, "elapsed_seconds": 40.50871260000031 }, { "process_rss_mb": 1838.47265625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 54.0, "gpu_power_w": 135.445, "elapsed_seconds": 41.522361300000284 }, { "process_rss_mb": 1838.47265625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 202.176, "elapsed_seconds": 42.68779230000018 }, { "process_rss_mb": 1838.4140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 189.931, "elapsed_seconds": 43.69476650000024 }, { "process_rss_mb": 1844.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 214.682, "elapsed_seconds": 44.70853510000006 }, { "process_rss_mb": 1838.4140625, "cuda_allocated_mb": 1977.5234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 130.19, "elapsed_seconds": 45.720372600000246 }, { "process_rss_mb": 1838.40234375, "cuda_allocated_mb": 2472.76513671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 216.066, "elapsed_seconds": 46.73164110000016 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 214.427, "elapsed_seconds": 47.74076220000006 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 56.0, "gpu_power_w": 161.577, "elapsed_seconds": 48.75268730000016 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 176.756, "elapsed_seconds": 49.766948400000274 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 214.42, "elapsed_seconds": 50.7803286000003 }, { "process_rss_mb": 1844.3828125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 212.94, "elapsed_seconds": 51.794866300000194 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 120.4111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 125.606, "elapsed_seconds": 52.80323970000018 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 213.22, "elapsed_seconds": 53.8170554000003 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 213.757, "elapsed_seconds": 54.83032309999999 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 57.0, "gpu_power_w": 179.581, "elapsed_seconds": 55.84177780000027 }, { "process_rss_mb": 1838.27734375, "cuda_allocated_mb": 138.1806640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 150.874, "elapsed_seconds": 56.85250010000027 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 214.11, "elapsed_seconds": 57.864999400000215 }, { "process_rss_mb": 1844.3671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 208.516, "elapsed_seconds": 58.877296300000125 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 120.643, "elapsed_seconds": 59.890534800000296 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 203.595, "elapsed_seconds": 60.895921199999975 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 92.0, "gpu_memory_util_percent": 64.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 218.123, "elapsed_seconds": 61.90245820000018 }, { "process_rss_mb": 1853.52734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 207.74, "elapsed_seconds": 62.912675800000216 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 1287.984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 131.178, "elapsed_seconds": 63.922915699999976 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 211.085, "elapsed_seconds": 64.93188240000018 }, { "process_rss_mb": 1844.3671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 210.091, "elapsed_seconds": 65.94270830000005 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 125.335, "elapsed_seconds": 66.95745720000014 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 191.101, "elapsed_seconds": 67.9721185000003 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 464.78857421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 213.363, "elapsed_seconds": 68.9761211 }, { "process_rss_mb": 1847.421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 58.0, "gpu_power_w": 174.729, "elapsed_seconds": 69.98649909999995 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 161.456, "elapsed_seconds": 70.99823620000006 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 219.671, "elapsed_seconds": 72.00650180000002 }, { "process_rss_mb": 1838.26171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 212.49, "elapsed_seconds": 73.01812670000027 }, { "process_rss_mb": 1838.26953125, "cuda_allocated_mb": 2345.0185546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 127.697, "elapsed_seconds": 74.03739610000002 }, { "process_rss_mb": 1838.26953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 213.308, "elapsed_seconds": 75.04154229999995 }, { "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 189.84, "elapsed_seconds": 76.06009650000033 }, { "process_rss_mb": 1850.26953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 47.0, "gpu_memory_util_percent": 33.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 64.0, "gpu_power_w": 211.756, "elapsed_seconds": 77.06903330000023 }, { "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 2268.978515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 123.322, "elapsed_seconds": 78.0798592000001 }, { "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 212.395, "elapsed_seconds": 79.09310159999995 }, { "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 211.607, "elapsed_seconds": 80.10264210000014 }, { "process_rss_mb": 1859.4296875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 58.0, "gpu_power_w": 163.102, "elapsed_seconds": 81.11667140000009 }, { "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 172.719, "elapsed_seconds": 82.12956880000002 }, { "process_rss_mb": 1844.1640625, "cuda_allocated_mb": 3121.9833984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 216.816, "elapsed_seconds": 83.1434704000003 }, { "process_rss_mb": 1839.234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 213.062, "elapsed_seconds": 84.15443390000019 }, { "process_rss_mb": 1839.234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 115.13, "elapsed_seconds": 85.16730959999995 }, { "process_rss_mb": 1839.234375, "cuda_allocated_mb": 2954.5068359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 215.341, "elapsed_seconds": 86.17566600000009 }, { "process_rss_mb": 1839.296875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 217.452, "elapsed_seconds": 87.19748819999995 }, { "process_rss_mb": 1854.5625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 191.629, "elapsed_seconds": 88.20032489999994 }, { "process_rss_mb": 1839.296875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 154.379, "elapsed_seconds": 89.21077860000014 }, { "process_rss_mb": 1839.296875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 209.776, "elapsed_seconds": 90.22235620000038 }, { "process_rss_mb": 1839.296875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 220.328, "elapsed_seconds": 91.24107380000032 }, { "process_rss_mb": 1839.30078125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 56.0, "gpu_memory_util_percent": 39.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 126.199, "elapsed_seconds": 92.24839190000012 }, { "process_rss_mb": 1839.30078125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 214.167, "elapsed_seconds": 93.26066340000034 }, { "process_rss_mb": 1839.30078125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 212.446, "elapsed_seconds": 94.27848900000026 }, { "process_rss_mb": 1848.47265625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 63.0, "gpu_memory_util_percent": 44.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 210.254, "elapsed_seconds": 95.28198280000015 }, { "process_rss_mb": 1839.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 126.329, "elapsed_seconds": 96.29313890000003 }, { "process_rss_mb": 1839.3125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.588, "elapsed_seconds": 97.3103258000001 }, { "process_rss_mb": 1839.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 216.545, "elapsed_seconds": 98.31946300000027 }, { "process_rss_mb": 1839.3125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 59.0, "gpu_power_w": 161.584, "elapsed_seconds": 99.33486640000001 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 185.637, "elapsed_seconds": 100.3429059 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 2345.0185546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 219.136, "elapsed_seconds": 101.35734610000009 }, { "process_rss_mb": 1845.36328125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 214.571, "elapsed_seconds": 102.36818310000035 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 123.948, "elapsed_seconds": 103.38242740000032 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 22.0, "gpu_memory_util_percent": 16.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 62.0, "gpu_power_w": 205.729, "elapsed_seconds": 104.51358600000003 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 1785.54638671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 190.011, "elapsed_seconds": 105.52302570000029 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 58.0, "gpu_power_w": 138.105, "elapsed_seconds": 106.53395450000016 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 965.98828125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 213.631, "elapsed_seconds": 107.5460320000002 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 221.227, "elapsed_seconds": 108.55737780000027 }, { "process_rss_mb": 1845.36328125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 24.0, "gpu_memory_util_percent": 17.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 202.639, "elapsed_seconds": 109.57012669999995 }, { "process_rss_mb": 1838.5078125, "cuda_allocated_mb": 1351.984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 123.69, "elapsed_seconds": 110.58259700000008 }, { "process_rss_mb": 1839.2578125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 211.505, "elapsed_seconds": 111.59079060000022 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 7.0, "gpu_memory_util_percent": 4.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 199.619, "elapsed_seconds": 112.60284530000035 }, { "process_rss_mb": 1859.64453125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 174.158, "elapsed_seconds": 113.61450360000026 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 169.537, "elapsed_seconds": 114.63217430000032 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.103, "elapsed_seconds": 115.63984410000012 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.18212890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.267, "elapsed_seconds": 116.65238799999997 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 129.018, "elapsed_seconds": 117.66626580000002 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 214.717, "elapsed_seconds": 118.68157210000027 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 1271.609375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.653, "elapsed_seconds": 119.68853050000007 }, { "process_rss_mb": 1850.484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 13.0, "gpu_memory_util_percent": 9.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 63.0, "gpu_power_w": 204.409, "elapsed_seconds": 120.69792800000005 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 115.004, "elapsed_seconds": 121.70812250000017 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 1351.984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 216.846, "elapsed_seconds": 122.7208028 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.035, "elapsed_seconds": 123.72851160000027 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 164.409, "elapsed_seconds": 124.74118339999995 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 173.754, "elapsed_seconds": 125.75384460000032 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 61.0, "gpu_power_w": 198.248, "elapsed_seconds": 126.76781329999994 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 210.4, "elapsed_seconds": 127.7751003000003 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 46.0, "gpu_memory_util_percent": 32.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 135.455, "elapsed_seconds": 128.78585160000011 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 207.509, "elapsed_seconds": 129.79613880000034 }, { "process_rss_mb": 1844.37890625, "cuda_allocated_mb": 2472.76416015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 92.0, "gpu_memory_util_percent": 65.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.983, "elapsed_seconds": 130.80985439999995 }, { "process_rss_mb": 1854.15234375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.502, "elapsed_seconds": 131.82053410000026 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 129.1, "elapsed_seconds": 132.8321261000001 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 220.211, "elapsed_seconds": 133.84237640000038 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 183.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.208, "elapsed_seconds": 134.8539165000002 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 151.081, "elapsed_seconds": 135.86233800000036 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 193.476, "elapsed_seconds": 136.87782650000008 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 1849.671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 218.239, "elapsed_seconds": 137.88827750000019 }, { "process_rss_mb": 1848.046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 219.801, "elapsed_seconds": 138.8996500000003 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 121.929, "elapsed_seconds": 139.9127314000002 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.322, "elapsed_seconds": 140.92550759999995 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 219.203, "elapsed_seconds": 141.93658559999994 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 130.445, "elapsed_seconds": 142.94908329999998 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 187.504, "elapsed_seconds": 143.96423390000018 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 2542.22998046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.598, "elapsed_seconds": 144.96501070000022 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.518, "elapsed_seconds": 145.97475780000013 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 140.272, "elapsed_seconds": 146.98776429999998 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 217.114, "elapsed_seconds": 147.99688400000014 }, { "process_rss_mb": 1838.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.769, "elapsed_seconds": 149.00561450000032 }, { "process_rss_mb": 1859.6484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 60.0, "gpu_power_w": 142.814, "elapsed_seconds": 150.01718870000013 }, { "process_rss_mb": 1839.390625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 185.066, "elapsed_seconds": 151.03133760000037 }, { "process_rss_mb": 1839.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.943, "elapsed_seconds": 152.03553839999995 }, { "process_rss_mb": 1839.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 222.382, "elapsed_seconds": 153.04940750000014 }, { "process_rss_mb": 1835.66015625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 136.963, "elapsed_seconds": 154.06368410000005 }, { "process_rss_mb": 1835.66015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 216.034, "elapsed_seconds": 155.07422840000027 }, { "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 218.598, "elapsed_seconds": 156.08648240000002 }, { "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 1785.54638671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 212.528, "elapsed_seconds": 157.10045780000019 }, { "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 221.245, "elapsed_seconds": 158.10951990000012 }, { "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.788, "elapsed_seconds": 159.1218428000002 }, { "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.073, "elapsed_seconds": 160.13319040000033 }, { "process_rss_mb": 1840.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.928, "elapsed_seconds": 161.14159410000002 }, { "process_rss_mb": 1840.42578125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 62.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 215.928, "elapsed_seconds": 162.1536804000002 }, { "process_rss_mb": 1840.45703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 217.846, "elapsed_seconds": 163.1626308 }, { "process_rss_mb": 1840.45703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 219.679, "elapsed_seconds": 164.1713896000001 }, { "process_rss_mb": 1840.45703125, "cuda_allocated_mb": 340.37451171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 59.0, "gpu_memory_util_percent": 41.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 213.262, "elapsed_seconds": 165.18709160000026 }, { "process_rss_mb": 1840.45703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.581, "elapsed_seconds": 166.1983335 }, { "process_rss_mb": 1840.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 222.034, "elapsed_seconds": 167.19902930000035 }, { "process_rss_mb": 1840.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 217.32, "elapsed_seconds": 168.21167160000005 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 3034.505859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 189.925, "elapsed_seconds": 169.2209581000002 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 218.585, "elapsed_seconds": 170.2345819000002 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.833, "elapsed_seconds": 171.2459374 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 1719.4755859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 217.266, "elapsed_seconds": 172.25774769999998 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 218.558, "elapsed_seconds": 173.267781 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 91.0, "gpu_memory_util_percent": 64.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 224.006, "elapsed_seconds": 174.28379270000005 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 247.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.845, "elapsed_seconds": 175.29767850000007 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.389, "elapsed_seconds": 176.3106426000004 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 217.066, "elapsed_seconds": 177.322365 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 1255.484375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 217.631, "elapsed_seconds": 178.3324676000002 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 219.936, "elapsed_seconds": 179.34354890000031 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 1735.4775390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.978, "elapsed_seconds": 180.35736440000028 }, { "process_rss_mb": 1839.95703125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 214.874, "elapsed_seconds": 181.37061710000035 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 194.055, "elapsed_seconds": 182.37993670000014 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 223.554, "elapsed_seconds": 183.39152150000018 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 2618.63818359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.516, "elapsed_seconds": 184.4005889 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 221.21, "elapsed_seconds": 185.40279439999995 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.282, "elapsed_seconds": 186.4115856000003 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.841, "elapsed_seconds": 187.42432589999999 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 218.723, "elapsed_seconds": 188.43876290000026 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 1703.47802734375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 194.752, "elapsed_seconds": 189.44853220000005 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.852, "elapsed_seconds": 190.46232559999999 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.332, "elapsed_seconds": 191.4799823000003 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.728, "elapsed_seconds": 192.4830433000002 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 1319.48046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 219.038, "elapsed_seconds": 193.49664410000014 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.11, "elapsed_seconds": 194.5088641000002 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 121.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.555, "elapsed_seconds": 195.5208365000003 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 546.9140625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.214, "elapsed_seconds": 196.53711020000037 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 220.641, "elapsed_seconds": 197.54884270000002 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 2472.7646484375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.549, "elapsed_seconds": 198.55483570000024 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 220.463, "elapsed_seconds": 199.56950470000038 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 1734.9775390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.51, "elapsed_seconds": 200.57942400000002 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.606, "elapsed_seconds": 201.59563500000013 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.699, "elapsed_seconds": 202.60650780000014 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 198.202, "elapsed_seconds": 203.61891960000003 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 196.1298828125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.571, "elapsed_seconds": 204.6308600000002 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.153, "elapsed_seconds": 205.64536859999998 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.768, "elapsed_seconds": 206.6572355000003 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 205.885, "elapsed_seconds": 207.66262430000006 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 372.5, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.145, "elapsed_seconds": 208.6700492 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 221.765, "elapsed_seconds": 209.67653390000032 }, { "process_rss_mb": 1855.2265625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.294, "elapsed_seconds": 210.6862218000001 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.432, "elapsed_seconds": 211.69980910000004 }, { "process_rss_mb": 1839.9609375, "cuda_allocated_mb": 980.41650390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.446, "elapsed_seconds": 212.7098145 }, { "process_rss_mb": 1840.34765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 204.889, "elapsed_seconds": 213.71793400000024 }, { "process_rss_mb": 1840.34765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.487, "elapsed_seconds": 214.72599880000007 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 57.0, "gpu_memory_util_percent": 40.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.961, "elapsed_seconds": 215.73639980000007 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 2408.89111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.054, "elapsed_seconds": 216.74369030000025 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.913, "elapsed_seconds": 217.76541480000014 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 220.415, "elapsed_seconds": 218.77746700000034 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.737, "elapsed_seconds": 219.7866479000004 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 2667.0126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 194.12, "elapsed_seconds": 220.80488690000038 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 217.135, "elapsed_seconds": 221.81125760000032 }, { "process_rss_mb": 1839.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 220.443, "elapsed_seconds": 222.82456580000007 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.217, "elapsed_seconds": 223.8348772000004 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 965.98876953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.605, "elapsed_seconds": 224.84631639999998 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.552, "elapsed_seconds": 225.86045190000004 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 213.975, "elapsed_seconds": 226.86996829999998 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.241, "elapsed_seconds": 227.88129560000016 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.235, "elapsed_seconds": 228.89242140000033 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 2041.525390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 219.175, "elapsed_seconds": 229.9038585000003 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 219.609, "elapsed_seconds": 230.91736570000012 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.091, "elapsed_seconds": 231.93118970000023 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.639, "elapsed_seconds": 232.94451630000003 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 121.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.746, "elapsed_seconds": 233.95737120000013 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.032, "elapsed_seconds": 234.96958320000022 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.082, "elapsed_seconds": 235.98378500000035 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 1093.0419921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.297, "elapsed_seconds": 236.99319790000027 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.956, "elapsed_seconds": 238.00485620000018 }, { "process_rss_mb": 1839.16796875, "cuda_allocated_mb": 550.12255859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.784, "elapsed_seconds": 239.01687110000012 }, { "process_rss_mb": 1839.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.764, "elapsed_seconds": 240.02870630000007 }, { "process_rss_mb": 1839.23046875, "cuda_allocated_mb": 1849.671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 192.845, "elapsed_seconds": 241.03887170000007 }, { "process_rss_mb": 1839.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.817, "elapsed_seconds": 242.05505250000033 }, { "process_rss_mb": 1839.23046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.64, "elapsed_seconds": 243.0647269000001 }, { "process_rss_mb": 1839.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.828, "elapsed_seconds": 244.0772895 }, { "process_rss_mb": 1844.98828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 204.502, "elapsed_seconds": 245.0889298000002 }, { "process_rss_mb": 1840.30859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.505, "elapsed_seconds": 246.1029893 }, { "process_rss_mb": 1840.30859375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.797, "elapsed_seconds": 247.1159348000001 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.876, "elapsed_seconds": 248.1217112000004 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 122.20751953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.295, "elapsed_seconds": 249.1313210000003 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.241, "elapsed_seconds": 250.1443578000003 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.337, "elapsed_seconds": 251.1554744 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.891, "elapsed_seconds": 252.1693908000002 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.545, "elapsed_seconds": 253.18066970000018 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 217.163, "elapsed_seconds": 254.19232760000023 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 2041.021484375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.598, "elapsed_seconds": 255.19750750000003 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 193.269, "elapsed_seconds": 256.20711329999995 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.829, "elapsed_seconds": 257.2204135000002 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 372.24951171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.135, "elapsed_seconds": 258.23283040000024 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 219.255, "elapsed_seconds": 259.24474690000034 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.729, "elapsed_seconds": 260.2550739000003 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 965.990234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.136, "elapsed_seconds": 261.2651278000003 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.474, "elapsed_seconds": 262.27735280000024 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.153, "elapsed_seconds": 263.288552 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 218.194, "elapsed_seconds": 264.30157290000034 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.338, "elapsed_seconds": 265.3125387 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 260.5029296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.039, "elapsed_seconds": 266.32375990000037 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.403, "elapsed_seconds": 267.3321369 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.506, "elapsed_seconds": 268.3452516000002 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.788, "elapsed_seconds": 269.35970780000025 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 200.759, "elapsed_seconds": 270.3706567000004 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.503, "elapsed_seconds": 271.38217910000003 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.759, "elapsed_seconds": 272.39134030000014 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.799, "elapsed_seconds": 273.4018860000001 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 1093.73681640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.892, "elapsed_seconds": 274.41036170000007 }, { "process_rss_mb": 1840.3125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.117, "elapsed_seconds": 275.4224543 }, { "process_rss_mb": 1840.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 196.857, "elapsed_seconds": 276.4357123 }, { "process_rss_mb": 1840.34375, "cuda_allocated_mb": 2762.5087890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.159, "elapsed_seconds": 277.4528031 }, { "process_rss_mb": 1840.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.539, "elapsed_seconds": 278.4592428000001 }, { "process_rss_mb": 1840.34375, "cuda_allocated_mb": 2446.1044921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 220.438, "elapsed_seconds": 279.4657371000003 }, { "process_rss_mb": 1840.34375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.853, "elapsed_seconds": 280.47953929999994 }, { "process_rss_mb": 1840.359375, "cuda_allocated_mb": 276.5009765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.967, "elapsed_seconds": 281.49212400000033 }, { "process_rss_mb": 1840.359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.571, "elapsed_seconds": 282.5037390000002 }, { "process_rss_mb": 1840.359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.525, "elapsed_seconds": 283.5141758 }, { "process_rss_mb": 1836.62890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.176, "elapsed_seconds": 284.51945939999996 }, { "process_rss_mb": 1835.87890625, "cuda_allocated_mb": 3050.52197265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 219.8, "elapsed_seconds": 285.5406912000003 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 59.0, "gpu_memory_util_percent": 40.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 208.992, "elapsed_seconds": 286.55678350000017 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.908, "elapsed_seconds": 287.5787755000001 }, { "process_rss_mb": 1841.484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 90.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.245, "elapsed_seconds": 288.58090820000007 }, { "process_rss_mb": 1841.484375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.418, "elapsed_seconds": 289.5943427000002 }, { "process_rss_mb": 1840.921875, "cuda_allocated_mb": 183.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.825, "elapsed_seconds": 290.6051215000002 }, { "process_rss_mb": 1840.921875, "cuda_allocated_mb": 1189.6103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.045, "elapsed_seconds": 291.6211166000003 }, { "process_rss_mb": 1840.921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 59.0, "gpu_memory_util_percent": 41.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.98, "elapsed_seconds": 292.63234090000014 }, { "process_rss_mb": 1840.921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 191.762, "elapsed_seconds": 293.6438393000003 }, { "process_rss_mb": 1840.921875, "cuda_allocated_mb": 138.2158203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.622, "elapsed_seconds": 294.6564797000001 }, { "process_rss_mb": 1840.921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.555, "elapsed_seconds": 295.6677732000003 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.742, "elapsed_seconds": 296.67651950000027 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 65.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.446, "elapsed_seconds": 297.6877891000004 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.018, "elapsed_seconds": 298.7001610000002 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.117, "elapsed_seconds": 299.7121258000002 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.328, "elapsed_seconds": 300.72533550000026 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.636, "elapsed_seconds": 301.73633640000025 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.243, "elapsed_seconds": 302.7472477000001 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 90.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.353, "elapsed_seconds": 303.75533510000014 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.397, "elapsed_seconds": 304.7653157 }, { "process_rss_mb": 1840.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.127, "elapsed_seconds": 305.7783437000003 }, { "process_rss_mb": 1841.51171875, "cuda_allocated_mb": 2075.9765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.071, "elapsed_seconds": 306.78846999999996 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 193.405, "elapsed_seconds": 307.79472460000034 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 2043.9765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.936, "elapsed_seconds": 308.80373320000035 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.969, "elapsed_seconds": 309.81166120000034 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.766, "elapsed_seconds": 310.8228670000003 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.725, "elapsed_seconds": 311.8306990000001 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.973, "elapsed_seconds": 312.8445578000001 }, { "process_rss_mb": 1839.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.992, "elapsed_seconds": 313.8584133000004 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.368, "elapsed_seconds": 314.87107200000037 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.267, "elapsed_seconds": 315.8827093 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.905, "elapsed_seconds": 316.89275 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.284, "elapsed_seconds": 317.9016144000002 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 436.24951171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.268, "elapsed_seconds": 318.91139580000026 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 55.0, "gpu_memory_util_percent": 38.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 187.569, "elapsed_seconds": 319.92474340000035 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 2954.5068359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.918, "elapsed_seconds": 320.9346936000002 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.759, "elapsed_seconds": 321.9440241000002 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.998, "elapsed_seconds": 322.9553242000002 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.542, "elapsed_seconds": 323.96932620000007 }, { "process_rss_mb": 1839.69921875, "cuda_allocated_mb": 120.4736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.317, "elapsed_seconds": 324.9799734000003 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.89, "elapsed_seconds": 325.98902769999995 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 218.769, "elapsed_seconds": 326.9976835000002 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 1913.79736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.753, "elapsed_seconds": 328.0103388000002 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.502, "elapsed_seconds": 329.0251817000003 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 218.878, "elapsed_seconds": 330.0324915000001 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 219.193, "elapsed_seconds": 331.04246230000035 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.743, "elapsed_seconds": 332.05610320000005 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.896, "elapsed_seconds": 333.0710280000003 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.225, "elapsed_seconds": 334.08138069999995 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.18603515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.249, "elapsed_seconds": 335.0930526000002 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 64.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.242, "elapsed_seconds": 336.1065201000001 }, { "process_rss_mb": 1839.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.861, "elapsed_seconds": 337.1178600000003 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 20.0, "gpu_memory_util_percent": 13.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 203.63, "elapsed_seconds": 338.1271772 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 3121.9833984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.224, "elapsed_seconds": 339.1412637000003 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.92, "elapsed_seconds": 340.1547800000003 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.064, "elapsed_seconds": 341.1657376000003 }, { "process_rss_mb": 1839.37890625, "cuda_allocated_mb": 400.6630859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.381, "elapsed_seconds": 342.1752566 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 1655.4775390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.137, "elapsed_seconds": 343.1854363000002 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.004, "elapsed_seconds": 344.19376090000014 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.493, "elapsed_seconds": 345.2012158000002 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 197.427, "elapsed_seconds": 346.2115312000001 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 220.5, "elapsed_seconds": 347.22531370000024 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.241, "elapsed_seconds": 348.2374777 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 210.518, "elapsed_seconds": 349.2517355 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.135, "elapsed_seconds": 350.25980720000007 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.353, "elapsed_seconds": 351.26820080000016 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 1223.6103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.59, "elapsed_seconds": 352.2813908000003 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.245, "elapsed_seconds": 353.2955637 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.892, "elapsed_seconds": 354.30806870000015 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.534, "elapsed_seconds": 355.3206061000001 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 1271.29296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.718, "elapsed_seconds": 356.33859329999996 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.798, "elapsed_seconds": 357.3464498000003 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 210.15, "elapsed_seconds": 358.3582759000001 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.567, "elapsed_seconds": 359.37055050000026 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 197.651, "elapsed_seconds": 360.38068569999996 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.158, "elapsed_seconds": 361.3917027000002 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.838, "elapsed_seconds": 362.4084335000002 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.263, "elapsed_seconds": 363.41205720000016 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.76, "elapsed_seconds": 364.4222014000002 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.702, "elapsed_seconds": 365.43300480000016 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.504, "elapsed_seconds": 366.44336220000014 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.72, "elapsed_seconds": 367.45494210000015 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 1221.29296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.28, "elapsed_seconds": 368.4642441000001 }, { "process_rss_mb": 1845.06640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 47.0, "gpu_memory_util_percent": 32.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 199.145, "elapsed_seconds": 369.47353880000037 }, { "process_rss_mb": 1845.06640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 210.238, "elapsed_seconds": 370.48518730000023 }, { "process_rss_mb": 1845.06640625, "cuda_allocated_mb": 183.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.088, "elapsed_seconds": 371.49731500000007 }, { "process_rss_mb": 1845.06640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.132, "elapsed_seconds": 372.5090035000003 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.83, "elapsed_seconds": 373.5188902 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.79, "elapsed_seconds": 374.5331433000001 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 191.987, "elapsed_seconds": 375.54428780000035 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.153, "elapsed_seconds": 376.5559599000003 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.042, "elapsed_seconds": 377.56753000000026 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.637, "elapsed_seconds": 378.5784805000003 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 125.97412109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.85, "elapsed_seconds": 379.58847620000006 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.292, "elapsed_seconds": 380.60400359999994 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 1029.990234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 91.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.18, "elapsed_seconds": 381.6162134000001 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 2656.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.011, "elapsed_seconds": 382.6293433000001 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.913, "elapsed_seconds": 383.6410331000002 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 125.72314453125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.191, "elapsed_seconds": 384.655166 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.835, "elapsed_seconds": 385.6665068000002 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.829, "elapsed_seconds": 386.6761845000001 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.573, "elapsed_seconds": 387.6923815 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 1575.478515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.63, "elapsed_seconds": 388.70113310000033 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.588, "elapsed_seconds": 389.7117304000003 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.663, "elapsed_seconds": 390.7231463000003 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.095, "elapsed_seconds": 391.7371111000002 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.433, "elapsed_seconds": 392.7477699000001 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 209.854, "elapsed_seconds": 393.7603865000001 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.979, "elapsed_seconds": 394.77301060000036 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.762, "elapsed_seconds": 395.7818688000002 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 203.77783203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 194.739, "elapsed_seconds": 396.8099671 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 965.990234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.039, "elapsed_seconds": 397.8284445000004 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.06, "elapsed_seconds": 398.83289319999994 }, { "process_rss_mb": 1840.13671875, "cuda_allocated_mb": 404.24951171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.324, "elapsed_seconds": 399.84345200000007 }, { "process_rss_mb": 1845.06640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.074, "elapsed_seconds": 400.85727160000033 }, { "process_rss_mb": 1845.125, "cuda_allocated_mb": 216.3291015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 205.417, "elapsed_seconds": 401.87105820000033 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 886.49072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.317, "elapsed_seconds": 402.8840344 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 1093.86083984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 209.262, "elapsed_seconds": 403.89791890000015 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.924, "elapsed_seconds": 404.9104265000001 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.073, "elapsed_seconds": 405.9225044 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.852, "elapsed_seconds": 406.92953130000024 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.695, "elapsed_seconds": 407.94077830000015 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 206.939, "elapsed_seconds": 408.95078030000013 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.623, "elapsed_seconds": 409.9565401 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.463, "elapsed_seconds": 410.9700087000001 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.892, "elapsed_seconds": 411.9842824000002 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.232, "elapsed_seconds": 412.9932852000002 }, { "process_rss_mb": 1839.7578125, "cuda_allocated_mb": 1576.42041015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.256, "elapsed_seconds": 414.0055179000001 }, { "process_rss_mb": 1836.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.191, "elapsed_seconds": 415.0167779000003 }, { "process_rss_mb": 1836.02734375, "cuda_allocated_mb": 1335.6103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.506, "elapsed_seconds": 416.02584260000003 }, { "process_rss_mb": 1840.28125, "cuda_allocated_mb": 376.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.75, "elapsed_seconds": 417.03638150000006 }, { "process_rss_mb": 1840.28125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.52, "elapsed_seconds": 418.0462854000002 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 196.1298828125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.08, "elapsed_seconds": 419.0568509 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.316, "elapsed_seconds": 420.0645351000003 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.807, "elapsed_seconds": 421.0777154000002 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.056, "elapsed_seconds": 422.0857919 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.659, "elapsed_seconds": 423.0969990000003 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 121.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 187.63, "elapsed_seconds": 424.1043761000001 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.07, "elapsed_seconds": 425.1169404000002 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 3034.505859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.356, "elapsed_seconds": 426.1285896000004 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.03, "elapsed_seconds": 427.1343667000001 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.637, "elapsed_seconds": 428.1452730000001 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 209.809, "elapsed_seconds": 429.1545132000001 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.526, "elapsed_seconds": 430.1630647000002 }, { "process_rss_mb": 1839.94140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.696, "elapsed_seconds": 431.1739467000002 }, { "process_rss_mb": 1845.3203125, "cuda_allocated_mb": 138.21044921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 18.0, "gpu_memory_util_percent": 11.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 205.445, "elapsed_seconds": 432.1852106000001 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 2155.2763671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 206.696, "elapsed_seconds": 433.1954155000003 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 63.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 210.94, "elapsed_seconds": 434.20571180000024 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.715, "elapsed_seconds": 435.21873570000025 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 125.22119140625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.573, "elapsed_seconds": 436.2281183 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.129, "elapsed_seconds": 437.239517 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.544, "elapsed_seconds": 438.25060170000006 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 2762.5087890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.307, "elapsed_seconds": 439.2590981000003 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.429, "elapsed_seconds": 440.2703911000003 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.661, "elapsed_seconds": 441.28295700000035 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 324.5029296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.508, "elapsed_seconds": 442.29629120000027 }, { "process_rss_mb": 1840.390625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 216.645, "elapsed_seconds": 443.3067597000004 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.634, "elapsed_seconds": 444.31029370000033 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 205.122, "elapsed_seconds": 445.3211865000003 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.08, "elapsed_seconds": 446.33293570000023 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.832, "elapsed_seconds": 447.34651600000007 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 500.24853515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.039, "elapsed_seconds": 448.3578673000002 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.677, "elapsed_seconds": 449.37132150000025 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.885, "elapsed_seconds": 450.38704000000007 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.324, "elapsed_seconds": 451.3950850000001 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.866, "elapsed_seconds": 452.4073978000001 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 91.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.768, "elapsed_seconds": 453.4214476000002 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 206.719, "elapsed_seconds": 454.43294600000036 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 191.637, "elapsed_seconds": 455.44304809999994 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.954, "elapsed_seconds": 456.4544016 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.079, "elapsed_seconds": 457.46710570000005 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.758, "elapsed_seconds": 458.47930940000015 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.378, "elapsed_seconds": 459.48005009999997 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.774, "elapsed_seconds": 460.49316309999995 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 2736.48095703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.773, "elapsed_seconds": 461.50147640000023 }, { "process_rss_mb": 1840.453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.182, "elapsed_seconds": 462.5134000000003 }, { "process_rss_mb": 1845.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 203.186, "elapsed_seconds": 463.523099 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.76, "elapsed_seconds": 464.5314777000003 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.348, "elapsed_seconds": 465.5427361000002 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.652, "elapsed_seconds": 466.5541095000003 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.975, "elapsed_seconds": 467.5630949000001 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.021, "elapsed_seconds": 468.57645660000026 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.166, "elapsed_seconds": 469.5913071 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 210.633, "elapsed_seconds": 470.6028785000003 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 885.9912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.66, "elapsed_seconds": 471.61455730000034 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.157, "elapsed_seconds": 472.62428260000024 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 76.0, "gpu_power_w": 219.512, "elapsed_seconds": 473.6382859 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 180.913, "elapsed_seconds": 474.65065730000015 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.912, "elapsed_seconds": 475.65909360000023 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.011, "elapsed_seconds": 476.6699856 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.20263671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.161, "elapsed_seconds": 477.68301480000036 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.896, "elapsed_seconds": 478.6921805000002 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.202, "elapsed_seconds": 479.70829040000035 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5536.04296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.46, "elapsed_seconds": 480.71434960000033 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5536.10546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.171, "elapsed_seconds": 481.7237177000002 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5536.10546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.946, "elapsed_seconds": 482.7330856000003 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 1815.22412109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.316, "elapsed_seconds": 483.74061870000014 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 2509.47900390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.824, "elapsed_seconds": 484.74848820000034 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.613, "elapsed_seconds": 485.7591249000002 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.376, "elapsed_seconds": 486.7689673 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 1093.86083984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.075, "elapsed_seconds": 487.78189710000015 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.854, "elapsed_seconds": 488.7872212000002 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.027, "elapsed_seconds": 489.7958230000004 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.523, "elapsed_seconds": 490.8040672000002 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 90.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.302, "elapsed_seconds": 491.8149721 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.655, "elapsed_seconds": 492.8219390000004 }, { "process_rss_mb": 1840.46875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.745, "elapsed_seconds": 493.83247960000017 }, { "process_rss_mb": 1840.5078125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 65.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 184.835, "elapsed_seconds": 494.8459122000004 }, { "process_rss_mb": 1840.99609375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 193.743, "elapsed_seconds": 495.8499084 }, { "process_rss_mb": 1840.99609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 218.243, "elapsed_seconds": 496.86095539999997 }, { "process_rss_mb": 1840.99609375, "cuda_allocated_mb": 3167.0068359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.003, "elapsed_seconds": 497.8722262000001 }, { "process_rss_mb": 1840.99609375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.822, "elapsed_seconds": 498.88583770000014 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.292, "elapsed_seconds": 499.89769509999996 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.816, "elapsed_seconds": 500.91051860000016 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 1961.15087890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.093, "elapsed_seconds": 501.92098350000015 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.557, "elapsed_seconds": 502.93234280000024 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.579, "elapsed_seconds": 503.9472490000003 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.352, "elapsed_seconds": 504.95835460000035 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.937, "elapsed_seconds": 505.96754750000036 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.423, "elapsed_seconds": 506.9811995 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.546, "elapsed_seconds": 507.99648830000024 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 195.274, "elapsed_seconds": 509.0107330000001 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.391, "elapsed_seconds": 510.02253169999994 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.822, "elapsed_seconds": 511.0353143000002 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.656, "elapsed_seconds": 512.0478395 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.89, "elapsed_seconds": 513.0601909000002 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.384, "elapsed_seconds": 514.0660199000004 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 3034.505859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.215, "elapsed_seconds": 515.0777302000001 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.164, "elapsed_seconds": 516.0909491000002 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.358, "elapsed_seconds": 517.1013853 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 222.284, "elapsed_seconds": 518.1027016000003 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.341, "elapsed_seconds": 519.1089676000001 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 312.037109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.062, "elapsed_seconds": 520.1231641000004 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 3033.0576171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 209.843, "elapsed_seconds": 521.1342946 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.903, "elapsed_seconds": 522.1462866000002 }, { "process_rss_mb": 1839.875, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 199.426, "elapsed_seconds": 523.1603349000002 }, { "process_rss_mb": 1839.9140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.794, "elapsed_seconds": 524.1733395000001 }, { "process_rss_mb": 1839.9140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 210.443, "elapsed_seconds": 525.1830217000002 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 1093.0419921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.527, "elapsed_seconds": 526.1965753 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 203.047, "elapsed_seconds": 527.2084580000001 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 582.1220703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.651, "elapsed_seconds": 528.2209506000004 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.326, "elapsed_seconds": 529.2309766000003 }, { "process_rss_mb": 1860.48046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.221, "elapsed_seconds": 530.2447119000003 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 209.656, "elapsed_seconds": 531.2568817000001 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.597, "elapsed_seconds": 532.2700803000002 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 211.198, "elapsed_seconds": 533.2834174 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.465, "elapsed_seconds": 534.2953749000003 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.842, "elapsed_seconds": 535.3084943000003 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 3114.505859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.082, "elapsed_seconds": 536.3167432 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.59, "elapsed_seconds": 537.3288521 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.291, "elapsed_seconds": 538.3390631000002 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.819, "elapsed_seconds": 539.3492443 }, { "process_rss_mb": 1845.21484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.022, "elapsed_seconds": 540.3562890000003 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.98046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.949, "elapsed_seconds": 541.3694130000003 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5536.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.451, "elapsed_seconds": 542.3866676000002 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5536.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 195.918, "elapsed_seconds": 543.3930318000002 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 2688.48095703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5536.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.26, "elapsed_seconds": 544.3980350000002 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.989, "elapsed_seconds": 545.4092077 }, { "process_rss_mb": 1841.5, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.871, "elapsed_seconds": 546.4251354000003 }, { "process_rss_mb": 1841.5, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.488, "elapsed_seconds": 547.4352252000003 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 2961.48291015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.141, "elapsed_seconds": 548.4508758000002 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.493, "elapsed_seconds": 549.4573101000001 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 93.0, "gpu_memory_util_percent": 65.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.152, "elapsed_seconds": 550.4658545000002 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 264.41943359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.099, "elapsed_seconds": 551.4770570000001 }, { "process_rss_mb": 1845.23046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.543, "elapsed_seconds": 552.4858904000002 }, { "process_rss_mb": 1841.30078125, "cuda_allocated_mb": 264.4814453125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.087, "elapsed_seconds": 553.4950103000001 }, { "process_rss_mb": 1841.30078125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.463, "elapsed_seconds": 554.5059708000003 }, { "process_rss_mb": 1841.30078125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.259, "elapsed_seconds": 555.5192316000002 }, { "process_rss_mb": 1841.30078125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.59, "elapsed_seconds": 556.5294706 }, { "process_rss_mb": 1846.2734375, "cuda_allocated_mb": 1977.525390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 201.565, "elapsed_seconds": 557.5320801000003 }, { "process_rss_mb": 1846.2734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.871, "elapsed_seconds": 558.6542543 }, { "process_rss_mb": 1846.2734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 193.421, "elapsed_seconds": 559.6635119000002 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.406, "elapsed_seconds": 560.6744084000002 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.966, "elapsed_seconds": 561.6873137000002 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.639, "elapsed_seconds": 562.6967718000001 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.801, "elapsed_seconds": 563.7088371 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 3114.4833984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.4, "elapsed_seconds": 564.7218467000002 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.937, "elapsed_seconds": 565.7340601000001 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 207.846, "elapsed_seconds": 566.7467588 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 1783.22412109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.383, "elapsed_seconds": 567.7623997000001 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.272, "elapsed_seconds": 568.7742145000002 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.413, "elapsed_seconds": 569.7884005000001 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.548, "elapsed_seconds": 570.8005249000003 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 2606.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.91796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 206.845, "elapsed_seconds": 571.8131025000002 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.377, "elapsed_seconds": 572.8310766 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.58, "elapsed_seconds": 573.8338863000004 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.632, "elapsed_seconds": 574.8453385000003 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.764, "elapsed_seconds": 575.8509613000001 }, { "process_rss_mb": 1841.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.919, "elapsed_seconds": 576.8612998000003 }, { "process_rss_mb": 1841.359375, "cuda_allocated_mb": 514.78857421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.45, "elapsed_seconds": 577.8755276000002 }, { "process_rss_mb": 1841.359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.915, "elapsed_seconds": 578.8911392 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 190.53, "elapsed_seconds": 579.9060113 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.736, "elapsed_seconds": 580.9165993000001 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 210.923, "elapsed_seconds": 581.9334419000002 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 92.0, "gpu_memory_util_percent": 65.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.29, "elapsed_seconds": 582.9433993000002 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.573, "elapsed_seconds": 583.9509503000004 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.112, "elapsed_seconds": 584.9646593000002 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 183.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.564, "elapsed_seconds": 585.9792580000003 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.084, "elapsed_seconds": 586.9903623 }, { "process_rss_mb": 1841.421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.451, "elapsed_seconds": 588.0018655000003 }, { "process_rss_mb": 1846.359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 201.064, "elapsed_seconds": 589.0163264000003 }, { "process_rss_mb": 1846.359375, "cuda_allocated_mb": 2478.1044921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.389, "elapsed_seconds": 590.0297275000003 }, { "process_rss_mb": 1846.359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.094, "elapsed_seconds": 591.0431607 }, { "process_rss_mb": 1846.359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.98, "elapsed_seconds": 592.0563482000002 }, { "process_rss_mb": 1846.359375, "cuda_allocated_mb": 138.1826171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.951, "elapsed_seconds": 593.0814061000001 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.988, "elapsed_seconds": 594.0838244000001 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.347, "elapsed_seconds": 595.0929983999999 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.861, "elapsed_seconds": 596.1067781000002 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.006, "elapsed_seconds": 597.1101798 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 2762.5087890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.274, "elapsed_seconds": 598.1193681 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.131, "elapsed_seconds": 599.1275938000003 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 1576.42041015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.194, "elapsed_seconds": 600.1391662000001 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.871, "elapsed_seconds": 601.1415622 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.437, "elapsed_seconds": 602.1532186 }, { "process_rss_mb": 1846.34375, "cuda_allocated_mb": 1719.35009765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.473, "elapsed_seconds": 603.1692654000003 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.412, "elapsed_seconds": 604.1806424000001 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 1751.35009765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.127, "elapsed_seconds": 605.1952271 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 3165.69921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 209.981, "elapsed_seconds": 606.2092227000003 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.281, "elapsed_seconds": 607.2181643000004 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.733, "elapsed_seconds": 608.2302297000001 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.42, "elapsed_seconds": 609.2397415 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 198.756, "elapsed_seconds": 610.2510128000004 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.387, "elapsed_seconds": 611.2636121 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.743, "elapsed_seconds": 612.2739422 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 1287.48046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.48, "elapsed_seconds": 613.2873838 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.22216796875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.009, "elapsed_seconds": 614.2951911 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.124, "elapsed_seconds": 615.3044481000002 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.709, "elapsed_seconds": 616.3187238 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.691, "elapsed_seconds": 617.3287474000003 }, { "process_rss_mb": 1840.109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.934, "elapsed_seconds": 618.3357655 }, { "process_rss_mb": 1853.3515625, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.293, "elapsed_seconds": 619.3477830000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 201.657, "elapsed_seconds": 620.3617773000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 63.0, "gpu_memory_util_percent": 44.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.399, "elapsed_seconds": 621.3739361000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 1963.85107421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.612, "elapsed_seconds": 622.3886713000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.51, "elapsed_seconds": 623.3989678000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 1995.9765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.721, "elapsed_seconds": 624.4196024000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.643, "elapsed_seconds": 625.4224631000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.943, "elapsed_seconds": 626.4346390000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.766, "elapsed_seconds": 627.4446229 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.471, "elapsed_seconds": 628.4578163000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.837, "elapsed_seconds": 629.4714111000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 190.484, "elapsed_seconds": 630.482974 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.883, "elapsed_seconds": 631.4947991000004 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.1826171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.762, "elapsed_seconds": 632.4960905000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.641, "elapsed_seconds": 633.5081658000004 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 1125.86181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.041, "elapsed_seconds": 634.5197627000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.316, "elapsed_seconds": 635.5283887 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.847, "elapsed_seconds": 636.5410871000004 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 1013.99072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.586, "elapsed_seconds": 637.5531581 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.402, "elapsed_seconds": 638.5647478000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.699, "elapsed_seconds": 639.5765623000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.292, "elapsed_seconds": 640.5882643 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.231, "elapsed_seconds": 641.5977737000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 1383.41845703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.564, "elapsed_seconds": 642.6045133000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.886, "elapsed_seconds": 643.6175979 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 194.573, "elapsed_seconds": 644.6287381000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.8, "elapsed_seconds": 645.6424860000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.691, "elapsed_seconds": 646.6572871000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 2440.890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.898, "elapsed_seconds": 647.6652360000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.064, "elapsed_seconds": 648.6750199000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.732, "elapsed_seconds": 649.6885853000003 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.693, "elapsed_seconds": 650.6990513000001 }, { "process_rss_mb": 1845.546875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 200.127, "elapsed_seconds": 651.7127154 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.871, "elapsed_seconds": 652.7241554000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.998, "elapsed_seconds": 653.7381359000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.941, "elapsed_seconds": 654.7636821000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 121.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.961, "elapsed_seconds": 655.7740806000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.978, "elapsed_seconds": 656.7882822000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 2408.890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.533, "elapsed_seconds": 657.8059951 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.355, "elapsed_seconds": 658.8124247000001 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 2446.1044921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.259, "elapsed_seconds": 659.8257876000002 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.082, "elapsed_seconds": 660.8322444 }, { "process_rss_mb": 1840.6171875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.935, "elapsed_seconds": 661.8448471000002 }, { "process_rss_mb": 1840.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.242, "elapsed_seconds": 662.8578355 }, { "process_rss_mb": 1840.6796875, "cuda_allocated_mb": 2618.63818359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.012, "elapsed_seconds": 663.8658579000003 }, { "process_rss_mb": 1840.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.383, "elapsed_seconds": 664.8780855 }, { "process_rss_mb": 1840.6796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.642, "elapsed_seconds": 665.8900151000003 }, { "process_rss_mb": 1840.6796875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.116, "elapsed_seconds": 666.8963223000001 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 198.13134765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.094, "elapsed_seconds": 667.9076055 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 121.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.114, "elapsed_seconds": 668.9175115000003 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.108, "elapsed_seconds": 669.9300341000003 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 1849.671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.961, "elapsed_seconds": 670.9460858000002 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 198.1328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 186.291, "elapsed_seconds": 671.9516973 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.3, "elapsed_seconds": 672.9617578000002 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.477, "elapsed_seconds": 673.9725472 }, { "process_rss_mb": 1840.7421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.683, "elapsed_seconds": 674.9855183 }, { "process_rss_mb": 1837.01171875, "cuda_allocated_mb": 196.2529296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.51, "elapsed_seconds": 675.996967 }, { "process_rss_mb": 1837.01171875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.182, "elapsed_seconds": 677.0144986 }, { "process_rss_mb": 1837.01171875, "cuda_allocated_mb": 122.20751953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.223, "elapsed_seconds": 678.0228242000003 }, { "process_rss_mb": 1841.1875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.536, "elapsed_seconds": 679.0385722000001 }, { "process_rss_mb": 1841.1875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.047, "elapsed_seconds": 680.0485186000001 }, { "process_rss_mb": 1841.1875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.196, "elapsed_seconds": 681.0646102000001 }, { "process_rss_mb": 1846.24609375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.938, "elapsed_seconds": 682.0752231000001 }, { "process_rss_mb": 1846.24609375, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 200.3, "elapsed_seconds": 683.0933275000002 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.486, "elapsed_seconds": 684.0939827000002 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.198, "elapsed_seconds": 685.1062856000003 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.88, "elapsed_seconds": 686.1198967 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.216, "elapsed_seconds": 687.1301277000002 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.615, "elapsed_seconds": 688.1444292000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.385, "elapsed_seconds": 689.1520762 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.519, "elapsed_seconds": 690.1589947000002 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.461, "elapsed_seconds": 691.1730818999999 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.152, "elapsed_seconds": 692.1877402 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.419, "elapsed_seconds": 693.1984609000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.811, "elapsed_seconds": 694.2091165000002 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.422, "elapsed_seconds": 695.2211261000002 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.009, "elapsed_seconds": 696.2341201000004 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.956, "elapsed_seconds": 697.2463715000004 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 550.12255859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 211.72, "elapsed_seconds": 698.2569290000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.237, "elapsed_seconds": 699.2697362000004 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.856, "elapsed_seconds": 700.2752651000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.139, "elapsed_seconds": 701.2893272000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.772, "elapsed_seconds": 702.3032431000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.186, "elapsed_seconds": 703.3145918 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.22216796875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.859, "elapsed_seconds": 704.3228346000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.901, "elapsed_seconds": 705.3337304000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 198.6318359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 44.0, "gpu_memory_util_percent": 31.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 193.037, "elapsed_seconds": 706.3435411 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.695, "elapsed_seconds": 707.3566877000003 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.131, "elapsed_seconds": 708.3693152000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 1013.99072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.25, "elapsed_seconds": 709.3821432 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.128, "elapsed_seconds": 710.3976222000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.414, "elapsed_seconds": 711.4074620000001 }, { "process_rss_mb": 1840.37109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.188, "elapsed_seconds": 712.4172432 }, { "process_rss_mb": 1845.87109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.764, "elapsed_seconds": 713.4284809000001 }, { "process_rss_mb": 1845.87109375, "cuda_allocated_mb": 3165.69921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 204.377, "elapsed_seconds": 714.4388130000002 }, { "process_rss_mb": 1845.87109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.325, "elapsed_seconds": 715.4503303000001 }, { "process_rss_mb": 1845.87109375, "cuda_allocated_mb": 2440.89111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.265, "elapsed_seconds": 716.4609218 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.064, "elapsed_seconds": 717.4714584000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.1826171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.766, "elapsed_seconds": 718.4777551000002 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.095, "elapsed_seconds": 719.4869271000002 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.738, "elapsed_seconds": 720.4983918000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 1287.984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 208.727, "elapsed_seconds": 721.5090529000004 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.627, "elapsed_seconds": 722.5175448 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 210.929, "elapsed_seconds": 723.5228637 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 192.552, "elapsed_seconds": 724.5366570000001 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 1977.525390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 199.127, "elapsed_seconds": 725.5488511000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.805, "elapsed_seconds": 726.5615633000002 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.934, "elapsed_seconds": 727.5757164000001 }, { "process_rss_mb": 1840.12890625, "cuda_allocated_mb": 1287.6103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.892, "elapsed_seconds": 728.5842597000001 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 208.915, "elapsed_seconds": 729.5967596 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.559, "elapsed_seconds": 730.6103018000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.949, "elapsed_seconds": 731.6169952 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.941, "elapsed_seconds": 732.6265907000002 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.005, "elapsed_seconds": 733.6386471999999 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 464.78857421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.735, "elapsed_seconds": 734.6463346000005 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.564, "elapsed_seconds": 735.6585418 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 500.248046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.057, "elapsed_seconds": 736.6686869 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.622, "elapsed_seconds": 737.6791128999998 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.144, "elapsed_seconds": 738.6893726000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 966.11279296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 191.11, "elapsed_seconds": 739.7010812000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.846, "elapsed_seconds": 740.7089037000001 }, { "process_rss_mb": 1839.8515625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.203, "elapsed_seconds": 741.7219735000003 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.222, "elapsed_seconds": 742.7348379999999 }, { "process_rss_mb": 1840.87890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.095, "elapsed_seconds": 743.7483112000004 }, { "process_rss_mb": 1845.8125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.144, "elapsed_seconds": 744.7578911000005 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 199.959, "elapsed_seconds": 745.7687066000003 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.065, "elapsed_seconds": 746.7808371000006 }, { "process_rss_mb": 1840.1328125, "cuda_allocated_mb": 1061.8623046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 62.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.682, "elapsed_seconds": 747.7916197 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 56.0, "gpu_memory_util_percent": 39.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 204.574, "elapsed_seconds": 748.8002650000003 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 2025.1513671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.773, "elapsed_seconds": 749.8067749000006 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.422, "elapsed_seconds": 750.8289688999998 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.028, "elapsed_seconds": 751.8316245999999 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.535, "elapsed_seconds": 752.8465891000001 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.847, "elapsed_seconds": 753.8528736000003 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 209.854, "elapsed_seconds": 754.8681612000005 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.181, "elapsed_seconds": 755.8820172000001 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 24.0, "gpu_memory_util_percent": 17.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 193.119, "elapsed_seconds": 756.8964933000002 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.545, "elapsed_seconds": 757.9118702999999 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 2656.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.474, "elapsed_seconds": 758.9201775000006 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.214, "elapsed_seconds": 759.9310485999999 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.205, "elapsed_seconds": 760.9438543000001 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.407, "elapsed_seconds": 761.9572450000001 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.801, "elapsed_seconds": 762.9670632000002 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.728, "elapsed_seconds": 763.9763476000003 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.21, "elapsed_seconds": 764.9891910000001 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.149, "elapsed_seconds": 766.0001744000006 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 550.12255859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.936, "elapsed_seconds": 767.0112289000003 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.912, "elapsed_seconds": 768.0224929000001 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.417, "elapsed_seconds": 769.0320498000006 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 3101.69921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.053, "elapsed_seconds": 770.0433605000003 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.914, "elapsed_seconds": 771.0572004999999 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.832, "elapsed_seconds": 772.0697034 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.371, "elapsed_seconds": 773.0782525 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.13, "elapsed_seconds": 774.0905020000005 }, { "process_rss_mb": 1840.8828125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 219.363, "elapsed_seconds": 775.1044015000002 }, { "process_rss_mb": 1845.875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 2.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 204.642, "elapsed_seconds": 776.1211021000004 }, { "process_rss_mb": 1846.0, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.52, "elapsed_seconds": 777.1289293000004 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 192.291, "elapsed_seconds": 778.1366025000002 }, { "process_rss_mb": 1839.6953125, "cuda_allocated_mb": 661.9931640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.697, "elapsed_seconds": 779.1548636000002 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 211.564, "elapsed_seconds": 780.1584110999997 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.993, "elapsed_seconds": 781.1674115000001 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.631, "elapsed_seconds": 782.1767412000004 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 120.4736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.148, "elapsed_seconds": 783.1897866000004 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 128.73486328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.383, "elapsed_seconds": 784.2083543000003 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.081, "elapsed_seconds": 785.2121843 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.804, "elapsed_seconds": 786.2217667999998 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.317, "elapsed_seconds": 787.2312980000002 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.829, "elapsed_seconds": 788.2427491000003 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 120.4111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.868, "elapsed_seconds": 789.2538366000003 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.54, "elapsed_seconds": 790.2633695 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 3151.126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.101, "elapsed_seconds": 791.2753451000003 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 2440.89111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 208.566, "elapsed_seconds": 792.2869421999999 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.057, "elapsed_seconds": 793.3119916 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.419, "elapsed_seconds": 794.3241060000005 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 1655.4775390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.184, "elapsed_seconds": 795.3468131000004 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.475, "elapsed_seconds": 796.3489730000006 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.504, "elapsed_seconds": 797.3616271000005 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.702, "elapsed_seconds": 798.3733248000003 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.764, "elapsed_seconds": 799.3838615 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.247, "elapsed_seconds": 800.399046 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 966.11279296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.829, "elapsed_seconds": 801.4094684000006 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.4, "elapsed_seconds": 802.4183609000006 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.37, "elapsed_seconds": 803.4322470000002 }, { "process_rss_mb": 1840.4453125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.482, "elapsed_seconds": 804.4419256000006 }, { "process_rss_mb": 1840.5078125, "cuda_allocated_mb": 1849.4208984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.659, "elapsed_seconds": 805.4533420000002 }, { "process_rss_mb": 1836.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.479, "elapsed_seconds": 806.4629983 }, { "process_rss_mb": 1842.26953125, "cuda_allocated_mb": 2504.63916015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 199.652, "elapsed_seconds": 807.4666139000005 }, { "process_rss_mb": 1842.26953125, "cuda_allocated_mb": 260.5029296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.705, "elapsed_seconds": 808.4764992000005 }, { "process_rss_mb": 1846.0, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.898, "elapsed_seconds": 809.4866132000002 }, { "process_rss_mb": 1846.0, "cuda_allocated_mb": 1045.990234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.757, "elapsed_seconds": 810.4983179000005 }, { "process_rss_mb": 1846.0, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.943, "elapsed_seconds": 811.5069815000002 }, { "process_rss_mb": 1846.0, "cuda_allocated_mb": 3082.50634765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 188.025, "elapsed_seconds": 812.5165720999998 }, { "process_rss_mb": 1846.0, "cuda_allocated_mb": 120.4111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.027, "elapsed_seconds": 813.5303115000002 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 57.0, "gpu_memory_util_percent": 40.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 208.05, "elapsed_seconds": 814.5436829000005 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.955, "elapsed_seconds": 815.5551345999997 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.562, "elapsed_seconds": 816.5671693999998 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 1736.9208984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 76.0, "gpu_power_w": 215.993, "elapsed_seconds": 817.5777932000005 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.05, "elapsed_seconds": 818.5853509000003 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 372.375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.977, "elapsed_seconds": 819.6004001000006 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.57, "elapsed_seconds": 820.6132061000003 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.931, "elapsed_seconds": 821.6254562999998 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.994, "elapsed_seconds": 822.6396625000002 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.231, "elapsed_seconds": 823.6483047000006 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 207.646, "elapsed_seconds": 824.6587481000001 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.837, "elapsed_seconds": 825.6648397000004 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.565, "elapsed_seconds": 826.675103 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.864, "elapsed_seconds": 827.6832172 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 2736.48095703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.436, "elapsed_seconds": 828.6982934000002 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 336.53759765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.766, "elapsed_seconds": 829.7076296 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.489, "elapsed_seconds": 830.7162609000002 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.971, "elapsed_seconds": 831.7298223000003 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 2397.47900390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.153, "elapsed_seconds": 832.7446333000003 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 1783.22412109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.163, "elapsed_seconds": 833.7556174000006 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.228, "elapsed_seconds": 834.7670678999998 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.176, "elapsed_seconds": 835.7784745000004 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.112, "elapsed_seconds": 836.7925250000003 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.865, "elapsed_seconds": 837.8047944000004 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 1271.29296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 34.0, "gpu_memory_util_percent": 23.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 206.552, "elapsed_seconds": 838.8193470000001 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 3055.1103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 205.74, "elapsed_seconds": 839.8312742000003 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.556, "elapsed_seconds": 840.8464694999998 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.66796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.136, "elapsed_seconds": 841.8586461999998 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5535.66796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.278, "elapsed_seconds": 842.8687671000002 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 191.765, "elapsed_seconds": 843.8815902000001 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.266, "elapsed_seconds": 844.8952297000001 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 264.41845703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.025, "elapsed_seconds": 845.9055858000006 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.186, "elapsed_seconds": 846.9192335000002 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.94, "elapsed_seconds": 847.9281276000006 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.18212890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.071, "elapsed_seconds": 848.9361650000005 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 1303.41845703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 211.36, "elapsed_seconds": 849.9500005000004 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 634.623046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.055, "elapsed_seconds": 850.9585769999999 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.35, "elapsed_seconds": 851.9785377000003 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.111, "elapsed_seconds": 852.9863649000004 }, { "process_rss_mb": 1846.9375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.965, "elapsed_seconds": 854.0016200000005 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 207.231, "elapsed_seconds": 855.0118045000004 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.72, "elapsed_seconds": 856.0226970999997 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 203.104, "elapsed_seconds": 857.0348373000002 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 206.646, "elapsed_seconds": 858.0489233000003 }, { "process_rss_mb": 1841.6953125, "cuda_allocated_mb": 885.9912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.557, "elapsed_seconds": 859.0761486000006 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 209.289, "elapsed_seconds": 860.0895163999999 }, { "process_rss_mb": 1841.8203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.973, "elapsed_seconds": 861.1054993000002 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.081, "elapsed_seconds": 862.1184825000005 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 598.4970703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.358, "elapsed_seconds": 863.1293285000006 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.847, "elapsed_seconds": 864.1444241000004 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.059, "elapsed_seconds": 865.1582751999999 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.911, "elapsed_seconds": 866.1712411000003 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 215.334, "elapsed_seconds": 867.1846298000005 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.421, "elapsed_seconds": 868.1992052 }, { "process_rss_mb": 1841.0546875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.852, "elapsed_seconds": 869.2091579000003 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 200.022, "elapsed_seconds": 870.2222801000003 }, { "process_rss_mb": 1841.8046875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 211.67, "elapsed_seconds": 871.2314439000006 }, { "process_rss_mb": 1840.4921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 189.172, "elapsed_seconds": 872.2411203000006 }, { "process_rss_mb": 1840.4921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.009, "elapsed_seconds": 873.2535896000004 }, { "process_rss_mb": 1840.4921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.722, "elapsed_seconds": 874.2673756999998 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.914, "elapsed_seconds": 875.2828556999998 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 208.525, "elapsed_seconds": 876.2945519999998 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 1751.35009765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.334, "elapsed_seconds": 877.3082801000005 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.427, "elapsed_seconds": 878.3217884999999 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.168, "elapsed_seconds": 879.3377198000003 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.865, "elapsed_seconds": 880.3477716000002 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.722, "elapsed_seconds": 881.3565985999999 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 260.5029296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.794, "elapsed_seconds": 882.3693175000003 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.417, "elapsed_seconds": 883.3703541 }, { "process_rss_mb": 1839.76171875, "cuda_allocated_mb": 634.62255859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.31, "elapsed_seconds": 884.3832692000001 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 63.0, "gpu_memory_util_percent": 44.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 202.418, "elapsed_seconds": 885.3958889 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.41, "elapsed_seconds": 886.4102182000001 }, { "process_rss_mb": 1840.51171875, "cuda_allocated_mb": 1783.2236328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.601, "elapsed_seconds": 887.4231592000001 }, { "process_rss_mb": 1839.76171875, "cuda_allocated_mb": 1013.99072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.874, "elapsed_seconds": 888.4389511000004 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.224, "elapsed_seconds": 889.4503195000002 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 210.36, "elapsed_seconds": 890.4592944999999 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.338, "elapsed_seconds": 891.4600980999999 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 247.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.135, "elapsed_seconds": 892.4746111 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.629, "elapsed_seconds": 893.4861968 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 198.6318359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.853, "elapsed_seconds": 894.5017702999999 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 598.4970703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.463, "elapsed_seconds": 895.5133016000004 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.49, "elapsed_seconds": 896.5263899000006 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.27, "elapsed_seconds": 897.5366760000002 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.414, "elapsed_seconds": 898.5520163000006 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 128.73583984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.258, "elapsed_seconds": 899.5611898000002 }, { "process_rss_mb": 1840.57421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5534.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 193.928, "elapsed_seconds": 900.576869 }, { "process_rss_mb": 1846.25390625, "cuda_allocated_mb": 2536.638671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5534.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 197.151, "elapsed_seconds": 901.5894761000004 }, { "process_rss_mb": 1846.25390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5534.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 218.676, "elapsed_seconds": 902.6040377000004 }, { "process_rss_mb": 1846.25390625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.934, "elapsed_seconds": 903.6140902000002 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.207, "elapsed_seconds": 904.6209426 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.882, "elapsed_seconds": 905.6219370000003 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.231, "elapsed_seconds": 906.6313174000002 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.261, "elapsed_seconds": 907.6468606000003 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.6, "elapsed_seconds": 908.6591942 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 65.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.244, "elapsed_seconds": 909.6727807000002 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.162, "elapsed_seconds": 910.6854425000006 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 1125.86181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.408, "elapsed_seconds": 911.6978890000005 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.126, "elapsed_seconds": 912.7172716 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 550.12255859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.445, "elapsed_seconds": 913.7228928 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.674, "elapsed_seconds": 914.7355616 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 211.6, "elapsed_seconds": 915.7447742999998 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.254, "elapsed_seconds": 916.7566171000003 }, { "process_rss_mb": 1841.07421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.619, "elapsed_seconds": 917.7727607000002 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.476, "elapsed_seconds": 918.7819017000006 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 1881.4208984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.363, "elapsed_seconds": 919.7957517 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.034, "elapsed_seconds": 920.9042732000003 }, { "process_rss_mb": 1840.34375, "cuda_allocated_mb": 693.9931640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 191.691, "elapsed_seconds": 921.9093182000001 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.69, "elapsed_seconds": 922.9251792 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.568, "elapsed_seconds": 923.9366377000001 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 219.751, "elapsed_seconds": 924.9444767999998 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 3151.126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.859, "elapsed_seconds": 925.9729017000004 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.532, "elapsed_seconds": 926.9855704000006 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 90.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.163, "elapsed_seconds": 927.9993183000001 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 2606.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.727, "elapsed_seconds": 929.0138726999999 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 58.0, "gpu_memory_util_percent": 41.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 212.885, "elapsed_seconds": 930.0311704000001 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 204.054, "elapsed_seconds": 931.0423663000006 }, { "process_rss_mb": 1841.09375, "cuda_allocated_mb": 3034.505859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.6, "elapsed_seconds": 932.0587553 }, { "process_rss_mb": 1846.0234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 204.864, "elapsed_seconds": 933.0696619000005 }, { "process_rss_mb": 1846.0234375, "cuda_allocated_mb": 1575.478515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.209, "elapsed_seconds": 934.0813695000002 }, { "process_rss_mb": 1846.0234375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.111, "elapsed_seconds": 935.0956846000004 }, { "process_rss_mb": 1846.0234375, "cuda_allocated_mb": 1029.8623046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.661, "elapsed_seconds": 936.1076767 }, { "process_rss_mb": 1846.0234375, "cuda_allocated_mb": 3046.93115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.303, "elapsed_seconds": 937.1203541999998 }, { "process_rss_mb": 1842.29296875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.406, "elapsed_seconds": 938.1352097000004 }, { "process_rss_mb": 1842.29296875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.06, "elapsed_seconds": 939.1498372000001 }, { "process_rss_mb": 1841.59375, "cuda_allocated_mb": 3151.126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.391, "elapsed_seconds": 940.1643251999999 }, { "process_rss_mb": 1841.59375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.818, "elapsed_seconds": 941.1654003999997 }, { "process_rss_mb": 1841.59375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.074, "elapsed_seconds": 942.1781016999998 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.083, "elapsed_seconds": 943.1924647000001 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 1287.6103515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.732, "elapsed_seconds": 944.1931134000001 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 205.245, "elapsed_seconds": 945.2045861000001 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 64.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 194.3, "elapsed_seconds": 946.2159526 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.535, "elapsed_seconds": 947.2273465000003 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.847, "elapsed_seconds": 948.2424802999999 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.309, "elapsed_seconds": 949.2532396000001 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 221.381, "elapsed_seconds": 950.2616671000001 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.874, "elapsed_seconds": 951.2749472999999 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 120.4111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.072, "elapsed_seconds": 952.2855795 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 92.0, "gpu_memory_util_percent": 64.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.917, "elapsed_seconds": 953.2973388000005 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.03, "elapsed_seconds": 954.3097692000006 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 582.1220703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.184, "elapsed_seconds": 955.3263812 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 598.123046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 208.629, "elapsed_seconds": 956.3360575000002 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.964, "elapsed_seconds": 957.3512104000006 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.802, "elapsed_seconds": 958.3597112000002 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 203.779296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.713, "elapsed_seconds": 959.3725594000002 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.886, "elapsed_seconds": 960.3810671000006 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.022, "elapsed_seconds": 961.3986467999998 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.232, "elapsed_seconds": 962.4126032999998 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 2345.0185546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 41.0, "gpu_memory_used_mb": 5533.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.421, "elapsed_seconds": 963.4259279000003 }, { "process_rss_mb": 1841.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 194.796, "elapsed_seconds": 964.4402875999999 }, { "process_rss_mb": 1841.93359375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.63, "elapsed_seconds": 965.4557122000001 }, { "process_rss_mb": 1840.62109375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 210.696, "elapsed_seconds": 966.4715413999998 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 197.027, "elapsed_seconds": 967.4810545000005 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 1719.4765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.237, "elapsed_seconds": 968.4911852999999 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.258, "elapsed_seconds": 969.5043455999999 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.01, "elapsed_seconds": 970.51496 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.061, "elapsed_seconds": 971.5200908000002 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.025, "elapsed_seconds": 972.5366010000002 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.213, "elapsed_seconds": 973.5444087000001 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 2396.978515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.035, "elapsed_seconds": 974.5581587000001 }, { "process_rss_mb": 1839.88671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.028, "elapsed_seconds": 975.5672717000002 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 208.231, "elapsed_seconds": 976.5681197999997 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.199, "elapsed_seconds": 977.5772615000001 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.367, "elapsed_seconds": 978.5851985000004 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.611, "elapsed_seconds": 979.5998855000003 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.483, "elapsed_seconds": 980.6130969000001 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 187.758, "elapsed_seconds": 981.6274698000002 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 1303.41845703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 221.726, "elapsed_seconds": 982.6401390000005 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.18212890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.703, "elapsed_seconds": 983.6519387000003 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 1977.1513671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.183, "elapsed_seconds": 984.6636162 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.142, "elapsed_seconds": 985.6745439000001 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.339, "elapsed_seconds": 986.6873873 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.214, "elapsed_seconds": 987.6956777 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 2268.978515625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.836, "elapsed_seconds": 988.7207939000004 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 94.0, "gpu_memory_util_percent": 66.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 211.659, "elapsed_seconds": 989.7362835999998 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 3266.4833984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.59, "elapsed_seconds": 990.7503774000002 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.039, "elapsed_seconds": 991.7632088000005 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 3082.50634765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.352, "elapsed_seconds": 992.7783788000002 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.02, "elapsed_seconds": 993.7937849000004 }, { "process_rss_mb": 1840.63671875, "cuda_allocated_mb": 203.779296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.404, "elapsed_seconds": 994.8031065 }, { "process_rss_mb": 1846.12890625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 208.3, "elapsed_seconds": 995.8235393999998 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 189.951, "elapsed_seconds": 996.8320153999998 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 206.532, "elapsed_seconds": 997.8422211000002 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 216.489, "elapsed_seconds": 998.8568675000001 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.28, "elapsed_seconds": 999.8661619999998 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 1028.41650390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.683, "elapsed_seconds": 1000.8680150000005 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 183.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.737, "elapsed_seconds": 1001.8827553000006 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.036, "elapsed_seconds": 1002.8941362 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 500.123046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.349, "elapsed_seconds": 1003.9087513999998 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.53, "elapsed_seconds": 1004.9204658000003 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.117, "elapsed_seconds": 1005.9356852000005 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 208.041, "elapsed_seconds": 1006.9481724000002 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.835, "elapsed_seconds": 1007.9596229000003 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.007, "elapsed_seconds": 1008.9731639000006 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 264.41650390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.881, "elapsed_seconds": 1009.9884422999999 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.147, "elapsed_seconds": 1011.0003272999998 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.963, "elapsed_seconds": 1012.0165066000004 }, { "process_rss_mb": 1841.13671875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 212.216, "elapsed_seconds": 1013.0281040000004 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.289, "elapsed_seconds": 1014.0375404000001 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 121.0361328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.574, "elapsed_seconds": 1015.0511529999999 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.772, "elapsed_seconds": 1016.0639410999997 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.194, "elapsed_seconds": 1017.0780971999998 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.414, "elapsed_seconds": 1018.0904862000002 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.595, "elapsed_seconds": 1019.0947748000003 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 3042.30615234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.298, "elapsed_seconds": 1020.1100911999997 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.16796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.102, "elapsed_seconds": 1021.1240109000005 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.369, "elapsed_seconds": 1022.1396196000001 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.889, "elapsed_seconds": 1023.1445237000003 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 120.4111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.29296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.798, "elapsed_seconds": 1024.1544520000002 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.095, "elapsed_seconds": 1025.1647609000006 }, { "process_rss_mb": 1846.12890625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.931, "elapsed_seconds": 1026.1793594000005 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 195.772, "elapsed_seconds": 1027.1939113000003 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.084, "elapsed_seconds": 1028.2029314000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.486, "elapsed_seconds": 1029.2155013000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 372.50048828125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 195.614, "elapsed_seconds": 1030.2290400000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.149, "elapsed_seconds": 1031.2303537000003 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.958, "elapsed_seconds": 1032.2395854000001 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.18212890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.172, "elapsed_seconds": 1033.2635440999998 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.377, "elapsed_seconds": 1034.2705089000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.305, "elapsed_seconds": 1035.2815848000005 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.669, "elapsed_seconds": 1036.2974899000005 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.714, "elapsed_seconds": 1037.3102589000005 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.848, "elapsed_seconds": 1038.3195654000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.766, "elapsed_seconds": 1039.3315770000004 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.632, "elapsed_seconds": 1040.3416542999998 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 91.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.022, "elapsed_seconds": 1041.3579691 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.43, "elapsed_seconds": 1042.3718496000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.568, "elapsed_seconds": 1043.3855269 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 189.952, "elapsed_seconds": 1044.3867964000005 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.471, "elapsed_seconds": 1045.3999168000005 }, { "process_rss_mb": 1856.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.235, "elapsed_seconds": 1046.4139946 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 66.0, "gpu_memory_util_percent": 46.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 212.241, "elapsed_seconds": 1047.4249209000004 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.79, "elapsed_seconds": 1048.4382203 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.346, "elapsed_seconds": 1049.4555984000003 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.795, "elapsed_seconds": 1050.4625052000006 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 1655.35009765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.853, "elapsed_seconds": 1051.4822387 }, { "process_rss_mb": 1856.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.786, "elapsed_seconds": 1052.4901640000003 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 120.95068359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.384, "elapsed_seconds": 1053.5038878999999 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.752, "elapsed_seconds": 1054.5176962 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 1221.29296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.061, "elapsed_seconds": 1055.529046 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 1995.9765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.744, "elapsed_seconds": 1056.5431417000004 }, { "process_rss_mb": 1841.19921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 2.0, "gpu_memory_util_percent": 2.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 202.148, "elapsed_seconds": 1057.5571289000004 }, { "process_rss_mb": 1846.12890625, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 193.224, "elapsed_seconds": 1058.5714570000005 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.764, "elapsed_seconds": 1059.5839679000005 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.862, "elapsed_seconds": 1060.5991215999998 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 220.927, "elapsed_seconds": 1061.6098458000001 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 209.466, "elapsed_seconds": 1062.6241450000002 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.486, "elapsed_seconds": 1063.6405991000006 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.897, "elapsed_seconds": 1064.6522725 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 400.6630859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.266, "elapsed_seconds": 1065.6688832000004 }, { "process_rss_mb": 1846.44140625, "cuda_allocated_mb": 3121.9833984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.398, "elapsed_seconds": 1066.681199 }, { "process_rss_mb": 1840.69921875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 65.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.547, "elapsed_seconds": 1067.6929418000004 }, { "process_rss_mb": 1836.96875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.025, "elapsed_seconds": 1068.7064376000003 }, { "process_rss_mb": 1836.96875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.21, "elapsed_seconds": 1069.7188357999999 }, { "process_rss_mb": 1841.7421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.35, "elapsed_seconds": 1070.7336277999998 }, { "process_rss_mb": 1841.7421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 219.088, "elapsed_seconds": 1071.7446925999998 }, { "process_rss_mb": 1841.7578125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 213.04, "elapsed_seconds": 1072.7591363000006 }, { "process_rss_mb": 1841.1484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.384, "elapsed_seconds": 1073.7739542000004 }, { "process_rss_mb": 1841.1484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 220.047, "elapsed_seconds": 1074.7855820999998 }, { "process_rss_mb": 1841.16015625, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 218.61, "elapsed_seconds": 1075.8074369 }, { "process_rss_mb": 1841.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.01, "elapsed_seconds": 1076.8130201999998 }, { "process_rss_mb": 1841.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.594, "elapsed_seconds": 1077.8224030000006 }, { "process_rss_mb": 1841.1640625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.501, "elapsed_seconds": 1078.8345352000001 }, { "process_rss_mb": 1841.1640625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.221, "elapsed_seconds": 1079.8474998000006 }, { "process_rss_mb": 1841.1796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.23046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.062, "elapsed_seconds": 1080.8573811 }, { "process_rss_mb": 1841.1796875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.31, "elapsed_seconds": 1081.8689436999998 }, { "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.046, "elapsed_seconds": 1082.8791258000006 }, { "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.62, "elapsed_seconds": 1083.8940521 }, { "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 2345.0185546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 192.806, "elapsed_seconds": 1084.9049563000003 }, { "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.543, "elapsed_seconds": 1085.9206278000006 }, { "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.049, "elapsed_seconds": 1086.9382695 }, { "process_rss_mb": 1841.2421875, "cuda_allocated_mb": 138.1884765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.385, "elapsed_seconds": 1087.9502884999997 }, { "process_rss_mb": 1841.52734375, "cuda_allocated_mb": 119.9111328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 198.776, "elapsed_seconds": 1088.9638097000002 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.1826171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.815, "elapsed_seconds": 1089.9723442 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 217.229, "elapsed_seconds": 1090.9862949000003 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.853, "elapsed_seconds": 1091.9990669000003 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.475, "elapsed_seconds": 1093.0106491000001 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 207.904, "elapsed_seconds": 1094.0240852000002 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.513, "elapsed_seconds": 1095.0380565 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 1815.2236328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.829, "elapsed_seconds": 1096.0559224000003 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.675, "elapsed_seconds": 1097.0683441000006 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 2265.01904296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 209.972, "elapsed_seconds": 1098.0739626 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.458, "elapsed_seconds": 1099.0848031 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.165, "elapsed_seconds": 1100.0991175000004 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.116, "elapsed_seconds": 1101.1136315999997 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.702, "elapsed_seconds": 1102.1219513 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.081, "elapsed_seconds": 1103.1327559000006 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 210.118, "elapsed_seconds": 1104.1443806000002 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 582.1220703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.21, "elapsed_seconds": 1105.1552091999997 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.781, "elapsed_seconds": 1106.1636432999999 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.397, "elapsed_seconds": 1107.1791233999998 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 211.571, "elapsed_seconds": 1108.1912459 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.035, "elapsed_seconds": 1109.2042486000005 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 2667.0126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.48, "elapsed_seconds": 1110.2168889999998 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.704, "elapsed_seconds": 1111.2192292000004 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.379, "elapsed_seconds": 1112.2317564999998 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 264.41845703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.695, "elapsed_seconds": 1113.2425706999998 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 2667.0126953125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.115, "elapsed_seconds": 1114.2536511000003 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 56.0, "gpu_memory_util_percent": 39.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.618, "elapsed_seconds": 1115.2647729000005 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.906, "elapsed_seconds": 1116.2792433 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.486, "elapsed_seconds": 1117.2920592000005 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.946, "elapsed_seconds": 1118.3073041000002 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 189.972, "elapsed_seconds": 1119.3158831000005 }, { "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.205, "elapsed_seconds": 1120.3296904000003 }, { "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 206.563, "elapsed_seconds": 1121.3442827000003 }, { "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.678, "elapsed_seconds": 1122.357461 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 55.0, "gpu_memory_util_percent": 38.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 211.136, "elapsed_seconds": 1123.3688868000004 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 514.78857421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.684, "elapsed_seconds": 1124.3805760999999 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.261, "elapsed_seconds": 1125.3925775000002 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.792, "elapsed_seconds": 1126.4064508000006 }, { "process_rss_mb": 1840.71484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.644, "elapsed_seconds": 1127.4190165000005 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.529, "elapsed_seconds": 1128.4315083000006 }, { "process_rss_mb": 1841.46484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 55.0, "gpu_memory_util_percent": 39.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 68.0, "gpu_power_w": 205.302, "elapsed_seconds": 1129.4467096000003 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.746, "elapsed_seconds": 1130.4606874000006 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.367, "elapsed_seconds": 1131.4755357999998 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.895, "elapsed_seconds": 1132.4896156 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 192.269, "elapsed_seconds": 1133.5011942000006 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.859, "elapsed_seconds": 1134.5025305000004 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.114, "elapsed_seconds": 1135.5167172000006 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.625, "elapsed_seconds": 1136.5312132000004 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.678, "elapsed_seconds": 1137.5436832 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 276.50146484375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.011, "elapsed_seconds": 1138.5624142000001 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 212.705, "elapsed_seconds": 1139.5708802000004 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.35546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.358, "elapsed_seconds": 1140.5862188000006 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.346, "elapsed_seconds": 1141.5965284 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.339, "elapsed_seconds": 1142.6047017000005 }, { "process_rss_mb": 1840.69921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.825, "elapsed_seconds": 1143.6092494999998 }, { "process_rss_mb": 1841.44921875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 208.388, "elapsed_seconds": 1144.6201846 }, { "process_rss_mb": 1841.46875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.538, "elapsed_seconds": 1145.6372274999999 }, { "process_rss_mb": 1841.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 67.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.108, "elapsed_seconds": 1146.6444124000004 }, { "process_rss_mb": 1841.46875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 194.33, "elapsed_seconds": 1147.6550615 }, { "process_rss_mb": 1841.46875, "cuda_allocated_mb": 598.4970703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.034, "elapsed_seconds": 1148.6560319000005 }, { "process_rss_mb": 1841.46875, "cuda_allocated_mb": 3034.505859375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 210.237, "elapsed_seconds": 1149.6664984000004 }, { "process_rss_mb": 1841.46875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.638, "elapsed_seconds": 1150.6810115000003 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 11.0, "gpu_memory_util_percent": 6.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 204.022, "elapsed_seconds": 1151.6888628000002 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 206.444, "elapsed_seconds": 1152.6996186000001 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 210.848, "elapsed_seconds": 1153.7133163000003 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 41.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 214.308, "elapsed_seconds": 1154.7286131000005 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.695, "elapsed_seconds": 1155.7306082 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 1287.984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.263, "elapsed_seconds": 1156.7463594000005 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.449, "elapsed_seconds": 1157.7531191000003 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.289, "elapsed_seconds": 1158.7677054 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 1013.99072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.997, "elapsed_seconds": 1159.7807585000005 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.981, "elapsed_seconds": 1160.7922032000001 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 1913.79736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 213.872, "elapsed_seconds": 1161.8046958 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.469, "elapsed_seconds": 1162.8191229 }, { "process_rss_mb": 1846.5859375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.699, "elapsed_seconds": 1163.8252286000002 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.567, "elapsed_seconds": 1164.8378924000003 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.264, "elapsed_seconds": 1165.8509942000005 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 23.0, "gpu_memory_util_percent": 16.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 191.276, "elapsed_seconds": 1166.8610113 }, { "process_rss_mb": 1849.9921875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 209.519, "elapsed_seconds": 1167.8629400000004 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 514.78857421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.586, "elapsed_seconds": 1168.8789424000001 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 208.836, "elapsed_seconds": 1169.8903470999999 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.579, "elapsed_seconds": 1170.9016550000001 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.452, "elapsed_seconds": 1171.9103756 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.453, "elapsed_seconds": 1172.9190362000004 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.741, "elapsed_seconds": 1173.9331994999998 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 336.53759765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.76, "elapsed_seconds": 1174.9452585000004 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.784, "elapsed_seconds": 1175.9603069 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.822, "elapsed_seconds": 1176.9728154 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 1045.990234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.671, "elapsed_seconds": 1177.9931474999998 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 1013.99072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.09, "elapsed_seconds": 1178.9947246000006 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.497, "elapsed_seconds": 1180.0072033000001 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 218.49, "elapsed_seconds": 1181.0167998000002 }, { "process_rss_mb": 1840.83203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.739, "elapsed_seconds": 1182.0191310000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 203.956, "elapsed_seconds": 1183.0283307000004 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 2954.5068359375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.361, "elapsed_seconds": 1184.0485164000006 }, { "process_rss_mb": 1841.33984375, "cuda_allocated_mb": 464.78857421875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 211.138, "elapsed_seconds": 1185.0510457000005 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.179, "elapsed_seconds": 1186.0597936999998 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 3033.05712890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.194, "elapsed_seconds": 1187.0738668 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.295, "elapsed_seconds": 1188.0882857000001 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.43, "elapsed_seconds": 1189.1040243000002 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.327, "elapsed_seconds": 1190.11292 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 190.657, "elapsed_seconds": 1191.1276641999998 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.733, "elapsed_seconds": 1192.1412022999998 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.684, "elapsed_seconds": 1193.1550002 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.789, "elapsed_seconds": 1194.1653493999997 }, { "process_rss_mb": 1856.66796875, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.782, "elapsed_seconds": 1195.1827769000006 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 1221.29296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.878, "elapsed_seconds": 1196.1929327000003 }, { "process_rss_mb": 1841.40234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.81, "elapsed_seconds": 1197.2041891000003 }, { "process_rss_mb": 1837.671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.47, "elapsed_seconds": 1198.2187782000005 }, { "process_rss_mb": 1837.671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 73.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.943, "elapsed_seconds": 1199.2284676000004 }, { "process_rss_mb": 1837.671875, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.31, "elapsed_seconds": 1200.2371174000004 }, { "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.48046875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.962, "elapsed_seconds": 1201.2528704000001 }, { "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.237, "elapsed_seconds": 1202.2630015 }, { "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.95, "elapsed_seconds": 1203.2783591000002 }, { "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.60546875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.195, "elapsed_seconds": 1204.2934424 }, { "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 220.347, "elapsed_seconds": 1205.3040210000004 }, { "process_rss_mb": 1842.41015625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 217.024, "elapsed_seconds": 1206.3178137999998 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.588, "elapsed_seconds": 1207.3301210000004 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 208.695, "elapsed_seconds": 1208.3382088000003 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 210.975, "elapsed_seconds": 1209.3533215000002 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 3033.0576171875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.664, "elapsed_seconds": 1210.3695524999998 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.366, "elapsed_seconds": 1211.3705186000002 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.172, "elapsed_seconds": 1212.3804676000004 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 1028.41650390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.522, "elapsed_seconds": 1213.3891792999998 }, { "process_rss_mb": 1846.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 199.338, "elapsed_seconds": 1214.4003914 }, { "process_rss_mb": 1846.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 192.349, "elapsed_seconds": 1215.4133329000001 }, { "process_rss_mb": 1846.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.809, "elapsed_seconds": 1216.4221442000003 }, { "process_rss_mb": 1846.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.314, "elapsed_seconds": 1217.4352810000005 }, { "process_rss_mb": 1846.77734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.34, "elapsed_seconds": 1218.4472392000002 }, { "process_rss_mb": 1846.77734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.133, "elapsed_seconds": 1219.4597608999998 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 213.105, "elapsed_seconds": 1220.4730998999999 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.706, "elapsed_seconds": 1221.4885496000002 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 549.9970703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 216.503, "elapsed_seconds": 1222.5004044000002 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 219.817, "elapsed_seconds": 1223.5116043000003 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 260.5029296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.038, "elapsed_seconds": 1224.5221815 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.056, "elapsed_seconds": 1225.5320071 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.144, "elapsed_seconds": 1226.5466121000004 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 65.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 211.166, "elapsed_seconds": 1227.5531258000005 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.87, "elapsed_seconds": 1228.5697931999998 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 194.948, "elapsed_seconds": 1229.5827624999997 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.611, "elapsed_seconds": 1230.5906637000003 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.886, "elapsed_seconds": 1231.6026243000001 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 1929.1513671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.641, "elapsed_seconds": 1232.6142602000004 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 121.03662109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.266, "elapsed_seconds": 1233.615851 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.451, "elapsed_seconds": 1234.6289014 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 90.0, "gpu_memory_util_percent": 63.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 218.679, "elapsed_seconds": 1235.6419851000005 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 217.152, "elapsed_seconds": 1236.6527903 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.565, "elapsed_seconds": 1237.6660758000003 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 219.032, "elapsed_seconds": 1238.6775271000001 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 456.537109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.289, "elapsed_seconds": 1239.6903914 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.938, "elapsed_seconds": 1240.7049700000002 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 210.844, "elapsed_seconds": 1241.7130164000005 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.735, "elapsed_seconds": 1242.7231980000001 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 0.0, "gpu_memory_util_percent": 0.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 65.0, "gpu_power_w": 194.098, "elapsed_seconds": 1243.7340934 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.009, "elapsed_seconds": 1244.7489248000006 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.18212890625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 200.411, "elapsed_seconds": 1245.7620517 }, { "process_rss_mb": 1841.84765625, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 212.19, "elapsed_seconds": 1246.7752602000005 }, { "process_rss_mb": 1842.21484375, "cuda_allocated_mb": 196.1279296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.125, "elapsed_seconds": 1247.7857305000002 }, { "process_rss_mb": 1842.21484375, "cuda_allocated_mb": 1351.41845703125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 79.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 219.244, "elapsed_seconds": 1248.8006399 }, { "process_rss_mb": 1842.21484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 214.48, "elapsed_seconds": 1249.8163034000004 }, { "process_rss_mb": 1842.21484375, "cuda_allocated_mb": 264.41259765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.269, "elapsed_seconds": 1250.8353658 }, { "process_rss_mb": 1842.21484375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.001, "elapsed_seconds": 1251.8437061 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 207.521, "elapsed_seconds": 1252.8575243 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.994, "elapsed_seconds": 1253.8677094 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.969, "elapsed_seconds": 1254.8820814000005 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.856, "elapsed_seconds": 1255.8904595 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.886, "elapsed_seconds": 1256.8917459999998 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 436.2490234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.035, "elapsed_seconds": 1257.9017595999999 }, { "process_rss_mb": 1840.90234375, "cuda_allocated_mb": 138.1806640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.491, "elapsed_seconds": 1258.9177715 }, { "process_rss_mb": 1840.96484375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 217.149, "elapsed_seconds": 1259.9299459 }, { "process_rss_mb": 1840.96484375, "cuda_allocated_mb": 119.912109375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.41796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.461, "elapsed_seconds": 1260.9404559999998 }, { "process_rss_mb": 1840.96484375, "cuda_allocated_mb": 2025.1513671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.7, "elapsed_seconds": 1261.9487586999999 }, { "process_rss_mb": 1840.96484375, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.654, "elapsed_seconds": 1262.9657645000002 }, { "process_rss_mb": 1840.96484375, "cuda_allocated_mb": 1013.99072265625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.573, "elapsed_seconds": 1263.9723465000002 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 212.485, "elapsed_seconds": 1264.9856353999999 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.622, "elapsed_seconds": 1265.9976565000002 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.152, "elapsed_seconds": 1267.0114508000001 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 216.7, "elapsed_seconds": 1268.0231040000003 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 1719.349609375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.992, "elapsed_seconds": 1269.0360038000003 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 1287.98046875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 206.154, "elapsed_seconds": 1270.0502338000001 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 2656.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 205.739, "elapsed_seconds": 1271.0627251 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 63.0, "gpu_memory_util_percent": 44.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 200.493, "elapsed_seconds": 1272.0640792 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.782, "elapsed_seconds": 1273.0755709999999 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.2158203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 208.113, "elapsed_seconds": 1274.0890063000002 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 204.932, "elapsed_seconds": 1275.0999309000003 }, { "process_rss_mb": 1841.02734375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 204.752, "elapsed_seconds": 1276.1146916000002 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 191.73, "elapsed_seconds": 1277.1279374 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 13.0, "gpu_memory_util_percent": 9.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 179.362, "elapsed_seconds": 1278.1420854000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 60.0, "gpu_memory_util_percent": 42.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 194.175, "elapsed_seconds": 1279.1568518 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 196.1279296875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 205.519, "elapsed_seconds": 1280.1691016 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 3121.9833984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 202.471, "elapsed_seconds": 1281.1825332000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 207.517, "elapsed_seconds": 1282.196789 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 83.0, "gpu_memory_util_percent": 58.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 210.72, "elapsed_seconds": 1283.2118925 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 1287.984375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 58.0, "gpu_memory_util_percent": 41.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 69.0, "gpu_power_w": 202.913, "elapsed_seconds": 1284.2259699 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 208.485, "elapsed_seconds": 1285.2361301000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 661.9931640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 89.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 209.874, "elapsed_seconds": 1286.2512567999997 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 203.272, "elapsed_seconds": 1287.2698270000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 64.0, "gpu_memory_util_percent": 45.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 200.68, "elapsed_seconds": 1288.2807503999998 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 1751.35009765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 206.98, "elapsed_seconds": 1289.2940727999999 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 2606.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 202.753, "elapsed_seconds": 1290.3085747 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 198.881, "elapsed_seconds": 1291.3192027999999 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 71.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 205.548, "elapsed_seconds": 1292.3334993000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 49.0, "gpu_memory_util_percent": 35.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 67.0, "gpu_power_w": 201.795, "elapsed_seconds": 1293.3348763999998 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 61.0, "gpu_memory_util_percent": 43.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 177.602, "elapsed_seconds": 1294.3548833999998 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 50.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 199.807, "elapsed_seconds": 1295.3704134000004 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 206.738, "elapsed_seconds": 1296.3859032000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.21533203125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 70.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 205.906, "elapsed_seconds": 1297.3959219000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 208.312, "elapsed_seconds": 1298.4106457000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 2606.35546875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.657, "elapsed_seconds": 1299.4221999000006 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.883, "elapsed_seconds": 1300.4350796000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 84.0, "gpu_memory_util_percent": 59.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 215.852, "elapsed_seconds": 1301.4490189000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 51.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.303, "elapsed_seconds": 1302.4636279000001 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.372, "elapsed_seconds": 1303.4741848999997 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.11, "elapsed_seconds": 1304.4853843000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 1977.1513671875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.504, "elapsed_seconds": 1305.5007537000006 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 216.243, "elapsed_seconds": 1306.5157543999999 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 18.0, "gpu_memory_util_percent": 13.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 66.0, "gpu_power_w": 203.958, "elapsed_seconds": 1307.5684643 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 192.617, "elapsed_seconds": 1308.5738556000001 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 82.0, "gpu_memory_util_percent": 57.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.101, "elapsed_seconds": 1309.5896733000004 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 198.857, "elapsed_seconds": 1310.6026394000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 215.053, "elapsed_seconds": 1311.6158981000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 76.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 215.48, "elapsed_seconds": 1312.6289609 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 2536.63916015625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 68.0, "gpu_memory_util_percent": 47.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 213.056, "elapsed_seconds": 1313.6399250999998 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 81.0, "gpu_memory_util_percent": 56.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.992, "elapsed_seconds": 1314.6536161999998 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 3050.50390625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.189, "elapsed_seconds": 1315.6682753 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 2478.1044921875, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 86.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.168, "elapsed_seconds": 1316.6817326000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.633, "elapsed_seconds": 1317.6931090000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 215.976, "elapsed_seconds": 1318.7076412000001 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 1093.736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 75.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 70.0, "gpu_power_w": 213.543, "elapsed_seconds": 1319.7154130000004 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 74.0, "gpu_memory_util_percent": 52.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 212.834, "elapsed_seconds": 1320.7281241000005 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.66796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 213.046, "elapsed_seconds": 1321.7451851 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 2140.4765625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 53.0, "gpu_memory_used_mb": 5533.66796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 211.227, "elapsed_seconds": 1322.7565440000003 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 85.0, "gpu_memory_util_percent": 60.0, "gpu_memory_used_mb": 5533.66796875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.52, "elapsed_seconds": 1323.7682647000006 }, { "process_rss_mb": 1846.51953125, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 216.552, "elapsed_seconds": 1324.7828401000002 }, { "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 77.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 72.0, "gpu_power_w": 214.746, "elapsed_seconds": 1325.7955084999999 }, { "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 88.0, "gpu_memory_util_percent": 62.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 218.564, "elapsed_seconds": 1326.8056793 }, { "process_rss_mb": 1846.58203125, "cuda_allocated_mb": 138.18115234375, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 78.0, "gpu_memory_util_percent": 54.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.856, "elapsed_seconds": 1327.8151984 }, { "process_rss_mb": 1841.58984375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 214.552, "elapsed_seconds": 1328.8311491999998 }, { "process_rss_mb": 1841.58984375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 69.0, "gpu_memory_util_percent": 48.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 71.0, "gpu_power_w": 209.702, "elapsed_seconds": 1329.8375396000006 }, { "process_rss_mb": 1837.859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 72.0, "gpu_memory_util_percent": 49.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 73.0, "gpu_power_w": 214.622, "elapsed_seconds": 1330.8513681 }, { "process_rss_mb": 1837.859375, "cuda_allocated_mb": 138.181640625, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 87.0, "gpu_memory_util_percent": 61.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 75.0, "gpu_power_w": 214.642, "elapsed_seconds": 1331.8625599000002 }, { "process_rss_mb": 1837.859375, "cuda_allocated_mb": 120.9736328125, "cuda_reserved_mb": 3520.0, "cuda_max_allocated_mb": 3266.4833984375, "cuda_max_reserved_mb": 3520.0, "gpu_util_percent": 80.0, "gpu_memory_util_percent": 55.0, "gpu_memory_used_mb": 5533.54296875, "gpu_memory_total_mb": 16303.0, "gpu_temperature_c": 74.0, "gpu_power_w": 215.807, "elapsed_seconds": 1332.8763401 } ], "step_samples_per_second_avg": 8044.423103969248, "step_samples_per_second_max": 8395.268938012663, "step_samples_per_second_min": 5750.578539649555, "step_tokens_per_second_avg": 1029686.1573080637, "step_tokens_per_second_max": 1074594.4240656209, "step_tokens_per_second_min": 736074.053075143, "step_process_rss_mb_avg": 1840.8937237079326, "step_process_rss_mb_max": 1846.9375, "step_process_rss_mb_min": 1816.14453125, "step_cuda_max_allocated_mb_avg": 3266.4833984375, "step_cuda_max_allocated_mb_max": 3266.4833984375, "step_cuda_max_allocated_mb_min": 3266.4833984375, "step_gpu_util_percent_avg": 76.95192307692308, "step_gpu_util_percent_max": 93.0, "step_gpu_util_percent_min": 0.0, "step_gpu_memory_util_percent_avg": 53.67788461538461, "step_gpu_memory_util_percent_max": 65.0, "step_gpu_memory_util_percent_min": 0.0, "step_gpu_power_w_avg": 210.98742307692308, "step_gpu_power_w_max": 221.726, "step_gpu_power_w_min": 106.578, "step_gpu_temperature_c_avg": 72.48076923076923, "step_gpu_temperature_c_max": 75.0, "step_gpu_temperature_c_min": 48.0, "background_process_rss_mb_avg": 1841.0384455675778, "background_process_rss_mb_max": 1860.48046875, "background_process_rss_mb_min": 1477.0, "background_cuda_max_allocated_mb_avg": 3266.4312880392226, "background_cuda_max_allocated_mb_max": 3266.4833984375, "background_cuda_max_allocated_mb_min": 3197.85400390625, "background_gpu_util_percent_avg": 76.40698557327259, "background_gpu_util_percent_max": 94.0, "background_gpu_util_percent_min": 0.0, "background_gpu_memory_util_percent_avg": 53.34396355353075, "background_gpu_memory_util_percent_max": 66.0, "background_gpu_memory_util_percent_min": 0.0, "background_gpu_power_w_avg": 210.0710820045558, "background_gpu_power_w_max": 224.006, "background_gpu_power_w_min": 39.826, "background_gpu_temperature_c_avg": 72.25816249050874, "background_gpu_temperature_c_max": 76.0, "background_gpu_temperature_c_min": 43.0, "samples_per_second_avg": 8044.423103969248, "samples_per_second_max": 8395.268938012663, "tokens_per_second_avg": 1029686.1573080637, "tokens_per_second_max": 1074594.4240656209, "process_rss_mb_avg": 1840.8937237079326, "process_rss_mb_max": 1846.9375, "cuda_max_allocated_mb_avg": 3266.4833984375, "cuda_max_allocated_mb_max": 3266.4833984375, "gpu_util_percent_avg": 76.95192307692308, "gpu_util_percent_max": 93.0, "gpu_memory_util_percent_avg": 53.67788461538461, "gpu_memory_util_percent_max": 65.0, "gpu_power_w_avg": 210.98742307692308, "gpu_power_w_max": 221.726, "gpu_temperature_c_avg": 72.48076923076923, "gpu_temperature_c_max": 75.0 }