remove training logs
Browse files- neurv/logs/rv1.log +0 -45
- neurv/logs/rv2.log +0 -179
- neurv/logs/rv3.log +0 -91
- neurv/logs/rv3_smoke.log +0 -10
- neurv/logs/rv4_smoke.log +0 -9
- neurv/logs/rvsmoke8.log +0 -9
- neurv/runs/rv1/log.jsonl +0 -4
- neurv/runs/rv2/log.jsonl +0 -92
- neurv/runs/rv3/log.jsonl +0 -48
- neurv/runs/rv4/log.jsonl +0 -135
neurv/logs/rv1.log
DELETED
|
@@ -1,45 +0,0 @@
|
|
| 1 |
-
17:41:00 RV start name=rv1 steps=150000 args=
|
| 2 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 49424, "name": "rv1", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "17:41:15"}
|
| 3 |
-
{"event": "train", "step": 50, "loss": 12.7005, "changed": 10.7102, "kept": 9.95181, "gn": 6.628, "lr": 1.5e-05, "s_per_step": 1.247, "mem_GB": 48.9, "t": "17:42:23"}
|
| 4 |
-
{"event": "train", "step": 100, "loss": 8.5641, "changed": 6.9295, "kept": 8.17321, "gn": 3.743, "lr": 3e-05, "s_per_step": 1.201, "mem_GB": 48.9, "t": "17:43:21"}
|
| 5 |
-
{"event": "train", "step": 150, "loss": 6.5321, "changed": 5.3917, "kept": 5.70183, "gn": 3.317, "lr": 4.5e-05, "s_per_step": 1.186, "mem_GB": 48.9, "t": "17:44:19"}
|
| 6 |
-
W1031 17:47:03.858000 3473380 torch/distributed/elastic/agent/server/api.py:725] Received 15 death signal, shutting down workers
|
| 7 |
-
W1031 17:47:03.859000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487649 closing signal SIGTERM
|
| 8 |
-
W1031 17:47:03.863000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487679 closing signal SIGTERM
|
| 9 |
-
W1031 17:47:03.868000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487718 closing signal SIGTERM
|
| 10 |
-
W1031 17:47:03.896000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487730 closing signal SIGTERM
|
| 11 |
-
W1031 17:47:03.945000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487732 closing signal SIGTERM
|
| 12 |
-
W1031 17:47:03.965000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487737 closing signal SIGTERM
|
| 13 |
-
W1031 17:47:03.971000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487742 closing signal SIGTERM
|
| 14 |
-
W1031 17:47:03.979000 3473380 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3487747 closing signal SIGTERM
|
| 15 |
-
Traceback (most recent call last):
|
| 16 |
-
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 17 |
-
File "<frozen runpy>", line 88, in _run_code
|
| 18 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 940, in <module>
|
| 19 |
-
main()
|
| 20 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
|
| 21 |
-
return f(*args, **kwargs)
|
| 22 |
-
^^^^^^^^^^^^^^^^^^
|
| 23 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main
|
| 24 |
-
run(args)
|
| 25 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run
|
| 26 |
-
elastic_launch(
|
| 27 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__
|
| 28 |
-
return launch_agent(self._config, self._entrypoint, list(args))
|
| 29 |
-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 30 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 284, in launch_agent
|
| 31 |
-
result = agent.run()
|
| 32 |
-
^^^^^^^^^^^
|
| 33 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 34 |
-
result = f(*args, **kwargs)
|
| 35 |
-
^^^^^^^^^^^^^^^^^^
|
| 36 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 717, in run
|
| 37 |
-
result = self._invoke_run(role)
|
| 38 |
-
^^^^^^^^^^^^^^^^^^^^^^
|
| 39 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 881, in _invoke_run
|
| 40 |
-
time.sleep(monitor_interval)
|
| 41 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 85, in _terminate_process_handler
|
| 42 |
-
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 43 |
-
torch.distributed.elastic.multiprocessing.api.SignalException: Process 3473380 got signal: 15
|
| 44 |
-
[W1031 17:47:11.250042609 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 45 |
-
17:47:12 TRAIN_EXIT rc=0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/logs/rv2.log
DELETED
|
@@ -1,179 +0,0 @@
|
|
| 1 |
-
18:23:36 RV start name=rv2 steps=150000 args=
|
| 2 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 49424, "name": "rv2", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "18:23:51"}
|
| 3 |
-
{"event": "train", "step": 50, "loss": 11.7748, "changed": 9.4664, "kept": 11.54169, "gn": 7.76, "lr": 1.5e-05, "s_per_step": 0.595, "mem_GB": 48.9, "t": "18:24:42"}
|
| 4 |
-
{"event": "train", "step": 100, "loss": 7.9366, "changed": 6.1683, "kept": 8.84151, "gn": 3.922, "lr": 3e-05, "s_per_step": 0.549, "mem_GB": 48.9, "t": "18:25:07"}
|
| 5 |
-
{"event": "train", "step": 150, "loss": 6.9692, "changed": 5.6799, "kept": 6.44635, "gn": 3.528, "lr": 4.5e-05, "s_per_step": 0.535, "mem_GB": 48.9, "t": "18:25:32"}
|
| 6 |
-
{"event": "train", "step": 200, "loss": 6.6736, "changed": 5.0267, "kept": 8.23482, "gn": 2.797, "lr": 6e-05, "s_per_step": 0.528, "mem_GB": 48.9, "t": "18:25:58"}
|
| 7 |
-
{"event": "train", "step": 250, "loss": 5.4879, "changed": 4.6458, "kept": 4.21092, "gn": 2.812, "lr": 7.5e-05, "s_per_step": 0.523, "mem_GB": 48.9, "t": "18:26:23"}
|
| 8 |
-
{"event": "train", "step": 300, "loss": 6.6982, "changed": 6.1344, "kept": 2.81894, "gn": 3.043, "lr": 9e-05, "s_per_step": 0.52, "mem_GB": 48.9, "t": "18:26:48"}
|
| 9 |
-
{"event": "train", "step": 350, "loss": 4.3961, "changed": 3.8394, "kept": 2.78348, "gn": 2.532, "lr": 0.000105, "s_per_step": 0.518, "mem_GB": 48.9, "t": "18:27:14"}
|
| 10 |
-
{"event": "train", "step": 400, "loss": 5.1912, "changed": 4.6676, "kept": 2.61762, "gn": 2.668, "lr": 0.00012, "s_per_step": 0.517, "mem_GB": 48.9, "t": "18:27:39"}
|
| 11 |
-
{"event": "train", "step": 450, "loss": 4.6512, "changed": 4.1101, "kept": 2.70544, "gn": 2.518, "lr": 0.000135, "s_per_step": 0.516, "mem_GB": 48.9, "t": "18:28:04"}
|
| 12 |
-
{"event": "train", "step": 500, "loss": 4.6321, "changed": 4.3733, "kept": 1.29404, "gn": 2.398, "lr": 0.00015, "s_per_step": 0.515, "mem_GB": 48.9, "t": "18:28:30"}
|
| 13 |
-
{"event": "train", "step": 550, "loss": 5.6399, "changed": 5.3611, "kept": 1.3941, "gn": 2.671, "lr": 0.000165, "s_per_step": 0.514, "mem_GB": 48.9, "t": "18:28:55"}
|
| 14 |
-
{"event": "train", "step": 600, "loss": 3.8454, "changed": 3.6633, "kept": 0.91083, "gn": 3.234, "lr": 0.00018, "s_per_step": 0.514, "mem_GB": 48.9, "t": "18:29:20"}
|
| 15 |
-
{"event": "train", "step": 650, "loss": 3.1944, "changed": 3.0502, "kept": 0.7206, "gn": 2.654, "lr": 0.000195, "s_per_step": 0.513, "mem_GB": 48.9, "t": "18:29:46"}
|
| 16 |
-
{"event": "train", "step": 700, "loss": 4.1536, "changed": 4.0144, "kept": 0.69581, "gn": 2.763, "lr": 0.00021, "s_per_step": 0.513, "mem_GB": 48.9, "t": "18:30:11"}
|
| 17 |
-
{"event": "train", "step": 750, "loss": 3.3662, "changed": 3.2073, "kept": 0.79431, "gn": 2.734, "lr": 0.000225, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:30:36"}
|
| 18 |
-
{"event": "train", "step": 800, "loss": 2.5462, "changed": 2.419, "kept": 0.63579, "gn": 2.177, "lr": 0.00024, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:31:02"}
|
| 19 |
-
{"event": "train", "step": 850, "loss": 5.2034, "changed": 5.0946, "kept": 0.54373, "gn": 5.122, "lr": 0.000255, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:31:27"}
|
| 20 |
-
{"event": "train", "step": 900, "loss": 5.8737, "changed": 5.7174, "kept": 0.78131, "gn": 2.632, "lr": 0.00027, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:31:52"}
|
| 21 |
-
{"event": "train", "step": 950, "loss": 5.9586, "changed": 5.8194, "kept": 0.69578, "gn": 2.097, "lr": 0.000285, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:32:18"}
|
| 22 |
-
{"event": "train", "step": 1000, "loss": 5.2801, "changed": 5.1506, "kept": 0.64768, "gn": 3.302, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:32:43"}
|
| 23 |
-
{"event": "train", "step": 1050, "loss": 3.992, "changed": 3.8973, "kept": 0.47376, "gn": 2.393, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:33:08"}
|
| 24 |
-
{"event": "train", "step": 1100, "loss": 4.7451, "changed": 4.6598, "kept": 0.42622, "gn": 2.15, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:33:34"}
|
| 25 |
-
{"event": "train", "step": 1150, "loss": 4.2295, "changed": 4.124, "kept": 0.52744, "gn": 2.425, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:33:59"}
|
| 26 |
-
{"event": "train", "step": 1200, "loss": 4.7774, "changed": 4.6802, "kept": 0.48632, "gn": 2.261, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:34:24"}
|
| 27 |
-
{"event": "train", "step": 1250, "loss": 4.3593, "changed": 4.283, "kept": 0.38117, "gn": 2.595, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:34:50"}
|
| 28 |
-
{"event": "train", "step": 1300, "loss": 3.9209, "changed": 3.8097, "kept": 0.55596, "gn": 2.708, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:35:15"}
|
| 29 |
-
{"event": "train", "step": 1350, "loss": 1.7429, "changed": 1.6935, "kept": 0.24687, "gn": 2.281, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:35:40"}
|
| 30 |
-
{"event": "train", "step": 1400, "loss": 2.6995, "changed": 2.5962, "kept": 0.51676, "gn": 2.391, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:36:06"}
|
| 31 |
-
{"event": "train", "step": 1450, "loss": 3.1172, "changed": 3.0313, "kept": 0.42948, "gn": 2.733, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:36:31"}
|
| 32 |
-
{"event": "train", "step": 1500, "loss": 2.2271, "changed": 2.1529, "kept": 0.371, "gn": 2.118, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:36:56"}
|
| 33 |
-
{"event": "train", "step": 1550, "loss": 3.9892, "changed": 3.9106, "kept": 0.39276, "gn": 2.448, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:37:22"}
|
| 34 |
-
{"event": "train", "step": 1600, "loss": 3.8216, "changed": 3.747, "kept": 0.37294, "gn": 2.167, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:37:47"}
|
| 35 |
-
{"event": "train", "step": 1650, "loss": 2.7949, "changed": 2.6818, "kept": 0.56543, "gn": 1.808, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:38:12"}
|
| 36 |
-
{"event": "train", "step": 1700, "loss": 4.5407, "changed": 4.4453, "kept": 0.47651, "gn": 1.998, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:38:38"}
|
| 37 |
-
{"event": "train", "step": 1750, "loss": 4.888, "changed": 4.8208, "kept": 0.33567, "gn": 2.301, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:39:03"}
|
| 38 |
-
{"event": "train", "step": 1800, "loss": 4.1498, "changed": 4.0518, "kept": 0.48998, "gn": 2.14, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:39:28"}
|
| 39 |
-
{"event": "train", "step": 1850, "loss": 1.8098, "changed": 1.7394, "kept": 0.35204, "gn": 2.407, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:39:54"}
|
| 40 |
-
{"event": "train", "step": 1900, "loss": 3.2753, "changed": 3.2154, "kept": 0.29973, "gn": 2.246, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:40:19"}
|
| 41 |
-
{"event": "train", "step": 1950, "loss": 4.1304, "changed": 4.0168, "kept": 0.56789, "gn": 2.511, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:40:44"}
|
| 42 |
-
{"event": "train", "step": 2000, "loss": 4.8259, "changed": 4.7308, "kept": 0.47558, "gn": 2.561, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:41:10"}
|
| 43 |
-
{"event": "eval", "step": 2000, "heldout_exact": 0.0, "heldout_changed_acc": 0.82805, "t": "18:41:12"}
|
| 44 |
-
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 45 |
-
warnings.warn( # warn only once
|
| 46 |
-
{"event": "train", "step": 2050, "loss": 3.616, "changed": 3.5198, "kept": 0.4809, "gn": 2.27, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:41:42"}
|
| 47 |
-
W1031 18:41:49.735000 2631494 torch/distributed/elastic/agent/server/api.py:725] Received 15 death signal, shutting down workers
|
| 48 |
-
W1031 18:41:49.743000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648218 closing signal SIGTERM
|
| 49 |
-
W1031 18:41:49.747000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648225 closing signal SIGTERM
|
| 50 |
-
W1031 18:41:49.752000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648230 closing signal SIGTERM
|
| 51 |
-
W1031 18:41:49.757000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648240 closing signal SIGTERM
|
| 52 |
-
W1031 18:41:49.762000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648266 closing signal SIGTERM
|
| 53 |
-
W1031 18:41:49.766000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648278 closing signal SIGTERM
|
| 54 |
-
W1031 18:41:49.772000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648281 closing signal SIGTERM
|
| 55 |
-
W1031 18:41:49.778000 2631494 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2648297 closing signal SIGTERM
|
| 56 |
-
Traceback (most recent call last):
|
| 57 |
-
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 58 |
-
File "<frozen runpy>", line 88, in _run_code
|
| 59 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 940, in <module>
|
| 60 |
-
main()
|
| 61 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
|
| 62 |
-
return f(*args, **kwargs)
|
| 63 |
-
^^^^^^^^^^^^^^^^^^
|
| 64 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main
|
| 65 |
-
run(args)
|
| 66 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run
|
| 67 |
-
elastic_launch(
|
| 68 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__
|
| 69 |
-
return launch_agent(self._config, self._entrypoint, list(args))
|
| 70 |
-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 71 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 284, in launch_agent
|
| 72 |
-
result = agent.run()
|
| 73 |
-
^^^^^^^^^^^
|
| 74 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 75 |
-
result = f(*args, **kwargs)
|
| 76 |
-
^^^^^^^^^^^^^^^^^^
|
| 77 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 717, in run
|
| 78 |
-
result = self._invoke_run(role)
|
| 79 |
-
^^^^^^^^^^^^^^^^^^^^^^
|
| 80 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 881, in _invoke_run
|
| 81 |
-
time.sleep(monitor_interval)
|
| 82 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 85, in _terminate_process_handler
|
| 83 |
-
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 84 |
-
torch.distributed.elastic.multiprocessing.api.SignalException: Process 2631494 got signal: 15
|
| 85 |
-
[W1031 18:41:56.238984710 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 86 |
-
rv1.sh: line 11: -workers: command not found
|
| 87 |
-
18:41:57 TRAIN_EXIT rc=0
|
| 88 |
-
18:42:37 RV start name=rv2 steps=150000 args=
|
| 89 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 437673, "name": "rv2", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split2/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "18:42:54"}
|
| 90 |
-
{"event": "resume", "step": 2000, "t": "18:43:04"}
|
| 91 |
-
{"event": "train", "step": 2050, "loss": 2.7133, "changed": 2.5944, "kept": 0.59474, "gn": 1.928, "lr": 0.0003, "s_per_step": 1.228, "mem_GB": 48.9, "t": "18:44:26"}
|
| 92 |
-
{"event": "train", "step": 2100, "loss": 3.9493, "changed": 3.8537, "kept": 0.47814, "gn": 2.551, "lr": 0.0003, "s_per_step": 1.187, "mem_GB": 48.9, "t": "18:45:23"}
|
| 93 |
-
{"event": "train", "step": 2150, "loss": 2.8071, "changed": 2.7405, "kept": 0.33299, "gn": 2.537, "lr": 0.0003, "s_per_step": 1.172, "mem_GB": 48.9, "t": "18:46:20"}
|
| 94 |
-
{"event": "train", "step": 2200, "loss": 4.5497, "changed": 4.4754, "kept": 0.37121, "gn": 2.366, "lr": 0.0003, "s_per_step": 1.158, "mem_GB": 48.9, "t": "18:47:16"}
|
| 95 |
-
{"event": "train", "step": 2250, "loss": 3.7875, "changed": 3.7097, "kept": 0.38858, "gn": 3.064, "lr": 0.0003, "s_per_step": 1.027, "mem_GB": 48.9, "t": "18:47:41"}
|
| 96 |
-
{"event": "train", "step": 2300, "loss": 4.2531, "changed": 4.1714, "kept": 0.40849, "gn": 2.148, "lr": 0.0003, "s_per_step": 0.94, "mem_GB": 48.9, "t": "18:48:06"}
|
| 97 |
-
{"event": "train", "step": 2350, "loss": 3.9655, "changed": 3.886, "kept": 0.39752, "gn": 2.057, "lr": 0.0003, "s_per_step": 0.878, "mem_GB": 48.9, "t": "18:48:32"}
|
| 98 |
-
{"event": "train", "step": 2400, "loss": 2.3351, "changed": 2.2686, "kept": 0.33231, "gn": 2.033, "lr": 0.0003, "s_per_step": 0.831, "mem_GB": 48.9, "t": "18:48:57"}
|
| 99 |
-
{"event": "train", "step": 2450, "loss": 2.1471, "changed": 2.0735, "kept": 0.36827, "gn": 2.205, "lr": 0.0003, "s_per_step": 0.795, "mem_GB": 48.9, "t": "18:49:22"}
|
| 100 |
-
{"event": "train", "step": 2500, "loss": 2.2578, "changed": 2.1937, "kept": 0.32098, "gn": 2.326, "lr": 0.0003, "s_per_step": 0.766, "mem_GB": 48.9, "t": "18:49:47"}
|
| 101 |
-
{"event": "train", "step": 2550, "loss": 4.0374, "changed": 3.9525, "kept": 0.42437, "gn": 2.169, "lr": 0.0003, "s_per_step": 0.743, "mem_GB": 48.9, "t": "18:50:13"}
|
| 102 |
-
{"event": "train", "step": 2600, "loss": 3.5357, "changed": 3.4498, "kept": 0.42942, "gn": 2.029, "lr": 0.0003, "s_per_step": 0.723, "mem_GB": 48.9, "t": "18:50:38"}
|
| 103 |
-
{"event": "train", "step": 2650, "loss": 4.0694, "changed": 3.9886, "kept": 0.40385, "gn": 2.243, "lr": 0.0003, "s_per_step": 0.706, "mem_GB": 48.9, "t": "18:51:03"}
|
| 104 |
-
{"event": "train", "step": 2700, "loss": 4.4654, "changed": 4.4034, "kept": 0.31002, "gn": 2.705, "lr": 0.0003, "s_per_step": 0.692, "mem_GB": 48.9, "t": "18:51:29"}
|
| 105 |
-
{"event": "train", "step": 2750, "loss": 3.746, "changed": 3.6522, "kept": 0.46905, "gn": 2.159, "lr": 0.0003, "s_per_step": 0.68, "mem_GB": 48.9, "t": "18:51:54"}
|
| 106 |
-
{"event": "train", "step": 2800, "loss": 2.7013, "changed": 2.6347, "kept": 0.33275, "gn": 2.571, "lr": 0.0003, "s_per_step": 0.669, "mem_GB": 48.9, "t": "18:52:19"}
|
| 107 |
-
{"event": "train", "step": 2850, "loss": 2.503, "changed": 2.4317, "kept": 0.35671, "gn": 2.348, "lr": 0.0003, "s_per_step": 0.659, "mem_GB": 48.9, "t": "18:52:45"}
|
| 108 |
-
{"event": "train", "step": 2900, "loss": 2.7059, "changed": 2.6287, "kept": 0.38607, "gn": 1.919, "lr": 0.0003, "s_per_step": 0.651, "mem_GB": 48.9, "t": "18:53:10"}
|
| 109 |
-
{"event": "train", "step": 2950, "loss": 2.9244, "changed": 2.8551, "kept": 0.3465, "gn": 1.715, "lr": 0.0003, "s_per_step": 0.643, "mem_GB": 48.9, "t": "18:53:35"}
|
| 110 |
-
{"event": "train", "step": 3000, "loss": 4.9648, "changed": 4.8712, "kept": 0.4683, "gn": 1.902, "lr": 0.0003, "s_per_step": 0.636, "mem_GB": 48.9, "t": "18:54:01"}
|
| 111 |
-
{"event": "train", "step": 3050, "loss": 5.9648, "changed": 5.8559, "kept": 0.5444, "gn": 2.137, "lr": 0.0003, "s_per_step": 0.63, "mem_GB": 48.9, "t": "18:54:26"}
|
| 112 |
-
{"event": "train", "step": 3100, "loss": 2.5671, "changed": 2.5028, "kept": 0.32181, "gn": 1.928, "lr": 0.0003, "s_per_step": 0.624, "mem_GB": 48.9, "t": "18:54:51"}
|
| 113 |
-
{"event": "train", "step": 3150, "loss": 3.7125, "changed": 3.6444, "kept": 0.3402, "gn": 1.824, "lr": 0.0003, "s_per_step": 0.619, "mem_GB": 48.9, "t": "18:55:17"}
|
| 114 |
-
{"event": "train", "step": 3200, "loss": 1.949, "changed": 1.891, "kept": 0.28978, "gn": 2.576, "lr": 0.0003, "s_per_step": 0.615, "mem_GB": 48.9, "t": "18:55:42"}
|
| 115 |
-
{"event": "train", "step": 3250, "loss": 4.8198, "changed": 4.7177, "kept": 0.51048, "gn": 2.497, "lr": 0.0003, "s_per_step": 0.61, "mem_GB": 48.9, "t": "18:56:07"}
|
| 116 |
-
{"event": "train", "step": 3300, "loss": 2.6943, "changed": 2.6303, "kept": 0.32025, "gn": 1.945, "lr": 0.0003, "s_per_step": 0.606, "mem_GB": 48.9, "t": "18:56:32"}
|
| 117 |
-
{"event": "train", "step": 3350, "loss": 1.7631, "changed": 1.6812, "kept": 0.40926, "gn": 2.234, "lr": 0.0003, "s_per_step": 0.603, "mem_GB": 48.9, "t": "18:56:58"}
|
| 118 |
-
{"event": "train", "step": 3400, "loss": 3.9119, "changed": 3.8542, "kept": 0.28862, "gn": 2.276, "lr": 0.0003, "s_per_step": 0.599, "mem_GB": 48.9, "t": "18:57:23"}
|
| 119 |
-
{"event": "train", "step": 3450, "loss": 4.0511, "changed": 3.9636, "kept": 0.43733, "gn": 2.401, "lr": 0.0003, "s_per_step": 0.596, "mem_GB": 48.9, "t": "18:57:48"}
|
| 120 |
-
{"event": "train", "step": 3500, "loss": 3.4284, "changed": 3.354, "kept": 0.37194, "gn": 2.198, "lr": 0.0003, "s_per_step": 0.593, "mem_GB": 48.9, "t": "18:58:14"}
|
| 121 |
-
{"event": "train", "step": 3550, "loss": 3.763, "changed": 3.6887, "kept": 0.37143, "gn": 1.932, "lr": 0.0003, "s_per_step": 0.59, "mem_GB": 48.9, "t": "18:58:39"}
|
| 122 |
-
{"event": "train", "step": 3600, "loss": 3.1771, "changed": 3.0943, "kept": 0.41384, "gn": 2.206, "lr": 0.0003, "s_per_step": 0.588, "mem_GB": 48.9, "t": "18:59:04"}
|
| 123 |
-
{"event": "train", "step": 3650, "loss": 2.1705, "changed": 2.1195, "kept": 0.25491, "gn": 2.341, "lr": 0.0003, "s_per_step": 0.585, "mem_GB": 48.9, "t": "18:59:30"}
|
| 124 |
-
{"event": "train", "step": 3700, "loss": 2.4223, "changed": 2.3521, "kept": 0.351, "gn": 1.926, "lr": 0.0003, "s_per_step": 0.583, "mem_GB": 48.9, "t": "18:59:55"}
|
| 125 |
-
{"event": "train", "step": 3750, "loss": 2.9837, "changed": 2.9013, "kept": 0.4119, "gn": 2.304, "lr": 0.0003, "s_per_step": 0.581, "mem_GB": 48.9, "t": "19:00:20"}
|
| 126 |
-
{"event": "train", "step": 3800, "loss": 2.5083, "changed": 2.4373, "kept": 0.35537, "gn": 5.765, "lr": 0.0003, "s_per_step": 0.579, "mem_GB": 48.9, "t": "19:00:46"}
|
| 127 |
-
{"event": "train", "step": 3850, "loss": 4.1169, "changed": 4.0521, "kept": 0.32404, "gn": 2.044, "lr": 0.0003, "s_per_step": 0.577, "mem_GB": 48.9, "t": "19:01:11"}
|
| 128 |
-
{"event": "train", "step": 3900, "loss": 2.2727, "changed": 2.185, "kept": 0.43866, "gn": 1.997, "lr": 0.0003, "s_per_step": 0.575, "mem_GB": 48.9, "t": "19:01:36"}
|
| 129 |
-
{"event": "train", "step": 3950, "loss": 3.4321, "changed": 3.3522, "kept": 0.39978, "gn": 1.9, "lr": 0.0003, "s_per_step": 0.573, "mem_GB": 48.9, "t": "19:02:02"}
|
| 130 |
-
{"event": "train", "step": 4000, "loss": 2.5479, "changed": 2.4759, "kept": 0.3602, "gn": 1.9, "lr": 0.0003, "s_per_step": 0.571, "mem_GB": 48.9, "t": "19:02:27"}
|
| 131 |
-
{"event": "eval", "step": 4000, "heldout_exact": 0.0, "heldout_changed_acc": 0.87195, "t": "19:02:29"}
|
| 132 |
-
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 133 |
-
warnings.warn( # warn only once
|
| 134 |
-
{"event": "train", "step": 4050, "loss": 3.9324, "changed": 3.8535, "kept": 0.39451, "gn": 1.819, "lr": 0.0003, "s_per_step": 0.573, "mem_GB": 48.9, "t": "19:02:59"}
|
| 135 |
-
{"event": "train", "step": 4100, "loss": 2.8907, "changed": 2.8276, "kept": 0.31542, "gn": 2.174, "lr": 0.0003, "s_per_step": 0.571, "mem_GB": 48.9, "t": "19:03:24"}
|
| 136 |
-
{"event": "train", "step": 4150, "loss": 4.9793, "changed": 4.881, "kept": 0.49165, "gn": 1.944, "lr": 0.0003, "s_per_step": 0.57, "mem_GB": 48.9, "t": "19:03:49"}
|
| 137 |
-
{"event": "train", "step": 4200, "loss": 2.8858, "changed": 2.8298, "kept": 0.27998, "gn": 1.763, "lr": 0.0003, "s_per_step": 0.57, "mem_GB": 48.9, "t": "19:04:18"}
|
| 138 |
-
{"event": "train", "step": 4250, "loss": 2.8387, "changed": 2.7611, "kept": 0.38791, "gn": 1.96, "lr": 0.0003, "s_per_step": 0.57, "mem_GB": 48.9, "t": "19:04:48"}
|
| 139 |
-
{"event": "train", "step": 4300, "loss": 3.7765, "changed": 3.6996, "kept": 0.38497, "gn": 2.422, "lr": 0.0003, "s_per_step": 0.569, "mem_GB": 48.9, "t": "19:05:13"}
|
| 140 |
-
W1031 19:05:33.685000 1741816 torch/distributed/elastic/agent/server/api.py:725] Received 15 death signal, shutting down workers
|
| 141 |
-
W1031 19:05:33.688000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774184 closing signal SIGTERM
|
| 142 |
-
W1031 19:05:33.688000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774188 closing signal SIGTERM
|
| 143 |
-
W1031 19:05:33.693000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774191 closing signal SIGTERM
|
| 144 |
-
W1031 19:05:33.700000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774205 closing signal SIGTERM
|
| 145 |
-
W1031 19:05:33.703000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774215 closing signal SIGTERM
|
| 146 |
-
W1031 19:05:33.710000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774242 closing signal SIGTERM
|
| 147 |
-
W1031 19:05:33.721000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774260 closing signal SIGTERM
|
| 148 |
-
W1031 19:05:33.725000 1741816 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 1774264 closing signal SIGTERM
|
| 149 |
-
Traceback (most recent call last):
|
| 150 |
-
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 151 |
-
File "<frozen runpy>", line 88, in _run_code
|
| 152 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 940, in <module>
|
| 153 |
-
main()
|
| 154 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
|
| 155 |
-
return f(*args, **kwargs)
|
| 156 |
-
^^^^^^^^^^^^^^^^^^
|
| 157 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main
|
| 158 |
-
run(args)
|
| 159 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run
|
| 160 |
-
elastic_launch(
|
| 161 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__
|
| 162 |
-
return launch_agent(self._config, self._entrypoint, list(args))
|
| 163 |
-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 164 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 284, in launch_agent
|
| 165 |
-
result = agent.run()
|
| 166 |
-
^^^^^^^^^^^
|
| 167 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 168 |
-
result = f(*args, **kwargs)
|
| 169 |
-
^^^^^^^^^^^^^^^^^^
|
| 170 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 717, in run
|
| 171 |
-
result = self._invoke_run(role)
|
| 172 |
-
^^^^^^^^^^^^^^^^^^^^^^
|
| 173 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 881, in _invoke_run
|
| 174 |
-
time.sleep(monitor_interval)
|
| 175 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 85, in _terminate_process_handler
|
| 176 |
-
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 177 |
-
torch.distributed.elastic.multiprocessing.api.SignalException: Process 1741816 got signal: 15
|
| 178 |
-
rv1.sh: line 10: 1741816 Killed CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 /dev/shm/prh/venv/bin/python -u -m torch.distributed.run --nproc_per_node 8 --master_port 29820 train_rv.py --name $NAME --programs ${PROGS:-$D/split2/train.jsonl} --heldout $D/split1/heldout.jsonl --steps $STEPS --batch 24 --d 1024 --layers 20 --lr 3e-4 --hard 0.5 --prog_share 0.6 --eval_every 2000 --workers 14 "$@"
|
| 179 |
-
19:05:42 TRAIN_EXIT rc=0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/logs/rv3.log
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
19:07:35 RV start name=rv3 steps=150000 args=--init /dev/shm/neurv/runs/rv2/step_4000.pt
|
| 2 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 437673, "name": "rv3", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split2/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "/dev/shm/neurv/runs/rv2/step_4000.pt", "world": 8, "t": "19:07:54"}
|
| 3 |
-
{"event": "init", "path": "/dev/shm/neurv/runs/rv2/step_4000.pt", "from_step": 4000, "missing": ["copy"], "t": "19:08:03"}
|
| 4 |
-
{"event": "train", "step": 50, "loss": 2.8755, "changed": 2.866, "kept": 0.04731, "gn": 3.621, "lr": 1.5e-05, "s_per_step": 0.598, "mem_GB": 49.1, "t": "19:08:57"}
|
| 5 |
-
{"event": "train", "step": 100, "loss": 2.2407, "changed": 2.2073, "kept": 0.16686, "gn": 1.992, "lr": 3e-05, "s_per_step": 0.551, "mem_GB": 49.1, "t": "19:09:22"}
|
| 6 |
-
{"event": "train", "step": 150, "loss": 4.1524, "changed": 4.116, "kept": 0.18215, "gn": 1.722, "lr": 4.5e-05, "s_per_step": 0.536, "mem_GB": 49.1, "t": "19:09:47"}
|
| 7 |
-
{"event": "train", "step": 200, "loss": 3.5992, "changed": 3.5536, "kept": 0.22796, "gn": 2.031, "lr": 6e-05, "s_per_step": 0.529, "mem_GB": 49.1, "t": "19:10:12"}
|
| 8 |
-
{"event": "train", "step": 250, "loss": 3.4841, "changed": 3.4229, "kept": 0.30623, "gn": 1.851, "lr": 7.5e-05, "s_per_step": 0.525, "mem_GB": 49.1, "t": "19:10:38"}
|
| 9 |
-
{"event": "train", "step": 300, "loss": 2.8161, "changed": 2.7802, "kept": 0.17927, "gn": 2.018, "lr": 9e-05, "s_per_step": 0.522, "mem_GB": 49.1, "t": "19:11:03"}
|
| 10 |
-
{"event": "train", "step": 350, "loss": 2.4208, "changed": 2.3765, "kept": 0.22153, "gn": 2.331, "lr": 0.000105, "s_per_step": 0.519, "mem_GB": 49.1, "t": "19:11:29"}
|
| 11 |
-
{"event": "train", "step": 400, "loss": 1.5732, "changed": 1.5426, "kept": 0.15311, "gn": 1.691, "lr": 0.00012, "s_per_step": 0.518, "mem_GB": 49.1, "t": "19:11:54"}
|
| 12 |
-
{"event": "train", "step": 450, "loss": 1.9167, "changed": 1.8857, "kept": 0.15485, "gn": 2.081, "lr": 0.000135, "s_per_step": 0.517, "mem_GB": 49.1, "t": "19:12:19"}
|
| 13 |
-
{"event": "train", "step": 500, "loss": 2.0773, "changed": 2.0467, "kept": 0.15301, "gn": 1.956, "lr": 0.00015, "s_per_step": 0.516, "mem_GB": 49.1, "t": "19:12:45"}
|
| 14 |
-
{"event": "train", "step": 550, "loss": 3.2985, "changed": 3.2484, "kept": 0.25055, "gn": 1.668, "lr": 0.000165, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:13:10"}
|
| 15 |
-
{"event": "train", "step": 600, "loss": 1.9771, "changed": 1.9358, "kept": 0.20646, "gn": 1.637, "lr": 0.00018, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:13:35"}
|
| 16 |
-
{"event": "train", "step": 650, "loss": 4.524, "changed": 4.4593, "kept": 0.32373, "gn": 1.881, "lr": 0.000195, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:14:01"}
|
| 17 |
-
{"event": "train", "step": 700, "loss": 5.2836, "changed": 5.2201, "kept": 0.31753, "gn": 1.889, "lr": 0.00021, "s_per_step": 0.513, "mem_GB": 49.1, "t": "19:14:26"}
|
| 18 |
-
{"event": "train", "step": 750, "loss": 3.1666, "changed": 3.1058, "kept": 0.30419, "gn": 1.881, "lr": 0.000225, "s_per_step": 0.513, "mem_GB": 49.1, "t": "19:14:51"}
|
| 19 |
-
{"event": "train", "step": 800, "loss": 4.3442, "changed": 4.2966, "kept": 0.23802, "gn": 1.734, "lr": 0.00024, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:15:17"}
|
| 20 |
-
{"event": "train", "step": 850, "loss": 3.5104, "changed": 3.4645, "kept": 0.22934, "gn": 2.172, "lr": 0.000255, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:15:42"}
|
| 21 |
-
{"event": "train", "step": 900, "loss": 2.076, "changed": 2.0109, "kept": 0.32553, "gn": 1.725, "lr": 0.00027, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:16:07"}
|
| 22 |
-
{"event": "train", "step": 950, "loss": 3.1892, "changed": 3.0878, "kept": 0.50684, "gn": 1.708, "lr": 0.000285, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:16:33"}
|
| 23 |
-
{"event": "train", "step": 1000, "loss": 2.9083, "changed": 2.8602, "kept": 0.24051, "gn": 2.668, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:16:58"}
|
| 24 |
-
{"event": "train", "step": 1050, "loss": 2.7945, "changed": 2.7029, "kept": 0.45812, "gn": 2.473, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:17:23"}
|
| 25 |
-
{"event": "train", "step": 1100, "loss": 2.8337, "changed": 2.8016, "kept": 0.16068, "gn": 2.375, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:17:49"}
|
| 26 |
-
{"event": "train", "step": 1150, "loss": 2.2013, "changed": 2.1573, "kept": 0.21998, "gn": 2.997, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:18:14"}
|
| 27 |
-
{"event": "train", "step": 1200, "loss": 2.2897, "changed": 2.2463, "kept": 0.21718, "gn": 2.294, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:18:40"}
|
| 28 |
-
{"event": "train", "step": 1250, "loss": 3.3919, "changed": 3.3456, "kept": 0.2315, "gn": 2.149, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:19:05"}
|
| 29 |
-
{"event": "train", "step": 1300, "loss": 3.3938, "changed": 3.3562, "kept": 0.18789, "gn": 2.142, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:19:30"}
|
| 30 |
-
{"event": "train", "step": 1350, "loss": 3.4609, "changed": 3.3931, "kept": 0.33874, "gn": 2.029, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:19:56"}
|
| 31 |
-
{"event": "train", "step": 1400, "loss": 2.6733, "changed": 2.6271, "kept": 0.23143, "gn": 1.915, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:20:21"}
|
| 32 |
-
{"event": "train", "step": 1450, "loss": 1.2932, "changed": 1.2441, "kept": 0.24585, "gn": 2.622, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:20:46"}
|
| 33 |
-
{"event": "train", "step": 1500, "loss": 1.7521, "changed": 1.7142, "kept": 0.18991, "gn": 1.717, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:21:12"}
|
| 34 |
-
{"event": "train", "step": 1550, "loss": 3.4166, "changed": 3.3492, "kept": 0.33694, "gn": 2.27, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:21:37"}
|
| 35 |
-
{"event": "train", "step": 1600, "loss": 3.6549, "changed": 3.5786, "kept": 0.3816, "gn": 2.227, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:22:02"}
|
| 36 |
-
{"event": "train", "step": 1650, "loss": 4.5485, "changed": 4.49, "kept": 0.29291, "gn": 2.532, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:22:28"}
|
| 37 |
-
{"event": "train", "step": 1700, "loss": 3.9856, "changed": 3.9253, "kept": 0.30118, "gn": 2.803, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:22:53"}
|
| 38 |
-
{"event": "train", "step": 1750, "loss": 4.3288, "changed": 4.2641, "kept": 0.32376, "gn": 2.175, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:23:18"}
|
| 39 |
-
{"event": "train", "step": 1800, "loss": 3.2129, "changed": 3.1568, "kept": 0.28076, "gn": 2.026, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:23:44"}
|
| 40 |
-
{"event": "train", "step": 1850, "loss": 2.2886, "changed": 2.2258, "kept": 0.31365, "gn": 2.041, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:24:09"}
|
| 41 |
-
{"event": "train", "step": 1900, "loss": 3.2922, "changed": 3.2283, "kept": 0.31976, "gn": 2.211, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:24:34"}
|
| 42 |
-
{"event": "train", "step": 1950, "loss": 4.1001, "changed": 4.0419, "kept": 0.29091, "gn": 2.198, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:25:00"}
|
| 43 |
-
{"event": "train", "step": 2000, "loss": 3.239, "changed": 3.1891, "kept": 0.24978, "gn": 2.093, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:25:25"}
|
| 44 |
-
{"event": "eval", "step": 2000, "heldout_exact": 0.0, "heldout_changed_acc": 0.88987, "t": "19:25:27"}
|
| 45 |
-
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 46 |
-
warnings.warn( # warn only once
|
| 47 |
-
{"event": "train", "step": 2050, "loss": 3.5524, "changed": 3.49, "kept": 0.31163, "gn": 1.846, "lr": 0.0003, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:26:01"}
|
| 48 |
-
{"event": "train", "step": 2100, "loss": 2.7671, "changed": 2.7177, "kept": 0.2474, "gn": 2.324, "lr": 0.0003, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:26:27"}
|
| 49 |
-
{"event": "train", "step": 2150, "loss": 3.2628, "changed": 3.1974, "kept": 0.32679, "gn": 1.688, "lr": 0.0003, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:26:54"}
|
| 50 |
-
{"event": "train", "step": 2200, "loss": 3.7346, "changed": 3.6684, "kept": 0.33086, "gn": 2.213, "lr": 0.0003, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:27:19"}
|
| 51 |
-
{"event": "train", "step": 2250, "loss": 3.9253, "changed": 3.8691, "kept": 0.28094, "gn": 2.031, "lr": 0.0003, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:27:45"}
|
| 52 |
-
W1031 19:27:51.473000 3081282 torch/distributed/elastic/agent/server/api.py:725] Received 15 death signal, shutting down workers
|
| 53 |
-
W1031 19:27:51.474000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096147 closing signal SIGTERM
|
| 54 |
-
W1031 19:27:51.474000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096169 closing signal SIGTERM
|
| 55 |
-
W1031 19:27:51.475000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096175 closing signal SIGTERM
|
| 56 |
-
W1031 19:27:51.476000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096180 closing signal SIGTERM
|
| 57 |
-
W1031 19:27:51.477000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096186 closing signal SIGTERM
|
| 58 |
-
W1031 19:27:51.478000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096196 closing signal SIGTERM
|
| 59 |
-
W1031 19:27:51.479000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096202 closing signal SIGTERM
|
| 60 |
-
W1031 19:27:51.480000 3081282 torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3096210 closing signal SIGTERM
|
| 61 |
-
Traceback (most recent call last):
|
| 62 |
-
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 63 |
-
File "<frozen runpy>", line 88, in _run_code
|
| 64 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 940, in <module>
|
| 65 |
-
main()
|
| 66 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
|
| 67 |
-
return f(*args, **kwargs)
|
| 68 |
-
^^^^^^^^^^^^^^^^^^
|
| 69 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main
|
| 70 |
-
run(args)
|
| 71 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run
|
| 72 |
-
elastic_launch(
|
| 73 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__
|
| 74 |
-
return launch_agent(self._config, self._entrypoint, list(args))
|
| 75 |
-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 76 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 284, in launch_agent
|
| 77 |
-
result = agent.run()
|
| 78 |
-
^^^^^^^^^^^
|
| 79 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 80 |
-
result = f(*args, **kwargs)
|
| 81 |
-
^^^^^^^^^^^^^^^^^^
|
| 82 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 717, in run
|
| 83 |
-
result = self._invoke_run(role)
|
| 84 |
-
^^^^^^^^^^^^^^^^^^^^^^
|
| 85 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 881, in _invoke_run
|
| 86 |
-
time.sleep(monitor_interval)
|
| 87 |
-
File "/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 85, in _terminate_process_handler
|
| 88 |
-
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 89 |
-
torch.distributed.elastic.multiprocessing.api.SignalException: Process 3081282 got signal: 15
|
| 90 |
-
[W1031 19:27:54.179714567 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 91 |
-
19:27:54 TRAIN_EXIT rc=0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/logs/rv3_smoke.log
DELETED
|
@@ -1,10 +0,0 @@
|
|
| 1 |
-
19:06:02 RV start name=rv3_smoke steps=60 args=--init /dev/shm/neurv/runs/rv2/step_4000.pt
|
| 2 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 437673, "name": "rv3_smoke", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split2/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "/dev/shm/neurv/runs/rv2/step_4000.pt", "world": 8, "t": "19:06:20"}
|
| 3 |
-
{"event": "init", "path": "/dev/shm/neurv/runs/rv2/step_4000.pt", "from_step": 4000, "missing": ["copy"], "t": "19:06:29"}
|
| 4 |
-
{"event": "train", "step": 50, "loss": 2.8772, "changed": 2.8678, "kept": 0.04732, "gn": 3.618, "lr": 1.5e-05, "s_per_step": 0.587, "mem_GB": 49.1, "t": "19:07:20"}
|
| 5 |
-
{"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.85974, "t": "19:07:27"}
|
| 6 |
-
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 7 |
-
warnings.warn( # warn only once
|
| 8 |
-
{"event": "done", "t": "19:07:32"}
|
| 9 |
-
[W1031 19:07:33.484611997 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 10 |
-
19:07:34 TRAIN_EXIT rc=0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/logs/rv4_smoke.log
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
23:01:51 RV start name=rv4_smoke steps=60 args=--mem_sum_w 0.1
|
| 2 |
-
{"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4_smoke", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 1.0, "reg_w": 1.0, "mem_sum_w": 0.1, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "", "world": 8, "t": "23:02:05"}
|
| 3 |
-
{"event": "train", "step": 50, "loss": 467.9702, "changed": 11.4167, "kept_reg": 2.64743, "gn": 370.671, "lr": 1.5e-05, "s_per_step": 0.535, "mem_GB": 49.1, "t": "23:02:44"}
|
| 4 |
-
{"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.02494, "t": "23:02:51"}
|
| 5 |
-
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 6 |
-
warnings.warn( # warn only once
|
| 7 |
-
{"event": "done", "t": "23:02:52"}
|
| 8 |
-
[W1031 23:02:53.367584628 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 9 |
-
23:02:53 TRAIN_EXIT rc=0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/logs/rvsmoke8.log
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
18:22:12 RV start name=rvsmoke8 steps=60 args=
|
| 2 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 49424, "name": "rvsmoke8", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "18:22:27"}
|
| 3 |
-
{"event": "train", "step": 50, "loss": 11.8035, "changed": 9.4437, "kept": 11.79893, "gn": 7.095, "lr": 1.5e-05, "s_per_step": 0.707, "mem_GB": 48.9, "t": "18:23:21"}
|
| 4 |
-
{"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.05429, "t": "18:23:28"}
|
| 5 |
-
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 6 |
-
warnings.warn( # warn only once
|
| 7 |
-
{"event": "done", "t": "18:23:33"}
|
| 8 |
-
[W1031 18:23:35.872293010 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 9 |
-
18:23:35 TRAIN_EXIT rc=0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/runs/rv1/log.jsonl
DELETED
|
@@ -1,4 +0,0 @@
|
|
| 1 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 49424, "name": "rv1", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "17:41:15"}
|
| 2 |
-
{"event": "train", "step": 50, "loss": 12.7005, "changed": 10.7102, "kept": 9.95181, "gn": 6.628, "lr": 1.5e-05, "s_per_step": 1.247, "mem_GB": 48.9, "t": "17:42:23"}
|
| 3 |
-
{"event": "train", "step": 100, "loss": 8.5641, "changed": 6.9295, "kept": 8.17321, "gn": 3.743, "lr": 3e-05, "s_per_step": 1.201, "mem_GB": 48.9, "t": "17:43:21"}
|
| 4 |
-
{"event": "train", "step": 150, "loss": 6.5321, "changed": 5.3917, "kept": 5.70183, "gn": 3.317, "lr": 4.5e-05, "s_per_step": 1.186, "mem_GB": 48.9, "t": "17:44:19"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/runs/rv2/log.jsonl
DELETED
|
@@ -1,92 +0,0 @@
|
|
| 1 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 49424, "name": "rv2", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "18:23:51"}
|
| 2 |
-
{"event": "train", "step": 50, "loss": 11.7748, "changed": 9.4664, "kept": 11.54169, "gn": 7.76, "lr": 1.5e-05, "s_per_step": 0.595, "mem_GB": 48.9, "t": "18:24:42"}
|
| 3 |
-
{"event": "train", "step": 100, "loss": 7.9366, "changed": 6.1683, "kept": 8.84151, "gn": 3.922, "lr": 3e-05, "s_per_step": 0.549, "mem_GB": 48.9, "t": "18:25:07"}
|
| 4 |
-
{"event": "train", "step": 150, "loss": 6.9692, "changed": 5.6799, "kept": 6.44635, "gn": 3.528, "lr": 4.5e-05, "s_per_step": 0.535, "mem_GB": 48.9, "t": "18:25:32"}
|
| 5 |
-
{"event": "train", "step": 200, "loss": 6.6736, "changed": 5.0267, "kept": 8.23482, "gn": 2.797, "lr": 6e-05, "s_per_step": 0.528, "mem_GB": 48.9, "t": "18:25:58"}
|
| 6 |
-
{"event": "train", "step": 250, "loss": 5.4879, "changed": 4.6458, "kept": 4.21092, "gn": 2.812, "lr": 7.5e-05, "s_per_step": 0.523, "mem_GB": 48.9, "t": "18:26:23"}
|
| 7 |
-
{"event": "train", "step": 300, "loss": 6.6982, "changed": 6.1344, "kept": 2.81894, "gn": 3.043, "lr": 9e-05, "s_per_step": 0.52, "mem_GB": 48.9, "t": "18:26:48"}
|
| 8 |
-
{"event": "train", "step": 350, "loss": 4.3961, "changed": 3.8394, "kept": 2.78348, "gn": 2.532, "lr": 0.000105, "s_per_step": 0.518, "mem_GB": 48.9, "t": "18:27:14"}
|
| 9 |
-
{"event": "train", "step": 400, "loss": 5.1912, "changed": 4.6676, "kept": 2.61762, "gn": 2.668, "lr": 0.00012, "s_per_step": 0.517, "mem_GB": 48.9, "t": "18:27:39"}
|
| 10 |
-
{"event": "train", "step": 450, "loss": 4.6512, "changed": 4.1101, "kept": 2.70544, "gn": 2.518, "lr": 0.000135, "s_per_step": 0.516, "mem_GB": 48.9, "t": "18:28:04"}
|
| 11 |
-
{"event": "train", "step": 500, "loss": 4.6321, "changed": 4.3733, "kept": 1.29404, "gn": 2.398, "lr": 0.00015, "s_per_step": 0.515, "mem_GB": 48.9, "t": "18:28:30"}
|
| 12 |
-
{"event": "train", "step": 550, "loss": 5.6399, "changed": 5.3611, "kept": 1.3941, "gn": 2.671, "lr": 0.000165, "s_per_step": 0.514, "mem_GB": 48.9, "t": "18:28:55"}
|
| 13 |
-
{"event": "train", "step": 600, "loss": 3.8454, "changed": 3.6633, "kept": 0.91083, "gn": 3.234, "lr": 0.00018, "s_per_step": 0.514, "mem_GB": 48.9, "t": "18:29:20"}
|
| 14 |
-
{"event": "train", "step": 650, "loss": 3.1944, "changed": 3.0502, "kept": 0.7206, "gn": 2.654, "lr": 0.000195, "s_per_step": 0.513, "mem_GB": 48.9, "t": "18:29:46"}
|
| 15 |
-
{"event": "train", "step": 700, "loss": 4.1536, "changed": 4.0144, "kept": 0.69581, "gn": 2.763, "lr": 0.00021, "s_per_step": 0.513, "mem_GB": 48.9, "t": "18:30:11"}
|
| 16 |
-
{"event": "train", "step": 750, "loss": 3.3662, "changed": 3.2073, "kept": 0.79431, "gn": 2.734, "lr": 0.000225, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:30:36"}
|
| 17 |
-
{"event": "train", "step": 800, "loss": 2.5462, "changed": 2.419, "kept": 0.63579, "gn": 2.177, "lr": 0.00024, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:31:02"}
|
| 18 |
-
{"event": "train", "step": 850, "loss": 5.2034, "changed": 5.0946, "kept": 0.54373, "gn": 5.122, "lr": 0.000255, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:31:27"}
|
| 19 |
-
{"event": "train", "step": 900, "loss": 5.8737, "changed": 5.7174, "kept": 0.78131, "gn": 2.632, "lr": 0.00027, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:31:52"}
|
| 20 |
-
{"event": "train", "step": 950, "loss": 5.9586, "changed": 5.8194, "kept": 0.69578, "gn": 2.097, "lr": 0.000285, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:32:18"}
|
| 21 |
-
{"event": "train", "step": 1000, "loss": 5.2801, "changed": 5.1506, "kept": 0.64768, "gn": 3.302, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:32:43"}
|
| 22 |
-
{"event": "train", "step": 1050, "loss": 3.992, "changed": 3.8973, "kept": 0.47376, "gn": 2.393, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:33:08"}
|
| 23 |
-
{"event": "train", "step": 1100, "loss": 4.7451, "changed": 4.6598, "kept": 0.42622, "gn": 2.15, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 48.9, "t": "18:33:34"}
|
| 24 |
-
{"event": "train", "step": 1150, "loss": 4.2295, "changed": 4.124, "kept": 0.52744, "gn": 2.425, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:33:59"}
|
| 25 |
-
{"event": "train", "step": 1200, "loss": 4.7774, "changed": 4.6802, "kept": 0.48632, "gn": 2.261, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:34:24"}
|
| 26 |
-
{"event": "train", "step": 1250, "loss": 4.3593, "changed": 4.283, "kept": 0.38117, "gn": 2.595, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:34:50"}
|
| 27 |
-
{"event": "train", "step": 1300, "loss": 3.9209, "changed": 3.8097, "kept": 0.55596, "gn": 2.708, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:35:15"}
|
| 28 |
-
{"event": "train", "step": 1350, "loss": 1.7429, "changed": 1.6935, "kept": 0.24687, "gn": 2.281, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:35:40"}
|
| 29 |
-
{"event": "train", "step": 1400, "loss": 2.6995, "changed": 2.5962, "kept": 0.51676, "gn": 2.391, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:36:06"}
|
| 30 |
-
{"event": "train", "step": 1450, "loss": 3.1172, "changed": 3.0313, "kept": 0.42948, "gn": 2.733, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:36:31"}
|
| 31 |
-
{"event": "train", "step": 1500, "loss": 2.2271, "changed": 2.1529, "kept": 0.371, "gn": 2.118, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 48.9, "t": "18:36:56"}
|
| 32 |
-
{"event": "train", "step": 1550, "loss": 3.9892, "changed": 3.9106, "kept": 0.39276, "gn": 2.448, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:37:22"}
|
| 33 |
-
{"event": "train", "step": 1600, "loss": 3.8216, "changed": 3.747, "kept": 0.37294, "gn": 2.167, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:37:47"}
|
| 34 |
-
{"event": "train", "step": 1650, "loss": 2.7949, "changed": 2.6818, "kept": 0.56543, "gn": 1.808, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:38:12"}
|
| 35 |
-
{"event": "train", "step": 1700, "loss": 4.5407, "changed": 4.4453, "kept": 0.47651, "gn": 1.998, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:38:38"}
|
| 36 |
-
{"event": "train", "step": 1750, "loss": 4.888, "changed": 4.8208, "kept": 0.33567, "gn": 2.301, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:39:03"}
|
| 37 |
-
{"event": "train", "step": 1800, "loss": 4.1498, "changed": 4.0518, "kept": 0.48998, "gn": 2.14, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:39:28"}
|
| 38 |
-
{"event": "train", "step": 1850, "loss": 1.8098, "changed": 1.7394, "kept": 0.35204, "gn": 2.407, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:39:54"}
|
| 39 |
-
{"event": "train", "step": 1900, "loss": 3.2753, "changed": 3.2154, "kept": 0.29973, "gn": 2.246, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:40:19"}
|
| 40 |
-
{"event": "train", "step": 1950, "loss": 4.1304, "changed": 4.0168, "kept": 0.56789, "gn": 2.511, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:40:44"}
|
| 41 |
-
{"event": "train", "step": 2000, "loss": 4.8259, "changed": 4.7308, "kept": 0.47558, "gn": 2.561, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 48.9, "t": "18:41:10"}
|
| 42 |
-
{"event": "eval", "step": 2000, "heldout_exact": 0.0, "heldout_changed_acc": 0.82805, "t": "18:41:12"}
|
| 43 |
-
{"event": "train", "step": 2050, "loss": 3.616, "changed": 3.5198, "kept": 0.4809, "gn": 2.27, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 48.9, "t": "18:41:42"}
|
| 44 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 437673, "name": "rv2", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split2/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "world": 8, "t": "18:42:54"}
|
| 45 |
-
{"event": "resume", "step": 2000, "t": "18:43:04"}
|
| 46 |
-
{"event": "train", "step": 2050, "loss": 2.7133, "changed": 2.5944, "kept": 0.59474, "gn": 1.928, "lr": 0.0003, "s_per_step": 1.228, "mem_GB": 48.9, "t": "18:44:26"}
|
| 47 |
-
{"event": "train", "step": 2100, "loss": 3.9493, "changed": 3.8537, "kept": 0.47814, "gn": 2.551, "lr": 0.0003, "s_per_step": 1.187, "mem_GB": 48.9, "t": "18:45:23"}
|
| 48 |
-
{"event": "train", "step": 2150, "loss": 2.8071, "changed": 2.7405, "kept": 0.33299, "gn": 2.537, "lr": 0.0003, "s_per_step": 1.172, "mem_GB": 48.9, "t": "18:46:20"}
|
| 49 |
-
{"event": "train", "step": 2200, "loss": 4.5497, "changed": 4.4754, "kept": 0.37121, "gn": 2.366, "lr": 0.0003, "s_per_step": 1.158, "mem_GB": 48.9, "t": "18:47:16"}
|
| 50 |
-
{"event": "train", "step": 2250, "loss": 3.7875, "changed": 3.7097, "kept": 0.38858, "gn": 3.064, "lr": 0.0003, "s_per_step": 1.027, "mem_GB": 48.9, "t": "18:47:41"}
|
| 51 |
-
{"event": "train", "step": 2300, "loss": 4.2531, "changed": 4.1714, "kept": 0.40849, "gn": 2.148, "lr": 0.0003, "s_per_step": 0.94, "mem_GB": 48.9, "t": "18:48:06"}
|
| 52 |
-
{"event": "train", "step": 2350, "loss": 3.9655, "changed": 3.886, "kept": 0.39752, "gn": 2.057, "lr": 0.0003, "s_per_step": 0.878, "mem_GB": 48.9, "t": "18:48:32"}
|
| 53 |
-
{"event": "train", "step": 2400, "loss": 2.3351, "changed": 2.2686, "kept": 0.33231, "gn": 2.033, "lr": 0.0003, "s_per_step": 0.831, "mem_GB": 48.9, "t": "18:48:57"}
|
| 54 |
-
{"event": "train", "step": 2450, "loss": 2.1471, "changed": 2.0735, "kept": 0.36827, "gn": 2.205, "lr": 0.0003, "s_per_step": 0.795, "mem_GB": 48.9, "t": "18:49:22"}
|
| 55 |
-
{"event": "train", "step": 2500, "loss": 2.2578, "changed": 2.1937, "kept": 0.32098, "gn": 2.326, "lr": 0.0003, "s_per_step": 0.766, "mem_GB": 48.9, "t": "18:49:47"}
|
| 56 |
-
{"event": "train", "step": 2550, "loss": 4.0374, "changed": 3.9525, "kept": 0.42437, "gn": 2.169, "lr": 0.0003, "s_per_step": 0.743, "mem_GB": 48.9, "t": "18:50:13"}
|
| 57 |
-
{"event": "train", "step": 2600, "loss": 3.5357, "changed": 3.4498, "kept": 0.42942, "gn": 2.029, "lr": 0.0003, "s_per_step": 0.723, "mem_GB": 48.9, "t": "18:50:38"}
|
| 58 |
-
{"event": "train", "step": 2650, "loss": 4.0694, "changed": 3.9886, "kept": 0.40385, "gn": 2.243, "lr": 0.0003, "s_per_step": 0.706, "mem_GB": 48.9, "t": "18:51:03"}
|
| 59 |
-
{"event": "train", "step": 2700, "loss": 4.4654, "changed": 4.4034, "kept": 0.31002, "gn": 2.705, "lr": 0.0003, "s_per_step": 0.692, "mem_GB": 48.9, "t": "18:51:29"}
|
| 60 |
-
{"event": "train", "step": 2750, "loss": 3.746, "changed": 3.6522, "kept": 0.46905, "gn": 2.159, "lr": 0.0003, "s_per_step": 0.68, "mem_GB": 48.9, "t": "18:51:54"}
|
| 61 |
-
{"event": "train", "step": 2800, "loss": 2.7013, "changed": 2.6347, "kept": 0.33275, "gn": 2.571, "lr": 0.0003, "s_per_step": 0.669, "mem_GB": 48.9, "t": "18:52:19"}
|
| 62 |
-
{"event": "train", "step": 2850, "loss": 2.503, "changed": 2.4317, "kept": 0.35671, "gn": 2.348, "lr": 0.0003, "s_per_step": 0.659, "mem_GB": 48.9, "t": "18:52:45"}
|
| 63 |
-
{"event": "train", "step": 2900, "loss": 2.7059, "changed": 2.6287, "kept": 0.38607, "gn": 1.919, "lr": 0.0003, "s_per_step": 0.651, "mem_GB": 48.9, "t": "18:53:10"}
|
| 64 |
-
{"event": "train", "step": 2950, "loss": 2.9244, "changed": 2.8551, "kept": 0.3465, "gn": 1.715, "lr": 0.0003, "s_per_step": 0.643, "mem_GB": 48.9, "t": "18:53:35"}
|
| 65 |
-
{"event": "train", "step": 3000, "loss": 4.9648, "changed": 4.8712, "kept": 0.4683, "gn": 1.902, "lr": 0.0003, "s_per_step": 0.636, "mem_GB": 48.9, "t": "18:54:01"}
|
| 66 |
-
{"event": "train", "step": 3050, "loss": 5.9648, "changed": 5.8559, "kept": 0.5444, "gn": 2.137, "lr": 0.0003, "s_per_step": 0.63, "mem_GB": 48.9, "t": "18:54:26"}
|
| 67 |
-
{"event": "train", "step": 3100, "loss": 2.5671, "changed": 2.5028, "kept": 0.32181, "gn": 1.928, "lr": 0.0003, "s_per_step": 0.624, "mem_GB": 48.9, "t": "18:54:51"}
|
| 68 |
-
{"event": "train", "step": 3150, "loss": 3.7125, "changed": 3.6444, "kept": 0.3402, "gn": 1.824, "lr": 0.0003, "s_per_step": 0.619, "mem_GB": 48.9, "t": "18:55:17"}
|
| 69 |
-
{"event": "train", "step": 3200, "loss": 1.949, "changed": 1.891, "kept": 0.28978, "gn": 2.576, "lr": 0.0003, "s_per_step": 0.615, "mem_GB": 48.9, "t": "18:55:42"}
|
| 70 |
-
{"event": "train", "step": 3250, "loss": 4.8198, "changed": 4.7177, "kept": 0.51048, "gn": 2.497, "lr": 0.0003, "s_per_step": 0.61, "mem_GB": 48.9, "t": "18:56:07"}
|
| 71 |
-
{"event": "train", "step": 3300, "loss": 2.6943, "changed": 2.6303, "kept": 0.32025, "gn": 1.945, "lr": 0.0003, "s_per_step": 0.606, "mem_GB": 48.9, "t": "18:56:32"}
|
| 72 |
-
{"event": "train", "step": 3350, "loss": 1.7631, "changed": 1.6812, "kept": 0.40926, "gn": 2.234, "lr": 0.0003, "s_per_step": 0.603, "mem_GB": 48.9, "t": "18:56:58"}
|
| 73 |
-
{"event": "train", "step": 3400, "loss": 3.9119, "changed": 3.8542, "kept": 0.28862, "gn": 2.276, "lr": 0.0003, "s_per_step": 0.599, "mem_GB": 48.9, "t": "18:57:23"}
|
| 74 |
-
{"event": "train", "step": 3450, "loss": 4.0511, "changed": 3.9636, "kept": 0.43733, "gn": 2.401, "lr": 0.0003, "s_per_step": 0.596, "mem_GB": 48.9, "t": "18:57:48"}
|
| 75 |
-
{"event": "train", "step": 3500, "loss": 3.4284, "changed": 3.354, "kept": 0.37194, "gn": 2.198, "lr": 0.0003, "s_per_step": 0.593, "mem_GB": 48.9, "t": "18:58:14"}
|
| 76 |
-
{"event": "train", "step": 3550, "loss": 3.763, "changed": 3.6887, "kept": 0.37143, "gn": 1.932, "lr": 0.0003, "s_per_step": 0.59, "mem_GB": 48.9, "t": "18:58:39"}
|
| 77 |
-
{"event": "train", "step": 3600, "loss": 3.1771, "changed": 3.0943, "kept": 0.41384, "gn": 2.206, "lr": 0.0003, "s_per_step": 0.588, "mem_GB": 48.9, "t": "18:59:04"}
|
| 78 |
-
{"event": "train", "step": 3650, "loss": 2.1705, "changed": 2.1195, "kept": 0.25491, "gn": 2.341, "lr": 0.0003, "s_per_step": 0.585, "mem_GB": 48.9, "t": "18:59:30"}
|
| 79 |
-
{"event": "train", "step": 3700, "loss": 2.4223, "changed": 2.3521, "kept": 0.351, "gn": 1.926, "lr": 0.0003, "s_per_step": 0.583, "mem_GB": 48.9, "t": "18:59:55"}
|
| 80 |
-
{"event": "train", "step": 3750, "loss": 2.9837, "changed": 2.9013, "kept": 0.4119, "gn": 2.304, "lr": 0.0003, "s_per_step": 0.581, "mem_GB": 48.9, "t": "19:00:20"}
|
| 81 |
-
{"event": "train", "step": 3800, "loss": 2.5083, "changed": 2.4373, "kept": 0.35537, "gn": 5.765, "lr": 0.0003, "s_per_step": 0.579, "mem_GB": 48.9, "t": "19:00:46"}
|
| 82 |
-
{"event": "train", "step": 3850, "loss": 4.1169, "changed": 4.0521, "kept": 0.32404, "gn": 2.044, "lr": 0.0003, "s_per_step": 0.577, "mem_GB": 48.9, "t": "19:01:11"}
|
| 83 |
-
{"event": "train", "step": 3900, "loss": 2.2727, "changed": 2.185, "kept": 0.43866, "gn": 1.997, "lr": 0.0003, "s_per_step": 0.575, "mem_GB": 48.9, "t": "19:01:36"}
|
| 84 |
-
{"event": "train", "step": 3950, "loss": 3.4321, "changed": 3.3522, "kept": 0.39978, "gn": 1.9, "lr": 0.0003, "s_per_step": 0.573, "mem_GB": 48.9, "t": "19:02:02"}
|
| 85 |
-
{"event": "train", "step": 4000, "loss": 2.5479, "changed": 2.4759, "kept": 0.3602, "gn": 1.9, "lr": 0.0003, "s_per_step": 0.571, "mem_GB": 48.9, "t": "19:02:27"}
|
| 86 |
-
{"event": "eval", "step": 4000, "heldout_exact": 0.0, "heldout_changed_acc": 0.87195, "t": "19:02:29"}
|
| 87 |
-
{"event": "train", "step": 4050, "loss": 3.9324, "changed": 3.8535, "kept": 0.39451, "gn": 1.819, "lr": 0.0003, "s_per_step": 0.573, "mem_GB": 48.9, "t": "19:02:59"}
|
| 88 |
-
{"event": "train", "step": 4100, "loss": 2.8907, "changed": 2.8276, "kept": 0.31542, "gn": 2.174, "lr": 0.0003, "s_per_step": 0.571, "mem_GB": 48.9, "t": "19:03:24"}
|
| 89 |
-
{"event": "train", "step": 4150, "loss": 4.9793, "changed": 4.881, "kept": 0.49165, "gn": 1.944, "lr": 0.0003, "s_per_step": 0.57, "mem_GB": 48.9, "t": "19:03:49"}
|
| 90 |
-
{"event": "train", "step": 4200, "loss": 2.8858, "changed": 2.8298, "kept": 0.27998, "gn": 1.763, "lr": 0.0003, "s_per_step": 0.57, "mem_GB": 48.9, "t": "19:04:18"}
|
| 91 |
-
{"event": "train", "step": 4250, "loss": 2.8387, "changed": 2.7611, "kept": 0.38791, "gn": 1.96, "lr": 0.0003, "s_per_step": 0.57, "mem_GB": 48.9, "t": "19:04:48"}
|
| 92 |
-
{"event": "train", "step": 4300, "loss": 3.7765, "changed": 3.6996, "kept": 0.38497, "gn": 2.422, "lr": 0.0003, "s_per_step": 0.569, "mem_GB": 48.9, "t": "19:05:13"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/runs/rv3/log.jsonl
DELETED
|
@@ -1,48 +0,0 @@
|
|
| 1 |
-
{"event": "model", "params_M": 255.9, "tokens": 2081, "n_programs": 437673, "name": "rv3", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split2/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "/dev/shm/neurv/runs/rv2/step_4000.pt", "world": 8, "t": "19:07:54"}
|
| 2 |
-
{"event": "init", "path": "/dev/shm/neurv/runs/rv2/step_4000.pt", "from_step": 4000, "missing": ["copy"], "t": "19:08:03"}
|
| 3 |
-
{"event": "train", "step": 50, "loss": 2.8755, "changed": 2.866, "kept": 0.04731, "gn": 3.621, "lr": 1.5e-05, "s_per_step": 0.598, "mem_GB": 49.1, "t": "19:08:57"}
|
| 4 |
-
{"event": "train", "step": 100, "loss": 2.2407, "changed": 2.2073, "kept": 0.16686, "gn": 1.992, "lr": 3e-05, "s_per_step": 0.551, "mem_GB": 49.1, "t": "19:09:22"}
|
| 5 |
-
{"event": "train", "step": 150, "loss": 4.1524, "changed": 4.116, "kept": 0.18215, "gn": 1.722, "lr": 4.5e-05, "s_per_step": 0.536, "mem_GB": 49.1, "t": "19:09:47"}
|
| 6 |
-
{"event": "train", "step": 200, "loss": 3.5992, "changed": 3.5536, "kept": 0.22796, "gn": 2.031, "lr": 6e-05, "s_per_step": 0.529, "mem_GB": 49.1, "t": "19:10:12"}
|
| 7 |
-
{"event": "train", "step": 250, "loss": 3.4841, "changed": 3.4229, "kept": 0.30623, "gn": 1.851, "lr": 7.5e-05, "s_per_step": 0.525, "mem_GB": 49.1, "t": "19:10:38"}
|
| 8 |
-
{"event": "train", "step": 300, "loss": 2.8161, "changed": 2.7802, "kept": 0.17927, "gn": 2.018, "lr": 9e-05, "s_per_step": 0.522, "mem_GB": 49.1, "t": "19:11:03"}
|
| 9 |
-
{"event": "train", "step": 350, "loss": 2.4208, "changed": 2.3765, "kept": 0.22153, "gn": 2.331, "lr": 0.000105, "s_per_step": 0.519, "mem_GB": 49.1, "t": "19:11:29"}
|
| 10 |
-
{"event": "train", "step": 400, "loss": 1.5732, "changed": 1.5426, "kept": 0.15311, "gn": 1.691, "lr": 0.00012, "s_per_step": 0.518, "mem_GB": 49.1, "t": "19:11:54"}
|
| 11 |
-
{"event": "train", "step": 450, "loss": 1.9167, "changed": 1.8857, "kept": 0.15485, "gn": 2.081, "lr": 0.000135, "s_per_step": 0.517, "mem_GB": 49.1, "t": "19:12:19"}
|
| 12 |
-
{"event": "train", "step": 500, "loss": 2.0773, "changed": 2.0467, "kept": 0.15301, "gn": 1.956, "lr": 0.00015, "s_per_step": 0.516, "mem_GB": 49.1, "t": "19:12:45"}
|
| 13 |
-
{"event": "train", "step": 550, "loss": 3.2985, "changed": 3.2484, "kept": 0.25055, "gn": 1.668, "lr": 0.000165, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:13:10"}
|
| 14 |
-
{"event": "train", "step": 600, "loss": 1.9771, "changed": 1.9358, "kept": 0.20646, "gn": 1.637, "lr": 0.00018, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:13:35"}
|
| 15 |
-
{"event": "train", "step": 650, "loss": 4.524, "changed": 4.4593, "kept": 0.32373, "gn": 1.881, "lr": 0.000195, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:14:01"}
|
| 16 |
-
{"event": "train", "step": 700, "loss": 5.2836, "changed": 5.2201, "kept": 0.31753, "gn": 1.889, "lr": 0.00021, "s_per_step": 0.513, "mem_GB": 49.1, "t": "19:14:26"}
|
| 17 |
-
{"event": "train", "step": 750, "loss": 3.1666, "changed": 3.1058, "kept": 0.30419, "gn": 1.881, "lr": 0.000225, "s_per_step": 0.513, "mem_GB": 49.1, "t": "19:14:51"}
|
| 18 |
-
{"event": "train", "step": 800, "loss": 4.3442, "changed": 4.2966, "kept": 0.23802, "gn": 1.734, "lr": 0.00024, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:15:17"}
|
| 19 |
-
{"event": "train", "step": 850, "loss": 3.5104, "changed": 3.4645, "kept": 0.22934, "gn": 2.172, "lr": 0.000255, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:15:42"}
|
| 20 |
-
{"event": "train", "step": 900, "loss": 2.076, "changed": 2.0109, "kept": 0.32553, "gn": 1.725, "lr": 0.00027, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:16:07"}
|
| 21 |
-
{"event": "train", "step": 950, "loss": 3.1892, "changed": 3.0878, "kept": 0.50684, "gn": 1.708, "lr": 0.000285, "s_per_step": 0.512, "mem_GB": 49.1, "t": "19:16:33"}
|
| 22 |
-
{"event": "train", "step": 1000, "loss": 2.9083, "changed": 2.8602, "kept": 0.24051, "gn": 2.668, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:16:58"}
|
| 23 |
-
{"event": "train", "step": 1050, "loss": 2.7945, "changed": 2.7029, "kept": 0.45812, "gn": 2.473, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:17:23"}
|
| 24 |
-
{"event": "train", "step": 1100, "loss": 2.8337, "changed": 2.8016, "kept": 0.16068, "gn": 2.375, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:17:49"}
|
| 25 |
-
{"event": "train", "step": 1150, "loss": 2.2013, "changed": 2.1573, "kept": 0.21998, "gn": 2.997, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:18:14"}
|
| 26 |
-
{"event": "train", "step": 1200, "loss": 2.2897, "changed": 2.2463, "kept": 0.21718, "gn": 2.294, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:18:40"}
|
| 27 |
-
{"event": "train", "step": 1250, "loss": 3.3919, "changed": 3.3456, "kept": 0.2315, "gn": 2.149, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 49.1, "t": "19:19:05"}
|
| 28 |
-
{"event": "train", "step": 1300, "loss": 3.3938, "changed": 3.3562, "kept": 0.18789, "gn": 2.142, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:19:30"}
|
| 29 |
-
{"event": "train", "step": 1350, "loss": 3.4609, "changed": 3.3931, "kept": 0.33874, "gn": 2.029, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:19:56"}
|
| 30 |
-
{"event": "train", "step": 1400, "loss": 2.6733, "changed": 2.6271, "kept": 0.23143, "gn": 1.915, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:20:21"}
|
| 31 |
-
{"event": "train", "step": 1450, "loss": 1.2932, "changed": 1.2441, "kept": 0.24585, "gn": 2.622, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:20:46"}
|
| 32 |
-
{"event": "train", "step": 1500, "loss": 1.7521, "changed": 1.7142, "kept": 0.18991, "gn": 1.717, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:21:12"}
|
| 33 |
-
{"event": "train", "step": 1550, "loss": 3.4166, "changed": 3.3492, "kept": 0.33694, "gn": 2.27, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:21:37"}
|
| 34 |
-
{"event": "train", "step": 1600, "loss": 3.6549, "changed": 3.5786, "kept": 0.3816, "gn": 2.227, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:22:02"}
|
| 35 |
-
{"event": "train", "step": 1650, "loss": 4.5485, "changed": 4.49, "kept": 0.29291, "gn": 2.532, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:22:28"}
|
| 36 |
-
{"event": "train", "step": 1700, "loss": 3.9856, "changed": 3.9253, "kept": 0.30118, "gn": 2.803, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:22:53"}
|
| 37 |
-
{"event": "train", "step": 1750, "loss": 4.3288, "changed": 4.2641, "kept": 0.32376, "gn": 2.175, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 49.1, "t": "19:23:18"}
|
| 38 |
-
{"event": "train", "step": 1800, "loss": 3.2129, "changed": 3.1568, "kept": 0.28076, "gn": 2.026, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:23:44"}
|
| 39 |
-
{"event": "train", "step": 1850, "loss": 2.2886, "changed": 2.2258, "kept": 0.31365, "gn": 2.041, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:24:09"}
|
| 40 |
-
{"event": "train", "step": 1900, "loss": 3.2922, "changed": 3.2283, "kept": 0.31976, "gn": 2.211, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:24:34"}
|
| 41 |
-
{"event": "train", "step": 1950, "loss": 4.1001, "changed": 4.0419, "kept": 0.29091, "gn": 2.198, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:25:00"}
|
| 42 |
-
{"event": "train", "step": 2000, "loss": 3.239, "changed": 3.1891, "kept": 0.24978, "gn": 2.093, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 49.1, "t": "19:25:25"}
|
| 43 |
-
{"event": "eval", "step": 2000, "heldout_exact": 0.0, "heldout_changed_acc": 0.88987, "t": "19:25:27"}
|
| 44 |
-
{"event": "train", "step": 2050, "loss": 3.5524, "changed": 3.49, "kept": 0.31163, "gn": 1.846, "lr": 0.0003, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:26:01"}
|
| 45 |
-
{"event": "train", "step": 2100, "loss": 2.7671, "changed": 2.7177, "kept": 0.2474, "gn": 2.324, "lr": 0.0003, "s_per_step": 0.514, "mem_GB": 49.1, "t": "19:26:27"}
|
| 46 |
-
{"event": "train", "step": 2150, "loss": 3.2628, "changed": 3.1974, "kept": 0.32679, "gn": 1.688, "lr": 0.0003, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:26:54"}
|
| 47 |
-
{"event": "train", "step": 2200, "loss": 3.7346, "changed": 3.6684, "kept": 0.33086, "gn": 2.213, "lr": 0.0003, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:27:19"}
|
| 48 |
-
{"event": "train", "step": 2250, "loss": 3.9253, "changed": 3.8691, "kept": 0.28094, "gn": 2.031, "lr": 0.0003, "s_per_step": 0.515, "mem_GB": 49.1, "t": "19:27:45"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
neurv/runs/rv4/log.jsonl
DELETED
|
@@ -1,135 +0,0 @@
|
|
| 1 |
-
{"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 437673, "name": "rv4", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split2/train.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "/dev/shm/neurv/runs/rv3/step_2000.pt", "world": 8, "t": "19:29:31"}
|
| 2 |
-
{"event": "init", "path": "/dev/shm/neurv/runs/rv3/step_2000.pt", "from_step": 2000, "missing": ["fbyte.0.weight", "fbyte.1.weight", "fbyte.2.weight", "fbyte.3.weight", "fbits.weight", "fvalid.weight"], "t": "19:29:36"}
|
| 3 |
-
{"event": "train", "step": 50, "loss": 2.1145, "changed": 2.0721, "kept": 0.21193, "gn": 1.887, "lr": 1.5e-05, "s_per_step": 0.56, "mem_GB": 50.1, "t": "19:30:16"}
|
| 4 |
-
{"event": "train", "step": 100, "loss": 2.1382, "changed": 2.0929, "kept": 0.22616, "gn": 1.822, "lr": 3e-05, "s_per_step": 0.532, "mem_GB": 50.1, "t": "19:30:42"}
|
| 5 |
-
{"event": "train", "step": 150, "loss": 3.4892, "changed": 3.4419, "kept": 0.23609, "gn": 1.679, "lr": 4.5e-05, "s_per_step": 0.524, "mem_GB": 50.1, "t": "19:31:07"}
|
| 6 |
-
{"event": "train", "step": 200, "loss": 3.3746, "changed": 3.3321, "kept": 0.21267, "gn": 1.794, "lr": 6e-05, "s_per_step": 0.52, "mem_GB": 50.1, "t": "19:31:32"}
|
| 7 |
-
{"event": "train", "step": 250, "loss": 2.7841, "changed": 2.7262, "kept": 0.28932, "gn": 1.458, "lr": 7.5e-05, "s_per_step": 0.517, "mem_GB": 50.1, "t": "19:31:58"}
|
| 8 |
-
{"event": "train", "step": 300, "loss": 2.0117, "changed": 1.983, "kept": 0.14346, "gn": 1.66, "lr": 9e-05, "s_per_step": 0.515, "mem_GB": 50.1, "t": "19:32:23"}
|
| 9 |
-
{"event": "train", "step": 350, "loss": 1.5497, "changed": 1.5195, "kept": 0.15131, "gn": 1.818, "lr": 0.000105, "s_per_step": 0.514, "mem_GB": 50.1, "t": "19:32:48"}
|
| 10 |
-
{"event": "train", "step": 400, "loss": 1.3557, "changed": 1.3317, "kept": 0.11993, "gn": 1.53, "lr": 0.00012, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:33:14"}
|
| 11 |
-
{"event": "train", "step": 450, "loss": 1.6156, "changed": 1.5911, "kept": 0.12249, "gn": 1.919, "lr": 0.000135, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:33:39"}
|
| 12 |
-
{"event": "train", "step": 500, "loss": 1.6783, "changed": 1.651, "kept": 0.13671, "gn": 1.703, "lr": 0.00015, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:34:04"}
|
| 13 |
-
{"event": "train", "step": 550, "loss": 2.6414, "changed": 2.5989, "kept": 0.21284, "gn": 1.598, "lr": 0.000165, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:34:30"}
|
| 14 |
-
{"event": "train", "step": 600, "loss": 1.6684, "changed": 1.6304, "kept": 0.1903, "gn": 1.524, "lr": 0.00018, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:34:55"}
|
| 15 |
-
{"event": "train", "step": 650, "loss": 3.7046, "changed": 3.6487, "kept": 0.27969, "gn": 1.573, "lr": 0.000195, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:35:21"}
|
| 16 |
-
{"event": "train", "step": 700, "loss": 4.2632, "changed": 4.2205, "kept": 0.21331, "gn": 2.009, "lr": 0.00021, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:35:46"}
|
| 17 |
-
{"event": "train", "step": 750, "loss": 2.3397, "changed": 2.2937, "kept": 0.22997, "gn": 1.958, "lr": 0.000225, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:36:11"}
|
| 18 |
-
{"event": "train", "step": 800, "loss": 2.7949, "changed": 2.7524, "kept": 0.21274, "gn": 1.792, "lr": 0.00024, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:36:37"}
|
| 19 |
-
{"event": "train", "step": 850, "loss": 2.2405, "changed": 2.2022, "kept": 0.1915, "gn": 2.495, "lr": 0.000255, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:37:02"}
|
| 20 |
-
{"event": "train", "step": 900, "loss": 1.4067, "changed": 1.3731, "kept": 0.1681, "gn": 1.999, "lr": 0.00027, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:37:27"}
|
| 21 |
-
{"event": "train", "step": 950, "loss": 2.3223, "changed": 2.226, "kept": 0.48165, "gn": 1.643, "lr": 0.000285, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:37:53"}
|
| 22 |
-
{"event": "train", "step": 1000, "loss": 1.9272, "changed": 1.886, "kept": 0.20611, "gn": 2.28, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:38:18"}
|
| 23 |
-
{"event": "train", "step": 1050, "loss": 2.2264, "changed": 2.1578, "kept": 0.34326, "gn": 1.635, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:38:43"}
|
| 24 |
-
{"event": "train", "step": 1100, "loss": 2.4851, "changed": 2.4506, "kept": 0.17292, "gn": 2.098, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:39:09"}
|
| 25 |
-
{"event": "train", "step": 1150, "loss": 1.6898, "changed": 1.6551, "kept": 0.17367, "gn": 2.419, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:39:34"}
|
| 26 |
-
{"event": "train", "step": 1200, "loss": 1.5916, "changed": 1.5646, "kept": 0.13493, "gn": 1.546, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:39:59"}
|
| 27 |
-
{"event": "train", "step": 1250, "loss": 1.8846, "changed": 1.8507, "kept": 0.16916, "gn": 1.712, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:40:25"}
|
| 28 |
-
{"event": "train", "step": 1300, "loss": 1.9157, "changed": 1.8829, "kept": 0.16395, "gn": 1.81, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:40:50"}
|
| 29 |
-
{"event": "train", "step": 1350, "loss": 1.9406, "changed": 1.8894, "kept": 0.25608, "gn": 1.559, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:41:15"}
|
| 30 |
-
{"event": "train", "step": 1400, "loss": 1.4978, "changed": 1.4316, "kept": 0.33071, "gn": 1.55, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:41:41"}
|
| 31 |
-
{"event": "train", "step": 1450, "loss": 0.9608, "changed": 0.9341, "kept": 0.13345, "gn": 2.137, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:42:06"}
|
| 32 |
-
{"event": "train", "step": 1500, "loss": 0.6552, "changed": 0.632, "kept": 0.11598, "gn": 1.471, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:42:31"}
|
| 33 |
-
{"event": "train", "step": 1550, "loss": 2.3094, "changed": 2.2582, "kept": 0.25561, "gn": 1.658, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:42:57"}
|
| 34 |
-
{"event": "train", "step": 1600, "loss": 1.8248, "changed": 1.7412, "kept": 0.41825, "gn": 1.477, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:43:22"}
|
| 35 |
-
{"event": "train", "step": 1650, "loss": 2.5578, "changed": 2.5099, "kept": 0.23974, "gn": 2.253, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "19:43:48"}
|
| 36 |
-
{"event": "train", "step": 1700, "loss": 2.55, "changed": 2.5056, "kept": 0.22212, "gn": 2.13, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:44:13"}
|
| 37 |
-
{"event": "train", "step": 1750, "loss": 1.6007, "changed": 1.562, "kept": 0.19338, "gn": 1.625, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:44:38"}
|
| 38 |
-
{"event": "train", "step": 1800, "loss": 1.2221, "changed": 1.1797, "kept": 0.21179, "gn": 1.753, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:45:04"}
|
| 39 |
-
{"event": "train", "step": 1850, "loss": 1.1697, "changed": 1.1174, "kept": 0.2614, "gn": 1.682, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:45:29"}
|
| 40 |
-
{"event": "train", "step": 1900, "loss": 2.5301, "changed": 2.4887, "kept": 0.20721, "gn": 1.443, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:45:54"}
|
| 41 |
-
{"event": "train", "step": 1950, "loss": 1.7904, "changed": 1.7516, "kept": 0.19397, "gn": 1.474, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:46:20"}
|
| 42 |
-
{"event": "train", "step": 2000, "loss": 2.424, "changed": 2.384, "kept": 0.20026, "gn": 1.619, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "19:46:45"}
|
| 43 |
-
{"event": "eval", "step": 2000, "heldout_exact": 0.0, "heldout_changed_acc": 0.92234, "t": "19:46:47"}
|
| 44 |
-
{"event": "train", "step": 2050, "loss": 2.7973, "changed": 2.7551, "kept": 0.21141, "gn": 1.569, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "19:47:14"}
|
| 45 |
-
{"event": "train", "step": 2100, "loss": 1.3026, "changed": 1.2459, "kept": 0.28337, "gn": 1.651, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:47:47"}
|
| 46 |
-
{"event": "train", "step": 2150, "loss": 1.6621, "changed": 1.6094, "kept": 0.26348, "gn": 1.614, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:48:12"}
|
| 47 |
-
{"event": "train", "step": 2200, "loss": 1.8742, "changed": 1.8408, "kept": 0.16679, "gn": 1.309, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:48:37"}
|
| 48 |
-
{"event": "train", "step": 2250, "loss": 2.7357, "changed": 2.6883, "kept": 0.23698, "gn": 1.321, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:49:03"}
|
| 49 |
-
{"event": "train", "step": 2300, "loss": 1.4207, "changed": 1.3655, "kept": 0.27605, "gn": 1.929, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:49:28"}
|
| 50 |
-
{"event": "train", "step": 2350, "loss": 1.9008, "changed": 1.8366, "kept": 0.32104, "gn": 1.294, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:49:53"}
|
| 51 |
-
{"event": "train", "step": 2400, "loss": 1.4944, "changed": 1.463, "kept": 0.157, "gn": 1.497, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "19:50:19"}
|
| 52 |
-
{"event": "train", "step": 2450, "loss": 1.9675, "changed": 1.9351, "kept": 0.16223, "gn": 1.361, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:50:44"}
|
| 53 |
-
{"event": "train", "step": 2500, "loss": 2.1257, "changed": 2.0563, "kept": 0.34712, "gn": 1.37, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:51:09"}
|
| 54 |
-
{"event": "train", "step": 2550, "loss": 1.7532, "changed": 1.7005, "kept": 0.26357, "gn": 1.399, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:51:35"}
|
| 55 |
-
{"event": "train", "step": 2600, "loss": 0.4549, "changed": 0.3992, "kept": 0.2784, "gn": 1.622, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:52:00"}
|
| 56 |
-
{"event": "train", "step": 2650, "loss": 1.4027, "changed": 1.3668, "kept": 0.17928, "gn": 1.504, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:52:25"}
|
| 57 |
-
{"event": "train", "step": 2700, "loss": 1.1241, "changed": 1.0893, "kept": 0.17423, "gn": 1.33, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:52:51"}
|
| 58 |
-
{"event": "train", "step": 2750, "loss": 1.6817, "changed": 1.6507, "kept": 0.15539, "gn": 1.167, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:53:16"}
|
| 59 |
-
{"event": "train", "step": 2800, "loss": 1.4913, "changed": 1.4431, "kept": 0.24106, "gn": 1.608, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:53:41"}
|
| 60 |
-
{"event": "train", "step": 2850, "loss": 0.9401, "changed": 0.9217, "kept": 0.09168, "gn": 1.551, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:54:07"}
|
| 61 |
-
{"event": "train", "step": 2900, "loss": 2.5446, "changed": 2.452, "kept": 0.463, "gn": 1.978, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "19:54:32"}
|
| 62 |
-
{"event": "train", "step": 2950, "loss": 1.9982, "changed": 1.9573, "kept": 0.20468, "gn": 1.403, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:54:57"}
|
| 63 |
-
{"event": "train", "step": 3000, "loss": 1.642, "changed": 1.6048, "kept": 0.18639, "gn": 1.68, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:55:23"}
|
| 64 |
-
{"event": "train", "step": 3050, "loss": 1.5514, "changed": 1.5301, "kept": 0.10649, "gn": 1.656, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:55:48"}
|
| 65 |
-
{"event": "train", "step": 3100, "loss": 3.462, "changed": 3.4324, "kept": 0.14794, "gn": 1.268, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:56:13"}
|
| 66 |
-
{"event": "train", "step": 3150, "loss": 1.1707, "changed": 1.1019, "kept": 0.34357, "gn": 1.505, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:56:39"}
|
| 67 |
-
{"event": "train", "step": 3200, "loss": 1.2883, "changed": 1.2589, "kept": 0.14691, "gn": 1.313, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:57:04"}
|
| 68 |
-
{"event": "train", "step": 3250, "loss": 1.1653, "changed": 1.1239, "kept": 0.20727, "gn": 1.347, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:57:29"}
|
| 69 |
-
{"event": "train", "step": 3300, "loss": 2.439, "changed": 2.4062, "kept": 0.16387, "gn": 1.806, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:57:55"}
|
| 70 |
-
{"event": "train", "step": 3350, "loss": 1.6349, "changed": 1.6093, "kept": 0.12818, "gn": 1.792, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:58:20"}
|
| 71 |
-
{"event": "train", "step": 3400, "loss": 2.5063, "changed": 2.4575, "kept": 0.24407, "gn": 1.616, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:58:45"}
|
| 72 |
-
{"event": "train", "step": 3450, "loss": 1.0628, "changed": 1.0318, "kept": 0.1552, "gn": 1.361, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:59:11"}
|
| 73 |
-
{"event": "train", "step": 3500, "loss": 1.3204, "changed": 1.2881, "kept": 0.16142, "gn": 1.43, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "19:59:36"}
|
| 74 |
-
{"event": "train", "step": 3550, "loss": 2.3513, "changed": 2.3124, "kept": 0.19434, "gn": 1.442, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:00:01"}
|
| 75 |
-
{"event": "train", "step": 3600, "loss": 2.1317, "changed": 2.0982, "kept": 0.16774, "gn": 1.392, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:00:27"}
|
| 76 |
-
{"event": "train", "step": 3650, "loss": 3.1606, "changed": 3.108, "kept": 0.26282, "gn": 1.999, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:00:52"}
|
| 77 |
-
{"event": "train", "step": 3700, "loss": 2.1722, "changed": 2.1294, "kept": 0.21404, "gn": 2.057, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:01:17"}
|
| 78 |
-
{"event": "train", "step": 3750, "loss": 2.1285, "changed": 2.1053, "kept": 0.11594, "gn": 1.284, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:01:43"}
|
| 79 |
-
{"event": "train", "step": 3800, "loss": 2.3169, "changed": 2.2884, "kept": 0.14236, "gn": 1.731, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:02:08"}
|
| 80 |
-
{"event": "train", "step": 3850, "loss": 3.6352, "changed": 3.5658, "kept": 0.3472, "gn": 1.415, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:02:33"}
|
| 81 |
-
{"event": "train", "step": 3900, "loss": 1.0041, "changed": 0.9355, "kept": 0.34306, "gn": 1.368, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:02:59"}
|
| 82 |
-
{"event": "train", "step": 3950, "loss": 1.2675, "changed": 1.23, "kept": 0.18775, "gn": 1.224, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:03:24"}
|
| 83 |
-
{"event": "train", "step": 4000, "loss": 1.5661, "changed": 1.5471, "kept": 0.09527, "gn": 2.426, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:03:49"}
|
| 84 |
-
{"event": "eval", "step": 4000, "heldout_exact": 0.00462, "heldout_changed_acc": 0.93195, "t": "20:03:51"}
|
| 85 |
-
{"event": "train", "step": 4050, "loss": 2.0236, "changed": 1.9332, "kept": 0.45237, "gn": 1.453, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:04:19"}
|
| 86 |
-
{"event": "train", "step": 4100, "loss": 1.1988, "changed": 1.1566, "kept": 0.21089, "gn": 1.117, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:04:44"}
|
| 87 |
-
{"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "", "world": 8, "t": "20:06:11"}
|
| 88 |
-
{"event": "resume", "step": 4000, "t": "20:06:16"}
|
| 89 |
-
{"event": "train", "step": 4050, "loss": 1.9552, "changed": 1.8977, "kept": 0.28763, "gn": 1.435, "lr": 0.0003, "s_per_step": 0.537, "mem_GB": 50.1, "t": "20:06:53"}
|
| 90 |
-
{"event": "train", "step": 4100, "loss": 1.4779, "changed": 1.449, "kept": 0.14438, "gn": 3.751, "lr": 0.0003, "s_per_step": 0.521, "mem_GB": 50.1, "t": "20:07:18"}
|
| 91 |
-
{"event": "train", "step": 4150, "loss": 2.3226, "changed": 2.2891, "kept": 0.16735, "gn": 1.557, "lr": 0.0003, "s_per_step": 0.516, "mem_GB": 50.1, "t": "20:07:44"}
|
| 92 |
-
{"event": "train", "step": 4200, "loss": 2.579, "changed": 2.5372, "kept": 0.20876, "gn": 1.358, "lr": 0.0003, "s_per_step": 0.514, "mem_GB": 50.1, "t": "20:08:09"}
|
| 93 |
-
{"event": "train", "step": 4250, "loss": 3.2667, "changed": 3.2116, "kept": 0.27513, "gn": 1.276, "lr": 0.0003, "s_per_step": 0.512, "mem_GB": 50.1, "t": "20:08:34"}
|
| 94 |
-
{"event": "train", "step": 4300, "loss": 1.8565, "changed": 1.823, "kept": 0.16758, "gn": 1.355, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:09:00"}
|
| 95 |
-
{"event": "train", "step": 4350, "loss": 2.5599, "changed": 2.538, "kept": 0.1094, "gn": 1.366, "lr": 0.0003, "s_per_step": 0.511, "mem_GB": 50.1, "t": "20:09:25"}
|
| 96 |
-
{"event": "train", "step": 4400, "loss": 1.6867, "changed": 1.6697, "kept": 0.08505, "gn": 1.575, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:09:50"}
|
| 97 |
-
{"event": "train", "step": 4450, "loss": 1.8416, "changed": 1.7968, "kept": 0.22384, "gn": 1.652, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:10:16"}
|
| 98 |
-
{"event": "train", "step": 4500, "loss": 2.2567, "changed": 2.2035, "kept": 0.26621, "gn": 1.415, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "20:10:41"}
|
| 99 |
-
{"event": "train", "step": 4550, "loss": 3.3773, "changed": 3.3471, "kept": 0.15119, "gn": 1.373, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "20:11:06"}
|
| 100 |
-
{"event": "train", "step": 4600, "loss": 2.698, "changed": 2.6512, "kept": 0.23435, "gn": 1.353, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "20:11:32"}
|
| 101 |
-
{"event": "train", "step": 4650, "loss": 1.3368, "changed": 1.2873, "kept": 0.24743, "gn": 1.488, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "20:11:57"}
|
| 102 |
-
{"event": "train", "step": 4700, "loss": 1.5479, "changed": 1.5139, "kept": 0.16963, "gn": 1.414, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "20:12:22"}
|
| 103 |
-
{"event": "train", "step": 4750, "loss": 1.6179, "changed": 1.5799, "kept": 0.19027, "gn": 1.841, "lr": 0.0003, "s_per_step": 0.509, "mem_GB": 50.1, "t": "20:12:48"}
|
| 104 |
-
{"event": "train", "step": 4800, "loss": 2.4878, "changed": 2.4471, "kept": 0.20396, "gn": 1.209, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:13:13"}
|
| 105 |
-
{"event": "train", "step": 4850, "loss": 2.6804, "changed": 2.6239, "kept": 0.28242, "gn": 1.533, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:13:38"}
|
| 106 |
-
{"event": "train", "step": 4900, "loss": 1.3756, "changed": 1.3113, "kept": 0.32179, "gn": 1.089, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:14:04"}
|
| 107 |
-
{"event": "train", "step": 4950, "loss": 2.1167, "changed": 2.0756, "kept": 0.20536, "gn": 1.886, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:14:29"}
|
| 108 |
-
{"event": "train", "step": 5000, "loss": 2.8841, "changed": 2.8217, "kept": 0.31223, "gn": 1.514, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:14:54"}
|
| 109 |
-
{"event": "train", "step": 5050, "loss": 1.6735, "changed": 1.6073, "kept": 0.33136, "gn": 1.484, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:15:20"}
|
| 110 |
-
{"event": "train", "step": 5100, "loss": 1.849, "changed": 1.8199, "kept": 0.14551, "gn": 2.065, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:15:45"}
|
| 111 |
-
{"event": "train", "step": 5150, "loss": 2.1811, "changed": 2.1215, "kept": 0.29763, "gn": 1.096, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:16:10"}
|
| 112 |
-
{"event": "train", "step": 5200, "loss": 0.9798, "changed": 0.939, "kept": 0.20402, "gn": 2.02, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:16:36"}
|
| 113 |
-
{"event": "train", "step": 5250, "loss": 2.7181, "changed": 2.6895, "kept": 0.14315, "gn": 1.432, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:17:01"}
|
| 114 |
-
{"event": "train", "step": 5300, "loss": 2.6942, "changed": 2.644, "kept": 0.2508, "gn": 1.459, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:17:26"}
|
| 115 |
-
{"event": "train", "step": 5350, "loss": 3.5573, "changed": 3.5296, "kept": 0.13835, "gn": 1.488, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:17:52"}
|
| 116 |
-
{"event": "train", "step": 5400, "loss": 1.1595, "changed": 1.1195, "kept": 0.20001, "gn": 1.624, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:18:17"}
|
| 117 |
-
{"event": "train", "step": 5450, "loss": 0.648, "changed": 0.6275, "kept": 0.10279, "gn": 1.1, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:18:42"}
|
| 118 |
-
{"event": "train", "step": 5500, "loss": 1.4768, "changed": 1.4284, "kept": 0.2418, "gn": 1.071, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:19:08"}
|
| 119 |
-
{"event": "train", "step": 5550, "loss": 1.8369, "changed": 1.8054, "kept": 0.15735, "gn": 1.119, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:19:33"}
|
| 120 |
-
{"event": "train", "step": 5600, "loss": 1.3039, "changed": 1.2652, "kept": 0.19339, "gn": 0.997, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:19:58"}
|
| 121 |
-
{"event": "train", "step": 5650, "loss": 2.4128, "changed": 2.3795, "kept": 0.16654, "gn": 1.254, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:20:24"}
|
| 122 |
-
{"event": "train", "step": 5700, "loss": 2.0634, "changed": 1.997, "kept": 0.33198, "gn": 1.278, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:20:49"}
|
| 123 |
-
{"event": "train", "step": 5750, "loss": 1.0581, "changed": 1.0367, "kept": 0.10714, "gn": 1.435, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:21:14"}
|
| 124 |
-
{"event": "train", "step": 5800, "loss": 1.3144, "changed": 1.2963, "kept": 0.09043, "gn": 1.538, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:21:40"}
|
| 125 |
-
{"event": "train", "step": 5850, "loss": 0.8853, "changed": 0.8122, "kept": 0.36564, "gn": 1.538, "lr": 0.0003, "s_per_step": 0.508, "mem_GB": 50.1, "t": "20:22:05"}
|
| 126 |
-
{"event": "train", "step": 5900, "loss": 1.6605, "changed": 1.6349, "kept": 0.1278, "gn": 1.199, "lr": 0.0003, "s_per_step": 0.507, "mem_GB": 50.1, "t": "20:22:30"}
|
| 127 |
-
{"event": "train", "step": 5950, "loss": 0.7796, "changed": 0.7322, "kept": 0.23697, "gn": 1.573, "lr": 0.0003, "s_per_step": 0.507, "mem_GB": 50.1, "t": "20:22:56"}
|
| 128 |
-
{"event": "train", "step": 6000, "loss": 2.739, "changed": 2.6956, "kept": 0.21723, "gn": 1.425, "lr": 0.0003, "s_per_step": 0.507, "mem_GB": 50.1, "t": "20:23:21"}
|
| 129 |
-
{"event": "eval", "step": 6000, "heldout_exact": 0.01762, "heldout_changed_acc": 0.93948, "t": "20:23:23"}
|
| 130 |
-
{"event": "train", "step": 6050, "loss": 3.0523, "changed": 2.9697, "kept": 0.41288, "gn": 1.369, "lr": 0.0003, "s_per_step": 0.51, "mem_GB": 50.1, "t": "20:23:53"}
|
| 131 |
-
{"event": "train", "step": 6100, "loss": 0.575, "changed": 0.5465, "kept": 0.14248, "gn": 1.69, "lr": 0.0003, "s_per_step": 0.514, "mem_GB": 50.1, "t": "20:24:25"}
|
| 132 |
-
{"event": "train", "step": 6150, "loss": 1.6463, "changed": 1.6098, "kept": 0.18237, "gn": 1.308, "lr": 0.0003, "s_per_step": 0.513, "mem_GB": 50.1, "t": "20:24:50"}
|
| 133 |
-
{"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 150000, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 1.0, "reg_w": 1.0, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "", "world": 8, "t": "20:26:29"}
|
| 134 |
-
{"event": "resume", "step": 6000, "t": "20:26:34"}
|
| 135 |
-
{"event": "train", "step": 6050, "loss": 1.3942, "changed": 1.0244, "kept_reg": 0.33075, "gn": 2.227, "lr": 0.0003, "s_per_step": 0.535, "mem_GB": 50.1, "t": "20:27:10"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|