sync learned_computer: 2 files from /dev/shm/neurv
Browse files- neurv/logs/rv4_smoke.log +7 -7
- neurv/logs/watch_rv4.log +6 -0
neurv/logs/rv4_smoke.log
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
-
20:
|
| 2 |
-
{"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4_smoke", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.
|
| 3 |
-
{"event": "train", "step": 50, "loss":
|
| 4 |
-
{"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.
|
| 5 |
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 6 |
warnings.warn( # warn only once
|
| 7 |
-
{"event": "done", "t": "20:
|
| 8 |
-
[W1031 20:
|
| 9 |
-
20:
|
|
|
|
| 1 |
+
20:25:14 RV start name=rv4_smoke steps=60 args=
|
| 2 |
+
{"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4_smoke", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 1.0, "reg_w": 1.0, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "", "world": 8, "t": "20:25:28"}
|
| 3 |
+
{"event": "train", "step": 50, "loss": 14.3675, "changed": 9.6041, "kept_reg": 1.99426, "gn": 6.526, "lr": 1.5e-05, "s_per_step": 0.535, "mem_GB": 49.1, "t": "20:26:07"}
|
| 4 |
+
{"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.00156, "t": "20:26:14"}
|
| 5 |
/dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 6 |
warnings.warn( # warn only once
|
| 7 |
+
{"event": "done", "t": "20:26:16"}
|
| 8 |
+
[W1031 20:26:16.570299434 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
|
| 9 |
+
20:26:17 TRAIN_EXIT rc=0
|
neurv/logs/watch_rv4.log
CHANGED
|
@@ -7,3 +7,9 @@ watchdogs left 0
|
|
| 7 |
20:04:56 WATCH start rv4 steps=150000 args=
|
| 8 |
20:05:58 smoke ok: {"event": "train", "step": 50, "loss": 9.5028, "changed": 8.82, "kept": 3.4144, "gn": 4.075, "lr": 1.5e-05, "s_per_step": 0.534, "mem_GB": 50.1, "t": "20:05:48"}
|
| 9 |
20:05:58 launch rv4 attempt 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
20:04:56 WATCH start rv4 steps=150000 args=
|
| 8 |
20:05:58 smoke ok: {"event": "train", "step": 50, "loss": 9.5028, "changed": 8.82, "kept": 3.4144, "gn": 4.075, "lr": 1.5e-05, "s_per_step": 0.534, "mem_GB": 50.1, "t": "20:05:48"}
|
| 9 |
20:05:58 launch rv4 attempt 1
|
| 10 |
+
20:25:03 swap_loss: restarting rv4 with registers scored on their own (reg_w 1.0, keep_w 1.0)
|
| 11 |
+
left 0
|
| 12 |
+
watchdogs left 0
|
| 13 |
+
20:25:14 WATCH start rv4 steps=150000 args=
|
| 14 |
+
20:26:17 smoke ok: {"event": "train", "step": 50, "loss": 14.3675, "changed": 9.6041, "kept_reg": 1.99426, "gn": 6.526, "lr": 1.5e-05, "s_per_step": 0.535, "mem_GB": 49.1, "t": "20:26:07"}
|
| 15 |
+
20:26:17 launch rv4 attempt 1
|