Prahaladha commited on
Commit
6a708ca
·
verified ·
1 Parent(s): fffd0e2

sync learned_computer: 2 files from /dev/shm/neurv

Browse files
neurv/logs/rv4_smoke.log CHANGED
@@ -1,9 +1,9 @@
1
- 20:04:56 RV start name=rv4_smoke steps=60 args=
2
- {"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4_smoke", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 0.2, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "", "world": 8, "t": "20:05:10"}
3
- {"event": "train", "step": 50, "loss": 9.5028, "changed": 8.82, "kept": 3.4144, "gn": 4.075, "lr": 1.5e-05, "s_per_step": 0.534, "mem_GB": 50.1, "t": "20:05:48"}
4
- {"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.00494, "t": "20:05:55"}
5
  /dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
6
  warnings.warn( # warn only once
7
- {"event": "done", "t": "20:05:57"}
8
- [W1031 20:05:58.261296561 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
9
- 20:05:58 TRAIN_EXIT rc=0
 
1
+ 20:25:14 RV start name=rv4_smoke steps=60 args=
2
+ {"event": "model", "params_M": 257.0, "tokens": 2081, "n_programs": 993053, "name": "rv4_smoke", "programs": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/train_all.jsonl", "heldout": "/config/ssh-users/maya-research-fsxznwc8-prahaladh/neurv_data/split1/heldout.jsonl", "steps": 60, "batch": 24, "lr": 0.0003, "d": 1024, "layers": 20, "keep_w": 1.0, "reg_w": 1.0, "hard": 0.5, "prog_share": 0.6, "eval_every": 2000, "workers": 14, "ckpt_grad": 0, "init": "", "world": 8, "t": "20:25:28"}
3
+ {"event": "train", "step": 50, "loss": 14.3675, "changed": 9.6041, "kept_reg": 1.99426, "gn": 6.526, "lr": 1.5e-05, "s_per_step": 0.535, "mem_GB": 49.1, "t": "20:26:07"}
4
+ {"event": "eval", "step": 60, "heldout_exact": 0.0, "heldout_changed_acc": 0.00156, "t": "20:26:14"}
5
  /dev/shm/prh/venv/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
6
  warnings.warn( # warn only once
7
+ {"event": "done", "t": "20:26:16"}
8
+ [W1031 20:26:16.570299434 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
9
+ 20:26:17 TRAIN_EXIT rc=0
neurv/logs/watch_rv4.log CHANGED
@@ -7,3 +7,9 @@ watchdogs left 0
7
  20:04:56 WATCH start rv4 steps=150000 args=
8
  20:05:58 smoke ok: {"event": "train", "step": 50, "loss": 9.5028, "changed": 8.82, "kept": 3.4144, "gn": 4.075, "lr": 1.5e-05, "s_per_step": 0.534, "mem_GB": 50.1, "t": "20:05:48"}
9
  20:05:58 launch rv4 attempt 1
 
 
 
 
 
 
 
7
  20:04:56 WATCH start rv4 steps=150000 args=
8
  20:05:58 smoke ok: {"event": "train", "step": 50, "loss": 9.5028, "changed": 8.82, "kept": 3.4144, "gn": 4.075, "lr": 1.5e-05, "s_per_step": 0.534, "mem_GB": 50.1, "t": "20:05:48"}
9
  20:05:58 launch rv4 attempt 1
10
+ 20:25:03 swap_loss: restarting rv4 with registers scored on their own (reg_w 1.0, keep_w 1.0)
11
+ left 0
12
+ watchdogs left 0
13
+ 20:25:14 WATCH start rv4 steps=150000 args=
14
+ 20:26:17 smoke ok: {"event": "train", "step": 50, "loss": 14.3675, "changed": 9.6041, "kept_reg": 1.99426, "gn": 6.526, "lr": 1.5e-05, "s_per_step": 0.535, "mem_GB": 49.1, "t": "20:26:07"}
15
+ 20:26:17 launch rv4 attempt 1