File size: 1,327 Bytes
0a5e7d6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
{
  "run": "G3A1 joint R2R + English RxR15, one-epoch decay (3,852 updates, warmup 116, cosine to 0.1 x 5e-6)",
  "global_step": 2100,
  "max_steps": 3852,
  "epoch": 0.5451713395638629,
  "percent": 54.52,
  "remaining_updates": 1752,
  "learning_rate_at_stop": 2.5327567618061965e-06,
  "loss_at_stop": 0.1690405160188675,
  "grad_norm_at_stop": 2.477595806121826,
  "mean_loss_last_100_updates": 0.15498921513557434,
  "aux_weight_at_stop": 0.1,
  "g3_state_loss_per_pair_last_100": 0.013914937848588878,
  "a1_relation_loss_per_pair_last_100": 0.3475379606630543,
  "exposures": 16800,
  "unique_episodes_seen": 16800,
  "duplicate_exposures": 0,
  "manifest_episodes": 30815,
  "epoch_exposures": 30816,
  "remaining_exposures": 14016,
  "actions_seen": 1708373,
  "manifest_actions": 3128624,
  "sampler": "EpisodeLengthSampler seed 429 (position restored from checkpoint trainer_state/RNG; do not change the manifest order)",
  "cuda_ooms": 0,
  "allocations": [
    {
      "job": "926610",
      "node": "evc104 (4 CPUs, 512G)",
      "updates": "1-501",
      "stop": "deadline checkpoint-stop"
    },
    {
      "job": "930602",
      "node": "evc101 (8 CPUs, 768G)",
      "updates": "502-2100",
      "stop": "user-requested clean stop at 2100"
    }
  ],
  "resume_from": "checkpoint-2100 (complete, 4 ranks)"
}