Hugging Face's logo

Chaenn
/
diffusion_so101_cube_multitask_0724

Robotics
LeRobot
Safetensors
diffusion
Model card Files
xet
 Community
Chaenn commited on
Commit
ffef4c8
·
verified ·
1 Parent(s): 21af9c0

Upload folder using huggingface_hub

Browse files
Files changed (12) hide show
  1. .gitattributes +1 -0
  2. wandb/debug-internal.log +168 -0
  3. wandb/debug.log +58 -0
  4. wandb/run-20260724_180847-lqqse51n/files/config.yaml +302 -0
  5. wandb/run-20260724_180847-lqqse51n/files/output.log +59 -0
  6. wandb/run-20260724_180847-lqqse51n/files/requirements.txt +121 -0
  7. wandb/run-20260724_180847-lqqse51n/files/wandb-metadata.json +100 -0
  8. wandb/run-20260724_180847-lqqse51n/files/wandb-summary.json +1 -0
  9. wandb/run-20260724_180847-lqqse51n/logs/debug-core.log +15 -0
  10. wandb/run-20260724_180847-lqqse51n/logs/debug-internal.log +168 -0
  11. wandb/run-20260724_180847-lqqse51n/logs/debug.log +58 -0
  12. wandb/run-20260724_180847-lqqse51n/run-lqqse51n.wandb +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ wandb/run-20260724_180847-lqqse51n/run-lqqse51n.wandb filter=lfs diff=lfs merge=lfs -text
wandb/debug-internal.log ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-07-24T18:08:47.254284096+09:00","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-07-24T18:08:47.254548228+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
3
+ {"time":"2026-07-24T18:08:47.624038812+09:00","level":"INFO","msg":"stream: created new stream","id":"lqqse51n"}
4
+ {"time":"2026-07-24T18:08:47.624181677+09:00","level":"INFO","msg":"stream: started"}
5
+ {"time":"2026-07-24T18:08:47.62426253+09:00","level":"INFO","msg":"handler: started"}
6
+ {"time":"2026-07-24T18:08:47.624237648+09:00","level":"INFO","msg":"writer: started","stream_id":"lqqse51n"}
7
+ {"time":"2026-07-24T18:08:47.62430234+09:00","level":"INFO","msg":"sender: started"}
8
+ {"time":"2026-07-24T18:08:48.27917746+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
9
+ {"time":"2026-07-24T18:08:48.813488143+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-07-24T18:09:03.27956354+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":9,"uploaded_len":2}
11
+ {"time":"2026-07-24T18:09:03.794827834+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-07-24T18:09:18.280282729+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
13
+ {"time":"2026-07-24T18:09:47.94579489+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-07-24T18:09:47.946202132+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
15
+ {"time":"2026-07-24T18:09:48.588767707+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
16
+ {"time":"2026-07-24T18:09:48.589135866+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
17
+ {"time":"2026-07-24T18:10:34.47613234+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
18
+ {"time":"2026-07-24T18:10:34.476309275+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
19
+ {"time":"2026-07-24T18:11:18.904926052+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
20
+ {"time":"2026-07-24T18:11:18.905374248+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":6}
21
+ {"time":"2026-07-24T18:11:19.302967496+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
22
+ {"time":"2026-07-24T18:11:19.303172178+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":6}
23
+ {"time":"2026-07-24T18:11:19.657566076+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
24
+ {"time":"2026-07-24T18:11:33.175855996+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":1}
25
+ {"time":"2026-07-24T18:11:33.652327028+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
26
+ {"time":"2026-07-24T18:11:48.175949191+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":1,"console_offset":8,"console_lines":13}
27
+ {"time":"2026-07-24T18:11:48.658684131+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
28
+ {"time":"2026-07-24T18:12:03.176010749+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":20,"console_lines":1}
29
+ {"time":"2026-07-24T18:12:03.733839312+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
30
+ {"time":"2026-07-24T18:12:18.176170454+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":20,"console_lines":1}
31
+ {"time":"2026-07-24T18:12:42.870254353+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
32
+ {"time":"2026-07-24T18:12:42.870492134+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":4,"console_offset":20,"console_lines":1}
33
+ {"time":"2026-07-24T18:12:43.43127583+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
34
+ {"time":"2026-07-24T18:12:48.17625266+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":20,"console_lines":1}
35
+ {"time":"2026-07-24T18:13:07.036513826+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
36
+ {"time":"2026-07-24T18:13:07.036915417+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":4,"console_offset":20,"console_lines":2}
37
+ {"time":"2026-07-24T18:13:41.843757118+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
38
+ {"time":"2026-07-24T18:13:41.844141038+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
39
+ {"time":"2026-07-24T18:13:42.262626677+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
40
+ {"time":"2026-07-24T18:13:42.262826904+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":4,"console_offset":21,"console_lines":1}
41
+ {"time":"2026-07-24T18:13:42.738785592+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
42
+ {"time":"2026-07-24T18:13:42.738950817+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":21,"console_lines":1}
43
+ {"time":"2026-07-24T18:14:01.760076376+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
44
+ {"time":"2026-07-24T18:14:01.760452991+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":21,"console_lines":1}
45
+ {"time":"2026-07-24T18:14:02.563374728+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
46
+ {"time":"2026-07-24T18:14:12.125137015+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":21,"console_lines":1}
47
+ {"time":"2026-07-24T18:14:12.587166023+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
48
+ {"time":"2026-07-24T18:14:27.12544843+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":21,"console_lines":2}
49
+ {"time":"2026-07-24T18:14:49.827004263+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
50
+ {"time":"2026-07-24T18:14:49.827440242+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":4,"console_offset":22,"console_lines":1}
51
+ {"time":"2026-07-24T18:14:50.288076577+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
52
+ {"time":"2026-07-24T18:14:57.125784841+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":22,"console_lines":1}
53
+ {"time":"2026-07-24T18:15:17.942419688+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
54
+ {"time":"2026-07-24T18:15:17.942915971+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":48,"events_lines":2,"console_offset":22,"console_lines":1}
55
+ {"time":"2026-07-24T18:16:13.183416877+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
56
+ {"time":"2026-07-24T18:16:15.836775763+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
57
+ {"time":"2026-07-24T18:16:15.837056849+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
58
+ {"time":"2026-07-24T18:16:53.34792422+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
59
+ {"time":"2026-07-24T18:16:53.348219955+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":50,"events_lines":8,"console_offset":22,"console_lines":2}
60
+ {"time":"2026-07-24T18:16:53.80424795+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
61
+ {"time":"2026-07-24T18:16:53.804470685+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":58,"events_lines":6,"console_offset":23,"console_lines":2}
62
+ {"time":"2026-07-24T18:16:54.280058023+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
63
+ {"time":"2026-07-24T18:16:54.280201972+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":24,"console_lines":1}
64
+ {"time":"2026-07-24T18:17:27.538437939+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
65
+ {"time":"2026-07-24T18:17:27.538704863+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":64,"events_lines":2,"console_offset":24,"console_lines":1}
66
+ {"time":"2026-07-24T18:17:27.951358644+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
67
+ {"time":"2026-07-24T18:17:27.951574285+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":2,"console_offset":24,"console_lines":1}
68
+ {"time":"2026-07-24T18:17:28.380216641+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
69
+ {"time":"2026-07-24T18:17:38.464509947+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":24,"console_lines":1}
70
+ {"time":"2026-07-24T18:18:13.538159311+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
71
+ {"time":"2026-07-24T18:18:13.538355539+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
72
+ {"time":"2026-07-24T18:18:13.881337903+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
73
+ {"time":"2026-07-24T18:18:13.881848161+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":70,"events_lines":4,"console_offset":24,"console_lines":2}
74
+ {"time":"2026-07-24T18:18:14.42368108+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
75
+ {"time":"2026-07-24T18:18:14.42383956+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":25,"console_lines":1}
76
+ {"time":"2026-07-24T18:18:28.312533921+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
77
+ {"time":"2026-07-24T18:18:28.780687664+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":2,"console_offset":25,"console_lines":1}
78
+ {"time":"2026-07-24T18:18:29.304411418+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
79
+ {"time":"2026-07-24T18:18:43.781233325+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":76,"events_lines":2,"console_offset":25,"console_lines":1}
80
+ {"time":"2026-07-24T18:18:44.242368805+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
81
+ {"time":"2026-07-24T18:18:58.780732119+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":2,"console_offset":25,"console_lines":1}
82
+ {"time":"2026-07-24T18:19:36.984024288+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
83
+ {"time":"2026-07-24T18:19:36.984170259+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
84
+ {"time":"2026-07-24T18:20:32.202849625+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
85
+ {"time":"2026-07-24T18:21:29.677654094+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
86
+ {"time":"2026-07-24T18:21:51.426842527+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
87
+ {"time":"2026-07-24T18:21:51.427078628+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":80,"events_lines":6,"console_offset":25,"console_lines":2}
88
+ {"time":"2026-07-24T18:21:57.471027315+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
89
+ {"time":"2026-07-24T18:21:57.472032098+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":2,"events_offset":86,"events_lines":18,"console_offset":26,"console_lines":3}
90
+ {"time":"2026-07-24T18:22:12.599743156+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
91
+ {"time":"2026-07-24T18:22:12.60040309+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":28,"console_lines":1}
92
+ {"time":"2026-07-24T18:22:37.055177145+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
93
+ {"time":"2026-07-24T18:22:37.055630719+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":104,"events_lines":2,"console_offset":28,"console_lines":1}
94
+ {"time":"2026-07-24T18:22:54.575480712+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
95
+ {"time":"2026-07-24T18:22:54.575773219+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":106,"events_lines":4,"console_offset":28,"console_lines":1}
96
+ {"time":"2026-07-24T18:23:30.993229555+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
97
+ {"time":"2026-07-24T18:23:30.993504636+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":110,"events_lines":2,"console_offset":28,"console_lines":1}
98
+ {"time":"2026-07-24T18:23:53.412378385+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
99
+ {"time":"2026-07-24T18:23:53.412676178+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":112,"events_lines":4,"console_offset":28,"console_lines":2}
100
+ {"time":"2026-07-24T18:23:53.748547151+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
101
+ {"time":"2026-07-24T18:23:53.748986127+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":116,"events_lines":4,"console_offset":29,"console_lines":1}
102
+ {"time":"2026-07-24T18:24:02.677765013+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
103
+ {"time":"2026-07-24T18:24:02.677940765+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":29,"console_lines":1}
104
+ {"time":"2026-07-24T18:24:33.655524831+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
105
+ {"time":"2026-07-24T18:24:33.655744638+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":120,"events_lines":2,"console_offset":29,"console_lines":1}
106
+ {"time":"2026-07-24T18:24:42.548010937+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
107
+ {"time":"2026-07-24T18:24:42.548295182+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":122,"events_lines":4,"console_offset":29,"console_lines":2}
108
+ {"time":"2026-07-24T18:24:43.048453683+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
109
+ {"time":"2026-07-24T18:24:43.048776179+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":30,"console_lines":1}
110
+ {"time":"2026-07-24T18:24:43.441389357+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
111
+ {"time":"2026-07-24T18:24:53.441813516+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":126,"events_lines":2,"console_offset":30,"console_lines":1}
112
+ {"time":"2026-07-24T18:24:53.774222275+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
113
+ {"time":"2026-07-24T18:25:08.441844848+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":128,"events_lines":2,"console_offset":30,"console_lines":1}
114
+ {"time":"2026-07-24T18:25:38.61799626+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
115
+ {"time":"2026-07-24T18:25:38.618266701+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
116
+ {"time":"2026-07-24T18:25:50.158578758+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
117
+ {"time":"2026-07-24T18:25:50.15905911+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":130,"events_lines":4,"console_offset":30,"console_lines":2}
118
+ {"time":"2026-07-24T18:26:18.847568607+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
119
+ {"time":"2026-07-24T18:26:18.847963389+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":134,"events_lines":2,"console_offset":31,"console_lines":1}
120
+ {"time":"2026-07-24T18:26:19.280360669+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
121
+ {"time":"2026-07-24T18:26:19.280535951+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":136,"events_lines":4,"console_offset":31,"console_lines":1}
122
+ {"time":"2026-07-24T18:26:19.721235518+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
123
+ {"time":"2026-07-24T18:26:19.721503555+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":31,"console_lines":1}
124
+ {"time":"2026-07-24T18:26:34.584513612+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
125
+ {"time":"2026-07-24T18:26:34.584757767+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":140,"events_lines":2,"console_offset":31,"console_lines":1}
126
+ {"time":"2026-07-24T18:26:34.982047375+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
127
+ {"time":"2026-07-24T18:26:49.069179368+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":142,"events_lines":2,"console_offset":31,"console_lines":2}
128
+ {"time":"2026-07-24T18:26:49.631643115+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
129
+ {"time":"2026-07-24T18:27:04.069139811+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":144,"events_lines":2,"console_offset":32,"console_lines":1}
130
+ {"time":"2026-07-24T18:27:04.636529545+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
131
+ {"time":"2026-07-24T18:27:19.069177113+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":146,"events_lines":2,"console_offset":32,"console_lines":1}
132
+ {"time":"2026-07-24T18:27:45.598569863+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
133
+ {"time":"2026-07-24T18:27:45.598871699+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":148,"events_lines":2,"console_offset":32,"console_lines":1}
134
+ {"time":"2026-07-24T18:28:09.746807243+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
135
+ {"time":"2026-07-24T18:28:09.74735117+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":150,"events_lines":4,"console_offset":32,"console_lines":2}
136
+ {"time":"2026-07-24T18:28:10.492444366+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
137
+ {"time":"2026-07-24T18:28:10.492644284+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":33,"console_lines":1}
138
+ {"time":"2026-07-24T18:28:47.438857468+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
139
+ {"time":"2026-07-24T18:28:47.439277332+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
140
+ {"time":"2026-07-24T18:29:14.445259123+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
141
+ {"time":"2026-07-24T18:29:14.445723893+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":154,"events_lines":4,"console_offset":33,"console_lines":1}
142
+ {"time":"2026-07-24T18:29:42.312532292+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
143
+ {"time":"2026-07-24T18:29:42.312888364+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":158,"events_lines":4,"console_offset":33,"console_lines":1}
144
+ {"time":"2026-07-24T18:29:42.692074397+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
145
+ {"time":"2026-07-24T18:29:42.692483095+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":162,"events_lines":4,"console_offset":33,"console_lines":2}
146
+ {"time":"2026-07-24T18:29:43.142557482+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
147
+ {"time":"2026-07-24T18:29:43.142937672+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":34,"console_lines":1}
148
+ {"time":"2026-07-24T18:29:43.485860995+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
149
+ {"time":"2026-07-24T18:29:57.435267774+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":166,"events_lines":2,"console_offset":34,"console_lines":1}
150
+ {"time":"2026-07-24T18:29:57.823778239+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
151
+ {"time":"2026-07-24T18:30:12.434417821+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":168,"events_lines":2,"console_offset":34,"console_lines":1}
152
+ {"time":"2026-07-24T18:30:12.905547751+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
153
+ {"time":"2026-07-24T18:30:27.434946109+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":170,"events_lines":2,"console_offset":34,"console_lines":1}
154
+ {"time":"2026-07-24T18:31:03.575974488+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
155
+ {"time":"2026-07-24T18:31:03.576121214+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
156
+ {"time":"2026-07-24T18:31:26.598960374+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
157
+ {"time":"2026-07-24T18:31:58.795263215+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
158
+ {"time":"2026-07-24T18:32:56.160021927+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"An internal error occurred. Please contact support.\"}"}
159
+ {"time":"2026-07-24T18:33:55.570472582+09:00","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"rate limit exceeded: per_run limit on filestream requests\"}"}
160
+ {"time":"2026-07-24T18:34:34.736913463+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
161
+ {"time":"2026-07-24T18:34:34.737586259+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":172,"events_lines":6,"console_offset":34,"console_lines":2}
162
+ {"time":"2026-07-24T18:34:52.619228935+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
163
+ {"time":"2026-07-24T18:34:52.619443696+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":178,"events_lines":2,"console_offset":35,"console_lines":24,"uploaded_len":3,"complete":true,"exit_code":1}
164
+ {"time":"2026-07-24T18:35:16.31221796+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
165
+ {"time":"2026-07-24T18:35:16.312540995+09:00","level":"INFO","msg":"stream: finishing up"}
166
+ {"time":"2026-07-24T18:35:16.312585192+09:00","level":"INFO","msg":"handler: closed"}
167
+ {"time":"2026-07-24T18:35:16.31264971+09:00","level":"INFO","msg":"sender: closed"}
168
+ {"time":"2026-07-24T18:35:16.312659323+09:00","level":"INFO","msg":"stream: all finished"}
wandb/debug.log ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
2
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_setup.py:_flush():81] Configure stats pid to 184127
3
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_init.py:setup_run_log_directory():723] Logging user logs to outputs/train/so101_multitask_diffusion_0724/wandb/run-20260724_180847-lqqse51n/logs/debug.log
5
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to outputs/train/so101_multitask_diffusion_0724/wandb/run-20260724_180847-lqqse51n/logs/debug-internal.log
6
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:init():851] calling init triggers
7
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Chaenn/so101_cube_multitask_0724', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'streaming': False}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': True, 'repo_id': 'Chaenn/diffusion_so101_cube_multitask_0724', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'horizon': 32, 'n_action_steps': 15, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet18', 'resize_shape': None, 'crop_ratio': 1.0, 'crop_shape': None, 'crop_is_random': True, 'pretrained_backbone_weights': None, 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDPM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': None, 'compile_model': False, 'compile_mode': 'reduce-overhead', 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500}, 'reward_model': None, 'output_dir': 'outputs/train/so101_multitask_diffusion_0724', 'job_name': 'so101_cube_diffusion', 'resume': False, 'seed': 1000, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 32, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 200000, 'eval_freq': 20000, 'log_freq': 200, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 10000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 44, 'use_async_envs': True}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'diffusion_so101_cube_multitask_0724_training', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
9
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:init():899] starting backend
10
+ 2026-07-24 18:08:47,245 INFO MainThread:184127 [wandb_init.py:init():914] sending inform_init request
11
+ 2026-07-24 18:08:47,625 INFO MainThread:184127 [wandb_init.py:init():919] backend started and connected
12
+ 2026-07-24 18:08:47,628 INFO MainThread:184127 [wandb_init.py:init():989] updated telemetry
13
+ 2026-07-24 18:08:47,629 INFO MainThread:184127 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
14
+ 2026-07-24 18:08:48,172 INFO MainThread:184127 [wandb_init.py:init():1057] starting run threads in backend
15
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_console_start():2509] atexit reg
16
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_redirect():2359] redirect: wrap_raw
17
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_redirect():2428] Wrapping output streams.
18
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_redirect():2451] Redirects installed.
19
+ 2026-07-24 18:08:48,278 INFO MainThread:184127 [wandb_init.py:init():1095] run started, returning control to user process
20
+ 2026-07-24 18:31:25,953 INFO wandb-AsyncioManager-main:184127 [service_client.py:_forward_responses():122] Reached EOF.
21
+ 2026-07-24 18:31:25,953 INFO wandb-AsyncioManager-main:184127 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles.
22
+ 2026-07-24 18:31:28,520 ERROR wandb-AsyncioManager-main:184127 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
23
+ Traceback (most recent call last):
24
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
25
+ await fn()
26
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 105, in loop
27
+ await self._poll_and_print(timeout=None)
28
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 133, in _poll_and_print
29
+ handle = await self._interface.deliver_async(request)
30
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
31
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
32
+ handle = await self._client.deliver(request)
33
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
34
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
35
+ handle = self._mailbox.require_response(request)
36
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
37
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
38
+ raise MailboxClosedError()
39
+ wandb.sdk.mailbox.mailbox.MailboxClosedError
40
+ 2026-07-24 18:31:33,481 ERROR wandb-AsyncioManager-main:184127 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
41
+ Traceback (most recent call last):
42
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
43
+ await fn()
44
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
45
+ should_stop = await self._check_should_stop()
46
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
47
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
48
+ handle = await self._interface.deliver_async(request)
49
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
50
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
51
+ handle = await self._client.deliver(request)
52
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
53
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
54
+ handle = self._mailbox.require_response(request)
55
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
56
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
57
+ raise MailboxClosedError()
58
+ wandb.sdk.mailbox.mailbox.MailboxClosedError
wandb/run-20260724_180847-lqqse51n/files/config.yaml ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.27.0
4
+ e:
5
+ kbcujy7q2qwe0epbhd9t5r5e1k7czxgk:
6
+ args:
7
+ - --dataset.repo_id=Chaenn/so101_cube_multitask_0724
8
+ - --policy.type=diffusion
9
+ - --policy.device=cuda
10
+ - --policy.n_obs_steps=2
11
+ - --policy.horizon=32
12
+ - --policy.n_action_steps=15
13
+ - --batch_size=32
14
+ - --steps=200000
15
+ - --output_dir=outputs/train/so101_multitask_diffusion_0724
16
+ - --job_name=so101_cube_diffusion
17
+ - --save_freq=10000
18
+ - --wandb.enable=true
19
+ - --wandb.project=diffusion_so101_cube_multitask_0724_training
20
+ - --policy.repo_id=Chaenn/diffusion_so101_cube_multitask_0724
21
+ codePath: miniforge3/envs/lerobot/bin/lerobot-train
22
+ codePathLocal: miniforge3/envs/lerobot/bin/lerobot-train
23
+ cpu_count: 32
24
+ cpu_count_logical: 64
25
+ cudaVersion: "12.4"
26
+ disk:
27
+ /:
28
+ total: "1856833818624"
29
+ used: "1025881407488"
30
+ executable: /home/grad26/miniforge3/envs/lerobot/bin/python
31
+ gpu: NVIDIA GeForce RTX 4090
32
+ gpu_count: 8
33
+ gpu_nvidia:
34
+ - architecture: Ada
35
+ cudaCores: 16384
36
+ memoryTotal: "25757220864"
37
+ name: NVIDIA GeForce RTX 4090
38
+ uuid: GPU-931c6609-b18a-5ca6-2cab-2a70f83dbd4e
39
+ - architecture: Ada
40
+ cudaCores: 16384
41
+ memoryTotal: "25757220864"
42
+ name: NVIDIA GeForce RTX 4090
43
+ uuid: GPU-18a447c8-3eea-c690-3f7f-adf0e9789624
44
+ - architecture: Ada
45
+ cudaCores: 16384
46
+ memoryTotal: "25757220864"
47
+ name: NVIDIA GeForce RTX 4090
48
+ uuid: GPU-813c33cd-3310-0056-f3cc-57b46c355014
49
+ - architecture: Ada
50
+ cudaCores: 16384
51
+ memoryTotal: "25757220864"
52
+ name: NVIDIA GeForce RTX 4090
53
+ uuid: GPU-3ee80ae5-6d0a-4301-50a1-ce1575c88ed7
54
+ - architecture: Ada
55
+ cudaCores: 16384
56
+ memoryTotal: "25757220864"
57
+ name: NVIDIA GeForce RTX 4090
58
+ uuid: GPU-31a9f659-e5d8-8c28-a9f5-24abdbc5114d
59
+ - architecture: Ada
60
+ cudaCores: 16384
61
+ memoryTotal: "25757220864"
62
+ name: NVIDIA GeForce RTX 4090
63
+ uuid: GPU-5d6b6e52-3dc9-7949-cf08-62e3a014648e
64
+ - architecture: Ada
65
+ cudaCores: 16384
66
+ memoryTotal: "25757220864"
67
+ name: NVIDIA GeForce RTX 4090
68
+ uuid: GPU-a30b0063-f4dc-d94c-19cc-905c5649fd8b
69
+ - architecture: Ada
70
+ cudaCores: 16384
71
+ memoryTotal: "25757220864"
72
+ name: NVIDIA GeForce RTX 4090
73
+ uuid: GPU-aaaee8f0-1b1e-743a-0ee0-6f4a0049d86c
74
+ host: user
75
+ memory:
76
+ total: "270098366464"
77
+ os: Linux-6.8.0-111-generic-x86_64-with-glibc2.35
78
+ program: /home/grad26/miniforge3/envs/lerobot/bin/lerobot-train
79
+ python: CPython 3.12.13
80
+ root: outputs/train/so101_multitask_diffusion_0724
81
+ startedAt: "2026-07-24T09:08:47.022744Z"
82
+ writerId: kbcujy7q2qwe0epbhd9t5r5e1k7czxgk
83
+ m: []
84
+ python_version: 3.12.13
85
+ t:
86
+ "1":
87
+ - 1
88
+ - 11
89
+ - 41
90
+ - 49
91
+ - 51
92
+ - 71
93
+ - 83
94
+ "2":
95
+ - 1
96
+ - 11
97
+ - 41
98
+ - 49
99
+ - 51
100
+ - 71
101
+ - 83
102
+ "3":
103
+ - 13
104
+ - 15
105
+ - 16
106
+ - 61
107
+ "4": 3.12.13
108
+ "5": 0.27.0
109
+ "6": 5.5.4
110
+ "10":
111
+ - 21
112
+ "12": 0.27.0
113
+ "13": linux-x86_64
114
+ batch_size:
115
+ value: 32
116
+ checkpoint_path:
117
+ value: null
118
+ cudnn_deterministic:
119
+ value: false
120
+ dataset:
121
+ value:
122
+ episodes: null
123
+ image_transforms:
124
+ enable: false
125
+ max_num_transforms: 3
126
+ random_order: false
127
+ tfs:
128
+ affine:
129
+ kwargs:
130
+ degrees:
131
+ - -5
132
+ - 5
133
+ translate:
134
+ - 0.05
135
+ - 0.05
136
+ type: RandomAffine
137
+ weight: 1
138
+ brightness:
139
+ kwargs:
140
+ brightness:
141
+ - 0.8
142
+ - 1.2
143
+ type: ColorJitter
144
+ weight: 1
145
+ contrast:
146
+ kwargs:
147
+ contrast:
148
+ - 0.8
149
+ - 1.2
150
+ type: ColorJitter
151
+ weight: 1
152
+ hue:
153
+ kwargs:
154
+ hue:
155
+ - -0.05
156
+ - 0.05
157
+ type: ColorJitter
158
+ weight: 1
159
+ saturation:
160
+ kwargs:
161
+ saturation:
162
+ - 0.5
163
+ - 1.5
164
+ type: ColorJitter
165
+ weight: 1
166
+ sharpness:
167
+ kwargs:
168
+ sharpness:
169
+ - 0.5
170
+ - 1.5
171
+ type: SharpnessJitter
172
+ weight: 1
173
+ repo_id: Chaenn/so101_cube_multitask_0724
174
+ return_uint8: false
175
+ revision: null
176
+ root: null
177
+ streaming: false
178
+ use_imagenet_stats: true
179
+ video_backend: torchcodec
180
+ env:
181
+ value: null
182
+ eval:
183
+ value:
184
+ batch_size: 44
185
+ n_episodes: 50
186
+ use_async_envs: true
187
+ eval_freq:
188
+ value: 20000
189
+ job_name:
190
+ value: so101_cube_diffusion
191
+ log_freq:
192
+ value: 200
193
+ num_workers:
194
+ value: 4
195
+ optimizer:
196
+ value:
197
+ betas:
198
+ - 0.95
199
+ - 0.999
200
+ eps: 1e-08
201
+ grad_clip_norm: 10
202
+ lr: 0.0001
203
+ type: adam
204
+ weight_decay: 1e-06
205
+ output_dir:
206
+ value: outputs/train/so101_multitask_diffusion_0724
207
+ peft:
208
+ value: null
209
+ persistent_workers:
210
+ value: true
211
+ policy:
212
+ value:
213
+ beta_end: 0.02
214
+ beta_schedule: squaredcos_cap_v2
215
+ beta_start: 0.0001
216
+ clip_sample: true
217
+ clip_sample_range: 1
218
+ compile_mode: reduce-overhead
219
+ compile_model: false
220
+ crop_is_random: true
221
+ crop_ratio: 1
222
+ crop_shape: null
223
+ device: cuda
224
+ diffusion_step_embed_dim: 128
225
+ do_mask_loss_for_padding: false
226
+ down_dims:
227
+ - 512
228
+ - 1024
229
+ - 2048
230
+ drop_n_last_frames: 7
231
+ horizon: 32
232
+ kernel_size: 5
233
+ license: null
234
+ n_action_steps: 15
235
+ n_groups: 8
236
+ n_obs_steps: 2
237
+ noise_scheduler_type: DDPM
238
+ normalization_mapping:
239
+ ACTION: MIN_MAX
240
+ STATE: MIN_MAX
241
+ VISUAL: MEAN_STD
242
+ num_inference_steps: null
243
+ num_train_timesteps: 100
244
+ optimizer_betas:
245
+ - 0.95
246
+ - 0.999
247
+ optimizer_eps: 1e-08
248
+ optimizer_lr: 0.0001
249
+ optimizer_weight_decay: 1e-06
250
+ prediction_type: epsilon
251
+ pretrained_backbone_weights: null
252
+ pretrained_path: null
253
+ private: null
254
+ push_to_hub: true
255
+ repo_id: Chaenn/diffusion_so101_cube_multitask_0724
256
+ resize_shape: null
257
+ scheduler_name: cosine
258
+ scheduler_warmup_steps: 500
259
+ spatial_softmax_num_keypoints: 32
260
+ tags: null
261
+ type: diffusion
262
+ use_amp: false
263
+ use_film_scale_modulation: true
264
+ use_group_norm: true
265
+ use_peft: false
266
+ use_separate_rgb_encoder_per_camera: false
267
+ vision_backbone: resnet18
268
+ prefetch_factor:
269
+ value: 4
270
+ resume:
271
+ value: false
272
+ reward_model:
273
+ value: null
274
+ sample_weighting:
275
+ value: null
276
+ save_checkpoint:
277
+ value: true
278
+ save_freq:
279
+ value: 10000
280
+ scheduler:
281
+ value:
282
+ name: cosine
283
+ num_warmup_steps: 500
284
+ type: diffuser
285
+ seed:
286
+ value: 1000
287
+ steps:
288
+ value: 200000
289
+ tolerance_s:
290
+ value: 0.0001
291
+ use_policy_training_preset:
292
+ value: true
293
+ wandb:
294
+ value:
295
+ add_tags: true
296
+ disable_artifact: false
297
+ enable: true
298
+ entity: null
299
+ mode: null
300
+ notes: null
301
+ project: diffusion_so101_cube_multitask_0724_training
302
+ run_id: null
wandb/run-20260724_180847-lqqse51n/files/output.log ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO 2026-07-24 18:08:48 db_utils.py:121 Logs will be synced with wandb.
2
+ INFO 2026-07-24 18:08:48 db_utils.py:122 Track this run --> https://wandb.ai/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/runs/lqqse51n
3
+ INFO 2026-07-24 18:08:48 ot_train.py:236 Creating dataset
4
+ Fetching 13 files: 100%|█████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 16588.36it/s]
5
+ Download complete: : 0.00B [00:00, ?B/s] | 0/13 [00:00<?, ?it/s]
6
+ INFO 2026-07-24 18:08:48 eo_utils.py:110 Using video codec: libsvtav1
7
+ Fetching 13 files: 100%|██████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 8226.61it/s]
8
+ Download complete: : 0.00B [00:00, ?B/s] | 0/13 [00:00<?, ?it/s]
9
+ Fetching 45 files: 100%|████████████████████████████████████████████████████████████████| 45/45 [02:45<00:00, 3.67s/it]
10
+ Download complete: : 0.00B [02:45, ?B/s] INFO 2026-07-24 18:11:35 ot_train.py:270 Creating policy 3.94s/it]
11
+ Download complete: : 0.00B [02:45, ?B/s]
12
+ INFO 2026-07-24 18:11:37 ot_train.py:347 Creating optimizer and scheduler
13
+ INFO 2026-07-24 18:11:37 ot_train.py:374 Output dir: outputs/train/so101_multitask_diffusion_0724
14
+ INFO 2026-07-24 18:11:37 ot_train.py:381 cfg.steps=200000 (200K)
15
+ INFO 2026-07-24 18:11:37 ot_train.py:382 dataset.num_frames=99340 (99K)
16
+ INFO 2026-07-24 18:11:37 ot_train.py:383 dataset.num_episodes=87
17
+ INFO 2026-07-24 18:11:37 ot_train.py:386 Effective batch size: 32 x 1 = 32
18
+ INFO 2026-07-24 18:11:37 ot_train.py:387 num_learnable_params=266622758 (267M)
19
+ INFO 2026-07-24 18:11:37 ot_train.py:388 num_total_params=266622758 (267M)
20
+ Training: 0%| | 0/200000 [00:00<?, ?step/s]INFO 2026-07-24 18:11:37 ot_train.py:453 Start offline training on a fixed dataset, with effective batch size: 32
21
+ Training: 0%| | 200/200000 [01:21<20:49:36, 2.66step/s]INFO 2026-07-24 18:12:59 ot_train.py:488 step:200 smpl:6K ep:6 epch:0.06 loss:0.517 grdn:3.968 lr:2.0e-05 updt_s:0.391 data_s:0.015
22
+ Training: 0%|▏ | 400/200000 [02:36<20:47:15, 2.67step/s]INFO 2026-07-24 18:14:14 ot_train.py:488 step:400 smpl:13K ep:11 epch:0.13 loss:0.081 grdn:2.298 lr:6.0e-05 updt_s:0.365 data_s:0.009
23
+ Training: 0%|▏ | 600/200000 [03:51<20:45:38, 2.67step/s]INFO 2026-07-24 18:15:29 ot_train.py:488 step:600 smpl:19K ep:17 epch:0.19 loss:0.053 grdn:1.338 lr:9.5e-05 updt_s:0.366 data_s:0.009
24
+ Training: 0%|▎ | 800/200000 [05:06<20:46:00, 2.66step/s]INFO 2026-07-24 18:16:44 ot_train.py:488 step:800 smpl:26K ep:22 epch:0.26 loss:0.041 grdn:0.930 lr:1.0e-04 updt_s:0.366 data_s:0.009
25
+ Training: 0%|▎ | 1000/200000 [06:21<20:44:54, 2.66step/s]INFO 2026-07-24 18:17:59 ot_train.py:488 step:1K smpl:32K ep:28 epch:0.32 loss:0.036 grdn:0.759 lr:1.0e-04 updt_s:0.366 data_s:0.009
26
+ Training: 1%|▎ | 1200/200000 [07:36<20:43:40, 2.66step/s]INFO 2026-07-24 18:19:14 ot_train.py:488 step:1K smpl:38K ep:34 epch:0.39 loss:0.033 grdn:0.698 lr:1.0e-04 updt_s:0.366 data_s:0.009
27
+ Training: 1%|▍ | 1400/200000 [08:51<20:42:36, 2.66step/s]INFO 2026-07-24 18:20:29 ot_train.py:488 step:1K smpl:45K ep:39 epch:0.45 loss:0.030 grdn:0.645 lr:1.0e-04 updt_s:0.366 data_s:0.009
28
+ Training: 1%|▍ | 1600/200000 [10:06<20:41:16, 2.66step/s]INFO 2026-07-24 18:21:44 ot_train.py:488 step:2K smpl:51K ep:45 epch:0.52 loss:0.027 grdn:0.547 lr:1.0e-04 updt_s:0.366 data_s:0.009
29
+ Training: 1%|▌ | 1800/200000 [11:21<20:38:55, 2.67step/s]INFO 2026-07-24 18:22:59 ot_train.py:488 step:2K smpl:58K ep:50 epch:0.58 loss:0.027 grdn:0.573 lr:1.0e-04 updt_s:0.366 data_s:0.009
30
+ Training: 1%|▌ | 2000/200000 [12:36<20:38:37, 2.66step/s]INFO 2026-07-24 18:24:14 ot_train.py:488 step:2K smpl:64K ep:56 epch:0.64 loss:0.025 grdn:0.494 lr:1.0e-04 updt_s:0.366 data_s:0.009
31
+ Training: 1%|▋ | 2200/200000 [13:51<20:36:08, 2.67step/s]INFO 2026-07-24 18:25:29 ot_train.py:488 step:2K smpl:70K ep:62 epch:0.71 loss:0.025 grdn:0.483 lr:1.0e-04 updt_s:0.366 data_s:0.009
32
+ Training: 1%|▋ | 2400/200000 [15:06<20:34:51, 2.67step/s]INFO 2026-07-24 18:26:44 ot_train.py:488 step:2K smpl:77K ep:67 epch:0.77 loss:0.024 grdn:0.468 lr:1.0e-04 updt_s:0.366 data_s:0.009
33
+ Training: 1%|▊ | 2600/200000 [16:21<20:34:27, 2.67step/s]INFO 2026-07-24 18:27:59 ot_train.py:488 step:3K smpl:83K ep:73 epch:0.84 loss:0.024 grdn:0.449 lr:1.0e-04 updt_s:0.366 data_s:0.009
34
+ Training: 1%|▊ | 2800/200000 [17:36<20:33:56, 2.66step/s]INFO 2026-07-24 18:29:14 ot_train.py:488 step:3K smpl:90K ep:78 epch:0.90 loss:0.023 grdn:0.435 lr:1.0e-04 updt_s:0.366 data_s:0.009
35
+ Training: 2%|▉ | 3000/200000 [18:52<20:31:09, 2.67step/s]INFO 2026-07-24 18:30:29 ot_train.py:488 step:3K smpl:96K ep:84 epch:0.97 loss:0.022 grdn:0.420 lr:1.0e-04 updt_s:0.366 data_s:0.009
36
+ Training: 2%|▉ | 3089/200000 [19:27<37:20:18, 1.46step/s]Traceback (most recent call last):
37
+ File "/home/grad26/miniforge3/envs/lerobot/bin/lerobot-train", line 6, in <module>
38
+ sys.exit(main())
39
+ ^^^^^^
40
+ File "/home/grad26/lerobot/src/lerobot/scripts/lerobot_train.py", line 595, in main
41
+ train()
42
+ File "/home/grad26/lerobot/src/lerobot/configs/parser.py", line 233, in wrapper_inner
43
+ response = fn(cfg, *args, **kwargs)
44
+ ^^^^^^^^^^^^^^^^^^^^^^^^
45
+ File "/home/grad26/lerobot/src/lerobot/scripts/lerobot_train.py", line 466, in train
46
+ train_tracker, output_dict = update_policy(
47
+ ^^^^^^^^^^^^^^
48
+ File "/home/grad26/lerobot/src/lerobot/scripts/lerobot_train.py", line 132, in update_policy
49
+ accelerator.backward(loss)
50
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/accelerate/accelerator.py", line 2838, in backward
51
+ loss.backward(**kwargs)
52
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/torch/_tensor.py", line 648, in backward
53
+ torch.autograd.backward(
54
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/torch/autograd/__init__.py", line 353, in backward
55
+ _engine_run_backward(
56
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
57
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
58
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
59
+ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.34 GiB. GPU 0 has a total capacity of 23.64 GiB of which 1.59 GiB is free. Including non-PyTorch memory, this process has 22.04 GiB memory in use. Of the allocated memory 11.77 GiB is allocated by PyTorch, and 9.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
wandb/run-20260724_180847-lqqse51n/files/requirements.txt ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ nvidia-cuda-nvrtc-cu12==12.6.77
2
+ aiohttp==3.13.5
3
+ httpcore==1.0.9
4
+ nvidia-cufile-cu12==1.11.1.6
5
+ docopt==0.6.2
6
+ Farama-Notifications==0.0.6
7
+ torchaudio==2.7.1+cu126
8
+ pandas==2.3.3
9
+ nvidia-cufft-cu12==11.3.0.4
10
+ Jinja2==3.1.6
11
+ gitdb==4.0.12
12
+ pip==26.1.2
13
+ contourpy==1.3.3
14
+ anyio==4.13.0
15
+ matplotlib==3.10.9
16
+ gymnasium==1.3.0
17
+ PyYAML==6.0.3
18
+ pyyaml-include==1.4.1
19
+ accelerate==1.13.0
20
+ multidict==6.7.1
21
+ diffusers==0.35.2
22
+ pillow==12.2.0
23
+ wheel==0.47.0
24
+ dill==0.4.1
25
+ nvidia-nccl-cu12==2.26.2
26
+ annotated-doc==0.0.4
27
+ lerobot==0.5.2
28
+ tokenizers==0.22.2
29
+ nvidia-cudnn-cu12==9.5.1.17
30
+ h11==0.16.0
31
+ huggingface_hub==1.20.1
32
+ propcache==0.4.1
33
+ nvidia-nvjitlink-cu12==12.6.85
34
+ wandb==0.27.0
35
+ nvidia-curand-cu12==10.3.7.77
36
+ toml==0.10.2
37
+ transformers==5.5.4
38
+ shellingham==1.5.4
39
+ packaging==25.0
40
+ nvidia-cusolver-cu12==11.7.1.2
41
+ cloudpickle==3.1.2
42
+ typing_extensions==4.15.0
43
+ yarl==1.23.0
44
+ click==8.4.1
45
+ av==15.1.0
46
+ draccus==0.10.0
47
+ regex==2026.4.4
48
+ mergedeep==1.3.4
49
+ mpmath==1.3.0
50
+ python-dateutil==2.9.0.post0
51
+ filelock==3.29.0
52
+ pytz==2026.2
53
+ typer==0.25.1
54
+ sentry-sdk==2.60.0
55
+ pyarrow==24.0.0
56
+ hf-xet==1.5.1
57
+ frozenlist==1.8.0
58
+ termcolor==3.3.0
59
+ httpx==0.28.1
60
+ setuptools==80.10.2
61
+ datasets==4.8.5
62
+ nvidia-nvshmem-cu12==3.4.5
63
+ GitPython==3.1.50
64
+ typing-inspect==0.9.0
65
+ multiprocess==0.70.19
66
+ nvidia-nvtx-cu12==12.6.77
67
+ safetensors==0.7.0
68
+ nvidia-cuda-cupti-cu12==12.6.80
69
+ aiosignal==1.4.0
70
+ importlib_metadata==9.0.0
71
+ sympy==1.14.0
72
+ xxhash==3.7.0
73
+ rich==15.0.0
74
+ grpcio==1.73.1
75
+ numpy==2.2.6
76
+ protobuf==6.31.1
77
+ mdurl==0.1.2
78
+ fsspec==2026.2.0
79
+ torchvision==0.22.1+cu126
80
+ nvidia-cusparselt-cu12==0.6.3
81
+ nvidia-cublas-cu12==12.6.4.1
82
+ six==1.17.0
83
+ attrs==26.1.0
84
+ jsonlines==4.0.0
85
+ nvidia-cuda-runtime-cu12==12.6.77
86
+ urllib3==2.6.3
87
+ cycler==0.12.1
88
+ zipp==4.1.0
89
+ opencv-python-headless==4.13.0.92
90
+ cmake==4.1.3
91
+ platformdirs==4.9.6
92
+ pyparsing==3.3.2
93
+ requests==2.33.1
94
+ fonttools==4.63.0
95
+ typing-inspection==0.4.2
96
+ kiwisolver==1.5.0
97
+ networkx==3.6.1
98
+ aiohappyeyeballs==2.6.1
99
+ mypy_extensions==1.1.0
100
+ torchcodec==0.3.0
101
+ torch==2.7.1+cu126
102
+ nvidia-cusparse-cu12==12.5.4.2
103
+ idna==3.13
104
+ pydantic_core==2.46.4
105
+ psutil==7.2.2
106
+ cuda-bindings==12.9.4
107
+ pydantic==2.13.4
108
+ triton==3.3.1
109
+ cuda-pathfinder==1.5.4
110
+ smmap==5.0.3
111
+ einops==0.8.2
112
+ MarkupSafe==3.0.3
113
+ tzdata==2026.2
114
+ markdown-it-py==4.1.0
115
+ charset-normalizer==3.4.7
116
+ Pygments==2.20.0
117
+ certifi==2026.4.22
118
+ annotated-types==0.7.0
119
+ num2words==0.5.14
120
+ tqdm==4.67.3
121
+ lerobot==0.5.2
wandb/run-20260724_180847-lqqse51n/files/wandb-metadata.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-111-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.12.13",
4
+ "startedAt": "2026-07-24T09:08:47.022744Z",
5
+ "args": [
6
+ "--dataset.repo_id=Chaenn/so101_cube_multitask_0724",
7
+ "--policy.type=diffusion",
8
+ "--policy.device=cuda",
9
+ "--policy.n_obs_steps=2",
10
+ "--policy.horizon=32",
11
+ "--policy.n_action_steps=15",
12
+ "--batch_size=32",
13
+ "--steps=200000",
14
+ "--output_dir=outputs/train/so101_multitask_diffusion_0724",
15
+ "--job_name=so101_cube_diffusion",
16
+ "--save_freq=10000",
17
+ "--wandb.enable=true",
18
+ "--wandb.project=diffusion_so101_cube_multitask_0724_training",
19
+ "--policy.repo_id=Chaenn/diffusion_so101_cube_multitask_0724"
20
+ ],
21
+ "program": "/home/grad26/miniforge3/envs/lerobot/bin/lerobot-train",
22
+ "codePath": "miniforge3/envs/lerobot/bin/lerobot-train",
23
+ "codePathLocal": "miniforge3/envs/lerobot/bin/lerobot-train",
24
+ "root": "outputs/train/so101_multitask_diffusion_0724",
25
+ "host": "user",
26
+ "executable": "/home/grad26/miniforge3/envs/lerobot/bin/python",
27
+ "cpu_count": 32,
28
+ "cpu_count_logical": 64,
29
+ "gpu": "NVIDIA GeForce RTX 4090",
30
+ "gpu_count": 8,
31
+ "disk": {
32
+ "/": {
33
+ "total": "1856833818624",
34
+ "used": "1025881407488"
35
+ }
36
+ },
37
+ "memory": {
38
+ "total": "270098366464"
39
+ },
40
+ "gpu_nvidia": [
41
+ {
42
+ "name": "NVIDIA GeForce RTX 4090",
43
+ "memoryTotal": "25757220864",
44
+ "cudaCores": 16384,
45
+ "architecture": "Ada",
46
+ "uuid": "GPU-931c6609-b18a-5ca6-2cab-2a70f83dbd4e"
47
+ },
48
+ {
49
+ "name": "NVIDIA GeForce RTX 4090",
50
+ "memoryTotal": "25757220864",
51
+ "cudaCores": 16384,
52
+ "architecture": "Ada",
53
+ "uuid": "GPU-18a447c8-3eea-c690-3f7f-adf0e9789624"
54
+ },
55
+ {
56
+ "name": "NVIDIA GeForce RTX 4090",
57
+ "memoryTotal": "25757220864",
58
+ "cudaCores": 16384,
59
+ "architecture": "Ada",
60
+ "uuid": "GPU-813c33cd-3310-0056-f3cc-57b46c355014"
61
+ },
62
+ {
63
+ "name": "NVIDIA GeForce RTX 4090",
64
+ "memoryTotal": "25757220864",
65
+ "cudaCores": 16384,
66
+ "architecture": "Ada",
67
+ "uuid": "GPU-3ee80ae5-6d0a-4301-50a1-ce1575c88ed7"
68
+ },
69
+ {
70
+ "name": "NVIDIA GeForce RTX 4090",
71
+ "memoryTotal": "25757220864",
72
+ "cudaCores": 16384,
73
+ "architecture": "Ada",
74
+ "uuid": "GPU-31a9f659-e5d8-8c28-a9f5-24abdbc5114d"
75
+ },
76
+ {
77
+ "name": "NVIDIA GeForce RTX 4090",
78
+ "memoryTotal": "25757220864",
79
+ "cudaCores": 16384,
80
+ "architecture": "Ada",
81
+ "uuid": "GPU-5d6b6e52-3dc9-7949-cf08-62e3a014648e"
82
+ },
83
+ {
84
+ "name": "NVIDIA GeForce RTX 4090",
85
+ "memoryTotal": "25757220864",
86
+ "cudaCores": 16384,
87
+ "architecture": "Ada",
88
+ "uuid": "GPU-a30b0063-f4dc-d94c-19cc-905c5649fd8b"
89
+ },
90
+ {
91
+ "name": "NVIDIA GeForce RTX 4090",
92
+ "memoryTotal": "25757220864",
93
+ "cudaCores": 16384,
94
+ "architecture": "Ada",
95
+ "uuid": "GPU-aaaee8f0-1b1e-743a-0ee0-6f4a0049d86c"
96
+ }
97
+ ],
98
+ "cudaVersion": "12.4",
99
+ "writerId": "kbcujy7q2qwe0epbhd9t5r5e1k7czxgk"
100
+ }
wandb/run-20260724_180847-lqqse51n/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"train/lr":9.996425982676285e-05,"_step":3000,"_timestamp":1.7848854297137678e+09,"_wandb":{"runtime":1357},"train/steps":3000,"_runtime":1357.779419268,"train/update_s":0.365601389571093,"train/episodes":84.07489430239582,"train/epochs":0.9663780954298369,"train/loss":0.022023913562297822,"train/grad_norm":0.42026087813079355,"train/dataloading_s":0.008973219408653676,"train/samples":96000}
wandb/run-20260724_180847-lqqse51n/logs/debug-core.log ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-07-24T18:08:47.063488724+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp3u9wjwud/port-184127.txt","pid":184127,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-07-24T18:08:47.065334261+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":184127}
3
+ {"time":"2026-07-24T18:08:47.065286891+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-184127-185545-14568444/socket","Net":"unix"}}
4
+ {"time":"2026-07-24T18:08:47.244589157+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-07-24T18:08:47.254086466+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lqqse51n","id":"1(@)"}
6
+ {"time":"2026-07-24T18:08:47.624186187+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lqqse51n","id":"1(@)"}
7
+ {"time":"2026-07-24T18:08:53.281511281+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"57vj5xwn5de1"}
8
+ {"time":"2026-07-24T18:31:25.953154955+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
9
+ {"time":"2026-07-24T18:31:25.953237395+09:00","level":"INFO","msg":"server is shutting down"}
10
+ {"time":"2026-07-24T18:31:25.953260873+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
11
+ {"time":"2026-07-24T18:31:25.953347175+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
12
+ {"time":"2026-07-24T18:31:25.953348297+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-184127-185545-14568444/socket","Net":"unix"}}
13
+ {"time":"2026-07-24T18:35:16.313513507+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
14
+ {"time":"2026-07-24T18:35:16.313565959+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
15
+ {"time":"2026-07-24T18:35:16.313581408+09:00","level":"INFO","msg":"server is closed"}
wandb/run-20260724_180847-lqqse51n/logs/debug-internal.log ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-07-24T18:08:47.254284096+09:00","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-07-24T18:08:47.254548228+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
3
+ {"time":"2026-07-24T18:08:47.624038812+09:00","level":"INFO","msg":"stream: created new stream","id":"lqqse51n"}
4
+ {"time":"2026-07-24T18:08:47.624181677+09:00","level":"INFO","msg":"stream: started"}
5
+ {"time":"2026-07-24T18:08:47.62426253+09:00","level":"INFO","msg":"handler: started"}
6
+ {"time":"2026-07-24T18:08:47.624237648+09:00","level":"INFO","msg":"writer: started","stream_id":"lqqse51n"}
7
+ {"time":"2026-07-24T18:08:47.62430234+09:00","level":"INFO","msg":"sender: started"}
8
+ {"time":"2026-07-24T18:08:48.27917746+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
9
+ {"time":"2026-07-24T18:08:48.813488143+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-07-24T18:09:03.27956354+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":9,"uploaded_len":2}
11
+ {"time":"2026-07-24T18:09:03.794827834+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-07-24T18:09:18.280282729+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
13
+ {"time":"2026-07-24T18:09:47.94579489+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-07-24T18:09:47.946202132+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
15
+ {"time":"2026-07-24T18:09:48.588767707+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
16
+ {"time":"2026-07-24T18:09:48.589135866+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
17
+ {"time":"2026-07-24T18:10:34.47613234+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
18
+ {"time":"2026-07-24T18:10:34.476309275+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
19
+ {"time":"2026-07-24T18:11:18.904926052+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
20
+ {"time":"2026-07-24T18:11:18.905374248+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":6}
21
+ {"time":"2026-07-24T18:11:19.302967496+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
22
+ {"time":"2026-07-24T18:11:19.303172178+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":6}
23
+ {"time":"2026-07-24T18:11:19.657566076+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
24
+ {"time":"2026-07-24T18:11:33.175855996+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":1}
25
+ {"time":"2026-07-24T18:11:33.652327028+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
26
+ {"time":"2026-07-24T18:11:48.175949191+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":1,"console_offset":8,"console_lines":13}
27
+ {"time":"2026-07-24T18:11:48.658684131+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
28
+ {"time":"2026-07-24T18:12:03.176010749+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":20,"console_lines":1}
29
+ {"time":"2026-07-24T18:12:03.733839312+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
30
+ {"time":"2026-07-24T18:12:18.176170454+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":20,"console_lines":1}
31
+ {"time":"2026-07-24T18:12:42.870254353+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
32
+ {"time":"2026-07-24T18:12:42.870492134+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":4,"console_offset":20,"console_lines":1}
33
+ {"time":"2026-07-24T18:12:43.43127583+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
34
+ {"time":"2026-07-24T18:12:48.17625266+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":20,"console_lines":1}
35
+ {"time":"2026-07-24T18:13:07.036513826+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
36
+ {"time":"2026-07-24T18:13:07.036915417+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":4,"console_offset":20,"console_lines":2}
37
+ {"time":"2026-07-24T18:13:41.843757118+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
38
+ {"time":"2026-07-24T18:13:41.844141038+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
39
+ {"time":"2026-07-24T18:13:42.262626677+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
40
+ {"time":"2026-07-24T18:13:42.262826904+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":4,"console_offset":21,"console_lines":1}
41
+ {"time":"2026-07-24T18:13:42.738785592+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
42
+ {"time":"2026-07-24T18:13:42.738950817+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":21,"console_lines":1}
43
+ {"time":"2026-07-24T18:14:01.760076376+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
44
+ {"time":"2026-07-24T18:14:01.760452991+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":21,"console_lines":1}
45
+ {"time":"2026-07-24T18:14:02.563374728+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
46
+ {"time":"2026-07-24T18:14:12.125137015+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":21,"console_lines":1}
47
+ {"time":"2026-07-24T18:14:12.587166023+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
48
+ {"time":"2026-07-24T18:14:27.12544843+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":21,"console_lines":2}
49
+ {"time":"2026-07-24T18:14:49.827004263+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
50
+ {"time":"2026-07-24T18:14:49.827440242+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":4,"console_offset":22,"console_lines":1}
51
+ {"time":"2026-07-24T18:14:50.288076577+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
52
+ {"time":"2026-07-24T18:14:57.125784841+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":22,"console_lines":1}
53
+ {"time":"2026-07-24T18:15:17.942419688+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
54
+ {"time":"2026-07-24T18:15:17.942915971+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":48,"events_lines":2,"console_offset":22,"console_lines":1}
55
+ {"time":"2026-07-24T18:16:13.183416877+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
56
+ {"time":"2026-07-24T18:16:15.836775763+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
57
+ {"time":"2026-07-24T18:16:15.837056849+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
58
+ {"time":"2026-07-24T18:16:53.34792422+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
59
+ {"time":"2026-07-24T18:16:53.348219955+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":50,"events_lines":8,"console_offset":22,"console_lines":2}
60
+ {"time":"2026-07-24T18:16:53.80424795+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
61
+ {"time":"2026-07-24T18:16:53.804470685+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":58,"events_lines":6,"console_offset":23,"console_lines":2}
62
+ {"time":"2026-07-24T18:16:54.280058023+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
63
+ {"time":"2026-07-24T18:16:54.280201972+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":24,"console_lines":1}
64
+ {"time":"2026-07-24T18:17:27.538437939+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
65
+ {"time":"2026-07-24T18:17:27.538704863+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":64,"events_lines":2,"console_offset":24,"console_lines":1}
66
+ {"time":"2026-07-24T18:17:27.951358644+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
67
+ {"time":"2026-07-24T18:17:27.951574285+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":2,"console_offset":24,"console_lines":1}
68
+ {"time":"2026-07-24T18:17:28.380216641+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
69
+ {"time":"2026-07-24T18:17:38.464509947+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":24,"console_lines":1}
70
+ {"time":"2026-07-24T18:18:13.538159311+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
71
+ {"time":"2026-07-24T18:18:13.538355539+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
72
+ {"time":"2026-07-24T18:18:13.881337903+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
73
+ {"time":"2026-07-24T18:18:13.881848161+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":70,"events_lines":4,"console_offset":24,"console_lines":2}
74
+ {"time":"2026-07-24T18:18:14.42368108+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
75
+ {"time":"2026-07-24T18:18:14.42383956+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":25,"console_lines":1}
76
+ {"time":"2026-07-24T18:18:28.312533921+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
77
+ {"time":"2026-07-24T18:18:28.780687664+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":2,"console_offset":25,"console_lines":1}
78
+ {"time":"2026-07-24T18:18:29.304411418+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
79
+ {"time":"2026-07-24T18:18:43.781233325+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":76,"events_lines":2,"console_offset":25,"console_lines":1}
80
+ {"time":"2026-07-24T18:18:44.242368805+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
81
+ {"time":"2026-07-24T18:18:58.780732119+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":2,"console_offset":25,"console_lines":1}
82
+ {"time":"2026-07-24T18:19:36.984024288+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
83
+ {"time":"2026-07-24T18:19:36.984170259+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
84
+ {"time":"2026-07-24T18:20:32.202849625+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
85
+ {"time":"2026-07-24T18:21:29.677654094+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
86
+ {"time":"2026-07-24T18:21:51.426842527+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
87
+ {"time":"2026-07-24T18:21:51.427078628+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":80,"events_lines":6,"console_offset":25,"console_lines":2}
88
+ {"time":"2026-07-24T18:21:57.471027315+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
89
+ {"time":"2026-07-24T18:21:57.472032098+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":2,"events_offset":86,"events_lines":18,"console_offset":26,"console_lines":3}
90
+ {"time":"2026-07-24T18:22:12.599743156+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
91
+ {"time":"2026-07-24T18:22:12.60040309+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":28,"console_lines":1}
92
+ {"time":"2026-07-24T18:22:37.055177145+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
93
+ {"time":"2026-07-24T18:22:37.055630719+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":104,"events_lines":2,"console_offset":28,"console_lines":1}
94
+ {"time":"2026-07-24T18:22:54.575480712+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
95
+ {"time":"2026-07-24T18:22:54.575773219+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":106,"events_lines":4,"console_offset":28,"console_lines":1}
96
+ {"time":"2026-07-24T18:23:30.993229555+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
97
+ {"time":"2026-07-24T18:23:30.993504636+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":110,"events_lines":2,"console_offset":28,"console_lines":1}
98
+ {"time":"2026-07-24T18:23:53.412378385+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
99
+ {"time":"2026-07-24T18:23:53.412676178+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":112,"events_lines":4,"console_offset":28,"console_lines":2}
100
+ {"time":"2026-07-24T18:23:53.748547151+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
101
+ {"time":"2026-07-24T18:23:53.748986127+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":116,"events_lines":4,"console_offset":29,"console_lines":1}
102
+ {"time":"2026-07-24T18:24:02.677765013+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
103
+ {"time":"2026-07-24T18:24:02.677940765+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":29,"console_lines":1}
104
+ {"time":"2026-07-24T18:24:33.655524831+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
105
+ {"time":"2026-07-24T18:24:33.655744638+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":120,"events_lines":2,"console_offset":29,"console_lines":1}
106
+ {"time":"2026-07-24T18:24:42.548010937+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
107
+ {"time":"2026-07-24T18:24:42.548295182+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":122,"events_lines":4,"console_offset":29,"console_lines":2}
108
+ {"time":"2026-07-24T18:24:43.048453683+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
109
+ {"time":"2026-07-24T18:24:43.048776179+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":30,"console_lines":1}
110
+ {"time":"2026-07-24T18:24:43.441389357+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
111
+ {"time":"2026-07-24T18:24:53.441813516+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":126,"events_lines":2,"console_offset":30,"console_lines":1}
112
+ {"time":"2026-07-24T18:24:53.774222275+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
113
+ {"time":"2026-07-24T18:25:08.441844848+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":128,"events_lines":2,"console_offset":30,"console_lines":1}
114
+ {"time":"2026-07-24T18:25:38.61799626+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
115
+ {"time":"2026-07-24T18:25:38.618266701+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
116
+ {"time":"2026-07-24T18:25:50.158578758+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
117
+ {"time":"2026-07-24T18:25:50.15905911+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":130,"events_lines":4,"console_offset":30,"console_lines":2}
118
+ {"time":"2026-07-24T18:26:18.847568607+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
119
+ {"time":"2026-07-24T18:26:18.847963389+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":134,"events_lines":2,"console_offset":31,"console_lines":1}
120
+ {"time":"2026-07-24T18:26:19.280360669+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
121
+ {"time":"2026-07-24T18:26:19.280535951+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":136,"events_lines":4,"console_offset":31,"console_lines":1}
122
+ {"time":"2026-07-24T18:26:19.721235518+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
123
+ {"time":"2026-07-24T18:26:19.721503555+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":31,"console_lines":1}
124
+ {"time":"2026-07-24T18:26:34.584513612+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
125
+ {"time":"2026-07-24T18:26:34.584757767+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":140,"events_lines":2,"console_offset":31,"console_lines":1}
126
+ {"time":"2026-07-24T18:26:34.982047375+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
127
+ {"time":"2026-07-24T18:26:49.069179368+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":142,"events_lines":2,"console_offset":31,"console_lines":2}
128
+ {"time":"2026-07-24T18:26:49.631643115+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
129
+ {"time":"2026-07-24T18:27:04.069139811+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":144,"events_lines":2,"console_offset":32,"console_lines":1}
130
+ {"time":"2026-07-24T18:27:04.636529545+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
131
+ {"time":"2026-07-24T18:27:19.069177113+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":146,"events_lines":2,"console_offset":32,"console_lines":1}
132
+ {"time":"2026-07-24T18:27:45.598569863+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
133
+ {"time":"2026-07-24T18:27:45.598871699+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":148,"events_lines":2,"console_offset":32,"console_lines":1}
134
+ {"time":"2026-07-24T18:28:09.746807243+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
135
+ {"time":"2026-07-24T18:28:09.74735117+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":150,"events_lines":4,"console_offset":32,"console_lines":2}
136
+ {"time":"2026-07-24T18:28:10.492444366+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
137
+ {"time":"2026-07-24T18:28:10.492644284+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":33,"console_lines":1}
138
+ {"time":"2026-07-24T18:28:47.438857468+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
139
+ {"time":"2026-07-24T18:28:47.439277332+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
140
+ {"time":"2026-07-24T18:29:14.445259123+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
141
+ {"time":"2026-07-24T18:29:14.445723893+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":154,"events_lines":4,"console_offset":33,"console_lines":1}
142
+ {"time":"2026-07-24T18:29:42.312532292+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
143
+ {"time":"2026-07-24T18:29:42.312888364+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":158,"events_lines":4,"console_offset":33,"console_lines":1}
144
+ {"time":"2026-07-24T18:29:42.692074397+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
145
+ {"time":"2026-07-24T18:29:42.692483095+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":162,"events_lines":4,"console_offset":33,"console_lines":2}
146
+ {"time":"2026-07-24T18:29:43.142557482+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
147
+ {"time":"2026-07-24T18:29:43.142937672+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":34,"console_lines":1}
148
+ {"time":"2026-07-24T18:29:43.485860995+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
149
+ {"time":"2026-07-24T18:29:57.435267774+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":166,"events_lines":2,"console_offset":34,"console_lines":1}
150
+ {"time":"2026-07-24T18:29:57.823778239+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
151
+ {"time":"2026-07-24T18:30:12.434417821+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":168,"events_lines":2,"console_offset":34,"console_lines":1}
152
+ {"time":"2026-07-24T18:30:12.905547751+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
153
+ {"time":"2026-07-24T18:30:27.434946109+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":170,"events_lines":2,"console_offset":34,"console_lines":1}
154
+ {"time":"2026-07-24T18:31:03.575974488+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
155
+ {"time":"2026-07-24T18:31:03.576121214+09:00","level":"INFO","msg":"filestream: sending request","total_files":0}
156
+ {"time":"2026-07-24T18:31:26.598960374+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
157
+ {"time":"2026-07-24T18:31:58.795263215+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
158
+ {"time":"2026-07-24T18:32:56.160021927+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"An internal error occurred. Please contact support.\"}"}
159
+ {"time":"2026-07-24T18:33:55.570472582+09:00","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/codjs2659-pusan-national-university/diffusion_so101_cube_multitask_0724_training/lqqse51n/file_stream","body":"{\"error\":\"rate limit exceeded: per_run limit on filestream requests\"}"}
160
+ {"time":"2026-07-24T18:34:34.736913463+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
161
+ {"time":"2026-07-24T18:34:34.737586259+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":172,"events_lines":6,"console_offset":34,"console_lines":2}
162
+ {"time":"2026-07-24T18:34:52.619228935+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
163
+ {"time":"2026-07-24T18:34:52.619443696+09:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":178,"events_lines":2,"console_offset":35,"console_lines":24,"uploaded_len":3,"complete":true,"exit_code":1}
164
+ {"time":"2026-07-24T18:35:16.31221796+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
165
+ {"time":"2026-07-24T18:35:16.312540995+09:00","level":"INFO","msg":"stream: finishing up"}
166
+ {"time":"2026-07-24T18:35:16.312585192+09:00","level":"INFO","msg":"handler: closed"}
167
+ {"time":"2026-07-24T18:35:16.31264971+09:00","level":"INFO","msg":"sender: closed"}
168
+ {"time":"2026-07-24T18:35:16.312659323+09:00","level":"INFO","msg":"stream: all finished"}
wandb/run-20260724_180847-lqqse51n/logs/debug.log ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
2
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_setup.py:_flush():81] Configure stats pid to 184127
3
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-24 18:08:47,023 INFO MainThread:184127 [wandb_init.py:setup_run_log_directory():723] Logging user logs to outputs/train/so101_multitask_diffusion_0724/wandb/run-20260724_180847-lqqse51n/logs/debug.log
5
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to outputs/train/so101_multitask_diffusion_0724/wandb/run-20260724_180847-lqqse51n/logs/debug-internal.log
6
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:init():851] calling init triggers
7
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Chaenn/so101_cube_multitask_0724', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'streaming': False}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': True, 'repo_id': 'Chaenn/diffusion_so101_cube_multitask_0724', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'horizon': 32, 'n_action_steps': 15, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet18', 'resize_shape': None, 'crop_ratio': 1.0, 'crop_shape': None, 'crop_is_random': True, 'pretrained_backbone_weights': None, 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDPM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': None, 'compile_model': False, 'compile_mode': 'reduce-overhead', 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500}, 'reward_model': None, 'output_dir': 'outputs/train/so101_multitask_diffusion_0724', 'job_name': 'so101_cube_diffusion', 'resume': False, 'seed': 1000, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 32, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 200000, 'eval_freq': 20000, 'log_freq': 200, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 10000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 44, 'use_async_envs': True}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'diffusion_so101_cube_multitask_0724_training', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
9
+ 2026-07-24 18:08:47,024 INFO MainThread:184127 [wandb_init.py:init():899] starting backend
10
+ 2026-07-24 18:08:47,245 INFO MainThread:184127 [wandb_init.py:init():914] sending inform_init request
11
+ 2026-07-24 18:08:47,625 INFO MainThread:184127 [wandb_init.py:init():919] backend started and connected
12
+ 2026-07-24 18:08:47,628 INFO MainThread:184127 [wandb_init.py:init():989] updated telemetry
13
+ 2026-07-24 18:08:47,629 INFO MainThread:184127 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
14
+ 2026-07-24 18:08:48,172 INFO MainThread:184127 [wandb_init.py:init():1057] starting run threads in backend
15
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_console_start():2509] atexit reg
16
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_redirect():2359] redirect: wrap_raw
17
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_redirect():2428] Wrapping output streams.
18
+ 2026-07-24 18:08:48,275 INFO MainThread:184127 [wandb_run.py:_redirect():2451] Redirects installed.
19
+ 2026-07-24 18:08:48,278 INFO MainThread:184127 [wandb_init.py:init():1095] run started, returning control to user process
20
+ 2026-07-24 18:31:25,953 INFO wandb-AsyncioManager-main:184127 [service_client.py:_forward_responses():122] Reached EOF.
21
+ 2026-07-24 18:31:25,953 INFO wandb-AsyncioManager-main:184127 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles.
22
+ 2026-07-24 18:31:28,520 ERROR wandb-AsyncioManager-main:184127 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
23
+ Traceback (most recent call last):
24
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
25
+ await fn()
26
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 105, in loop
27
+ await self._poll_and_print(timeout=None)
28
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 133, in _poll_and_print
29
+ handle = await self._interface.deliver_async(request)
30
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
31
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
32
+ handle = await self._client.deliver(request)
33
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
34
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
35
+ handle = self._mailbox.require_response(request)
36
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
37
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
38
+ raise MailboxClosedError()
39
+ wandb.sdk.mailbox.mailbox.MailboxClosedError
40
+ 2026-07-24 18:31:33,481 ERROR wandb-AsyncioManager-main:184127 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
41
+ Traceback (most recent call last):
42
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
43
+ await fn()
44
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
45
+ should_stop = await self._check_should_stop()
46
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
47
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
48
+ handle = await self._interface.deliver_async(request)
49
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
50
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
51
+ handle = await self._client.deliver(request)
52
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
53
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
54
+ handle = self._mailbox.require_response(request)
55
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
56
+ File "/home/grad26/miniforge3/envs/lerobot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
57
+ raise MailboxClosedError()
58
+ wandb.sdk.mailbox.mailbox.MailboxClosedError
wandb/run-20260724_180847-lqqse51n/run-lqqse51n.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01ccae6fb449387a5e4573de3df57ca1f0a0fac77f2f4fc248e1a97f7d5644f2
3
+ size 822046