log

logs/train_s1.log

@@ -0,0 +1,75 @@
+building scenes 6468 tree scenes 5990 ground donors 5537
+earthview singles (4136, 384, 384, 3) real pairs (1619, 2, 384, 384, 3)
+step 100 67s loss 1.2406 l2sp 4.45
+step 200 103s loss 1.1174 l2sp 18.39
+step 300 130s loss 0.7714 l2sp 32.45
+step 400 160s loss 0.7715 l2sp 42.43
+step 500 187s loss 0.6585 l2sp 51.54
+step 600 223s loss 0.6234 l2sp 57.77
+step 700 251s loss 0.6430 l2sp 60.68
+step 800 277s loss 0.5319 l2sp 64.78
+snapshot 5
+step 900 307s loss 0.5194 l2sp 66.48
+step 1000 334s loss 0.4526 l2sp 69.17
+step 1100 372s loss 0.6404 l2sp 70.41
+step 1200 404s loss 0.8859 l2sp 73.77
+step 1300 432s loss 0.5095 l2sp 77.63
+step 1400 458s loss 0.5001 l2sp 80.27
+step 1500 487s loss 0.5588 l2sp 79.66
+step 1600 514s loss 0.5840 l2sp 82.55
+step 1700 540s loss 0.4543 l2sp 82.80
+step 1800 567s loss 0.5304 l2sp 85.65
+step 1900 592s loss 0.4350 l2sp 88.41
+snapshot 10
+step 2000 619s loss 0.5535 l2sp 89.17
+step 2100 643s loss 0.5253 l2sp 92.85
+step 2200 678s loss 0.5951 l2sp 91.09
+step 2300 702s loss 0.5194 l2sp 90.69
+step 2400 726s loss 0.7137 l2sp 90.49
+step 2500 753s loss 0.4488 l2sp 91.55
+step 2600 783s loss 0.4640 l2sp 95.08
+step 2700 817s loss 0.6564 l2sp 96.49
+step 2800 847s loss 0.4316 l2sp 98.89
+step 2900 885s loss 0.4319 l2sp 100.45
+snapshot 15
+step 3000 918s loss 0.5337 l2sp 99.97
+step 3100 948s loss 0.4463 l2sp 99.96
+step 3200 974s loss 0.4486 l2sp 99.71
+step 3300 1002s loss 0.4392 l2sp 100.50
+step 3400 1036s loss 0.3755 l2sp 100.68
+step 3500 1066s loss 0.4239 l2sp 101.85
+step 3600 1099s loss 0.5196 l2sp 101.12
+step 3700 1133s loss 0.4081 l2sp 103.43
+step 3800 1160s loss 0.3870 l2sp 102.82
+step 3900 1193s loss 0.4453 l2sp 102.47
+snapshot 20
+TRAIN_DONE 3929
+Exception in thread Thread-1 (_pin_memory_loop):
+Traceback (most recent call last):
+  File "/opt/conda/lib/python3.11/threading.py", line 1045, in _bootstrap_inner
+    self.run()
+  File "/opt/conda/lib/python3.11/threading.py", line 982, in run
+    self._target(*self._args, **self._kwargs)
+  File "/opt/conda/lib/python3.11/site-packages/torch/utils/data/_utils/pin_memory.py", line 59, in _pin_memory_loop
+    do_one_step()
+  File "/opt/conda/lib/python3.11/site-packages/torch/utils/data/_utils/pin_memory.py", line 35, in do_one_step
+    r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/lib/python3.11/multiprocessing/queues.py", line 122, in get
+    return _ForkingPickler.loads(res)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/lib/python3.11/site-packages/torch/multiprocessing/reductions.py", line 541, in rebuild_storage_fd
+    fd = df.detach()
+         ^^^^^^^^^^^
+  File "/opt/conda/lib/python3.11/multiprocessing/resource_sharer.py", line 57, in detach
+    with _resource_sharer.get_connection(self._id) as conn:
+         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/lib/python3.11/multiprocessing/resource_sharer.py", line 86, in get_connection
+    c = Client(address, authkey=process.current_process().authkey)
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/lib/python3.11/multiprocessing/connection.py", line 519, in Client
+    c = SocketClient(address)
+        ^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/lib/python3.11/multiprocessing/connection.py", line 647, in SocketClient
+    s.connect(address)
+ConnectionRefusedError: [Errno 111] Connection refused