Clean crash logs before fleet relaunch

runs/ablation_cartoon_isolated/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on ablation_cartoon_isolated at 2026-10-20T23:44:56.038975+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 256, 256] at entry 0 and [1, 384, 384] at entry 1
-
-

runs/ablation_global_avg_only/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on ablation_global_avg_only at 2026-10-20T23:43:41.088856+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_22/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_22/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 384, 384] at entry 0 and [1, 256, 384] at entry 1
-
-

runs/ablation_no_fga/logs/crash.log

@@ -1,39 +0,0 @@
-Crash on ablation_no_fga at 2026-10-20T23:45:26.179445+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_22/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_22/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 274, in collate_tensor_fn
-    out = elem.new(storage).resize_(len(batch), *list(elem.size()))
-RuntimeError: Trying to resize storage that is not resizable
-
-

runs/ablation_spatial_only/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on ablation_spatial_only at 2026-10-20T23:43:56.379506+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 256, 384] at entry 0 and [1, 384, 384] at entry 1
-
-

runs/baseline_efficientnet_b0/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on baseline_efficientnet_b0 at 2026-10-20T23:45:52.611936+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_22/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_22/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 384, 384] at entry 0 and [1, 1300, 1040] at entry 3
-
-

runs/baseline_resnet50/logs/crash.log

@@ -1,39 +0,0 @@
-Crash on baseline_resnet50 at 2026-10-20T23:44:55.989713+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 274, in collate_tensor_fn
-    out = elem.new(storage).resize_(len(batch), *list(elem.size()))
-RuntimeError: Trying to resize storage that is not resizable
-
-

runs/baseline_swin_t/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on baseline_swin_t at 2026-10-20T23:45:07.444654+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 256, 384] at entry 0 and [1, 384, 384] at entry 1
-
-

runs/baseline_vit_b16/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on baseline_vit_b16 at 2026-10-20T23:45:29.450408+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 384, 384] at entry 0 and [1, 256, 384] at entry 1
-
-

runs/logo_holdout_cartoon/logs/crash.log

@@ -1,39 +0,0 @@
-Crash on logo_holdout_cartoon at 2026-10-20T23:46:00.624846+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 274, in collate_tensor_fn
-    out = elem.new(storage).resize_(len(batch), *list(elem.size()))
-RuntimeError: Trying to resize storage that is not resizable
-
-

runs/logo_holdout_inpainting/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on logo_holdout_inpainting at 2026-10-20T23:45:41.612795+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 384, 384] at entry 0 and [1, 256, 256] at entry 1
-
-

runs/patch_mfft_base/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on patch_mfft_base at 2026-10-20T23:43:38.922634+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 217, 500] at entry 0 and [1, 256, 256] at entry 1
-
-

runs/patch_mfft_large/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on patch_mfft_large at 2026-10-20T23:41:19.274947+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 256, 384] at entry 0 and [1, 384, 384] at entry 1
-
-

runs/patch_mfft_large_seed999/logs/crash.log

@@ -1,39 +0,0 @@
-Crash on patch_mfft_large_seed999 at 2026-10-20T23:43:29.426791+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_22/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_22/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 274, in collate_tensor_fn
-    out = elem.new(storage).resize_(len(batch), *list(elem.size()))
-RuntimeError: Trying to resize storage that is not resizable
-
-

runs/patch_mfft_tiny/logs/crash.log

@@ -1,40 +0,0 @@
-Crash on patch_mfft_tiny at 2026-10-20T23:43:26.196297+00:00:
-
-Traceback (most recent call last):
-  File "/tmp/ipykernel_23/3889735120.py", line 27, in <cell line: 0>
-    tr_loss, tr_acc = train_epoch(model, train_loader, optimizer, scheduler, scaler, DEVICE, CONFIG["gradient_accumulation_steps"])
-                      ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/tmp/ipykernel_23/3470172129.py", line 14, in train_epoch
-    for step, (imgs, lbls, masks) in enumerate(loader):
-                                     ~~~~~~~~~^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 741, in __next__
-    data = self._next_data()
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
-    return self._process_data(data, worker_id)
-           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
-    data.reraise()
-    ~~~~~~~~~~~~^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/_utils.py", line 785, in reraise
-    raise exception
-RuntimeError: Caught RuntimeError in DataLoader worker process 0.
-Original Traceback (most recent call last):
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
-    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
-    return self.collate_fn(data)
-           ~~~~~~~~~~~~~~~^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
-    return collate(batch, collate_fn_map=default_collate_fn_map)
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 215, in collate
-    collate(samples, collate_fn_map=collate_fn_map)
-    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 155, in collate
-    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/dist-packages/torch/utils/data/_utils/collate.py", line 275, in collate_tensor_fn
-    return torch.stack(batch, 0, out=out)
-           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
-RuntimeError: stack expects each tensor to be equal size, but got [1, 384, 256] at entry 0 and [1, 363, 500] at entry 1
-
-