arrow-free3d-multik / scripts /debug_rig_load.py
smerkd's picture
scripts
443ddb7 verified
Raw History Blame Contribute Delete
2.1 kB
"""Reproduce rig-dataset load failures (IndexError / TemporalContextError) in-process with a traceback.
python debug_rig_load.py <order.jsonl> [n]"""
import sys, json, traceback
from omegaconf import OmegaConf
def main():
W = '/workspace/free3d_multik_hr'; order = sys.argv[1]; n = int(sys.argv[2]) if len(sys.argv) > 2 else 12
cfg = OmegaConf.load(W + '/config.yaml'); cfg.run.out_dir = W + '/overfit/scratch_debug_rig'
from arrow_train import train as T
from arrow_train.data import OrderedClips, SampleConfig
from arrow_train.sampling import SamplingConfig
s = OmegaConf.to_container(cfg.sample, resolve=True); s['sampling'] = SamplingConfig(**s['sampling']); s = SampleConfig(**s)
ds = OrderedClips(order, s, rank=0, world=1, seed=cfg.run.seed, samples_per_clip=1, with_queries=False,
modalities=list(dict.fromkeys(T.needed_modalities(cfg, False) + ['depth_status', 'depth_confidence', 'depth_reliability', 'transmittance'])),
start_sample=0, strict_decode=True, load_failure_policy='raise', load_failure_log_dir=cfg.run.out_dir,
sources=dict(cfg.data.sources), source_revisions=dict(cfg.data.source_revisions), default_source=cfg.data.default_source,
qc_flags=cfg.data.qc_flags)
it = iter(ds); seen = {}
for i in range(n):
try:
x = next(it); print('OK', x['dataset'], x['key'], 'views', sorted(set(map(int, x.get('observation_view', [])))) if x.get('observation_view') is not None else None, 'times', [int(v) for v in x.get('observation_time', [])] if x.get('observation_time') is not None else None, {k: tuple(v.shape) for k, v in x.items() if hasattr(v, 'shape') and k in ('images', 'depth')}, flush=True)
except StopIteration:break
except Exception as e:
tb = traceback.format_exc(); key = type(e).__name__
if seen.get(key, 0) < 2:print('FAIL', key, tb[-2500:], flush=True)
seen[key] = seen.get(key, 0) + 1
print('DEBUG_DONE', seen, flush=True)
if __name__ == '__main__':
main()