File size: 2,097 Bytes
443ddb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
"""Reproduce rig-dataset load failures (IndexError / TemporalContextError) in-process with a traceback.
  python debug_rig_load.py <order.jsonl> [n]"""
import sys, json, traceback
from omegaconf import OmegaConf


def main():
    W = '/workspace/free3d_multik_hr'; order = sys.argv[1]; n = int(sys.argv[2]) if len(sys.argv) > 2 else 12
    cfg = OmegaConf.load(W + '/config.yaml'); cfg.run.out_dir = W + '/overfit/scratch_debug_rig'
    from arrow_train import train as T
    from arrow_train.data import OrderedClips, SampleConfig
    from arrow_train.sampling import SamplingConfig
    s = OmegaConf.to_container(cfg.sample, resolve=True); s['sampling'] = SamplingConfig(**s['sampling']); s = SampleConfig(**s)
    ds = OrderedClips(order, s, rank=0, world=1, seed=cfg.run.seed, samples_per_clip=1, with_queries=False,
                      modalities=list(dict.fromkeys(T.needed_modalities(cfg, False) + ['depth_status', 'depth_confidence', 'depth_reliability', 'transmittance'])),
                      start_sample=0, strict_decode=True, load_failure_policy='raise', load_failure_log_dir=cfg.run.out_dir,
                      sources=dict(cfg.data.sources), source_revisions=dict(cfg.data.source_revisions), default_source=cfg.data.default_source,
                      qc_flags=cfg.data.qc_flags)
    it = iter(ds); seen = {}
    for i in range(n):
        try:
            x = next(it); print('OK', x['dataset'], x['key'], 'views', sorted(set(map(int, x.get('observation_view', [])))) if x.get('observation_view') is not None else None, 'times', [int(v) for v in x.get('observation_time', [])] if x.get('observation_time') is not None else None, {k: tuple(v.shape) for k, v in x.items() if hasattr(v, 'shape') and k in ('images', 'depth')}, flush=True)
        except StopIteration:break
        except Exception as e:
            tb = traceback.format_exc(); key = type(e).__name__
            if seen.get(key, 0) < 2:print('FAIL', key, tb[-2500:], flush=True)
            seen[key] = seen.get(key, 0) + 1
    print('DEBUG_DONE', seen, flush=True)


if __name__ == '__main__':
    main()