Ronaldo-GOAT commited on
Commit
5c63cb7
·
verified ·
1 Parent(s): c24253e

SLAT loader: pooled-anchor conditioning (white-bg cond + RecGen images; anchor source decides all aux views), RecGen camera QC, recgen/<id>/ tar join; ON in train_slatflow_prod.yaml. Verified: 2500 real draws, 0 mixed-source samples.

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow.yaml.bak_predatveiw +169 -0
  3. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml +11 -11
  4. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml.bak_predatveiw +177 -0
  5. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_2gpu.yaml.bak_predatveiw +177 -0
  6. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_pslaug_2gpu.yaml.bak_predatveiw +186 -0
  7. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslfinal_2gpu.yaml.bak_predatveiw +188 -0
  8. migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslocc_2gpu.yaml.bak_predatveiw +187 -0
  9. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_003.pt +3 -0
  10. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_008.pt +3 -0
  11. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/chair_026.pt +3 -0
  12. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ours_both.log +51 -0
  13. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe2_ref.pt +3 -0
  14. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe_ref.pt +3 -0
  15. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ref.log +73 -0
  16. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_003.pt +3 -0
  17. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_008.pt +3 -0
  18. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/chair_026.pt +3 -0
  19. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/ref.log +73 -0
  20. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/smoke_slatflow.log +19 -0
  21. migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/train_slatflow_smoke.log +28 -0
  22. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-310.pyc +0 -0
  23. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-311.pyc +0 -0
  24. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-39.pyc +0 -0
  25. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-310.pyc +0 -0
  26. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-311.pyc +0 -0
  27. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-39.pyc +0 -0
  28. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-310.pyc +0 -0
  29. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-311.pyc +0 -0
  30. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-39.pyc +0 -0
  31. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/adapters.cpython-311.pyc +3 -0
  32. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/base_dataset.cpython-311.pyc +0 -0
  33. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/canonical.cpython-311.pyc +0 -0
  34. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_calib.cpython-311.pyc +0 -0
  35. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_dataset.cpython-311.pyc +0 -0
  36. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/integrated.cpython-311.pyc +0 -0
  37. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slat_preprocess.cpython-311.pyc +0 -0
  38. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-310.pyc +0 -0
  39. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-311.pyc +3 -0
  40. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-310.pyc +0 -0
  41. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-311.pyc +0 -0
  42. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_preprocess.cpython-311.pyc +0 -0
  43. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-310.pyc +0 -0
  44. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-311.pyc +0 -0
  45. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-39.pyc +0 -0
  46. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/v_bucket_sampler.cpython-311.pyc +0 -0
  47. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py +152 -41
  48. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py.bak_predatveiw +804 -0
  49. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-310.pyc +0 -0
  50. migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-311.pyc +0 -0
.gitattributes CHANGED
@@ -36,3 +36,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
36
  migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/Cell-DINO.png filter=lfs diff=lfs merge=lfs -text
37
  migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/ChannelAdaptiveDINO.png filter=lfs diff=lfs merge=lfs -text
38
  migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/__pycache__/adapters.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
 
 
 
36
  migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/Cell-DINO.png filter=lfs diff=lfs merge=lfs -text
37
  migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/ChannelAdaptiveDINO.png filter=lfs diff=lfs merge=lfs -text
38
  migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/__pycache__/adapters.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
39
+ migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/adapters.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
40
+ migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow.yaml.bak_predatveiw ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
2
+ # ALL training hyperparameters live here; train_slat_flow.py reads them and
3
+ # nothing training-related is hardcoded in Python. Launch:
4
+ # SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
5
+ # python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
6
+ # (multi-GPU: torchrun --nproc_per_node=N ...)
7
+ # Per-key overrides: --set key.subkey=value (repeatable).
8
+
9
+ # --------------------------------------------------------------------------- #
10
+ # data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
11
+ # Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
12
+ # RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
13
+ # stored views.npz). Production roots get views/rgb + views/bbox.npy +
14
+ # slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
15
+ # --------------------------------------------------------------------------- #
16
+ data:
17
+ roots:
18
+ - /lp-dev/jonghoon/mv-sam3d-6d-code/smoke_data/toys4k200
19
+ # Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
20
+ # val = ALL objects under val_roots, train = roots minus those ids.
21
+ # NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
22
+ # set is toys4k-only until the production rgb/bbox precompute lands.
23
+ val_roots:
24
+ - /lp-dev/jonghoon/mv-sam3d-6d-code/smoke_data/val_toys4k100
25
+ # --- view-subset augmentation (the ONLY train-time stochasticity besides
26
+ # the fresh invisible-row noise in x0) ---
27
+ # smoke data has 4 views/object; production renders 24 (paper §3.1.1).
28
+ # TRELLIS-style per-object voxel FILTER (structured_latent.py
29
+ # filter_metadata): objects with more active 64^3 voxels are DROPPED at
30
+ # index-build time (never downsampled). 20000 = the single-object SLAT
31
+ # OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
32
+ # 0 disables the filter.
33
+ max_num_voxels: 20000
34
+ min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
35
+ max_views: 4 # clamped per object to its available view count
36
+ val_num_views: 2 # deterministic view count for validation samples
37
+ aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
38
+ # --- legacy hash split (used only when val_roots is empty) ---
39
+ val_fraction: 0.05
40
+ split_seed: 0
41
+
42
+ # --------------------------------------------------------------------------- #
43
+ # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
44
+ # SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
45
+ # with the pipeline's effective inference config (steps=25, cfg_strength=1,
46
+ # cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
47
+ # --------------------------------------------------------------------------- #
48
+ model:
49
+ # Warm-start from the deployed SLAT generator (526/526 tensors, verified).
50
+ pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
51
+ # Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
52
+ # PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
53
+ # 3 toys4k objects), so the bias is ENABLED for training. Set false to
54
+ # reproduce the gating configuration.
55
+ enable_soft_mask: true # local -10 soft-mask in cross-attention
56
+ enable_plucker: true # Plücker ray embedding on non-anchor views
57
+ bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
58
+ # Backbone param dtype: false = lossless fp16->fp32 upcast for training
59
+ # (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
60
+ fp16_torso: false
61
+ # Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
62
+ # .use_checkpoint flip, same toggle the batch-size bench used): recompute
63
+ # activations in backward -> big memory cut, ~modest step-time cost.
64
+ # false = current behavior (no checkpointing).
65
+ use_checkpoint: false
66
+ # Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
67
+ # towers inside are always frozen). false freezes the whole fuser so only
68
+ # the backbone + Plücker FC train.
69
+ train_condition_embedder: true
70
+
71
+ # --------------------------------------------------------------------------- #
72
+ # optim + train loop
73
+ # --------------------------------------------------------------------------- #
74
+ optim:
75
+ lr: 3.0e-5
76
+ weight_decay: 0.0 # AdamW
77
+ grad_clip: 1.0 # max grad norm
78
+
79
+ # --------------------------------------------------------------------------- #
80
+ # ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
81
+ # updated after every optimizer step; frozen DINO towers excluded).
82
+ # Disabled by default = existing behavior unchanged. State is saved as a
83
+ # per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
84
+ # and restored automatically on resume_from.
85
+ # --------------------------------------------------------------------------- #
86
+ ema:
87
+ enable: true
88
+ rate: 0.999 # ema = rate*ema + (1-rate)*param
89
+ eval_with_ema: true # swap EMA weights in for the in-loop validation
90
+
91
+ train:
92
+ # Effective batch = batch_objects x grad_accum_steps x world_size objects
93
+ # per optimizer step.
94
+ # batch_objects = 1 (default): loader bs=1 + collate_single, one object per
95
+ # backbone forward (batching purely via grad_accum_steps).
96
+ # batch_objects > 1: TRUE multi-object batching (collate_batched +
97
+ # SlatFlowModel.flow_step_batched) — B objects concatenated into ONE
98
+ # sparse coord set with the batch-index column, ONE backbone forward,
99
+ # independent t per object. Requires grad_accum_steps == 1.
100
+ steps: 20000
101
+ num_workers: 16 # CPU work: crops + projections + Plücker per view
102
+ batch_objects: 1 # objects per GPU per forward (true batching if >1)
103
+ grad_accum_steps: 8
104
+ # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
105
+ scheduler: cosine # cosine | constant
106
+ warmup_steps: 500
107
+ min_lr: 1.0e-6
108
+ # --- resume ---
109
+ resume_from: "" # checkpoint path to resume (empty = fresh start)
110
+ log_every: 20
111
+ ckpt_every: 5000
112
+ ckpt_dir: ${MIGRATOR_CACHE}/ckpt_slatflow
113
+ amp: true
114
+ amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
115
+ seed: 0
116
+ # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
117
+ backend: ddp # ddp | deepspeed
118
+ deepspeed_config: ""
119
+ # --- validation: held-out flow v-MSE only (fixed RNG). Decode +
120
+ # faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
121
+ val_every: 5000 # optimizer steps between validation passes (0 = never)
122
+ val_max_batches: 8 # val objects per rank per pass
123
+
124
+ # --------------------------------------------------------------------------- #
125
+ # wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
126
+ # val/* per validation pass). enabled: false makes every wandb call a no-op.
127
+ # Extra metric dicts can be pushed from anywhere via
128
+ # from mvsam3d.train.train_slat_flow import log_metrics
129
+ # log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
130
+ # --------------------------------------------------------------------------- #
131
+ wandb:
132
+ enabled: true
133
+ project: "mvsam3d-slatflow"
134
+ entity: "alphabet1"
135
+ name: null # null -> W&B autogenerates the run name
136
+ dir: "/lp-dev/jonghoon/mv-mesh/wandb"
137
+
138
+ # --------------------------------------------------------------------------- #
139
+ # val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
140
+ # the trained flow through the exact reported protocol (batch_appforce_sam3d
141
+ # sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
142
+ # OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
143
+ # weights, ALL ranks sample their shard then run the reference-env decode +
144
+ # eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
145
+ # Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
146
+ # tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
147
+ # <stage1_cache>/<name>/2v
148
+ # --------------------------------------------------------------------------- #
149
+ val_appforce:
150
+ enabled: true
151
+ every: 5000
152
+ at_step0: true
153
+ views: 2
154
+ schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
155
+ seed: 42
156
+ limit: 0 # debug: cap objects per dataset
157
+ batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
158
+ single: false # A/B: force the one-object-per-solve path
159
+ decode_procs: 2 # concurrent reference-env decode shards per rank
160
+ ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
161
+ stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
162
+ out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
163
+ ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
164
+ eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
165
+ datasets:
166
+ - name: toys4k100_tex
167
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
168
+ - name: omni3d
169
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml CHANGED
@@ -63,20 +63,20 @@ data:
63
  # Non-empty -> TarSlatFlowDataset reads latents/cond/cameras straight from
64
  # the .tar shards (GL->CV flip + alpha-bbox crop); `roots` is ignored. ---
65
  tar_roots: []
66
- # --- condition variants for the .tar path (2026-09-24; all default OFF) ---
67
  # use_recgen_images: index recgen/<id>/{cameras.json,NN.jpg,NN_mask.png}
68
- # (RecGen's own background image + mask + rigid cube cameras) from any tar
69
- # under tar_roots, joined to latents/<id>.npz by id; latent-only slats WITH
70
- # RecGen images become trainable (else they are skipped).
71
- # p_recgen_image: P(RecGen image instead of the white cond render) when an
72
- # object has both; single-source objects always use what they have.
 
 
 
73
  # cond_cameras_fallback: white-cond objects without cameras/<id>/cond_cameras.json
74
  # (24-view slat5/slat5rs batches, 22,755 objs) read cond/<id>/transforms.json.
75
- # For the Hub flat layout (slat_train_flat/) the intended setting is
76
- # use_recgen_images: true, p_recgen_image: 0.5, cond_cameras_fallback: true
77
- use_recgen_images: false
78
- p_recgen_image: 0.0
79
- cond_cameras_fallback: false
80
 
81
  # --------------------------------------------------------------------------- #
82
  # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
 
63
  # Non-empty -> TarSlatFlowDataset reads latents/cond/cameras straight from
64
  # the .tar shards (GL->CV flip + alpha-bbox crop); `roots` is ignored. ---
65
  tar_roots: []
66
+ # --- condition sources for the .tar path (2026-09-24) — ON for production ---
67
  # use_recgen_images: index recgen/<id>/{cameras.json,NN.jpg,NN_mask.png}
68
+ # (RecGen's own background image + visible mask + rigid cube cameras) from
69
+ # ANY tar under tar_roots, joined to latents/<id>.npz by the same id string.
70
+ # ONE POOL: an object with both sources draws its anchor from the union of
71
+ # its white cond views and RecGen views; the anchor's source decides ALL aux
72
+ # views (never mixed). RecGen anchors go through the same deployed
73
+ # preprocess_slat_image recipe into item["image"] (background kept, alpha =
74
+ # mask). Latent-only RecGen slats become trainable; white-only objects are
75
+ # drawn exactly as before.
76
  # cond_cameras_fallback: white-cond objects without cameras/<id>/cond_cameras.json
77
  # (24-view slat5/slat5rs batches, 22,755 objs) read cond/<id>/transforms.json.
78
+ use_recgen_images: true
79
+ cond_cameras_fallback: true
 
 
 
80
 
81
  # --------------------------------------------------------------------------- #
82
  # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml.bak_predatveiw ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
2
+ # ALL training hyperparameters live here; train_slat_flow.py reads them and
3
+ # nothing training-related is hardcoded in Python. Launch:
4
+ # SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
5
+ # python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
6
+ # (multi-GPU: torchrun --nproc_per_node=N ...)
7
+ # Per-key overrides: --set key.subkey=value (repeatable).
8
+
9
+ # --------------------------------------------------------------------------- #
10
+ # data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
11
+ # Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
12
+ # RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
13
+ # stored views.npz). Production roots get views/rgb + views/bbox.npy +
14
+ # slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
15
+ # --------------------------------------------------------------------------- #
16
+ data:
17
+ roots: # merged production set (remote 29.4k + local, DATA_FORMAT via tools/adapt_slat50k.py)
18
+ - /data/mv_mesh_data/slat_train/dataset
19
+ # Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
20
+ # val = ALL objects under val_roots, train = roots minus those ids.
21
+ # NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
22
+ # set is toys4k-only until the production rgb/bbox precompute lands.
23
+ val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
24
+ # --- view-subset augmentation (the ONLY train-time stochasticity besides
25
+ # the fresh invisible-row noise in x0) ---
26
+ # smoke data has 4 views/object; production renders 24 (paper §3.1.1).
27
+ # TRELLIS-style per-object voxel FILTER (structured_latent.py
28
+ # filter_metadata): objects with more active 64^3 voxels are DROPPED at
29
+ # index-build time (never downsampled). 20000 = the single-object SLAT
30
+ # OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
31
+ # 0 disables the filter.
32
+ max_num_voxels: 20000
33
+ min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
34
+ max_views: 4 # clamped per object to its available view count
35
+ val_num_views: 2 # deterministic view count for validation samples
36
+ aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
37
+ # --- legacy hash split (used only when val_roots is empty) ---
38
+ val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
39
+ split_seed: 0
40
+ # --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
41
+ # p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
42
+ # p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
43
+ # BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
44
+ p_seed_none: 0.0
45
+ p_seed_single: 0.0
46
+ # --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
47
+ # (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
48
+ # metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
49
+ # GPU decode-compat test before training on RecGen's RELEASED latents. ---
50
+ recgen_roots: []
51
+ # encoded_store recgen root with canonical cube cameras (pose_cube.npz /
52
+ # views.npz) looked up by sha; needed for CORRECT recgen voxel
53
+ # visibility (raw view_metadata alone is geometrically approximate).
54
+ recgen_pose_store: null
55
+
56
+ # --------------------------------------------------------------------------- #
57
+ # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
58
+ # SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
59
+ # with the pipeline's effective inference config (steps=25, cfg_strength=1,
60
+ # cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
61
+ # --------------------------------------------------------------------------- #
62
+ model:
63
+ # Warm-start from the deployed SLAT generator (526/526 tensors, verified).
64
+ pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
65
+ # Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
66
+ # PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
67
+ # 3 toys4k objects), so the bias is ENABLED for training. Set false to
68
+ # reproduce the gating configuration.
69
+ enable_soft_mask: true # local -10 soft-mask in cross-attention
70
+ enable_plucker: true # Plücker ray embedding on non-anchor views
71
+ bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
72
+ # Backbone param dtype: false = lossless fp16->fp32 upcast for training
73
+ # (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
74
+ fp16_torso: false
75
+ # Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
76
+ # .use_checkpoint flip, same toggle the batch-size bench used): recompute
77
+ # activations in backward -> big memory cut, ~modest step-time cost.
78
+ # false = current behavior (no checkpointing).
79
+ use_checkpoint: false # user choice 2026-09-03: +20% speed; worst case (3x20k-voxel objects) peaks ~69 GB, watchdog resumes on OOM
80
+ # Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
81
+ # towers inside are always frozen). false freezes the whole fuser so only
82
+ # the backbone + Plücker FC train.
83
+ train_condition_embedder: true
84
+
85
+ # --------------------------------------------------------------------------- #
86
+ # optim + train loop
87
+ # --------------------------------------------------------------------------- #
88
+ optim:
89
+ lr: 3.0e-5
90
+ weight_decay: 0.0 # AdamW
91
+ grad_clip: 1.0 # max grad norm
92
+
93
+ # --------------------------------------------------------------------------- #
94
+ # ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
95
+ # updated after every optimizer step; frozen DINO towers excluded).
96
+ # Disabled by default = existing behavior unchanged. State is saved as a
97
+ # per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
98
+ # and restored automatically on resume_from.
99
+ # --------------------------------------------------------------------------- #
100
+ ema:
101
+ enable: true
102
+ rate: 0.999 # ema = rate*ema + (1-rate)*param
103
+ eval_with_ema: true # swap EMA weights in for the in-loop validation
104
+
105
+ train:
106
+ # Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
107
+ # the effective batch is grad_accum_steps x world_size objects per step.
108
+ steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
109
+ num_workers: 16 # CPU work: crops + projections + Plücker per view
110
+ grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
111
+ batch_objects: 3 # objects per GPU per optimizer step (true batched flow step)
112
+ # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
113
+ scheduler: cosine # cosine | constant
114
+ warmup_steps: 500
115
+ min_lr: 1.0e-6
116
+ # --- resume ---
117
+ resume_from: "" # checkpoint path to resume (empty = fresh start)
118
+ log_every: 20
119
+ ckpt_every: 2500
120
+ ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_prod_20260902 # /data (1.1T free); 40 ckpts x 5 GB model+ema, optimizer sidecars pruned to the last 2
121
+ amp: true
122
+ amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
123
+ seed: 0
124
+ # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
125
+ backend: ddp # ddp | deepspeed
126
+ deepspeed_config: ""
127
+ # --- validation: held-out flow v-MSE only (fixed RNG). Decode +
128
+ # faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
129
+ val_every: 5000 # optimizer steps between validation passes (0 = never)
130
+ val_max_batches: 8 # val objects per rank per pass
131
+
132
+ # --------------------------------------------------------------------------- #
133
+ # wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
134
+ # val/* per validation pass). enabled: false makes every wandb call a no-op.
135
+ # Extra metric dicts can be pushed from anywhere via
136
+ # from mvsam3d.train.train_slat_flow import log_metrics
137
+ # log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
138
+ # --------------------------------------------------------------------------- #
139
+ wandb:
140
+ enabled: true
141
+ project: "mvsam3d-slatflow"
142
+ entity: "alphabet1"
143
+ name: "slatflow_prod_bs12_lr3e-5_ema0.999_100k"
144
+ dir: "/lp-dev/jonghoon/mv-mesh/wandb"
145
+
146
+ # --------------------------------------------------------------------------- #
147
+ # val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
148
+ # the trained flow through the exact reported protocol (batch_appforce_sam3d
149
+ # sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
150
+ # OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
151
+ # weights, ALL ranks sample their shard then run the reference-env decode +
152
+ # eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
153
+ # Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
154
+ # tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
155
+ # <stage1_cache>/<name>/2v
156
+ # --------------------------------------------------------------------------- #
157
+ val_appforce:
158
+ enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
159
+ every: 5000
160
+ at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
161
+ views: 2
162
+ schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
163
+ seed: 42
164
+ limit: 0 # debug: cap objects per dataset
165
+ batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
166
+ single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
167
+ decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
168
+ ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
169
+ stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
170
+ out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
171
+ ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
172
+ eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
173
+ datasets:
174
+ - name: toys4k100_tex
175
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
176
+ - name: omni3d
177
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_2gpu.yaml.bak_predatveiw ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
2
+ # ALL training hyperparameters live here; train_slat_flow.py reads them and
3
+ # nothing training-related is hardcoded in Python. Launch:
4
+ # SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
5
+ # python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
6
+ # (multi-GPU: torchrun --nproc_per_node=N ...)
7
+ # Per-key overrides: --set key.subkey=value (repeatable).
8
+
9
+ # --------------------------------------------------------------------------- #
10
+ # data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
11
+ # Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
12
+ # RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
13
+ # stored views.npz). Production roots get views/rgb + views/bbox.npy +
14
+ # slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
15
+ # --------------------------------------------------------------------------- #
16
+ data:
17
+ roots: # POOL for the 20k->30k resume (user 2026-09-07): objects present at the 20k stop + 5,000 randomly sampled NEW slats (local+HF); lists in .debug/slat_prod/pool_{base,new5k}_shas.txt
18
+ - /data/mv_mesh_data/slat_train/dataset_pool_20k_plus5k
19
+ # Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
20
+ # val = ALL objects under val_roots, train = roots minus those ids.
21
+ # NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
22
+ # set is toys4k-only until the production rgb/bbox precompute lands.
23
+ val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
24
+ # --- view-subset augmentation (the ONLY train-time stochasticity besides
25
+ # the fresh invisible-row noise in x0) ---
26
+ # smoke data has 4 views/object; production renders 24 (paper §3.1.1).
27
+ # TRELLIS-style per-object voxel FILTER (structured_latent.py
28
+ # filter_metadata): objects with more active 64^3 voxels are DROPPED at
29
+ # index-build time (never downsampled). 20000 = the single-object SLAT
30
+ # OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
31
+ # 0 disables the filter.
32
+ max_num_voxels: 20000
33
+ min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
34
+ max_views: 4 # clamped per object to its available view count
35
+ val_num_views: 2 # deterministic view count for validation samples
36
+ aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
37
+ # --- legacy hash split (used only when val_roots is empty) ---
38
+ val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
39
+ split_seed: 0
40
+ # --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
41
+ # p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
42
+ # p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
43
+ # BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
44
+ p_seed_none: 0.0
45
+ p_seed_single: 0.0
46
+ # --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
47
+ # (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
48
+ # metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
49
+ # GPU decode-compat test before training on RecGen's RELEASED latents. ---
50
+ recgen_roots: []
51
+ # encoded_store recgen root with canonical cube cameras (pose_cube.npz /
52
+ # views.npz) looked up by sha; needed for CORRECT recgen voxel
53
+ # visibility (raw view_metadata alone is geometrically approximate).
54
+ recgen_pose_store: null
55
+
56
+ # --------------------------------------------------------------------------- #
57
+ # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
58
+ # SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
59
+ # with the pipeline's effective inference config (steps=25, cfg_strength=1,
60
+ # cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
61
+ # --------------------------------------------------------------------------- #
62
+ model:
63
+ # Warm-start from the deployed SLAT generator (526/526 tensors, verified).
64
+ pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
65
+ # Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
66
+ # PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
67
+ # 3 toys4k objects), so the bias is ENABLED for training. Set false to
68
+ # reproduce the gating configuration.
69
+ enable_soft_mask: true # local -10 soft-mask in cross-attention
70
+ enable_plucker: true # Plücker ray embedding on non-anchor views
71
+ bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
72
+ # Backbone param dtype: false = lossless fp16->fp32 upcast for training
73
+ # (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
74
+ fp16_torso: false
75
+ # Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
76
+ # .use_checkpoint flip, same toggle the batch-size bench used): recompute
77
+ # activations in backward -> big memory cut, ~modest step-time cost.
78
+ # false = current behavior (no checkpointing).
79
+ use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
80
+ # Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
81
+ # towers inside are always frozen). false freezes the whole fuser so only
82
+ # the backbone + Plücker FC train.
83
+ train_condition_embedder: true
84
+
85
+ # --------------------------------------------------------------------------- #
86
+ # optim + train loop
87
+ # --------------------------------------------------------------------------- #
88
+ optim:
89
+ lr: 3.0e-5
90
+ weight_decay: 0.0 # AdamW
91
+ grad_clip: 1.0 # max grad norm
92
+
93
+ # --------------------------------------------------------------------------- #
94
+ # ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
95
+ # updated after every optimizer step; frozen DINO towers excluded).
96
+ # Disabled by default = existing behavior unchanged. State is saved as a
97
+ # per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
98
+ # and restored automatically on resume_from.
99
+ # --------------------------------------------------------------------------- #
100
+ ema:
101
+ enable: true
102
+ rate: 0.999 # ema = rate*ema + (1-rate)*param
103
+ eval_with_ema: true # swap EMA weights in for the in-loop validation
104
+
105
+ train:
106
+ # Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
107
+ # the effective batch is grad_accum_steps x world_size objects per step.
108
+ steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
109
+ num_workers: 16 # CPU work: crops + projections + Plücker per view
110
+ grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
111
+ batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
112
+ # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
113
+ scheduler: cosine # cosine | constant
114
+ warmup_steps: 500
115
+ min_lr: 1.0e-6
116
+ # --- resume ---
117
+ resume_from: "/data/mv_mesh_data/ckpt/slatflow_prod_20260902/step_0020000.pt" # resume the 20k prod ckpt (+ema/optimizer sidecars)
118
+ log_every: 20
119
+ ckpt_every: 2500
120
+ ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume20k_2gpu
121
+ amp: true
122
+ amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
123
+ seed: 0
124
+ # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
125
+ backend: ddp # ddp | deepspeed
126
+ deepspeed_config: ""
127
+ # --- validation: held-out flow v-MSE only (fixed RNG). Decode +
128
+ # faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
129
+ val_every: 5000 # optimizer steps between validation passes (0 = never)
130
+ val_max_batches: 8 # val objects per rank per pass
131
+
132
+ # --------------------------------------------------------------------------- #
133
+ # wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
134
+ # val/* per validation pass). enabled: false makes every wandb call a no-op.
135
+ # Extra metric dicts can be pushed from anywhere via
136
+ # from mvsam3d.train.train_slat_flow import log_metrics
137
+ # log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
138
+ # --------------------------------------------------------------------------- #
139
+ wandb:
140
+ enabled: true
141
+ project: "mvsam3d-slatflow"
142
+ entity: "alphabet1"
143
+ name: "slatflow_resume20k_2gpu_bs12_pool55k"
144
+ dir: "/lp-dev/jonghoon/mv-mesh/wandb"
145
+
146
+ # --------------------------------------------------------------------------- #
147
+ # val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
148
+ # the trained flow through the exact reported protocol (batch_appforce_sam3d
149
+ # sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
150
+ # OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
151
+ # weights, ALL ranks sample their shard then run the reference-env decode +
152
+ # eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
153
+ # Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
154
+ # tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
155
+ # <stage1_cache>/<name>/2v
156
+ # --------------------------------------------------------------------------- #
157
+ val_appforce:
158
+ enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
159
+ every: 5000
160
+ at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
161
+ views: 2
162
+ schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
163
+ seed: 42
164
+ limit: 0 # debug: cap objects per dataset
165
+ batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
166
+ single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
167
+ decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
168
+ ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
169
+ stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
170
+ out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
171
+ ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
172
+ eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
173
+ datasets:
174
+ - name: toys4k100_tex
175
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
176
+ - name: omni3d
177
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_pslaug_2gpu.yaml.bak_predatveiw ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
2
+ # ALL training hyperparameters live here; train_slat_flow.py reads them and
3
+ # nothing training-related is hardcoded in Python. Launch:
4
+ # SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
5
+ # python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
6
+ # (multi-GPU: torchrun --nproc_per_node=N ...)
7
+ # Per-key overrides: --set key.subkey=value (repeatable).
8
+
9
+ # --------------------------------------------------------------------------- #
10
+ # data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
11
+ # Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
12
+ # RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
13
+ # stored views.npz). Production roots get views/rgb + views/bbox.npy +
14
+ # slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
15
+ # --------------------------------------------------------------------------- #
16
+ data:
17
+ roots: # POOL for the 20k->30k PSL-AUG resume (user 2026-09-07):
18
+ # (a) the 50,621 base-pool objects present at the 20k stop
19
+ # (.debug/slat_prod/pool_base_shas.txt), and
20
+ # (b) ALL new PSL-augmentation objects from
21
+ # /data/mv_mesh_data/psl_aug/objects (psl_aug_shas.txt).
22
+ # The pool root is symlink-only, built by
23
+ # /lp-dev/jonghoon/mv-sam3d-6d-code/build_pool_base_pslaug.sh
24
+ # as <root>/{remote,local,pslaug}/objects/<sha[:2]>/<sha>.
25
+ # psl_aug objects carry NO special weighting: scan_objects
26
+ # flattens all three sources into one uniformly-sampled index.
27
+ - /data/mv_mesh_data/slat_train/dataset_pool_base_pslaug
28
+ # Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
29
+ # val = ALL objects under val_roots, train = roots minus those ids.
30
+ # NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
31
+ # set is toys4k-only until the production rgb/bbox precompute lands.
32
+ val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
33
+ # --- view-subset augmentation (the ONLY train-time stochasticity besides
34
+ # the fresh invisible-row noise in x0) ---
35
+ # smoke data has 4 views/object; production renders 24 (paper §3.1.1).
36
+ # TRELLIS-style per-object voxel FILTER (structured_latent.py
37
+ # filter_metadata): objects with more active 64^3 voxels are DROPPED at
38
+ # index-build time (never downsampled). 20000 = the single-object SLAT
39
+ # OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
40
+ # 0 disables the filter.
41
+ max_num_voxels: 20000
42
+ min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
43
+ max_views: 4 # clamped per object to its available view count
44
+ val_num_views: 2 # deterministic view count for validation samples
45
+ aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
46
+ # --- legacy hash split (used only when val_roots is empty) ---
47
+ val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
48
+ split_seed: 0
49
+ # --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
50
+ # p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
51
+ # p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
52
+ # BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
53
+ p_seed_none: 0.0
54
+ p_seed_single: 0.0
55
+ # --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
56
+ # (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
57
+ # metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
58
+ # GPU decode-compat test before training on RecGen's RELEASED latents. ---
59
+ recgen_roots: []
60
+ # encoded_store recgen root with canonical cube cameras (pose_cube.npz /
61
+ # views.npz) looked up by sha; needed for CORRECT recgen voxel
62
+ # visibility (raw view_metadata alone is geometrically approximate).
63
+ recgen_pose_store: null
64
+
65
+ # --------------------------------------------------------------------------- #
66
+ # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
67
+ # SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
68
+ # with the pipeline's effective inference config (steps=25, cfg_strength=1,
69
+ # cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
70
+ # --------------------------------------------------------------------------- #
71
+ model:
72
+ # Warm-start from the deployed SLAT generator (526/526 tensors, verified).
73
+ pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
74
+ # Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
75
+ # PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
76
+ # 3 toys4k objects), so the bias is ENABLED for training. Set false to
77
+ # reproduce the gating configuration.
78
+ enable_soft_mask: true # local -10 soft-mask in cross-attention
79
+ enable_plucker: true # Plücker ray embedding on non-anchor views
80
+ bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
81
+ # Backbone param dtype: false = lossless fp16->fp32 upcast for training
82
+ # (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
83
+ fp16_torso: false
84
+ # Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
85
+ # .use_checkpoint flip, same toggle the batch-size bench used): recompute
86
+ # activations in backward -> big memory cut, ~modest step-time cost.
87
+ # false = current behavior (no checkpointing).
88
+ use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
89
+ # Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
90
+ # towers inside are always frozen). false freezes the whole fuser so only
91
+ # the backbone + Plücker FC train.
92
+ train_condition_embedder: true
93
+
94
+ # --------------------------------------------------------------------------- #
95
+ # optim + train loop
96
+ # --------------------------------------------------------------------------- #
97
+ optim:
98
+ lr: 3.0e-5
99
+ weight_decay: 0.0 # AdamW
100
+ grad_clip: 1.0 # max grad norm
101
+
102
+ # --------------------------------------------------------------------------- #
103
+ # ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
104
+ # updated after every optimizer step; frozen DINO towers excluded).
105
+ # Disabled by default = existing behavior unchanged. State is saved as a
106
+ # per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
107
+ # and restored automatically on resume_from.
108
+ # --------------------------------------------------------------------------- #
109
+ ema:
110
+ enable: true
111
+ rate: 0.999 # ema = rate*ema + (1-rate)*param
112
+ eval_with_ema: true # swap EMA weights in for the in-loop validation
113
+
114
+ train:
115
+ # Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
116
+ # the effective batch is grad_accum_steps x world_size objects per step.
117
+ steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
118
+ num_workers: 16 # CPU work: crops + projections + Plücker per view
119
+ grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
120
+ batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
121
+ # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
122
+ scheduler: cosine # cosine | constant
123
+ warmup_steps: 500
124
+ min_lr: 1.0e-6
125
+ # --- resume ---
126
+ resume_from: "/data/mv_mesh_data/ckpt/slatflow_prod_20260902/step_0020000.pt" # resume the 20k prod ckpt (+ema/optimizer sidecars)
127
+ log_every: 20
128
+ ckpt_every: 2500
129
+ ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume20k_pslaug_2gpu
130
+ amp: true
131
+ amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
132
+ seed: 0
133
+ # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
134
+ backend: ddp # ddp | deepspeed
135
+ deepspeed_config: ""
136
+ # --- validation: held-out flow v-MSE only (fixed RNG). Decode +
137
+ # faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
138
+ val_every: 5000 # optimizer steps between validation passes (0 = never)
139
+ val_max_batches: 8 # val objects per rank per pass
140
+
141
+ # --------------------------------------------------------------------------- #
142
+ # wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
143
+ # val/* per validation pass). enabled: false makes every wandb call a no-op.
144
+ # Extra metric dicts can be pushed from anywhere via
145
+ # from mvsam3d.train.train_slat_flow import log_metrics
146
+ # log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
147
+ # --------------------------------------------------------------------------- #
148
+ wandb:
149
+ enabled: true
150
+ project: "mvsam3d-slatflow"
151
+ entity: "alphabet1"
152
+ name: "slatflow_resume20k_pslaug_bs12"
153
+ dir: "/lp-dev/jonghoon/mv-mesh/wandb"
154
+
155
+ # --------------------------------------------------------------------------- #
156
+ # val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
157
+ # the trained flow through the exact reported protocol (batch_appforce_sam3d
158
+ # sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
159
+ # OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
160
+ # weights, ALL ranks sample their shard then run the reference-env decode +
161
+ # eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
162
+ # Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
163
+ # tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
164
+ # <stage1_cache>/<name>/2v
165
+ # --------------------------------------------------------------------------- #
166
+ val_appforce:
167
+ enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
168
+ every: 5000
169
+ at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
170
+ views: 2
171
+ schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
172
+ seed: 42
173
+ limit: 0 # debug: cap objects per dataset
174
+ batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
175
+ single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
176
+ decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
177
+ ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
178
+ stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
179
+ out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
180
+ ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
181
+ eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
182
+ datasets:
183
+ - name: toys4k100_tex
184
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
185
+ - name: omni3d
186
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslfinal_2gpu.yaml.bak_predatveiw ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
2
+ # ALL training hyperparameters live here; train_slat_flow.py reads them and
3
+ # nothing training-related is hardcoded in Python. Launch:
4
+ # SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
5
+ # python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
6
+ # (multi-GPU: torchrun --nproc_per_node=N ...)
7
+ # Per-key overrides: --set key.subkey=value (repeatable).
8
+
9
+ # --------------------------------------------------------------------------- #
10
+ # data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
11
+ # Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
12
+ # RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
13
+ # stored views.npz). Production roots get views/rgb + views/bbox.npy +
14
+ # slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
15
+ # --------------------------------------------------------------------------- #
16
+ data:
17
+ roots: # POOL for the 22.5k->30k PSL-FINAL resume (user 2026-09-08):
18
+ # (a) the 50,621 base-pool objects present at the 20k stop
19
+ # (.debug/slat_prod/pool_base_shas.txt), and
20
+ # (b) the MERGED PSL set listed in
21
+ # /data/mv_mesh_data/psl_aug_occ/psl_final_shas.txt
22
+ # ('<set>\t<sha>': occ -> psl_aug_occ/objects, the
23
+ # occlusion-heavy replacement set; aug -> psl_aug/objects,
24
+ # the earlier whole-view set).
25
+ # Symlink-only root built by
26
+ # /lp-dev/jonghoon/mv-sam3d-6d-code/build_pool_base_pslfinal.sh
27
+ # as <root>/{remote,local,pslocc,pslaug}/objects/<sha[:2]>/<sha>.
28
+ # No special weighting: one uniformly-sampled index.
29
+ - /data/mv_mesh_data/slat_train/dataset_pool_base_pslfinal
30
+ # Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
31
+ # val = ALL objects under val_roots, train = roots minus those ids.
32
+ # NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
33
+ # set is toys4k-only until the production rgb/bbox precompute lands.
34
+ val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
35
+ # --- view-subset augmentation (the ONLY train-time stochasticity besides
36
+ # the fresh invisible-row noise in x0) ---
37
+ # smoke data has 4 views/object; production renders 24 (paper §3.1.1).
38
+ # TRELLIS-style per-object voxel FILTER (structured_latent.py
39
+ # filter_metadata): objects with more active 64^3 voxels are DROPPED at
40
+ # index-build time (never downsampled). 20000 = the single-object SLAT
41
+ # OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
42
+ # 0 disables the filter.
43
+ max_num_voxels: 20000
44
+ min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
45
+ max_views: 4 # clamped per object to its available view count
46
+ val_num_views: 2 # deterministic view count for validation samples
47
+ aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
48
+ # --- legacy hash split (used only when val_roots is empty) ---
49
+ val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
50
+ split_seed: 0
51
+ # --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
52
+ # p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
53
+ # p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
54
+ # BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
55
+ p_seed_none: 0.0
56
+ p_seed_single: 0.0
57
+ # --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
58
+ # (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
59
+ # metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
60
+ # GPU decode-compat test before training on RecGen's RELEASED latents. ---
61
+ recgen_roots: []
62
+ # encoded_store recgen root with canonical cube cameras (pose_cube.npz /
63
+ # views.npz) looked up by sha; needed for CORRECT recgen voxel
64
+ # visibility (raw view_metadata alone is geometrically approximate).
65
+ recgen_pose_store: null
66
+
67
+ # --------------------------------------------------------------------------- #
68
+ # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
69
+ # SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
70
+ # with the pipeline's effective inference config (steps=25, cfg_strength=1,
71
+ # cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
72
+ # --------------------------------------------------------------------------- #
73
+ model:
74
+ # Warm-start from the deployed SLAT generator (526/526 tensors, verified).
75
+ pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
76
+ # Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
77
+ # PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
78
+ # 3 toys4k objects), so the bias is ENABLED for training. Set false to
79
+ # reproduce the gating configuration.
80
+ enable_soft_mask: true # local -10 soft-mask in cross-attention
81
+ enable_plucker: true # Plücker ray embedding on non-anchor views
82
+ bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
83
+ # Backbone param dtype: false = lossless fp16->fp32 upcast for training
84
+ # (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
85
+ fp16_torso: false
86
+ # Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
87
+ # .use_checkpoint flip, same toggle the batch-size bench used): recompute
88
+ # activations in backward -> big memory cut, ~modest step-time cost.
89
+ # false = current behavior (no checkpointing).
90
+ use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
91
+ # Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
92
+ # towers inside are always frozen). false freezes the whole fuser so only
93
+ # the backbone + Plücker FC train.
94
+ train_condition_embedder: true
95
+
96
+ # --------------------------------------------------------------------------- #
97
+ # optim + train loop
98
+ # --------------------------------------------------------------------------- #
99
+ optim:
100
+ lr: 3.0e-5
101
+ weight_decay: 0.0 # AdamW
102
+ grad_clip: 1.0 # max grad norm
103
+
104
+ # --------------------------------------------------------------------------- #
105
+ # ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
106
+ # updated after every optimizer step; frozen DINO towers excluded).
107
+ # Disabled by default = existing behavior unchanged. State is saved as a
108
+ # per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
109
+ # and restored automatically on resume_from.
110
+ # --------------------------------------------------------------------------- #
111
+ ema:
112
+ enable: true
113
+ rate: 0.999 # ema = rate*ema + (1-rate)*param
114
+ eval_with_ema: true # swap EMA weights in for the in-loop validation
115
+
116
+ train:
117
+ # Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
118
+ # the effective batch is grad_accum_steps x world_size objects per step.
119
+ steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
120
+ num_workers: 16 # CPU work: crops + projections + Plücker per view
121
+ grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
122
+ batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
123
+ # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
124
+ scheduler: cosine # cosine | constant
125
+ warmup_steps: 500
126
+ min_lr: 1.0e-6
127
+ # --- resume ---
128
+ resume_from: "/data/mv_mesh_data/ckpt/slatflow_resume20k_pslaug_2gpu/step_0022500.pt" # resume the 22.5k pslaug ckpt (+ema/optimizer sidecars); pool swapped to base + merged PSL final set
129
+ log_every: 20
130
+ ckpt_every: 2500
131
+ ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume22p5k_pslfinal_2gpu
132
+ amp: true
133
+ amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
134
+ seed: 0
135
+ # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
136
+ backend: ddp # ddp | deepspeed
137
+ deepspeed_config: ""
138
+ # --- validation: held-out flow v-MSE only (fixed RNG). Decode +
139
+ # faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
140
+ val_every: 5000 # optimizer steps between validation passes (0 = never)
141
+ val_max_batches: 8 # val objects per rank per pass
142
+
143
+ # --------------------------------------------------------------------------- #
144
+ # wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
145
+ # val/* per validation pass). enabled: false makes every wandb call a no-op.
146
+ # Extra metric dicts can be pushed from anywhere via
147
+ # from mvsam3d.train.train_slat_flow import log_metrics
148
+ # log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
149
+ # --------------------------------------------------------------------------- #
150
+ wandb:
151
+ enabled: true
152
+ project: "mvsam3d-slatflow"
153
+ entity: "alphabet1"
154
+ name: "slatflow_resume22p5k_pslfinal_bs12"
155
+ dir: "/lp-dev/jonghoon/mv-mesh/wandb"
156
+
157
+ # --------------------------------------------------------------------------- #
158
+ # val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
159
+ # the trained flow through the exact reported protocol (batch_appforce_sam3d
160
+ # sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
161
+ # OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
162
+ # weights, ALL ranks sample their shard then run the reference-env decode +
163
+ # eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
164
+ # Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
165
+ # tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
166
+ # <stage1_cache>/<name>/2v
167
+ # --------------------------------------------------------------------------- #
168
+ val_appforce:
169
+ enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
170
+ every: 5000
171
+ at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
172
+ views: 2
173
+ schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
174
+ seed: 42
175
+ limit: 0 # debug: cap objects per dataset
176
+ batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
177
+ single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
178
+ decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
179
+ ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
180
+ stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
181
+ out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
182
+ ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
183
+ eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
184
+ datasets:
185
+ - name: toys4k100_tex
186
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
187
+ - name: omni3d
188
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslocc_2gpu.yaml.bak_predatveiw ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
2
+ # ALL training hyperparameters live here; train_slat_flow.py reads them and
3
+ # nothing training-related is hardcoded in Python. Launch:
4
+ # SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
5
+ # python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
6
+ # (multi-GPU: torchrun --nproc_per_node=N ...)
7
+ # Per-key overrides: --set key.subkey=value (repeatable).
8
+
9
+ # --------------------------------------------------------------------------- #
10
+ # data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
11
+ # Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
12
+ # RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
13
+ # stored views.npz). Production roots get views/rgb + views/bbox.npy +
14
+ # slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
15
+ # --------------------------------------------------------------------------- #
16
+ data:
17
+ roots: # POOL for the 22.5k->30k PSL-OCCLUSION resume (user 2026-09-08):
18
+ # (a) the 50,621 base-pool objects present at the 20k stop
19
+ # (.debug/slat_prod/pool_base_shas.txt), and
20
+ # (b) ALL objects of the REPLACEMENT occlusion set
21
+ # /data/mv_mesh_data/psl_aug_occ/objects (3-4 whole +
22
+ # 4-5 occluded views per sample; psl_aug_occ_shas.txt).
23
+ # The earlier /data/mv_mesh_data/psl_aug set is NOT included.
24
+ # Symlink-only root built by
25
+ # /lp-dev/jonghoon/mv-sam3d-6d-code/build_pool_base_pslocc.sh
26
+ # as <root>/{remote,local,pslocc}/objects/<sha[:2]>/<sha>.
27
+ # No special weighting: one uniformly-sampled index.
28
+ - /data/mv_mesh_data/slat_train/dataset_pool_base_pslocc
29
+ # Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
30
+ # val = ALL objects under val_roots, train = roots minus those ids.
31
+ # NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
32
+ # set is toys4k-only until the production rgb/bbox precompute lands.
33
+ val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
34
+ # --- view-subset augmentation (the ONLY train-time stochasticity besides
35
+ # the fresh invisible-row noise in x0) ---
36
+ # smoke data has 4 views/object; production renders 24 (paper §3.1.1).
37
+ # TRELLIS-style per-object voxel FILTER (structured_latent.py
38
+ # filter_metadata): objects with more active 64^3 voxels are DROPPED at
39
+ # index-build time (never downsampled). 20000 = the single-object SLAT
40
+ # OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
41
+ # 0 disables the filter.
42
+ max_num_voxels: 20000
43
+ min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
44
+ max_views: 4 # clamped per object to its available view count
45
+ val_num_views: 2 # deterministic view count for validation samples
46
+ aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
47
+ # --- legacy hash split (used only when val_roots is empty) ---
48
+ val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
49
+ split_seed: 0
50
+ # --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
51
+ # p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
52
+ # p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
53
+ # BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
54
+ p_seed_none: 0.0
55
+ p_seed_single: 0.0
56
+ # --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
57
+ # (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
58
+ # metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
59
+ # GPU decode-compat test before training on RecGen's RELEASED latents. ---
60
+ recgen_roots: []
61
+ # encoded_store recgen root with canonical cube cameras (pose_cube.npz /
62
+ # views.npz) looked up by sha; needed for CORRECT recgen voxel
63
+ # visibility (raw view_metadata alone is geometrically approximate).
64
+ recgen_pose_store: null
65
+
66
+ # --------------------------------------------------------------------------- #
67
+ # model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
68
+ # SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
69
+ # with the pipeline's effective inference config (steps=25, cfg_strength=1,
70
+ # cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
71
+ # --------------------------------------------------------------------------- #
72
+ model:
73
+ # Warm-start from the deployed SLAT generator (526/526 tensors, verified).
74
+ pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
75
+ # Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
76
+ # PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
77
+ # 3 toys4k objects), so the bias is ENABLED for training. Set false to
78
+ # reproduce the gating configuration.
79
+ enable_soft_mask: true # local -10 soft-mask in cross-attention
80
+ enable_plucker: true # Plücker ray embedding on non-anchor views
81
+ bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
82
+ # Backbone param dtype: false = lossless fp16->fp32 upcast for training
83
+ # (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
84
+ fp16_torso: false
85
+ # Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
86
+ # .use_checkpoint flip, same toggle the batch-size bench used): recompute
87
+ # activations in backward -> big memory cut, ~modest step-time cost.
88
+ # false = current behavior (no checkpointing).
89
+ use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
90
+ # Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
91
+ # towers inside are always frozen). false freezes the whole fuser so only
92
+ # the backbone + Plücker FC train.
93
+ train_condition_embedder: true
94
+
95
+ # --------------------------------------------------------------------------- #
96
+ # optim + train loop
97
+ # --------------------------------------------------------------------------- #
98
+ optim:
99
+ lr: 3.0e-5
100
+ weight_decay: 0.0 # AdamW
101
+ grad_clip: 1.0 # max grad norm
102
+
103
+ # --------------------------------------------------------------------------- #
104
+ # ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
105
+ # updated after every optimizer step; frozen DINO towers excluded).
106
+ # Disabled by default = existing behavior unchanged. State is saved as a
107
+ # per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
108
+ # and restored automatically on resume_from.
109
+ # --------------------------------------------------------------------------- #
110
+ ema:
111
+ enable: true
112
+ rate: 0.999 # ema = rate*ema + (1-rate)*param
113
+ eval_with_ema: true # swap EMA weights in for the in-loop validation
114
+
115
+ train:
116
+ # Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
117
+ # the effective batch is grad_accum_steps x world_size objects per step.
118
+ steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
119
+ num_workers: 16 # CPU work: crops + projections + Plücker per view
120
+ grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
121
+ batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
122
+ # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
123
+ scheduler: cosine # cosine | constant
124
+ warmup_steps: 500
125
+ min_lr: 1.0e-6
126
+ # --- resume ---
127
+ resume_from: "/data/mv_mesh_data/ckpt/slatflow_resume20k_pslaug_2gpu/step_0022500.pt" # resume the 22.5k pslaug ckpt (+ema/optimizer sidecars); pool swapped to base+psl_aug_occ
128
+ log_every: 20
129
+ ckpt_every: 2500
130
+ ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume22p5k_pslocc_2gpu
131
+ amp: true
132
+ amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
133
+ seed: 0
134
+ # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
135
+ backend: ddp # ddp | deepspeed
136
+ deepspeed_config: ""
137
+ # --- validation: held-out flow v-MSE only (fixed RNG). Decode +
138
+ # faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
139
+ val_every: 5000 # optimizer steps between validation passes (0 = never)
140
+ val_max_batches: 8 # val objects per rank per pass
141
+
142
+ # --------------------------------------------------------------------------- #
143
+ # wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
144
+ # val/* per validation pass). enabled: false makes every wandb call a no-op.
145
+ # Extra metric dicts can be pushed from anywhere via
146
+ # from mvsam3d.train.train_slat_flow import log_metrics
147
+ # log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
148
+ # --------------------------------------------------------------------------- #
149
+ wandb:
150
+ enabled: true
151
+ project: "mvsam3d-slatflow"
152
+ entity: "alphabet1"
153
+ name: "slatflow_resume22p5k_pslocc_bs12"
154
+ dir: "/lp-dev/jonghoon/mv-mesh/wandb"
155
+
156
+ # --------------------------------------------------------------------------- #
157
+ # val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
158
+ # the trained flow through the exact reported protocol (batch_appforce_sam3d
159
+ # sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
160
+ # OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
161
+ # weights, ALL ranks sample their shard then run the reference-env decode +
162
+ # eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
163
+ # Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
164
+ # tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
165
+ # <stage1_cache>/<name>/2v
166
+ # --------------------------------------------------------------------------- #
167
+ val_appforce:
168
+ enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
169
+ every: 5000
170
+ at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
171
+ views: 2
172
+ schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
173
+ seed: 42
174
+ limit: 0 # debug: cap objects per dataset
175
+ batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
176
+ single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
177
+ decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
178
+ ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
179
+ stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
180
+ out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
181
+ ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
182
+ eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
183
+ datasets:
184
+ - name: toys4k100_tex
185
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
186
+ - name: omni3d
187
+ exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_003.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14502f014a45dee714dbd919e394b419a72e95419897303bc7386eed8e0a5508
3
+ size 20996178
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_008.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92e4ff59ac962b00dc5a5975913c5e952ece70ab3a640d25c0727bd68265ae08
3
+ size 20511442
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/chair_026.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdf5dc60ce5b61d76bb78762bd5895cdb21ff3a49252f63d9ef8007c73ed3ffd
3
+ size 20690002
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ours_both.log ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
2
+ import pynvml # type: ignore[import]
3
+ 2026-09-01 01:49:09.196 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
4
+ 2026-09-01 01:49:12.938 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
5
+ [SPARSE][CONV] spconv algo: native
6
+ [slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
7
+ 2026-09-01 01:49:25.245 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
8
+ 2026-09-01 01:49:29.957 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
9
+ 2026-09-01 01:49:29.961 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
10
+ 2026-09-01 01:49:33.784 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
11
+ [slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
12
+ [slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
13
+ 2026-09-01 01:49:44.041 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
14
+ 2026-09-01 01:49:47.428 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
15
+ 2026-09-01 01:49:47.431 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
16
+ 2026-09-01 01:49:51.389 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
17
+ [slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
18
+ [ours] bunny_003 (N=7984, ref steps=25 strength=1.0 rescale_t=1.0 | deployed self-noise floor=0.000e+00 tol=5.000e-02)
19
+ [raw]
20
+ cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
21
+ slat_feats_noise WITHIN-FLOOR max|d|=6.896e-03 (tol 5.000e-02)
22
+ slat_feats_forced WITHIN-FLOOR max|d|=9.228e-03 (tol 5.000e-02)
23
+ [wrapped/zero-bias]
24
+ cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
25
+ slat_feats_noise WITHIN-FLOOR max|d|=6.896e-03 (tol 5.000e-02)
26
+ wrapped==raw (noise) BIT-IDENTICAL shape=(7984, 8) dtype=torch.float32 max|d|=0.000e+00
27
+ slat_feats_forced WITHIN-FLOOR max|d|=9.228e-03 (tol 5.000e-02)
28
+ wrapped==raw (forced) BIT-IDENTICAL shape=(7984, 8) dtype=torch.float32 max|d|=0.000e+00
29
+ [ours] bunny_008 (N=4619, ref steps=25 strength=1.0 rescale_t=1.0 | deployed self-noise floor=0.000e+00 tol=5.000e-02)
30
+ [raw]
31
+ cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
32
+ slat_feats_noise WITHIN-FLOOR max|d|=3.383e-03 (tol 5.000e-02)
33
+ slat_feats_forced WITHIN-FLOOR max|d|=4.269e-03 (tol 5.000e-02)
34
+ [wrapped/zero-bias]
35
+ cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
36
+ slat_feats_noise WITHIN-FLOOR max|d|=3.383e-03 (tol 5.000e-02)
37
+ wrapped==raw (noise) BIT-IDENTICAL shape=(4619, 8) dtype=torch.float32 max|d|=0.000e+00
38
+ slat_feats_forced WITHIN-FLOOR max|d|=4.269e-03 (tol 5.000e-02)
39
+ wrapped==raw (forced) BIT-IDENTICAL shape=(4619, 8) dtype=torch.float32 max|d|=0.000e+00
40
+ [ours] chair_026 (N=5858, ref steps=25 strength=1.0 rescale_t=1.0 | deployed self-noise floor=0.000e+00 tol=5.000e-02)
41
+ [raw]
42
+ cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
43
+ slat_feats_noise WITHIN-FLOOR max|d|=5.830e-03 (tol 5.000e-02)
44
+ slat_feats_forced WITHIN-FLOOR max|d|=1.043e-02 (tol 5.000e-02)
45
+ [wrapped/zero-bias]
46
+ cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
47
+ slat_feats_noise WITHIN-FLOOR max|d|=5.830e-03 (tol 5.000e-02)
48
+ wrapped==raw (noise) BIT-IDENTICAL shape=(5858, 8) dtype=torch.float32 max|d|=0.000e+00
49
+ slat_feats_forced WITHIN-FLOOR max|d|=1.043e-02 (tol 5.000e-02)
50
+ wrapped==raw (forced) BIT-IDENTICAL shape=(5858, 8) dtype=torch.float32 max|d|=0.000e+00
51
+ [ours] PARITY PASS
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe2_ref.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9518028c75f9bb983889d290b85f83e15fc8435e2072127aab57e4c7d1a4f53
3
+ size 10685283
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe_ref.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:033cd057451f09ac3883647d92a9685d10ca7c0d81f87dc9f1ce94724caf402d
3
+ size 43640209
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ref.log ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
2
+ import pynvml # type: ignore[import]
3
+ 2026-08-31 18:47:31.995 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
4
+ 2026-08-31 18:47:38.280 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
5
+ [SPARSE][CONV] spconv algo: native
6
+ Warp 1.12.1 initialized:
7
+ CUDA Toolkit 12.9, Driver 13.0
8
+ Devices:
9
+ "cpu" : "x86_64"
10
+ "cuda:0" : "NVIDIA A100-SXM4-80GB" (79 GiB, sm_80, mempool enabled)
11
+ Kernel cache:
12
+ /home/nvidia/.cache/warp/1.12.1
13
+ 2026-08-31 18:47:49.271 | INFO | sam3d_objects.pipeline.inference_pipeline:set_attention_backend:15 - GPU name is NVIDIA A100-SXM4-80GB
14
+ 2026-08-31 18:47:53.396 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
15
+ 2026-08-31 18:47:53.397 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
16
+ [ref] building deployed pipeline ...
17
+ 2026-08-31 18:47:53.453 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
18
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/moge/model/v1.py:172: UserWarning: The following deprecated/invalid arguments are ignored: {'output_mask': True, 'split_head': True}
19
+ warnings.warn(f"The following deprecated/invalid arguments are ignored: {deprecated_kwargs}")
20
+ 2026-08-31 18:48:02.212 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
21
+ 2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:98 - self.device: cuda
22
+ 2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:99 - CUDA_VISIBLE_DEVICES: 1
23
+ 2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:100 - Actually using GPU: 0
24
+ 2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:init_pose_decoder:295 - Using pose decoder: ScaleShiftInvariant
25
+ 2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:131 - Loading model weights...
26
+ 2026-08-31 18:48:02.515 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
27
+ 2026-08-31 18:48:13.901 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
28
+ 2026-08-31 18:48:19.400 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_decoder.ckpt
29
+ 2026-08-31 18:48:19.979 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs.ckpt
30
+ 2026-08-31 18:48:20.456 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs_4.ckpt
31
+ 2026-08-31 18:48:21.159 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_mesh.ckpt
32
+ 2026-08-31 18:48:21.987 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
33
+ 2026-08-31 18:48:23.321 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
34
+ 2026-08-31 18:48:23.330 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
35
+ 2026-08-31 18:48:24.723 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
36
+ 2026-08-31 18:48:24.776 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
37
+ 2026-08-31 18:48:31.503 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
38
+ 2026-08-31 18:48:32.940 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
39
+ 2026-08-31 18:48:32.949 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
40
+ 2026-08-31 18:48:34.490 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
41
+ 2026-08-31 18:48:34.519 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
42
+ 2026-08-31 18:48:40.669 | INFO | sam3d_objects.pipeline.inference_pipeline:override_ss_generator_cfg_config:520 - ss_generator parameters: inference_steps=25, cfg_strength=7, cfg_interval=[0, 500], rescale_t=3, cfg_strength_pm=0.0
43
+ 2026-08-31 18:48:40.669 | INFO | sam3d_objects.pipeline.inference_pipeline:override_slat_generator_cfg_config:542 - slat_generator parameters: inference_steps=25, cfg_strength=1, cfg_interval=[0, 500], rescale_t=1
44
+ 2026-08-31 18:48:40.669 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:196 - Loading model weights completed!
45
+ [ref] pipeline ready: slat steps=25 strength=1 interval=[0, 500] rescale_t=1
46
+ 2026-08-31 18:48:42.723 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
47
+ 2026-08-31 18:48:43.063 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
48
+ 2026-08-31 18:48:43.065 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
49
+ 2026-08-31 18:48:43.065 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
50
+ 2026-08-31 18:48:43.246 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
51
+ 2026-08-31 18:48:51.925 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
52
+ 2026-08-31 18:48:51.925 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
53
+ 2026-08-31 18:48:52.142 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
54
+ [ref] chair_026: N=5858 cond=(1, 5496, 1024)/torch.float16 feats=(5858, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen/chair_026.pt
55
+ 2026-08-31 18:49:00.060 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
56
+ 2026-08-31 18:49:00.226 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
57
+ 2026-08-31 18:49:00.227 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
58
+ 2026-08-31 18:49:00.227 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
59
+ 2026-08-31 18:49:00.389 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
60
+ 2026-08-31 18:49:09.140 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
61
+ 2026-08-31 18:49:09.140 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
62
+ 2026-08-31 18:49:09.316 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
63
+ [ref] bunny_003: N=7984 cond=(1, 5496, 1024)/torch.float16 feats=(7984, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen/bunny_003.pt
64
+ 2026-08-31 18:49:18.333 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
65
+ 2026-08-31 18:49:18.496 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
66
+ 2026-08-31 18:49:18.497 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
67
+ 2026-08-31 18:49:18.497 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
68
+ 2026-08-31 18:49:18.660 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
69
+ 2026-08-31 18:49:26.196 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
70
+ 2026-08-31 18:49:26.196 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
71
+ 2026-08-31 18:49:26.401 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
72
+ [ref] bunny_008: N=4619 cond=(1, 5496, 1024)/torch.float16 feats=(4619, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen/bunny_008.pt
73
+ [ref] DONE
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_003.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14502f014a45dee714dbd919e394b419a72e95419897303bc7386eed8e0a5508
3
+ size 20996178
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_008.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92e4ff59ac962b00dc5a5975913c5e952ece70ab3a640d25c0727bd68265ae08
3
+ size 20511442
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/chair_026.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdf5dc60ce5b61d76bb78762bd5895cdb21ff3a49252f63d9ef8007c73ed3ffd
3
+ size 20690002
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/ref.log ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
2
+ import pynvml # type: ignore[import]
3
+ 2026-08-31 19:04:42.766 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
4
+ 2026-08-31 19:04:47.903 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
5
+ [SPARSE][CONV] spconv algo: native
6
+ Warp 1.12.1 initialized:
7
+ CUDA Toolkit 12.9, Driver 13.0
8
+ Devices:
9
+ "cpu" : "x86_64"
10
+ "cuda:0" : "NVIDIA A100-SXM4-80GB" (79 GiB, sm_80, mempool enabled)
11
+ Kernel cache:
12
+ /home/nvidia/.cache/warp/1.12.1
13
+ 2026-08-31 19:04:56.352 | INFO | sam3d_objects.pipeline.inference_pipeline:set_attention_backend:15 - GPU name is NVIDIA A100-SXM4-80GB
14
+ 2026-08-31 19:04:58.555 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
15
+ 2026-08-31 19:04:58.555 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
16
+ [ref] building deployed pipeline ...
17
+ 2026-08-31 19:04:58.594 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
18
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/moge/model/v1.py:172: UserWarning: The following deprecated/invalid arguments are ignored: {'output_mask': True, 'split_head': True}
19
+ warnings.warn(f"The following deprecated/invalid arguments are ignored: {deprecated_kwargs}")
20
+ 2026-08-31 19:05:08.207 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
21
+ 2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:98 - self.device: cuda
22
+ 2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:99 - CUDA_VISIBLE_DEVICES: 1
23
+ 2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:100 - Actually using GPU: 0
24
+ 2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:init_pose_decoder:295 - Using pose decoder: ScaleShiftInvariant
25
+ 2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:131 - Loading model weights...
26
+ 2026-08-31 19:05:08.653 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
27
+ 2026-08-31 19:05:20.732 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
28
+ 2026-08-31 19:05:25.889 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_decoder.ckpt
29
+ 2026-08-31 19:05:26.449 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs.ckpt
30
+ 2026-08-31 19:05:26.881 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs_4.ckpt
31
+ 2026-08-31 19:05:27.467 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_mesh.ckpt
32
+ 2026-08-31 19:05:28.226 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
33
+ 2026-08-31 19:05:30.056 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
34
+ 2026-08-31 19:05:30.065 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
35
+ 2026-08-31 19:05:31.477 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
36
+ 2026-08-31 19:05:31.522 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
37
+ 2026-08-31 19:05:39.237 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
38
+ 2026-08-31 19:05:40.567 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
39
+ 2026-08-31 19:05:40.576 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
40
+ 2026-08-31 19:05:41.857 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
41
+ 2026-08-31 19:05:41.882 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
42
+ 2026-08-31 19:05:48.113 | INFO | sam3d_objects.pipeline.inference_pipeline:override_ss_generator_cfg_config:520 - ss_generator parameters: inference_steps=25, cfg_strength=7, cfg_interval=[0, 500], rescale_t=3, cfg_strength_pm=0.0
43
+ 2026-08-31 19:05:48.114 | INFO | sam3d_objects.pipeline.inference_pipeline:override_slat_generator_cfg_config:542 - slat_generator parameters: inference_steps=25, cfg_strength=1, cfg_interval=[0, 500], rescale_t=1
44
+ 2026-08-31 19:05:48.114 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:196 - Loading model weights completed!
45
+ [ref] pipeline ready: slat steps=25 strength=1 interval=[0, 500] rescale_t=1
46
+ 2026-08-31 19:05:48.971 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
47
+ 2026-08-31 19:05:49.265 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
48
+ 2026-08-31 19:05:49.267 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
49
+ 2026-08-31 19:05:49.267 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
50
+ 2026-08-31 19:05:49.431 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
51
+ 2026-08-31 19:05:57.635 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
52
+ 2026-08-31 19:05:57.635 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
53
+ 2026-08-31 19:05:57.803 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
54
+ [ref] chair_026: N=5858 cond=(1, 5496, 1024)/torch.float16 feats=(5858, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen_b/chair_026.pt
55
+ 2026-08-31 19:06:05.500 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
56
+ 2026-08-31 19:06:05.675 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
57
+ 2026-08-31 19:06:05.676 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
58
+ 2026-08-31 19:06:05.676 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
59
+ 2026-08-31 19:06:05.827 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
60
+ 2026-08-31 19:06:14.623 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
61
+ 2026-08-31 19:06:14.623 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
62
+ 2026-08-31 19:06:14.843 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
63
+ [ref] bunny_003: N=7984 cond=(1, 5496, 1024)/torch.float16 feats=(7984, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen_b/bunny_003.pt
64
+ 2026-08-31 19:06:22.977 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
65
+ 2026-08-31 19:06:23.201 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
66
+ 2026-08-31 19:06:23.202 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
67
+ 2026-08-31 19:06:23.202 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
68
+ 2026-08-31 19:06:23.378 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
69
+ 2026-08-31 19:06:30.672 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
70
+ 2026-08-31 19:06:30.672 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
71
+ 2026-08-31 19:06:30.837 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
72
+ [ref] bunny_008: N=4619 cond=(1, 5496, 1024)/torch.float16 feats=(4619, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen_b/bunny_008.pt
73
+ [ref] DONE
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/smoke_slatflow.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
2
+ import pynvml # type: ignore[import]
3
+ 2026-09-01 01:55:05.629 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
4
+ 2026-09-01 01:55:09.659 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
5
+ 2026-09-01 01:55:15.109 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
6
+ [SPARSE][CONV] spconv algo: native
7
+ [smoke] object=chair_026 N=5858 n_subset=2 views=[0, 1, 2, 3] crops=(4, 3, 518, 518) uv=(4, 5858, 2) vis=(4, 5858) plucker=(1, 1369, 6) bias_views=1
8
+ [slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
9
+ 2026-09-01 01:55:25.930 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
10
+ 2026-09-01 01:55:31.714 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
11
+ 2026-09-01 01:55:31.718 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
12
+ 2026-09-01 01:55:36.823 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
13
+ [slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
14
+ [smoke] model built: trainable=617.7M bias_enabled=True
15
+ [smoke] flow_step: loss=0.7099 frac_visible_x0=0.510 frac_target=0.702
16
+ [smoke] grad norms: backbone=6.378e+01 fuser=2.272e+00 plucker_fc=3.835e-02
17
+ [smoke] bias: shape=(1394, 6865) anchor_max|.|=0.0 nonanchor_vals=[-10.0, 0.0] frac_open=0.0066
18
+ [smoke] sample: cond=(1, 6865, 1024) base=(1, 5858, 8) slat feats=(5858, 8) finite=True std=4.557
19
+ [smoke] SELF-CHECK PASS
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/train_slatflow_smoke.log ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
2
+ import pynvml # type: ignore[import]
3
+ 2026-09-01 01:56:18.594 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
4
+ 2026-09-01 01:56:22.431 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
5
+ [SPARSE][CONV] spconv algo: native
6
+ [train] train samples: 201
7
+ [train] val samples: 100
8
+ [slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
9
+ 2026-09-01 01:56:35.604 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
10
+ 2026-09-01 01:56:40.418 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
11
+ 2026-09-01 01:56:40.422 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
12
+ 2026-09-01 01:56:45.101 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
13
+ [slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
14
+ [setup] trainable params: 617.7M / 1226.5M (enable_bias=True, fp16_torso=False)
15
+ /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/mvsam3d/train/train_slat_flow.py:277: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
16
+ scaler = torch.cuda.amp.GradScaler(enabled=use_scaler)
17
+ [setup] backend=ddp world=1 global_batch=2 (objects/optim-step) steps=2 grad_accum=2 amp=True/bf16 val_every=2
18
+ 2026-09-01 01:56:54.846 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
19
+ 2026-09-01 01:56:54.854 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
20
+ 2026-09-01 01:56:54.893 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
21
+ 2026-09-01 01:56:54.945 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
22
+ [train] step 1 lr=4.000000e-07 frac_target=0.7416 frac_visible_x0=0.6616 loss=0.9653 loss/slat=0.9653 (0.14 it/s)
23
+ [train] step 2 lr=6.000000e-07 frac_target=0.7212 frac_visible_x0=0.4989 loss=1.9295 loss/slat=1.9295 (0.48 it/s)
24
+ 2026-09-01 01:57:04.080 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
25
+ 2026-09-01 01:57:04.091 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
26
+ [val] step 2 frac_target=0.5975 frac_visible_x0=0.4419 loss=0.9582 loss/slat=0.9582 (n=2)
27
+ [train] saved /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/cache/ckpt_slatflow/step_0000002.pt
28
+ [train] done.
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-310.pyc ADDED
Binary file (673 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (750 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-39.pyc ADDED
Binary file (671 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-310.pyc ADDED
Binary file (4.09 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-311.pyc ADDED
Binary file (6.59 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-39.pyc ADDED
Binary file (4.07 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-310.pyc ADDED
Binary file (184 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (200 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-39.pyc ADDED
Binary file (182 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/adapters.cpython-311.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:859d14326540e24088b672bc15940079ab435832247d5921114087cdbcb3a342
3
+ size 118636
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/base_dataset.cpython-311.pyc ADDED
Binary file (16.8 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/canonical.cpython-311.pyc ADDED
Binary file (3.42 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_calib.cpython-311.pyc ADDED
Binary file (5.06 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_dataset.cpython-311.pyc ADDED
Binary file (35.6 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/integrated.cpython-311.pyc ADDED
Binary file (8.68 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slat_preprocess.cpython-311.pyc ADDED
Binary file (4.44 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-310.pyc ADDED
Binary file (46.7 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-311.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:920741be4f6ca85689c3629a6d65f832b0a060dbc701db71a0a7bcfd131e179e
3
+ size 126300
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-310.pyc ADDED
Binary file (17.9 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-311.pyc ADDED
Binary file (52.9 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_preprocess.cpython-311.pyc ADDED
Binary file (12 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-310.pyc ADDED
Binary file (15.9 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-311.pyc ADDED
Binary file (36.9 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-39.pyc ADDED
Binary file (9.78 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/v_bucket_sampler.cpython-311.pyc ADDED
Binary file (6.04 kB). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py CHANGED
@@ -903,6 +903,53 @@ class TrellisSlatFlowDataset(TrellisSSFlowDataset):
903
  }
904
 
905
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
906
  DEFAULT_TAR_ROOTS = ("/data/mv_mesh_data/slat_tars/processed",)
907
  DEFAULT_TAR_MANIFESTS = "/data/mv_mesh_data/DATASET_INDEX/hubman/processed/manifests"
908
 
@@ -960,30 +1007,33 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
960
  shards: Optional[Sequence[str]] = None,
961
  max_objects: Optional[int] = None,
962
  use_recgen_images: bool = False,
963
- p_recgen_image: float = 0.0,
964
  cond_cameras_fallback: bool = False):
965
- # ---- condition-variant knobs (2026-09-24; all OFF by default -> the
966
- # index, the draws and every item are byte-identical to the legacy path)
 
967
  # use_recgen_images : also index ``recgen/<id>/{cameras.json,NN.jpg,
968
  # NN_mask.png}`` members (RecGen's own background-baked RGB + its
969
  # visible mask + per-view RIGID OpenCV c2w in the SLAT cube) from ANY
970
- # tar under tar_roots, joined to ``latents/<id>.npz`` by id. Objects
971
- # with a latent + RecGen images but NO white cond (latent-only new
972
- # slats) become trainable instead of being silently skipped.
973
- # p_recgen_image : when an object has BOTH sources, P(use RecGen) per
974
- # (sample, epoch) on its own rng stream (offset 9191). Objects with
975
- # only one source always use it.
976
- # RecGen variant = the SAME default path with the RecGen full-frame
977
- # image (background kept, alpha = RecGen mask) substituted for the
978
- # white cond render in the preprocess_slat_image slot (item["image"]);
 
 
 
 
979
  # DINO seed crops / uv / vis use the mask-matted tight crop exactly
980
- # like the white path, cameras = RecGen per-view c2w.
981
  # cond_cameras_fallback : white-cond objects lacking
982
  # ``cameras/<id>/cond_cameras.json`` (the 24-view slat5/slat5rs
983
  # batches) use ``cond/<id>/transforms.json`` (same transform_matrix;
984
  # fx=fy=W/(2 tan(camera_angle_x/2)), cx=cy=W/2 from the PNG width).
985
  self.use_recgen_images = bool(use_recgen_images)
986
- self.p_recgen_image = float(p_recgen_image)
987
  self.cond_cameras_fallback = bool(cond_cameras_fallback)
988
  # NOTE: deliberately does NOT call TrellisSlatFlowDataset.__init__ (that
989
  # scans object DIRS). We set the attributes its shared methods
@@ -1062,7 +1112,7 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
1062
  # new index semantics -> distinct cache signature (legacy sig
1063
  # unchanged when both knobs are off, so old caches stay valid)
1064
  parts.append(f"rg={int(self.use_recgen_images)}"
1065
- f":fb={int(self.cond_cameras_fallback)}:v1")
1066
  return _hashlib.sha1("|".join(parts).encode()).hexdigest()
1067
  _sig = _tar_sig(tar_files)
1068
  _cache = os.environ.get("TAR_INDEX_CACHE") or os.path.join(
@@ -1290,9 +1340,27 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
1290
  views_all.append(dict(depth=None, fx=float(K[0, 0]), fy=float(K[1, 1]),
1291
  cx=float(K[0, 2]), cy=float(K[1, 2]),
1292
  c2w=np.asarray(c["c2w"], dtype=np.float64),
1293
- res=int(max(int(c["width"]), int(c["height"])))))
 
1294
  return views_all, len(views_all)
1295
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1296
  def _load_view_rgba(self, m: Dict, src: str, i: int) -> np.ndarray:
1297
  """FULL-FRAME RGBA uint8 of view i. white: the cond render PNG. recgen:
1298
  RecGen's RGB (background kept) + its visible mask as alpha (>127)."""
@@ -1307,18 +1375,6 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
1307
  return np.concatenate([rgb, a], axis=2)
1308
  return self._load_png_rgba(self._get_tar(m["tar"]), m["cond_pngs"][i])
1309
 
1310
- def _cond_source(self, idx: int, m: Dict) -> str:
1311
- """'white' | 'recgen' for this (sample, epoch). Own rng stream (9191)."""
1312
- has_w = bool(m.get("cond_pngs"))
1313
- has_r = m.get("rg") is not None
1314
- if has_r and not has_w:
1315
- return "recgen"
1316
- if has_r and has_w and self.p_recgen_image > 0.0:
1317
- r = float(np.random.default_rng(
1318
- [self.seed, int(self._epoch), int(idx), 9191]).random())
1319
- return "recgen" if r < self.p_recgen_image else "white"
1320
- return "white"
1321
-
1322
  # ---- OPT-IN bad-view gate (tar variant, keyed by sha) -------------- #
1323
  def _tar_valid_views(self, sha: str, tar, views_all, sc, centers, V,
1324
  src: str = "white") -> List[int]:
@@ -1381,11 +1437,17 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
1381
  tar = self._get_tar(m["tar"])
1382
  rng = np.random.default_rng([self.seed, self._epoch, idx])
1383
 
1384
- src = self._cond_source(idx, m) # 'white' unless recgen knobs on
1385
- if src == "recgen":
1386
- views_all, V = self._rg_cameras(m)
1387
- else:
1388
- views_all, V = self._tar_cameras(tar, m)
 
 
 
 
 
 
1389
 
1390
  # x1 = released SLAT: coords 0..63 used DIRECTLY, feats row-aligned.
1391
  zl = np.load(io.BytesIO(tar.extractfile(m["latent"]).read()))
@@ -1398,15 +1460,64 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
1398
  torch.from_numpy(sc).int()], dim=1) # (N,4) int32
1399
  centers = (sc.astype(np.float64) + 0.5) / VOX - 0.5
1400
 
1401
- n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
1402
- if self.filter_bad_views: # OPT-IN: draw only from valid
1403
- valid_arr = np.asarray(
1404
- self._tar_valid_views(sha, tar, views_all, sc, centers, V, src),
1405
- dtype=int)
1406
- n_sub = min(int(n_sub), len(valid_arr))
1407
- sub = rng.choice(valid_arr, size=n_sub, replace=False) # anchor in valid
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1408
  else:
1409
- sub = rng.choice(V, size=n_sub, replace=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1410
  order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
1411
 
1412
  crops, uvs, viss, fulls = [], [], [], []
 
903
  }
904
 
905
 
906
+ # ---- RecGen camera QC (2026-09-24) ------------------------------------------ #
907
+ # A RecGen view's camera is VALID iff the object's OWN SLAT voxels, projected with
908
+ # that view's camera, land inside RecGen's (modal) mask in proportion to how much
909
+ # of the object RecGen says is visible:
910
+ # p = #voxels projecting inside the mask / #voxels (all voxels)
911
+ # vf = RecGen view_metadata visible_fraction (modal / amodal)
912
+ # valid <=> vf >= RG_VF_MIN and p >= RG_RATIO * vf
913
+ # With a correct pose p ~= vf (occluders remove voxels from the modal mask exactly
914
+ # as they remove visible area). Calibrated on 4,000 random RecGen sets against our
915
+ # latents: keeps 99.0% of views of trusted analytic poses (p/vf p01 = 0.72), 0% of
916
+ # the analytic poses that failed their own seed test. Object guard: if fewer than
917
+ # RG_OBJ_MIN_FRAC of the object's verifiable (vf >= RG_VF_MIN) views pass, the pose
918
+ # is treated as systematically wrong and ALL its RecGen views are dropped (the
919
+ # object falls back to white-only, or is skipped if it has no white cond).
920
+ RG_VF_MIN = 0.10
921
+ RG_RATIO = 0.60
922
+ RG_OBJ_MIN_FRAC = 0.50
923
+
924
+
925
+ def recgen_view_qc(coords3: np.ndarray, views_all: List[Dict], masks: List[np.ndarray],
926
+ vfs: Sequence[float], vf_min: float = RG_VF_MIN,
927
+ ratio: float = RG_RATIO, obj_min_frac: float = RG_OBJ_MIN_FRAC):
928
+ """-> (valid view indices, per-view p). coords3 (N,3) int voxel coords of the
929
+ object's SLAT latent; views_all[i] has fx,fy,cx,cy,c2w (rigid, SLAT cube);
930
+ masks[i] (H,W) bool; vfs[i] RecGen visible_fraction."""
931
+ pts = (np.asarray(coords3, dtype=np.float64) + 0.5) / VOX - 0.5
932
+ N = max(len(pts), 1)
933
+ ps, ok = [], []
934
+ for v, m, vf in zip(views_all, masks, vfs):
935
+ w2c = np.linalg.inv(v["c2w"])
936
+ xc = pts @ w2c[:3, :3].T + w2c[:3, 3]
937
+ z = xc[:, 2]
938
+ zs = np.where(z == 0, 1e-12, z)
939
+ u = np.round(v["fx"] * xc[:, 0] / zs + v["cx"]).astype(np.int64)
940
+ q = np.round(v["fy"] * xc[:, 1] / zs + v["cy"]).astype(np.int64)
941
+ H, W = m.shape[:2]
942
+ inf = (z > 0) & (u >= 0) & (u < W) & (q >= 0) & (q < H)
943
+ p = float(m[q[inf], u[inf]].sum()) / N
944
+ ps.append(p)
945
+ ok.append(bool(vf >= vf_min and p >= ratio * vf))
946
+ verifiable = sum(1 for vf in vfs if vf >= vf_min)
947
+ valid = [i for i, o in enumerate(ok) if o]
948
+ if not valid or len(valid) < obj_min_frac * verifiable:
949
+ valid = []
950
+ return valid, ps
951
+
952
+
953
  DEFAULT_TAR_ROOTS = ("/data/mv_mesh_data/slat_tars/processed",)
954
  DEFAULT_TAR_MANIFESTS = "/data/mv_mesh_data/DATASET_INDEX/hubman/processed/manifests"
955
 
 
1007
  shards: Optional[Sequence[str]] = None,
1008
  max_objects: Optional[int] = None,
1009
  use_recgen_images: bool = False,
 
1010
  cond_cameras_fallback: bool = False):
1011
+ # ---- condition-source knobs (2026-09-24). Class defaults OFF -> index,
1012
+ # draws and every item byte-identical to the legacy loader. The PROD
1013
+ # config (train_slatflow_prod.yaml) turns BOTH ON.
1014
  # use_recgen_images : also index ``recgen/<id>/{cameras.json,NN.jpg,
1015
  # NN_mask.png}`` members (RecGen's own background-baked RGB + its
1016
  # visible mask + per-view RIGID OpenCV c2w in the SLAT cube) from ANY
1017
+ # tar under tar_roots, joined to ``latents/<id>.npz`` by the SAME id
1018
+ # string. Conditioning then follows the ONE-POOL rule:
1019
+ # * an object with BOTH sources draws its anchor uniformly from the
1020
+ # union of its (valid) white cond views and (valid) RecGen views;
1021
+ # * the anchor's source decides everything: ALL auxiliary views of
1022
+ # the sample come from that same source (and its cameras) — a
1023
+ # sample never mixes white and RecGen views;
1024
+ # * single-source objects draw exactly as the legacy loader does on
1025
+ # the source they have (white-only items are byte-identical to
1026
+ # legacy; latent-only RecGen slats become trainable).
1027
+ # A RecGen anchor goes through the SAME default path: the full-frame
1028
+ # RecGen image (background kept, alpha = its mask) is fed to
1029
+ # preprocess_slat_image -> item["image"] (+ mask/rgb_image/...);
1030
  # DINO seed crops / uv / vis use the mask-matted tight crop exactly
1031
+ # like the white path.
1032
  # cond_cameras_fallback : white-cond objects lacking
1033
  # ``cameras/<id>/cond_cameras.json`` (the 24-view slat5/slat5rs
1034
  # batches) use ``cond/<id>/transforms.json`` (same transform_matrix;
1035
  # fx=fy=W/(2 tan(camera_angle_x/2)), cx=cy=W/2 from the PNG width).
1036
  self.use_recgen_images = bool(use_recgen_images)
 
1037
  self.cond_cameras_fallback = bool(cond_cameras_fallback)
1038
  # NOTE: deliberately does NOT call TrellisSlatFlowDataset.__init__ (that
1039
  # scans object DIRS). We set the attributes its shared methods
 
1112
  # new index semantics -> distinct cache signature (legacy sig
1113
  # unchanged when both knobs are off, so old caches stay valid)
1114
  parts.append(f"rg={int(self.use_recgen_images)}"
1115
+ f":fb={int(self.cond_cameras_fallback)}:v2")
1116
  return _hashlib.sha1("|".join(parts).encode()).hexdigest()
1117
  _sig = _tar_sig(tar_files)
1118
  _cache = os.environ.get("TAR_INDEX_CACHE") or os.path.join(
 
1340
  views_all.append(dict(depth=None, fx=float(K[0, 0]), fy=float(K[1, 1]),
1341
  cx=float(K[0, 2]), cy=float(K[1, 2]),
1342
  c2w=np.asarray(c["c2w"], dtype=np.float64),
1343
+ res=int(max(int(c["width"]), int(c["height"]))),
1344
+ vf=float(c.get("visible_fraction", -1.0))))
1345
  return views_all, len(views_all)
1346
 
1347
+ def _rg_qc_valid(self, sha: str, m: Dict, views_all: List[Dict],
1348
+ sc: np.ndarray) -> List[int]:
1349
+ """RecGen views passing the camera QC (recgen_view_qc), cached per object."""
1350
+ key = (sha, "rgqc")
1351
+ v = self._valid_cache.get(key)
1352
+ if v is not None:
1353
+ return v
1354
+ r = m["rg"]
1355
+ tar = self._get_tar(r["tar"])
1356
+ masks = [np.array(Image.open(io.BytesIO(tar.extractfile(n).read())).convert("L")) > 127
1357
+ for n in r["masks"]]
1358
+ valid, _ = recgen_view_qc(sc, views_all, masks, [vw["vf"] for vw in views_all])
1359
+ if len(self._valid_cache) > 20000:
1360
+ self._valid_cache.clear()
1361
+ self._valid_cache[key] = valid
1362
+ return valid
1363
+
1364
  def _load_view_rgba(self, m: Dict, src: str, i: int) -> np.ndarray:
1365
  """FULL-FRAME RGBA uint8 of view i. white: the cond render PNG. recgen:
1366
  RecGen's RGB (background kept) + its visible mask as alpha (>127)."""
 
1375
  return np.concatenate([rgb, a], axis=2)
1376
  return self._load_png_rgba(self._get_tar(m["tar"]), m["cond_pngs"][i])
1377
 
 
 
 
 
 
 
 
 
 
 
 
 
1378
  # ---- OPT-IN bad-view gate (tar variant, keyed by sha) -------------- #
1379
  def _tar_valid_views(self, sha: str, tar, views_all, sc, centers, V,
1380
  src: str = "white") -> List[int]:
 
1437
  tar = self._get_tar(m["tar"])
1438
  rng = np.random.default_rng([self.seed, self._epoch, idx])
1439
 
1440
+ # ---- condition sources present for this object ------------------- #
1441
+ # white = cond/<id>/NNN.png + its cameras (always, legacy objects)
1442
+ # recgen = recgen/<id>/NN.jpg + NN_mask.png + cameras.json
1443
+ # (only indexed when use_recgen_images=True)
1444
+ has_w = bool(m.get("cond_pngs"))
1445
+ has_r = m.get("rg") is not None
1446
+ cams = {}
1447
+ if has_w:
1448
+ cams["white"] = self._tar_cameras(tar, m)
1449
+ if has_r:
1450
+ cams["recgen"] = self._rg_cameras(m)
1451
 
1452
  # x1 = released SLAT: coords 0..63 used DIRECTLY, feats row-aligned.
1453
  zl = np.load(io.BytesIO(tar.extractfile(m["latent"]).read()))
 
1460
  torch.from_numpy(sc).int()], dim=1) # (N,4) int32
1461
  centers = (sc.astype(np.float64) + 0.5) / VOX - 0.5
1462
 
1463
+ # RecGen camera QC (always on for RecGen views): only views whose camera
1464
+ # projects this object's own voxels into RecGen's mask consistently with
1465
+ # RecGen's visible_fraction may enter the pool. None valid -> the
1466
+ # object's RecGen set is dropped (white-only fallback, or skipped).
1467
+ rg_ok = None
1468
+ if has_r:
1469
+ rg_ok = self._rg_qc_valid(sha, m, cams["recgen"][0], sc)
1470
+ if not rg_ok:
1471
+ has_r = False
1472
+ if not (has_w or has_r):
1473
+ raise ValueError(f"{sha}: no condition view passes (white cond absent, "
1474
+ f"all RecGen views fail camera QC)")
1475
+
1476
+ def _valid_for(s):
1477
+ va, Vs = cams[s]
1478
+ fb = (np.asarray(self._tar_valid_views(sha, tar, va, sc, centers, Vs, s),
1479
+ dtype=int) if self.filter_bad_views
1480
+ else np.arange(Vs, dtype=int))
1481
+ if s == "recgen": # QC is the hard rule
1482
+ both = np.asarray([i for i in fb if i in set(rg_ok)], dtype=int)
1483
+ fb = both if len(both) else np.asarray(rg_ok, dtype=int)
1484
+ return fb
1485
+
1486
+ if not (has_w and has_r):
1487
+ # SINGLE source (white-only = every legacy object; recgen-only = the
1488
+ # latent-only new slats): the exact legacy draw on that source.
1489
+ src = "white" if has_w else "recgen"
1490
+ views_all, V = cams[src]
1491
+ n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
1492
+ if src == "recgen": # QC-valid (+ bad-view gate) only
1493
+ valid_arr = _valid_for("recgen")
1494
+ n_sub = min(int(n_sub), len(valid_arr))
1495
+ sub = rng.choice(valid_arr, size=n_sub, replace=False)
1496
+ elif self.filter_bad_views: # OPT-IN: draw only from valid
1497
+ valid_arr = np.asarray(
1498
+ self._tar_valid_views(sha, tar, views_all, sc, centers, V, src),
1499
+ dtype=int)
1500
+ n_sub = min(int(n_sub), len(valid_arr))
1501
+ sub = rng.choice(valid_arr, size=n_sub, replace=False) # anchor in valid
1502
+ else:
1503
+ sub = rng.choice(V, size=n_sub, replace=False)
1504
  else:
1505
+ # BOTH sources -> ONE POOL. Candidate anchors = the union of the
1506
+ # object's (valid) white cond views and (QC-valid) RecGen views; the
1507
+ # anchor is drawn uniformly from the pool, and the ANCHOR'S SOURCE
1508
+ # DECIDES EVERYTHING: all auxiliary views are drawn from that same
1509
+ # source only (with that source's cameras) — never mixed.
1510
+ valid = {s: _valid_for(s) for s in ("white", "recgen")}
1511
+ pool = ([("white", int(i)) for i in valid["white"]]
1512
+ + [("recgen", int(i)) for i in valid["recgen"]])
1513
+ n_sub, no_seed, mode = self._mixture_n_sub(idx, len(pool), rng)
1514
+ src, a_i = pool[int(rng.integers(len(pool)))]
1515
+ views_all, V = cams[src]
1516
+ rest = np.asarray([i for i in valid[src] if i != a_i], dtype=int)
1517
+ n_sub = min(int(n_sub), 1 + len(rest))
1518
+ aux = (rng.choice(rest, size=n_sub - 1, replace=False) if n_sub > 1
1519
+ else np.zeros(0, dtype=int))
1520
+ sub = np.concatenate([[a_i], aux]).astype(int)
1521
  order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
1522
 
1523
  crops, uvs, viss, fulls = [], [], [], []
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py.bak_predatveiw ADDED
@@ -0,0 +1,804 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """TrellisSlatFlowDataset — per-view DINO inputs + cameras + SLAT target on top
2
+ of TrellisSSFlowDataset (SLAT_GEN_PLAN §8).
3
+
4
+ Reused verbatim from the SS dataset: object scan / meta.json completion marker /
5
+ split logic, the per-(sample, epoch) view-subset RNG, and the views.npz loader
6
+ (per-view K + c2w_cv — those cameras cover BOTH the Plücker rays and the
7
+ voxel->patch projection; no new camera fields).
8
+
9
+ Per __getitem__ (CPU only — all frozen-GPU work happens in the train step):
10
+ * coords_slat = downsample(prune(voxels.npz coords, dist=1)) — the exact
11
+ inference coordinate ops (pure coordinate transforms, run on CPU torch);
12
+ * a view subset S (|S| ~ U{min_views..max_views}); anchor = S[0]; view order
13
+ of every per-view tensor = [S..., complement...] so the model can slice
14
+ the subset (x0/cond) vs all views;
15
+ * per view: the RGBA crop in the STORED bbox frame -> 518 LANCZOS ->
16
+ premult-alpha-on-black float [3,518,518] (dino_features convention,
17
+ batch_appforce:253-264) — feeds the shared frozen DINO forward;
18
+ * per view: (uv, vis) — voxel-center projections through the ABSOLUTE camera
19
+ (project_visible: nearest-pixel depth test, TOL=0.02) with uv already in
20
+ the crop-frame [-1,1] grid_sample convention (sample_feats :267-275),
21
+ computed in float64 and cast to float32;
22
+ * anchor: the 4 slat condition tensors via the DEPLOYED slat_preprocessor
23
+ recipe (OQ-6) on the raw RGBA input image;
24
+ * non-anchor: per-patch anchor-relative Plücker rays [1369,6] (§4a) and the
25
+ soft-mask camera dicts (§4b);
26
+ * x1: ``slat/slat_official_sam3d.npz`` — the OFFICIAL TRELLIS-toolkit
27
+ latent with SAM3D's slat_encoder (RAW, all active voxels; produced by
28
+ ``tools/gen_x1_official.py``). Rows are coord-hash-matched onto
29
+ coords_slat (a prune-subset of the voxel set) and full coverage is
30
+ asserted. There is NO on-the-fly fallback: the former full-view
31
+ visible-mean target (reimplemented, non-official) was deleted — the
32
+ train step errors if the file is absent.
33
+
34
+ RGB SOURCE: the production layout stores ``views/rgb/{i:03d}.webp`` + a
35
+ ``views/bbox.npy`` sidecar. The toys4k SMOKE objects pre-date that layout, so
36
+ when those files are missing the loader falls back to the toys4k1k render
37
+ assets (inputs/<obj>_<view>.png = the bbox crop, renders/<obj>_<view>.npz for
38
+ bbox/res) with the fixed front/side/oside/back view order — the exact files
39
+ batch_appforce consumed.
40
+ """
41
+ from __future__ import annotations
42
+
43
+ import os
44
+ from typing import Dict, List, Optional, Sequence
45
+
46
+ import numpy as np
47
+ import torch
48
+ from PIL import Image
49
+
50
+ from mvsam3d.data.ssflow_dataset import TrellisSSFlowDataset, load_views_npz
51
+ from mvsam3d.data.slat_preprocess import preprocess_slat_image
52
+ from mvsam3d.model.mv_slat_condition import plucker_rays_for_view
53
+
54
+ TOL = 0.02 # visibility depth tolerance (batch_appforce)
55
+ VOX = 64
56
+ # SS-stage seed-mixture mode -> integer code carried in the item/batch (mirrors
57
+ # ss_imgcond_dataset._MODE_CODE; 'val' shares 'single'==1 as an eval label).
58
+ _SEED_MODE_CODE = {"none": 0, "single": 1, "multi": 2, "legacy": 3, "val": 1}
59
+ TOYS4K1K = "/lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k1k"
60
+ TOYS4K_VIEW_ORDER = ("front", "side", "oside", "back")
61
+ # TRELLIS-style per-object voxel cap (structured_latent.py filter_metadata):
62
+ # objects with MORE active 64^3 voxels than this are DROPPED at index-build
63
+ # time (never downsampled — downsample_sparse_structure is inference-only and
64
+ # corrupts training targets). 20000 = the single-object SLAT OOM-safe bound
65
+ # (PLAN/BATCH_SIZE_BENCH.md; OOM at N~24000). Drops ~4.8% of trellis500k.
66
+ MAX_NUM_VOXELS_DEFAULT = 20000
67
+ # visibility source when a view has no rendered depth (production data):
68
+ # raymarch = exact self-occlusion through the 64^3 grid (default)
69
+ # zbuffer = image-space min-splat (biased ~1 voxel near; see data/vis_check)
70
+ # Calibrated against rendered-depth GT on 301 toys4k objects x 4 views
71
+ # (data/vis_check/vis_check4.py, run4.log; agreement / IoU vs project_visible_np):
72
+ # zbuffer tol .03 .8762 / .6692 raymarch skip 1.5 .8113 / .4355
73
+ # raymarch skip 2.0 .8742 / .6646 raymarch skip 2.5 .8755 / .6724 <- default
74
+ # raymarch skip 3.0 .8519 / .6472 ray2.0 OR zbuf .8757 / .6729
75
+ # skip_vox 2.5 mirrors the GT's own TOL=0.02 slack (~1.3 voxels) plus the
76
+ # half-voxel centre offset; it matches the z-buffer's accuracy without the
77
+ # z-buffer's splat-radius/resolution heuristic. ~40 ms/view (100 ms under a
78
+ # 48-way loaded box) at N=20000.
79
+ VIS_METHOD = os.environ.get("MVSAM3D_VIS_METHOD", "raymarch")
80
+ VIS_SKIP_VOX = float(os.environ.get("MVSAM3D_VIS_SKIP_VOX", "2.5"))
81
+ VIS_ZBUF_TOL = float(os.environ.get("MVSAM3D_VIS_ZBUF_TOL", "0.03"))
82
+
83
+
84
+ def num_active_voxels(voxels_npz: str) -> int:
85
+ """Active 64^3 voxel count = #rows of voxels.npz 'coords'.
86
+
87
+ Reads only the .npy header inside the zip archive (streaming decompression
88
+ of a few hundred bytes) — cheap enough to run over the whole index at
89
+ dataset-build time. Falls back to a full np.load on any header oddity."""
90
+ import zipfile
91
+ from numpy.lib import format as npfmt
92
+ try:
93
+ with zipfile.ZipFile(voxels_npz) as zf:
94
+ with zf.open("coords.npy") as f:
95
+ version = npfmt.read_magic(f)
96
+ shape, _, _ = npfmt._read_array_header(f, version)
97
+ return int(shape[0])
98
+ except Exception:
99
+ return int(np.load(voxels_npz)["coords"].shape[0])
100
+
101
+
102
+ # --------------------------------------------------------------------------- #
103
+ def slat_coords_from_voxels(coords3: np.ndarray) -> torch.Tensor:
104
+ """voxels.npz coords (N,3) -> the SLAT coord set (M,4) int32 (batch col 0):
105
+ prune_sparse_structure(dist=1) + downsample_sparse_structure — the exact
106
+ inference ops (inference_pipeline.py:818-833), on CPU."""
107
+ from sam3d_objects.pipeline.inference_utils import (
108
+ downsample_sparse_structure, prune_sparse_structure)
109
+ c = torch.from_numpy(np.asarray(coords3, dtype=np.int64)).int()
110
+ c4 = torch.cat([torch.zeros(len(c), 1, dtype=torch.int32), c], dim=1)
111
+ c4 = prune_sparse_structure(c4, max_neighbor_axes_dist=1)
112
+ c4, _ = downsample_sparse_structure(c4)
113
+ return c4.int()
114
+
115
+
116
+ def project_visible_np(centers: np.ndarray, depth: np.ndarray, fx, fy, cx, cy,
117
+ c2w: np.ndarray, res: int):
118
+ """batch_appforce_sam3d.project_visible (:236-250), verbatim numpy port.
119
+ centers (N,3) float64 canonical -> (u, v, z, visible)."""
120
+ w2c = np.linalg.inv(c2w)
121
+ R, t = w2c[:3, :3], w2c[:3, 3]
122
+ xc = centers @ R.T + t
123
+ z = xc[:, 2]
124
+ u = fx * xc[:, 0] / z + cx
125
+ vv = fy * xc[:, 1] / z + cy
126
+ ui = np.round(u).astype(int)
127
+ vi = np.round(vv).astype(int)
128
+ inframe = (z > 0) & (ui >= 0) & (ui < res) & (vi >= 0) & (vi < res)
129
+ dep = np.zeros(len(centers))
130
+ dep[inframe] = depth[vi[inframe], ui[inframe]]
131
+ visible = inframe & (dep > 0) & (np.abs(z - dep) < TOL)
132
+ return u, vv, z, visible
133
+
134
+
135
+ def zbuffer_visible(centers: np.ndarray, fx, fy, cx, cy, c2w: np.ndarray,
136
+ res: int, tol: float = TOL):
137
+ """Per-voxel visibility from the object's OWN voxels (no depth image).
138
+
139
+ Self-occlusion test: every voxel is splatted into a per-view z-buffer over
140
+ its projected footprint (min planar depth per pixel); a voxel is visible
141
+ iff its own depth is within ``tol`` of the buffer at its centre pixel —
142
+ i.e. nothing closer covers it. This is the same decision rule as
143
+ ``project_visible_np`` with the rendered depth map replaced by the depth
144
+ of the voxelised surface itself (exact up to the 64^3 voxel size, which is
145
+ below ``tol`` = 0.02 ~ 1.3 voxels). Returns (u, v, z, visible) like
146
+ ``project_visible_np``. Production renders (slat50k cond views) ship no
147
+ depth pass, so this is the visibility source for training; the eval sets
148
+ keep their rendered GT depth."""
149
+ w2c = np.linalg.inv(c2w)
150
+ R, t = w2c[:3, :3], w2c[:3, 3]
151
+ xc = centers @ R.T + t
152
+ z = xc[:, 2]
153
+ u = fx * xc[:, 0] / z + cx
154
+ vv = fy * xc[:, 1] / z + cy
155
+ ui = np.round(u).astype(int)
156
+ vi = np.round(vv).astype(int)
157
+ inframe = (z > 0) & (ui >= 0) & (ui < res) & (vi >= 0) & (vi < res)
158
+ visible = np.zeros(len(centers), dtype=bool)
159
+ if not inframe.any():
160
+ return u, vv, z, visible
161
+ # projected voxel footprint (pixels): voxel edge 1/64 at depth z
162
+ zf = z[inframe]
163
+ r = int(np.clip(np.ceil(0.5 * fx / VOX / float(np.median(zf))), 1, 16))
164
+ buf = np.full((res, res), np.inf, dtype=np.float64)
165
+ ui_f, vi_f = ui[inframe], vi[inframe]
166
+ for dy in range(-r, r + 1):
167
+ yy = vi_f + dy
168
+ oky = (yy >= 0) & (yy < res)
169
+ for dx in range(-r, r + 1):
170
+ xx = ui_f + dx
171
+ ok = oky & (xx >= 0) & (xx < res)
172
+ np.minimum.at(buf, (yy[ok], xx[ok]), zf[ok])
173
+ dep = buf[vi_f, ui_f]
174
+ visible[inframe] = np.abs(zf - dep) < tol
175
+ return u, vv, z, visible
176
+
177
+
178
+ def raymarch_visible(centers: np.ndarray, coords3: np.ndarray, fx, fy, cx, cy,
179
+ c2w: np.ndarray, res: int, skip_vox: float = 2.5,
180
+ step_vox: float = 0.5):
181
+ """Per-voxel visibility by EXACT ray marching through the 64^3 occupancy
182
+ grid (no image-space splatting, hence none of the z-buffer's near-bias):
183
+ a voxel is visible iff the ray from its centre to the camera centre meets
184
+ no occupied voxel beyond ``skip_vox`` voxels from itself (the skip keeps
185
+ the voxel's own surface neighbours from blocking grazing rays, and matches
186
+ the rendered-depth reference's TOL=0.02 depth slack; see VIS_SKIP_VOX for
187
+ the calibration). Returns (u, v, z, visible) like ``project_visible_np``."""
188
+ occ = np.zeros((VOX, VOX, VOX), dtype=bool)
189
+ c3 = np.asarray(coords3, dtype=np.int64)
190
+ occ[c3[:, 0], c3[:, 1], c3[:, 2]] = True
191
+ w2c = np.linalg.inv(c2w)
192
+ R, t = w2c[:3, :3], w2c[:3, 3]
193
+ xc = centers @ R.T + t
194
+ z = xc[:, 2]
195
+ u = fx * xc[:, 0] / z + cx
196
+ vv = fy * xc[:, 1] / z + cy
197
+ ui = np.round(u).astype(int)
198
+ vi = np.round(vv).astype(int)
199
+ inframe = (z > 0) & (ui >= 0) & (ui < res) & (vi >= 0) & (vi < res)
200
+ cam = c2w[:3, 3]
201
+ d = cam[None, :] - centers # (N,3) towards the camera
202
+ dist = np.linalg.norm(d, axis=1)
203
+ d = d / np.maximum(dist[:, None], 1e-9)
204
+ # march only while still inside the cube: t_max = exit distance of the ray
205
+ # from [-0.5,0.5]^3 (voxels outside are never occupied)
206
+ with np.errstate(divide="ignore", invalid="ignore"):
207
+ t1 = (-0.5 - centers) / d
208
+ t2 = (0.5 - centers) / d
209
+ t_exit = np.nanmin(np.where(d != 0, np.maximum(t1, t2), np.inf), axis=1)
210
+ t_max = np.minimum(dist, t_exit)
211
+ # march the still-unblocked voxels only (active set shrinks fast: most
212
+ # voxels are occluded within a few steps), 16 steps per vectorised chunk
213
+ occf = occ.reshape(-1)
214
+ step = step_vox / VOX
215
+ t = skip_vox / VOX
216
+ blocked = np.zeros(len(centers), dtype=bool)
217
+ act = np.flatnonzero(inframe)
218
+ CH = 16
219
+ while act.size:
220
+ tt = t + step * np.arange(CH) # (CH,)
221
+ ca, da, tm = centers[act], d[act], t_max[act]
222
+ pts = ca[:, None, :] + tt[None, :, None] * da[:, None, :] # (A,CH,3)
223
+ valid = tt[None, :] <= tm[:, None]
224
+ gi = np.floor((pts + 0.5) * VOX).astype(np.int64)
225
+ inside = valid & np.all((gi >= 0) & (gi < VOX), axis=2)
226
+ np.clip(gi, 0, VOX - 1, out=gi)
227
+ flat = (gi[..., 0] * VOX + gi[..., 1]) * VOX + gi[..., 2]
228
+ hit = (occf[flat] & inside).any(axis=1)
229
+ blocked[act[hit]] = True
230
+ t = float(tt[-1]) + step
231
+ act = act[~hit & (tm >= t)]
232
+ visible = inframe & ~blocked
233
+ return u, vv, z, visible
234
+
235
+
236
+ def crop_uv_norm(u: np.ndarray, v: np.ndarray, bbox) -> np.ndarray:
237
+ """sample_feats' grid_sample coords (:267-275): render-frame (u,v) -> the
238
+ crop-frame [-1,1] convention, float64 -> float32. (N,2)."""
239
+ y0, y1, x0, x1 = [float(b) for b in bbox]
240
+ # PER-AXIS normalisation: the stored crop is rgba[y0:y1, x0:x1] (no square
241
+ # padding, adapt_slat50k.crop_bbox clamps at the frame edge) and is resized
242
+ # anisotropically to 518x518, so u maps by the crop WIDTH and v by its
243
+ # HEIGHT. Identical to the old single-`side` formula whenever w == h.
244
+ w = x1 - x0
245
+ h = y1 - y0
246
+ un = (u - x0 + 0.5) / w * 2 - 1
247
+ vn = (v - y0 + 0.5) / h * 2 - 1
248
+ return np.stack([un, vn], -1).astype(np.float32)
249
+
250
+
251
+ def rgba_to_crop(rgba_uint8: np.ndarray) -> Dict[str, np.ndarray]:
252
+ """In-memory RGBA uint8 (H,W,4) -> {'premult' [3,518,518] float32
253
+ (dino_features recipe: 518 LANCZOS + alpha-premult on black),
254
+ 'rgba_uint8' (H,W,4)}. Shared by the clean-RGBA file loader and the
255
+ RecGen matting path (which has no on-disk RGBA)."""
256
+ im = Image.fromarray(np.ascontiguousarray(rgba_uint8), mode="RGBA")
257
+ im518 = im.resize((518, 518), Image.Resampling.LANCZOS)
258
+ a = np.array(im518).astype(np.float32) / 255.0
259
+ rgb = a[:, :, :3] * a[:, :, 3:4] # premult alpha on BLACK
260
+ return {"premult": rgb.transpose(2, 0, 1), "rgba_uint8": rgba_uint8}
261
+
262
+
263
+ def load_crop_rgba(path: str) -> Dict[str, np.ndarray]:
264
+ """RGBA crop file -> {'premult' [3,518,518] float32 (dino_features recipe:
265
+ 518 LANCZOS + alpha-premult on black), 'rgba_uint8' (H,W,4)} ."""
266
+ return rgba_to_crop(np.array(Image.open(path).convert("RGBA")))
267
+
268
+
269
+ def matte_rgba(rgb_uint8: np.ndarray, mask_uint8: np.ndarray) -> np.ndarray:
270
+ """RGB (H,W,3) + binary/greyscale mask (H,W) -> RGBA (H,W,4) uint8 CROPPED
271
+ to the mask's bounding box. The RecGen ALPHA the SLAT loader's visibility
272
+ gate needs: cond_image has BACKGROUND baked in and no alpha, so the object
273
+ mask (prefer cond_mask_sam2, fallback cond_mask) becomes the alpha channel.
274
+ Returns the tight-bbox crop so ``crop_uv_norm`` / the alpha gate work exactly
275
+ as they do on the clean transparent-background renders."""
276
+ rgb = np.asarray(rgb_uint8)
277
+ m = np.asarray(mask_uint8)
278
+ if m.ndim == 3:
279
+ m = m[..., 0]
280
+ a = (m > 127).astype(np.uint8) * 255
281
+ ys, xs = np.nonzero(a)
282
+ if len(ys) == 0: # empty mask: whole frame
283
+ y0, y1, x0, x1 = 0, rgb.shape[0], 0, rgb.shape[1]
284
+ else:
285
+ y0, y1 = int(ys.min()), int(ys.max()) + 1
286
+ x0, x1 = int(xs.min()), int(xs.max()) + 1
287
+ rgba = np.concatenate([rgb, a[..., None]], axis=2) # (H,W,4)
288
+ crop = rgba[y0:y1, x0:x1]
289
+ return crop, (float(y0), float(y1), float(x0), float(x1))
290
+
291
+
292
+ # --------------------------------------------------------------------------- #
293
+ class TrellisSlatFlowDataset(TrellisSSFlowDataset):
294
+ """SLAT-flow training samples: SS dataset + per-view RGB/DINO inputs,
295
+ cameras, and the x1 target hookup."""
296
+
297
+ def __init__(self, *args, toys4k1k_root: str = TOYS4K1K,
298
+ max_num_voxels: Optional[int] = MAX_NUM_VOXELS_DEFAULT,
299
+ p_seed_none: float = 0.0, # unified mixture: P(EMPTY seed)
300
+ p_seed_single: float = 0.0, # P(single-view seed); rest=multi
301
+ recgen_roots: Optional[Sequence[str]] = None,
302
+ recgen_pose_store: Optional[str] = None,
303
+ **kwargs):
304
+ super().__init__(*args, **kwargs)
305
+ self.toys4k1k_root = toys4k1k_root
306
+ # encoded_store recgen root holding the canonical cube cameras
307
+ # (pose_cube.npz / views.npz) looked up by sha; None -> analytic fallback.
308
+ self._recgen_pose_store = recgen_pose_store
309
+ # ---- SS-STAGE seed MIXTURE (mirror of ss_imgcond_dataset.seed_mode) --
310
+ # p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
311
+ # p_seed_single= P(single-view seed -> n_sub == 1)
312
+ # remainder = multi (n_sub ~ U{max(2,min_views)..max_views})
313
+ # BOTH == 0.0 -> the LEGACY path (n_sub ~ U{min_views..max_views}), which
314
+ # is byte-identical to the pre-mixture behaviour. VAL (fixed_num_views
315
+ # set) is never randomised. Own rng stream (offset 4242 for the mode,
316
+ # 7717 for the multi view-count) so the mixture never perturbs the
317
+ # existing view-subset / crop draws on the main [seed,epoch,idx] stream.
318
+ self.p_seed_none = float(p_seed_none)
319
+ self.p_seed_single = float(p_seed_single)
320
+ assert self.p_seed_none + self.p_seed_single <= 1.0 + 1e-9, (
321
+ self.p_seed_none, self.p_seed_single)
322
+ # TRELLIS-style max_num_voxels FILTER (mirror of datasets/
323
+ # structured_latent.py filter_metadata: metadata['num_voxels'] <= cap):
324
+ # applied at index-build time so oversized objects never reach the
325
+ # loader. None / 0 disables the filter.
326
+ self.max_num_voxels = int(max_num_voxels) if max_num_voxels else None
327
+ if self.max_num_voxels:
328
+ n_before = len(self.dirs)
329
+ self.dirs = [
330
+ d for d in self.dirs
331
+ if num_active_voxels(os.path.join(d, "geometry", "voxels.npz"))
332
+ <= self.max_num_voxels
333
+ ]
334
+ n_drop = n_before - len(self.dirs)
335
+ print(f"[slatflow-data] split={self.split} max_num_voxels="
336
+ f"{self.max_num_voxels}: kept {len(self.dirs)}/{n_before} "
337
+ f"objects (dropped {n_drop})", flush=True)
338
+ if not self.dirs:
339
+ raise RuntimeError(
340
+ f"max_num_voxels={self.max_num_voxels} dropped every object "
341
+ f"in split={self.split}")
342
+
343
+ # ---- RecGen roots (ADDITIVE, opt-in) --------------------------------
344
+ # RecGen sample objects (<root>/<id>/ with slat_coords.npy + slat_feats.npy
345
+ # + cond_image_*.jpg + cond_mask*.png + view_metadata.json) do NOT carry a
346
+ # meta.json / geometry/voxels.npz, so they are invisible to scan_objects.
347
+ # Scan them separately and APPEND to self.dirs, tracking which dirs are
348
+ # recgen in self._recgen so __getitem__ can route them to the matting /
349
+ # released-SLAT loader. Non-recgen behaviour is untouched.
350
+ self._recgen = set()
351
+ if recgen_roots:
352
+ import glob as _glob
353
+ if isinstance(recgen_roots, (str, os.PathLike)):
354
+ recgen_roots = [recgen_roots]
355
+ rg = []
356
+ for root in recgen_roots:
357
+ for sc in _glob.iglob(os.path.join(root, "*", "slat_coords.npy")):
358
+ rg.append(os.path.dirname(sc))
359
+ rg = sorted(set(rg))
360
+ self._recgen = set(rg)
361
+ self.dirs = list(self.dirs) + rg
362
+ print(f"[slatflow-data] split={self.split} recgen_roots: added "
363
+ f"{len(rg)} recgen objects (total {len(self.dirs)})", flush=True)
364
+
365
+ # ---- SS-stage seed mixture (mirror of ss_imgcond_dataset.seed_mode) -- #
366
+ def seed_mode(self, idx: int, epoch: int) -> str:
367
+ """'none' | 'single' | 'multi' | 'legacy' for this (object, epoch).
368
+
369
+ Own rng stream (offset 4242) so the mode decision never perturbs the
370
+ [seed,epoch,idx] view-subset / crop draws. VAL (fixed_num_views set)
371
+ keeps the fixed eval view count (returns 'single' as an unused label —
372
+ __getitem__ pins n_sub to fixed_num_views in that case). With BOTH
373
+ knobs 0.0 the mixture is OFF -> 'legacy' (no draw, main stream
374
+ untouched)."""
375
+ if self.fixed_num_views is not None: # val: unchanged
376
+ return "single"
377
+ if self.p_seed_none <= 0.0 and self.p_seed_single <= 0.0:
378
+ return "legacy"
379
+ r = float(np.random.default_rng(
380
+ [self.seed, int(epoch), int(idx), 4242]).random())
381
+ if r < self.p_seed_none:
382
+ return "none"
383
+ if r < self.p_seed_none + self.p_seed_single:
384
+ return "single"
385
+ return "multi"
386
+
387
+ # ---- per-object RGB/bbox adapter ---------------------------------- #
388
+ def _rgb_assets(self, obj_dir: str, name: str, n_views: int) -> List[Dict]:
389
+ """-> per-view dict(png=..., bbox=(4,), res=int). Production layout
390
+ first (views/rgb + views/bbox.npy); toys4k1k fallback for smoke."""
391
+ rgb_dir = os.path.join(obj_dir, "views", "rgb")
392
+ bbox_path = os.path.join(obj_dir, "views", "bbox.npy")
393
+ if os.path.isdir(rgb_dir) and os.path.isfile(bbox_path):
394
+ bboxes = np.load(bbox_path)
395
+ out = []
396
+ for i in range(n_views):
397
+ png = os.path.join(rgb_dir, f"{i:03d}.png") # lossless (adapter)
398
+ if not os.path.isfile(png):
399
+ png = os.path.join(rgb_dir, f"{i:03d}.webp")
400
+ out.append(dict(png=png, bbox=bboxes[i].astype(np.float64), res=None))
401
+ return out
402
+ # toys4k smoke fallback (bbox/res from the render npz; png = the crop)
403
+ out = []
404
+ for i in range(n_views):
405
+ tag = TOYS4K_VIEW_ORDER[i]
406
+ png = os.path.join(self.toys4k1k_root, "inputs", f"{name}_{tag}.png")
407
+ rz = np.load(os.path.join(self.toys4k1k_root, "renders",
408
+ f"{name}_{tag}.npz"))
409
+ out.append(dict(png=png, bbox=rz["bbox"].astype(np.float64),
410
+ res=int(rz["res"])))
411
+ return out
412
+
413
+ # ---- SS-stage seed-mixture view-count selection --------------------- #
414
+ def _mixture_n_sub(self, idx: int, V: int, rng) -> tuple:
415
+ """Return (n_sub, no_seed, mode). With the mixture OFF (both knobs 0.0) this
416
+ is the LEGACY draw — ``rng.integers(min_views, max_views+1)`` off the
417
+ MAIN [seed,epoch,idx] stream — byte-identical to the pre-mixture path.
418
+ With the mixture ON the mode comes from the independent 4242 stream
419
+ (seed_mode), the multi view-count from the independent 7717 stream, so
420
+ the main stream (and thus the ``sub`` view choice) is only advanced by
421
+ the legacy branch — exactly as before."""
422
+ if self.fixed_num_views is not None: # VAL: never randomised
423
+ return min(int(self.fixed_num_views), V), False, "val"
424
+ mode = self.seed_mode(idx, self._epoch)
425
+ if mode == "legacy":
426
+ return (min(int(rng.integers(self.min_views, self.max_views + 1)), V),
427
+ False, "legacy")
428
+ if mode == "none": # EMPTY appearance seed: 1 cond view, x0 all-noise
429
+ return min(1, V), True, "none"
430
+ if mode == "single":
431
+ return min(1, V), False, "single"
432
+ # multi: n_sub ~ U{max(2,min_views) .. max_views}
433
+ lo = max(2, int(self.min_views))
434
+ hi = max(lo, int(self.max_views))
435
+ k = int(np.random.default_rng(
436
+ [self.seed, int(self._epoch), int(idx), 7717]).integers(lo, hi + 1))
437
+ return min(k, V), False, "multi"
438
+
439
+ # ---- item ----------------------------------------------------------- #
440
+ def __getitem__(self, idx: int) -> Dict:
441
+ d = self.dirs[idx]
442
+ if d in self._recgen: # RecGen loader (additive)
443
+ return self._getitem_recgen(idx)
444
+ name = os.path.basename(d)
445
+ views_path = os.path.join(d, "views", "views.npz")
446
+
447
+ # per-(sample, epoch) deterministic RNG — same recipe as SSFlow
448
+ rng = np.random.default_rng([self.seed, self._epoch, idx])
449
+ views_all = load_views_npz(views_path) # K, c2w, depth per view
450
+ V = len(views_all)
451
+ n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
452
+ sub = rng.choice(V, size=n_sub, replace=False)
453
+ order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
454
+
455
+ # SLAT coord set (exact inference coordinate ops)
456
+ vox = np.load(os.path.join(d, "geometry", "voxels.npz"))
457
+ coords = slat_coords_from_voxels(vox["coords"]) # (N,4) int32
458
+ centers = (coords[:, 1:].numpy().astype(np.float64) + 0.5) / VOX - 0.5
459
+
460
+ assets = self._rgb_assets(d, name, V)
461
+
462
+ # Only the SAMPLED subset S feeds the model (cond = anchor + S[1:],
463
+ # x0 seed = visible-mean over S, x1 = the precomputed official target),
464
+ # so crops/uv/vis are built for S ONLY: an 8-view object costs |S| <= 4
465
+ # DINO forwards + projections, not 8. view_order still lists all views.
466
+ crops, uvs, viss = [], [], []
467
+ for i in order[:n_sub]:
468
+ vw = views_all[i]
469
+ a = assets[i]
470
+ if a["res"] is not None:
471
+ res = a["res"]
472
+ elif vw["depth"] is not None:
473
+ res = vw["depth"].shape[0]
474
+ else:
475
+ res = vw["res"]
476
+ if vw["depth"] is not None: # rendered GT depth (eval sets)
477
+ u, vv, z, vis = project_visible_np(
478
+ centers, vw["depth"], vw["fx"], vw["fy"], vw["cx"], vw["cy"],
479
+ vw["c2w"], res)
480
+ elif VIS_METHOD == "zbuffer": # production: from the voxels
481
+ u, vv, z, vis = zbuffer_visible(
482
+ centers, vw["fx"], vw["fy"], vw["cx"], vw["cy"], vw["c2w"], res,
483
+ tol=VIS_ZBUF_TOL)
484
+ else: # production: exact grid ray march
485
+ u, vv, z, vis = raymarch_visible(
486
+ centers, vox["coords"], vw["fx"], vw["fy"], vw["cx"], vw["cy"],
487
+ vw["c2w"], res, skip_vox=VIS_SKIP_VOX)
488
+ crop = load_crop_rgba(a["png"])
489
+ uvn = crop_uv_norm(u, vv, a["bbox"])
490
+ # --- MODAL-MASK (alpha) VISIBILITY GATE ------------------------ #
491
+ # The geometric tests above (rendered depth / z-buffer / ray march)
492
+ # only know about the OBJECT's own surface, so they call a voxel
493
+ # visible even when the stored view actually shows something else
494
+ # in front of it (PSL occlusion crops) or when the projection falls
495
+ # outside the stored crop. The stored RGBA's alpha channel IS the
496
+ # per-view MODAL mask (what is really seen), so gate `vis` by it:
497
+ # nearest-pixel lookup in the crop frame (grid_sample's
498
+ # align_corners=False convention, i.e. px = (un+1)/2*W - 0.5), NO
499
+ # dilation. On occluder-free renders (the base pool) alpha covers
500
+ # the whole silhouette, so this is a near no-op (edge pixels only).
501
+ alpha = crop["rgba_uint8"][..., 3]
502
+ Hc, Wc = alpha.shape[0], alpha.shape[1]
503
+ px = np.rint((uvn[:, 0].astype(np.float64) + 1.0) * 0.5 * Wc - 0.5).astype(np.int64)
504
+ py = np.rint((uvn[:, 1].astype(np.float64) + 1.0) * 0.5 * Hc - 0.5).astype(np.int64)
505
+ inside = (px >= 0) & (px < Wc) & (py >= 0) & (py < Hc)
506
+ vis = np.asarray(vis) & inside
507
+ if inside.any():
508
+ vis[inside] &= alpha[py[inside], px[inside]] > 0
509
+ # --- EMPTY-SEED ('none') gate: contribute ZERO forced x0 rows ---
510
+ # The x0 appearance seed is built model-side as the visible-mean of
511
+ # the rows any view marks visible; an all-False `vis` therefore
512
+ # forces no rows (x0 stays pure noise) while leaving the condition
513
+ # (DINO crops / cameras) untouched. Belt-and-suspenders with the
514
+ # `no_seed` batch flag honoured in slatflow_model.flow_step.
515
+ if no_seed:
516
+ vis = np.zeros_like(np.asarray(vis), dtype=bool)
517
+ # --------------------------------------------------------------- #
518
+ crops.append(crop)
519
+ uvs.append(uvn)
520
+ viss.append(vis)
521
+
522
+ # anchor condition inputs (deployed slat_preprocessor recipe, OQ-6)
523
+ slat_input = preprocess_slat_image(crops[0]["rgba_uint8"])
524
+
525
+ # non-anchor Plücker + soft-mask camera dicts (subset views S[1:])
526
+ anchor_c2w = torch.from_numpy(views_all[order[0]]["c2w"])
527
+ pluckers, bias_views = [], []
528
+ for k in range(1, n_sub):
529
+ i = order[k]
530
+ vw = views_all[i]
531
+ a = assets[i]
532
+ res = (a["res"] if a["res"] is not None else
533
+ (vw["depth"].shape[0] if vw["depth"] is not None else vw["res"]))
534
+ pluckers.append(plucker_rays_for_view(
535
+ anchor_c2w, torch.from_numpy(vw["c2w"]),
536
+ vw["fx"], vw["fy"], vw["cx"], vw["cy"], a["bbox"]))
537
+ bias_views.append(dict(
538
+ w2c=torch.from_numpy(np.linalg.inv(vw["c2w"])).float(),
539
+ fx=vw["fx"], fy=vw["fy"], cx=vw["cx"], cy=vw["cy"],
540
+ bbox=[float(b) for b in a["bbox"]], res=float(res)))
541
+ plucker = (torch.stack(pluckers) if pluckers
542
+ else torch.zeros(0, 1369, 6))
543
+
544
+ item = {
545
+ "name": name,
546
+ "coords": coords, # (N,4) int32
547
+ "slat_input": slat_input, # dict of 4 CPU tensors
548
+ "crops": torch.from_numpy(np.stack([c["premult"] for c in crops])),
549
+ "uv": torch.from_numpy(np.stack(uvs)), # (V,N,2) f32
550
+ "vis": torch.from_numpy(np.stack(viss)), # (V,N) bool
551
+ "plucker": plucker, # (n_sub-1,1369,6)
552
+ "bias_views": bias_views,
553
+ "n_subset": n_sub,
554
+ "view_order": [int(i) for i in order],
555
+ # SS-stage seed-mixture flags (default: legacy/no-op). no_seed=True
556
+ # => EMPTY appearance seed (x0 pure noise, honoured in flow_step).
557
+ "no_seed": torch.tensor(bool(no_seed)),
558
+ "seed_mode": torch.tensor(float(_SEED_MODE_CODE[mode])),
559
+ }
560
+
561
+ # OFFICIAL precomputed x1 target (tools/gen_x1_official.py output):
562
+ # the exact TRELLIS-toolkit latent (SAM3D slat_encoder) on the FULL
563
+ # active-voxel set — coord-hash-match its rows onto coords_slat (a
564
+ # prune-subset of the voxels) and assert every row is covered.
565
+ slat_npz = os.path.join(d, "slat", "slat_official_sam3d.npz")
566
+ if os.path.isfile(slat_npz):
567
+ z = np.load(slat_npz)
568
+ oc = z["coords"].astype(np.int64) # (M,3) all voxels
569
+ ok = (oc[:, 0] * VOX + oc[:, 1]) * VOX + oc[:, 2]
570
+ order = np.argsort(ok)
571
+ want_c = coords[:, 1:].numpy().astype(np.int64)
572
+ want = (want_c[:, 0] * VOX + want_c[:, 1]) * VOX + want_c[:, 2]
573
+ pos = np.searchsorted(ok[order], want)
574
+ sel = order[np.clip(pos, 0, len(order) - 1)]
575
+ assert len(ok) > 0 and np.array_equal(ok[sel], want), \
576
+ (f"{name}: slat_official_sam3d.npz does not cover coords_slat "
577
+ f"(official voxel set mismatch — regenerate with "
578
+ f"tools/gen_x1_official.py)")
579
+ item["x1_feats_raw"] = torch.from_numpy(
580
+ z["feats"][sel].astype(np.float32)) # (N,8) RAW
581
+ return item
582
+
583
+ # ---- RecGen matting loader (additive) ------------------------------- #
584
+ def _recgen_cube_c2w(self, d: str, n: int):
585
+ """Locate the CORRECT camera->cube poses (c2w in the [-0.5,0.5] voxel
586
+ frame the SLAT loader projects into). RecGen's cube cameras come from
587
+ the ANALYTIC ``pose_cube`` conversion (pose_scale / orientation), NOT a
588
+ simple inv(model2world)@cam2world of the raw view_metadata — that lands
589
+ the object BEHIND the camera (z<0). So prefer, in order:
590
+ 1. ``pose_cube.npz`` (c2w_cube) inside the object dir or a views20/;
591
+ 2. the encoded_store's pose_cube.npz / views.npz(c2w_cv), looked up by
592
+ sha under ``recgen_pose_store`` when set.
593
+ Returns (c2w array (n,4,4)) or None if no canonical source is found."""
594
+ import glob as _glob
595
+ cands = [os.path.join(d, "pose_cube.npz"),
596
+ os.path.join(d, "views20", "pose_cube.npz"),
597
+ os.path.join(d, "views", "pose_cube.npz")]
598
+ store = getattr(self, "_recgen_pose_store", None)
599
+ if store:
600
+ sha = os.path.basename(d)
601
+ sf = os.path.join(d, "sha256.txt")
602
+ if os.path.isfile(sf):
603
+ sha = open(sf).read().strip() or sha
604
+ base = os.path.join(store, "objects", sha[:2], sha)
605
+ cands += [os.path.join(base, "views20", "pose_cube.npz"),
606
+ os.path.join(base, "views", "pose_cube.npz")]
607
+ for vp in (os.path.join(base, "views20", "views.npz"),
608
+ os.path.join(base, "views", "views.npz")):
609
+ if os.path.isfile(vp):
610
+ z = np.load(vp)
611
+ if "c2w_cv" in z.files:
612
+ c = np.asarray(z["c2w_cv"], dtype=np.float64)
613
+ if c.shape[0] >= n:
614
+ return c[:n]
615
+ for p in cands:
616
+ if os.path.isfile(p):
617
+ z = np.load(p)
618
+ key = "c2w_cube" if "c2w_cube" in z.files else (
619
+ "c2w_cv" if "c2w_cv" in z.files else None)
620
+ if key is not None:
621
+ c = np.asarray(z[key], dtype=np.float64)
622
+ if c.shape[0] >= n:
623
+ return c[:n]
624
+ return None
625
+
626
+ def _recgen_cameras(self, d: str):
627
+ """RecGen view_metadata.json -> (views_all, V) in the SAME per-view dict
628
+ shape ``load_views_npz`` produces (depth=None, fx/fy/cx/cy, c2w, res).
629
+
630
+ Intrinsics come from view_metadata.json[i]["intrinsics"] (fx=fy=610,
631
+ cx=320, cy=240, 640x480). The camera->cube EXTRINSIC prefers the
632
+ canonical pose_cube (see _recgen_cube_c2w); if none is found it falls
633
+ back to the ANALYTIC inv(model2world)@cam2world, which is format-correct
634
+ but geometrically APPROXIMATE (raw view_metadata alone does not yield the
635
+ cube frame — see report caveat #2). transforms.json is empty ({}) and
636
+ pose_data.json holds the OBJECT 6D pose, not cameras, in these samples."""
637
+ import json
638
+ vm_path = os.path.join(d, "view_metadata.json")
639
+ vm = json.load(open(vm_path))
640
+ vm = [json.loads(s) if isinstance(s, str) else s for s in vm]
641
+ V = len(vm)
642
+ cube = self._recgen_cube_c2w(d, V) # (V,4,4) canonical, or None
643
+ views_all = []
644
+ for i, m in enumerate(vm):
645
+ K = np.asarray(m["intrinsics"], dtype=np.float64)
646
+ H = int(m.get("height", 480)); W = int(m.get("width", 640))
647
+ if cube is not None:
648
+ c2w = cube[i]
649
+ else: # approximate fallback
650
+ c2w = (np.linalg.inv(np.asarray(m["model2world"], dtype=np.float64))
651
+ @ np.asarray(m["cam2world"], dtype=np.float64))
652
+ views_all.append(dict(
653
+ depth=None,
654
+ fx=float(K[0, 0]), fy=float(K[1, 1]),
655
+ cx=float(K[0, 2]), cy=float(K[1, 2]),
656
+ c2w=c2w, res=int(max(H, W))))
657
+ return views_all, V
658
+
659
+ def _recgen_asset(self, d: str, i: int):
660
+ """View i -> {'rgba': matted RGBA crop, 'bbox': (y0,y1,x0,x1), 'res': int}.
661
+ cond_image_i (RGB, background baked in) x cond_mask (prefer the SAM2 mask,
662
+ fallback the plain mask) -> RGBA so the SLAT alpha-visibility gate works."""
663
+ img_p = os.path.join(d, f"cond_image_{i:02d}.jpg")
664
+ rgb = np.array(Image.open(img_p).convert("RGB"))
665
+ mask_p = os.path.join(d, f"cond_mask_sam2_{i:02d}.png")
666
+ if not os.path.isfile(mask_p):
667
+ mask_p = os.path.join(d, f"cond_mask_{i:02d}.png")
668
+ mask = np.array(Image.open(mask_p).convert("L"))
669
+ crop_rgba, bbox = matte_rgba(rgb, mask)
670
+ return dict(rgba=crop_rgba, bbox=bbox, res=int(max(rgb.shape[0], rgb.shape[1])))
671
+
672
+ def _getitem_recgen(self, idx: int) -> Dict:
673
+ d = self.dirs[idx]
674
+ name = os.path.basename(d)
675
+
676
+ rng = np.random.default_rng([self.seed, self._epoch, idx])
677
+ views_all, V = self._recgen_cameras(d)
678
+ n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
679
+ sub = rng.choice(V, size=n_sub, replace=False)
680
+ order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
681
+
682
+ # x1 = RecGen's RELEASED SLAT (slat_coords 0..63 int + slat_feats [N,8]).
683
+ # The released coords ARE the final SLAT coord set (1:1 with slat_feats),
684
+ # so they are used directly as `coords` (no prune/downsample: RecGen ships
685
+ # no voxels.npz) and slat_feats is the row-aligned x1 target. Coverage
686
+ # is 1:1 by construction; assert the row counts match.
687
+ sc = np.load(os.path.join(d, "slat_coords.npy")).astype(np.int64) # (N,3)
688
+ feats = np.load(os.path.join(d, "slat_feats.npy")).astype(np.float32) # (N,8)
689
+ assert sc.ndim == 2 and sc.shape[1] == 3 and feats.shape[0] == sc.shape[0], \
690
+ (f"{name}: recgen slat_coords {sc.shape} / slat_feats {feats.shape} "
691
+ f"mismatch (x1 rows must align 1:1 with coords)")
692
+ assert sc.min() >= 0 and sc.max() < VOX, (name, sc.min(), sc.max())
693
+ coords = torch.cat([torch.zeros(len(sc), 1, dtype=torch.int32),
694
+ torch.from_numpy(sc).int()], dim=1) # (N,4) int32
695
+ centers = (sc.astype(np.float64) + 0.5) / VOX - 0.5
696
+
697
+ crops, uvs, viss = [], [], []
698
+ for i in order[:n_sub]:
699
+ vw = views_all[i]
700
+ a = self._recgen_asset(d, i)
701
+ res = a["res"]
702
+ u, vv, z, vis = raymarch_visible(
703
+ centers, sc, vw["fx"], vw["fy"], vw["cx"], vw["cy"],
704
+ vw["c2w"], res, skip_vox=VIS_SKIP_VOX)
705
+ crop = rgba_to_crop(a["rgba"])
706
+ uvn = crop_uv_norm(u, vv, a["bbox"])
707
+ # modal-mask (alpha) gate — identical to the clean-RGBA path
708
+ alpha = crop["rgba_uint8"][..., 3]
709
+ Hc, Wc = alpha.shape[0], alpha.shape[1]
710
+ px = np.rint((uvn[:, 0].astype(np.float64) + 1.0) * 0.5 * Wc - 0.5).astype(np.int64)
711
+ py = np.rint((uvn[:, 1].astype(np.float64) + 1.0) * 0.5 * Hc - 0.5).astype(np.int64)
712
+ inside = (px >= 0) & (px < Wc) & (py >= 0) & (py < Hc)
713
+ vis = np.asarray(vis) & inside
714
+ if inside.any():
715
+ vis[inside] &= alpha[py[inside], px[inside]] > 0
716
+ if no_seed:
717
+ vis = np.zeros_like(np.asarray(vis), dtype=bool)
718
+ crops.append(crop)
719
+ uvs.append(uvn)
720
+ viss.append(vis)
721
+ a["_bbox"] = a["bbox"] # keep for plucker reuse
722
+ views_all[i]["_asset"] = a
723
+
724
+ slat_input = preprocess_slat_image(crops[0]["rgba_uint8"])
725
+
726
+ anchor_c2w = torch.from_numpy(views_all[order[0]]["c2w"])
727
+ pluckers, bias_views = [], []
728
+ for k in range(1, n_sub):
729
+ i = order[k]
730
+ vw = views_all[i]
731
+ a = vw["_asset"]
732
+ res = a["res"]
733
+ pluckers.append(plucker_rays_for_view(
734
+ anchor_c2w, torch.from_numpy(vw["c2w"]),
735
+ vw["fx"], vw["fy"], vw["cx"], vw["cy"], a["bbox"]))
736
+ bias_views.append(dict(
737
+ w2c=torch.from_numpy(np.linalg.inv(vw["c2w"])).float(),
738
+ fx=vw["fx"], fy=vw["fy"], cx=vw["cx"], cy=vw["cy"],
739
+ bbox=[float(b) for b in a["bbox"]], res=float(res)))
740
+ plucker = (torch.stack(pluckers) if pluckers else torch.zeros(0, 1369, 6))
741
+
742
+ return {
743
+ "name": name,
744
+ "coords": coords,
745
+ "slat_input": slat_input,
746
+ "crops": torch.from_numpy(np.stack([c["premult"] for c in crops])),
747
+ "uv": torch.from_numpy(np.stack(uvs)),
748
+ "vis": torch.from_numpy(np.stack(viss)),
749
+ "plucker": plucker,
750
+ "bias_views": bias_views,
751
+ "n_subset": n_sub,
752
+ "view_order": [int(i) for i in order],
753
+ "no_seed": torch.tensor(bool(no_seed)),
754
+ "seed_mode": torch.tensor(float(_SEED_MODE_CODE[mode])),
755
+ "x1_feats_raw": torch.from_numpy(feats), # (N,8) RAW released SLAT
756
+ }
757
+
758
+
759
+ def collate_single(batch: Sequence[Dict]) -> Dict:
760
+ """bs=1 collate (variable N per object => per-object steps + grad accum)."""
761
+ assert len(batch) == 1, \
762
+ "SLAT flow trains bs=1 per object (grad accum for larger batches)"
763
+ return batch[0]
764
+
765
+
766
+ def collate_batched(batch: Sequence[Dict]) -> Dict:
767
+ """Multi-object collate mirroring TRELLIS ``SLat.collate_fn``: the B
768
+ objects' SLAT coords are CONCATENATED into one coordinate set whose batch
769
+ column ([b,x,y,z]) is the object index, with per-object ``layout`` slices
770
+ recorded. Everything with a per-object variable shape (crops, uv, vis,
771
+ plucker, condition inputs, targets) stays a python list — the batched
772
+ model path builds each object's own condition and seed from them.
773
+
774
+ Consumed by ``SlatFlowModel.sample_batch`` (true batched inference:
775
+ one forward per solver step for all B objects)."""
776
+ coords_parts: List[torch.Tensor] = []
777
+ layout: List[slice] = []
778
+ ofs = 0
779
+ for i, it in enumerate(batch):
780
+ c = it["coords"].clone()
781
+ assert int(c[:, 0].max()) == 0, "per-object coords must have batch col 0"
782
+ c[:, 0] = i # batch-index column
783
+ coords_parts.append(c)
784
+ layout.append(slice(ofs, ofs + c.shape[0]))
785
+ ofs += c.shape[0]
786
+ return {
787
+ "coords": torch.cat(coords_parts, dim=0), # (T_total,4) int32
788
+ "layout": layout, # per-object slices
789
+ "names": [it["name"] for it in batch],
790
+ "slat_input": [it["slat_input"] for it in batch],
791
+ "crops": [it["crops"] for it in batch],
792
+ "uv": [it["uv"] for it in batch],
793
+ "vis": [it["vis"] for it in batch],
794
+ "plucker": [it["plucker"] for it in batch],
795
+ "bias_views": [it["bias_views"] for it in batch],
796
+ "n_subset": [int(it["n_subset"]) for it in batch],
797
+ "view_order": [it["view_order"] for it in batch],
798
+ "x1_feats_raw": [it.get("x1_feats_raw") for it in batch],
799
+ # SS-stage seed-mixture flags (per object; default legacy/no-op)
800
+ "no_seed": [bool(it["no_seed"]) if it.get("no_seed") is not None else False
801
+ for it in batch],
802
+ "seed_mode": [float(it["seed_mode"]) if it.get("seed_mode") is not None else 3.0
803
+ for it in batch],
804
+ }
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-310.pyc ADDED
Binary file (185 Bytes). View file
 
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (201 Bytes). View file