SLAT loader: pooled-anchor conditioning (white-bg cond + RecGen images; anchor source decides all aux views), RecGen camera QC, recgen/<id>/ tar join; ON in train_slatflow_prod.yaml. Verified: 2500 real draws, 0 mixed-source samples.
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow.yaml.bak_predatveiw +169 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml +11 -11
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml.bak_predatveiw +177 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_2gpu.yaml.bak_predatveiw +177 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_pslaug_2gpu.yaml.bak_predatveiw +186 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslfinal_2gpu.yaml.bak_predatveiw +188 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslocc_2gpu.yaml.bak_predatveiw +187 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_003.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_008.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/chair_026.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ours_both.log +51 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe2_ref.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe_ref.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ref.log +73 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_003.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_008.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/chair_026.pt +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/ref.log +73 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/smoke_slatflow.log +19 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/train_slatflow_smoke.log +28 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-39.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-39.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-39.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/adapters.cpython-311.pyc +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/base_dataset.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/canonical.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_calib.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_dataset.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/integrated.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slat_preprocess.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-311.pyc +3 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_preprocess.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-39.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/v_bucket_sampler.cpython-311.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py +152 -41
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py.bak_predatveiw +804 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-310.pyc +0 -0
- migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-311.pyc +0 -0
.gitattributes
CHANGED
|
@@ -36,3 +36,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 36 |
migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/Cell-DINO.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/ChannelAdaptiveDINO.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/__pycache__/adapters.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 36 |
migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/Cell-DINO.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
migrator/checkpoints/hub/facebookresearch_dinov2_main/docs/ChannelAdaptiveDINO.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/__pycache__/adapters.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/adapters.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow.yaml.bak_predatveiw
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
|
| 2 |
+
# ALL training hyperparameters live here; train_slat_flow.py reads them and
|
| 3 |
+
# nothing training-related is hardcoded in Python. Launch:
|
| 4 |
+
# SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
|
| 5 |
+
# python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
|
| 6 |
+
# (multi-GPU: torchrun --nproc_per_node=N ...)
|
| 7 |
+
# Per-key overrides: --set key.subkey=value (repeatable).
|
| 8 |
+
|
| 9 |
+
# --------------------------------------------------------------------------- #
|
| 10 |
+
# data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
|
| 11 |
+
# Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
|
| 12 |
+
# RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
|
| 13 |
+
# stored views.npz). Production roots get views/rgb + views/bbox.npy +
|
| 14 |
+
# slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
|
| 15 |
+
# --------------------------------------------------------------------------- #
|
| 16 |
+
data:
|
| 17 |
+
roots:
|
| 18 |
+
- /lp-dev/jonghoon/mv-sam3d-6d-code/smoke_data/toys4k200
|
| 19 |
+
# Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
|
| 20 |
+
# val = ALL objects under val_roots, train = roots minus those ids.
|
| 21 |
+
# NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
|
| 22 |
+
# set is toys4k-only until the production rgb/bbox precompute lands.
|
| 23 |
+
val_roots:
|
| 24 |
+
- /lp-dev/jonghoon/mv-sam3d-6d-code/smoke_data/val_toys4k100
|
| 25 |
+
# --- view-subset augmentation (the ONLY train-time stochasticity besides
|
| 26 |
+
# the fresh invisible-row noise in x0) ---
|
| 27 |
+
# smoke data has 4 views/object; production renders 24 (paper §3.1.1).
|
| 28 |
+
# TRELLIS-style per-object voxel FILTER (structured_latent.py
|
| 29 |
+
# filter_metadata): objects with more active 64^3 voxels are DROPPED at
|
| 30 |
+
# index-build time (never downsampled). 20000 = the single-object SLAT
|
| 31 |
+
# OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
|
| 32 |
+
# 0 disables the filter.
|
| 33 |
+
max_num_voxels: 20000
|
| 34 |
+
min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
|
| 35 |
+
max_views: 4 # clamped per object to its available view count
|
| 36 |
+
val_num_views: 2 # deterministic view count for validation samples
|
| 37 |
+
aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
|
| 38 |
+
# --- legacy hash split (used only when val_roots is empty) ---
|
| 39 |
+
val_fraction: 0.05
|
| 40 |
+
split_seed: 0
|
| 41 |
+
|
| 42 |
+
# --------------------------------------------------------------------------- #
|
| 43 |
+
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
| 44 |
+
# SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
|
| 45 |
+
# with the pipeline's effective inference config (steps=25, cfg_strength=1,
|
| 46 |
+
# cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
|
| 47 |
+
# --------------------------------------------------------------------------- #
|
| 48 |
+
model:
|
| 49 |
+
# Warm-start from the deployed SLAT generator (526/526 tensors, verified).
|
| 50 |
+
pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
|
| 51 |
+
# Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
|
| 52 |
+
# PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
|
| 53 |
+
# 3 toys4k objects), so the bias is ENABLED for training. Set false to
|
| 54 |
+
# reproduce the gating configuration.
|
| 55 |
+
enable_soft_mask: true # local -10 soft-mask in cross-attention
|
| 56 |
+
enable_plucker: true # Plücker ray embedding on non-anchor views
|
| 57 |
+
bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
|
| 58 |
+
# Backbone param dtype: false = lossless fp16->fp32 upcast for training
|
| 59 |
+
# (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
|
| 60 |
+
fp16_torso: false
|
| 61 |
+
# Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
|
| 62 |
+
# .use_checkpoint flip, same toggle the batch-size bench used): recompute
|
| 63 |
+
# activations in backward -> big memory cut, ~modest step-time cost.
|
| 64 |
+
# false = current behavior (no checkpointing).
|
| 65 |
+
use_checkpoint: false
|
| 66 |
+
# Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
|
| 67 |
+
# towers inside are always frozen). false freezes the whole fuser so only
|
| 68 |
+
# the backbone + Plücker FC train.
|
| 69 |
+
train_condition_embedder: true
|
| 70 |
+
|
| 71 |
+
# --------------------------------------------------------------------------- #
|
| 72 |
+
# optim + train loop
|
| 73 |
+
# --------------------------------------------------------------------------- #
|
| 74 |
+
optim:
|
| 75 |
+
lr: 3.0e-5
|
| 76 |
+
weight_decay: 0.0 # AdamW
|
| 77 |
+
grad_clip: 1.0 # max grad norm
|
| 78 |
+
|
| 79 |
+
# --------------------------------------------------------------------------- #
|
| 80 |
+
# ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
|
| 81 |
+
# updated after every optimizer step; frozen DINO towers excluded).
|
| 82 |
+
# Disabled by default = existing behavior unchanged. State is saved as a
|
| 83 |
+
# per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
|
| 84 |
+
# and restored automatically on resume_from.
|
| 85 |
+
# --------------------------------------------------------------------------- #
|
| 86 |
+
ema:
|
| 87 |
+
enable: true
|
| 88 |
+
rate: 0.999 # ema = rate*ema + (1-rate)*param
|
| 89 |
+
eval_with_ema: true # swap EMA weights in for the in-loop validation
|
| 90 |
+
|
| 91 |
+
train:
|
| 92 |
+
# Effective batch = batch_objects x grad_accum_steps x world_size objects
|
| 93 |
+
# per optimizer step.
|
| 94 |
+
# batch_objects = 1 (default): loader bs=1 + collate_single, one object per
|
| 95 |
+
# backbone forward (batching purely via grad_accum_steps).
|
| 96 |
+
# batch_objects > 1: TRUE multi-object batching (collate_batched +
|
| 97 |
+
# SlatFlowModel.flow_step_batched) — B objects concatenated into ONE
|
| 98 |
+
# sparse coord set with the batch-index column, ONE backbone forward,
|
| 99 |
+
# independent t per object. Requires grad_accum_steps == 1.
|
| 100 |
+
steps: 20000
|
| 101 |
+
num_workers: 16 # CPU work: crops + projections + Plücker per view
|
| 102 |
+
batch_objects: 1 # objects per GPU per forward (true batching if >1)
|
| 103 |
+
grad_accum_steps: 8
|
| 104 |
+
# --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
|
| 105 |
+
scheduler: cosine # cosine | constant
|
| 106 |
+
warmup_steps: 500
|
| 107 |
+
min_lr: 1.0e-6
|
| 108 |
+
# --- resume ---
|
| 109 |
+
resume_from: "" # checkpoint path to resume (empty = fresh start)
|
| 110 |
+
log_every: 20
|
| 111 |
+
ckpt_every: 5000
|
| 112 |
+
ckpt_dir: ${MIGRATOR_CACHE}/ckpt_slatflow
|
| 113 |
+
amp: true
|
| 114 |
+
amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
|
| 115 |
+
seed: 0
|
| 116 |
+
# --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
|
| 117 |
+
backend: ddp # ddp | deepspeed
|
| 118 |
+
deepspeed_config: ""
|
| 119 |
+
# --- validation: held-out flow v-MSE only (fixed RNG). Decode +
|
| 120 |
+
# faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
|
| 121 |
+
val_every: 5000 # optimizer steps between validation passes (0 = never)
|
| 122 |
+
val_max_batches: 8 # val objects per rank per pass
|
| 123 |
+
|
| 124 |
+
# --------------------------------------------------------------------------- #
|
| 125 |
+
# wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
|
| 126 |
+
# val/* per validation pass). enabled: false makes every wandb call a no-op.
|
| 127 |
+
# Extra metric dicts can be pushed from anywhere via
|
| 128 |
+
# from mvsam3d.train.train_slat_flow import log_metrics
|
| 129 |
+
# log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
|
| 130 |
+
# --------------------------------------------------------------------------- #
|
| 131 |
+
wandb:
|
| 132 |
+
enabled: true
|
| 133 |
+
project: "mvsam3d-slatflow"
|
| 134 |
+
entity: "alphabet1"
|
| 135 |
+
name: null # null -> W&B autogenerates the run name
|
| 136 |
+
dir: "/lp-dev/jonghoon/mv-mesh/wandb"
|
| 137 |
+
|
| 138 |
+
# --------------------------------------------------------------------------- #
|
| 139 |
+
# val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
|
| 140 |
+
# the trained flow through the exact reported protocol (batch_appforce_sam3d
|
| 141 |
+
# sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
|
| 142 |
+
# OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
|
| 143 |
+
# weights, ALL ranks sample their shard then run the reference-env decode +
|
| 144 |
+
# eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
|
| 145 |
+
# Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
|
| 146 |
+
# tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
|
| 147 |
+
# <stage1_cache>/<name>/2v
|
| 148 |
+
# --------------------------------------------------------------------------- #
|
| 149 |
+
val_appforce:
|
| 150 |
+
enabled: true
|
| 151 |
+
every: 5000
|
| 152 |
+
at_step0: true
|
| 153 |
+
views: 2
|
| 154 |
+
schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
|
| 155 |
+
seed: 42
|
| 156 |
+
limit: 0 # debug: cap objects per dataset
|
| 157 |
+
batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
|
| 158 |
+
single: false # A/B: force the one-object-per-solve path
|
| 159 |
+
decode_procs: 2 # concurrent reference-env decode shards per rank
|
| 160 |
+
ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
|
| 161 |
+
stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
|
| 162 |
+
out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
|
| 163 |
+
ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
|
| 164 |
+
eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
|
| 165 |
+
datasets:
|
| 166 |
+
- name: toys4k100_tex
|
| 167 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
|
| 168 |
+
- name: omni3d
|
| 169 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml
CHANGED
|
@@ -63,20 +63,20 @@ data:
|
|
| 63 |
# Non-empty -> TarSlatFlowDataset reads latents/cond/cameras straight from
|
| 64 |
# the .tar shards (GL->CV flip + alpha-bbox crop); `roots` is ignored. ---
|
| 65 |
tar_roots: []
|
| 66 |
-
# --- condition
|
| 67 |
# use_recgen_images: index recgen/<id>/{cameras.json,NN.jpg,NN_mask.png}
|
| 68 |
-
# (RecGen's own background image + mask + rigid cube cameras) from
|
| 69 |
-
# under tar_roots, joined to latents/<id>.npz by
|
| 70 |
-
#
|
| 71 |
-
#
|
| 72 |
-
#
|
|
|
|
|
|
|
|
|
|
| 73 |
# cond_cameras_fallback: white-cond objects without cameras/<id>/cond_cameras.json
|
| 74 |
# (24-view slat5/slat5rs batches, 22,755 objs) read cond/<id>/transforms.json.
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
use_recgen_images: false
|
| 78 |
-
p_recgen_image: 0.0
|
| 79 |
-
cond_cameras_fallback: false
|
| 80 |
|
| 81 |
# --------------------------------------------------------------------------- #
|
| 82 |
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
|
|
|
| 63 |
# Non-empty -> TarSlatFlowDataset reads latents/cond/cameras straight from
|
| 64 |
# the .tar shards (GL->CV flip + alpha-bbox crop); `roots` is ignored. ---
|
| 65 |
tar_roots: []
|
| 66 |
+
# --- condition sources for the .tar path (2026-09-24) — ON for production ---
|
| 67 |
# use_recgen_images: index recgen/<id>/{cameras.json,NN.jpg,NN_mask.png}
|
| 68 |
+
# (RecGen's own background image + visible mask + rigid cube cameras) from
|
| 69 |
+
# ANY tar under tar_roots, joined to latents/<id>.npz by the same id string.
|
| 70 |
+
# ONE POOL: an object with both sources draws its anchor from the union of
|
| 71 |
+
# its white cond views and RecGen views; the anchor's source decides ALL aux
|
| 72 |
+
# views (never mixed). RecGen anchors go through the same deployed
|
| 73 |
+
# preprocess_slat_image recipe into item["image"] (background kept, alpha =
|
| 74 |
+
# mask). Latent-only RecGen slats become trainable; white-only objects are
|
| 75 |
+
# drawn exactly as before.
|
| 76 |
# cond_cameras_fallback: white-cond objects without cameras/<id>/cond_cameras.json
|
| 77 |
# (24-view slat5/slat5rs batches, 22,755 objs) read cond/<id>/transforms.json.
|
| 78 |
+
use_recgen_images: true
|
| 79 |
+
cond_cameras_fallback: true
|
|
|
|
|
|
|
|
|
|
| 80 |
|
| 81 |
# --------------------------------------------------------------------------- #
|
| 82 |
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_prod.yaml.bak_predatveiw
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
|
| 2 |
+
# ALL training hyperparameters live here; train_slat_flow.py reads them and
|
| 3 |
+
# nothing training-related is hardcoded in Python. Launch:
|
| 4 |
+
# SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
|
| 5 |
+
# python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
|
| 6 |
+
# (multi-GPU: torchrun --nproc_per_node=N ...)
|
| 7 |
+
# Per-key overrides: --set key.subkey=value (repeatable).
|
| 8 |
+
|
| 9 |
+
# --------------------------------------------------------------------------- #
|
| 10 |
+
# data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
|
| 11 |
+
# Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
|
| 12 |
+
# RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
|
| 13 |
+
# stored views.npz). Production roots get views/rgb + views/bbox.npy +
|
| 14 |
+
# slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
|
| 15 |
+
# --------------------------------------------------------------------------- #
|
| 16 |
+
data:
|
| 17 |
+
roots: # merged production set (remote 29.4k + local, DATA_FORMAT via tools/adapt_slat50k.py)
|
| 18 |
+
- /data/mv_mesh_data/slat_train/dataset
|
| 19 |
+
# Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
|
| 20 |
+
# val = ALL objects under val_roots, train = roots minus those ids.
|
| 21 |
+
# NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
|
| 22 |
+
# set is toys4k-only until the production rgb/bbox precompute lands.
|
| 23 |
+
val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
|
| 24 |
+
# --- view-subset augmentation (the ONLY train-time stochasticity besides
|
| 25 |
+
# the fresh invisible-row noise in x0) ---
|
| 26 |
+
# smoke data has 4 views/object; production renders 24 (paper §3.1.1).
|
| 27 |
+
# TRELLIS-style per-object voxel FILTER (structured_latent.py
|
| 28 |
+
# filter_metadata): objects with more active 64^3 voxels are DROPPED at
|
| 29 |
+
# index-build time (never downsampled). 20000 = the single-object SLAT
|
| 30 |
+
# OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
|
| 31 |
+
# 0 disables the filter.
|
| 32 |
+
max_num_voxels: 20000
|
| 33 |
+
min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
|
| 34 |
+
max_views: 4 # clamped per object to its available view count
|
| 35 |
+
val_num_views: 2 # deterministic view count for validation samples
|
| 36 |
+
aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
|
| 37 |
+
# --- legacy hash split (used only when val_roots is empty) ---
|
| 38 |
+
val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
|
| 39 |
+
split_seed: 0
|
| 40 |
+
# --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
|
| 41 |
+
# p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
|
| 42 |
+
# p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
|
| 43 |
+
# BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
|
| 44 |
+
p_seed_none: 0.0
|
| 45 |
+
p_seed_single: 0.0
|
| 46 |
+
# --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
|
| 47 |
+
# (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
|
| 48 |
+
# metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
|
| 49 |
+
# GPU decode-compat test before training on RecGen's RELEASED latents. ---
|
| 50 |
+
recgen_roots: []
|
| 51 |
+
# encoded_store recgen root with canonical cube cameras (pose_cube.npz /
|
| 52 |
+
# views.npz) looked up by sha; needed for CORRECT recgen voxel
|
| 53 |
+
# visibility (raw view_metadata alone is geometrically approximate).
|
| 54 |
+
recgen_pose_store: null
|
| 55 |
+
|
| 56 |
+
# --------------------------------------------------------------------------- #
|
| 57 |
+
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
| 58 |
+
# SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
|
| 59 |
+
# with the pipeline's effective inference config (steps=25, cfg_strength=1,
|
| 60 |
+
# cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
|
| 61 |
+
# --------------------------------------------------------------------------- #
|
| 62 |
+
model:
|
| 63 |
+
# Warm-start from the deployed SLAT generator (526/526 tensors, verified).
|
| 64 |
+
pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
|
| 65 |
+
# Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
|
| 66 |
+
# PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
|
| 67 |
+
# 3 toys4k objects), so the bias is ENABLED for training. Set false to
|
| 68 |
+
# reproduce the gating configuration.
|
| 69 |
+
enable_soft_mask: true # local -10 soft-mask in cross-attention
|
| 70 |
+
enable_plucker: true # Plücker ray embedding on non-anchor views
|
| 71 |
+
bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
|
| 72 |
+
# Backbone param dtype: false = lossless fp16->fp32 upcast for training
|
| 73 |
+
# (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
|
| 74 |
+
fp16_torso: false
|
| 75 |
+
# Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
|
| 76 |
+
# .use_checkpoint flip, same toggle the batch-size bench used): recompute
|
| 77 |
+
# activations in backward -> big memory cut, ~modest step-time cost.
|
| 78 |
+
# false = current behavior (no checkpointing).
|
| 79 |
+
use_checkpoint: false # user choice 2026-09-03: +20% speed; worst case (3x20k-voxel objects) peaks ~69 GB, watchdog resumes on OOM
|
| 80 |
+
# Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
|
| 81 |
+
# towers inside are always frozen). false freezes the whole fuser so only
|
| 82 |
+
# the backbone + Plücker FC train.
|
| 83 |
+
train_condition_embedder: true
|
| 84 |
+
|
| 85 |
+
# --------------------------------------------------------------------------- #
|
| 86 |
+
# optim + train loop
|
| 87 |
+
# --------------------------------------------------------------------------- #
|
| 88 |
+
optim:
|
| 89 |
+
lr: 3.0e-5
|
| 90 |
+
weight_decay: 0.0 # AdamW
|
| 91 |
+
grad_clip: 1.0 # max grad norm
|
| 92 |
+
|
| 93 |
+
# --------------------------------------------------------------------------- #
|
| 94 |
+
# ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
|
| 95 |
+
# updated after every optimizer step; frozen DINO towers excluded).
|
| 96 |
+
# Disabled by default = existing behavior unchanged. State is saved as a
|
| 97 |
+
# per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
|
| 98 |
+
# and restored automatically on resume_from.
|
| 99 |
+
# --------------------------------------------------------------------------- #
|
| 100 |
+
ema:
|
| 101 |
+
enable: true
|
| 102 |
+
rate: 0.999 # ema = rate*ema + (1-rate)*param
|
| 103 |
+
eval_with_ema: true # swap EMA weights in for the in-loop validation
|
| 104 |
+
|
| 105 |
+
train:
|
| 106 |
+
# Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
|
| 107 |
+
# the effective batch is grad_accum_steps x world_size objects per step.
|
| 108 |
+
steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
|
| 109 |
+
num_workers: 16 # CPU work: crops + projections + Plücker per view
|
| 110 |
+
grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
|
| 111 |
+
batch_objects: 3 # objects per GPU per optimizer step (true batched flow step)
|
| 112 |
+
# --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
|
| 113 |
+
scheduler: cosine # cosine | constant
|
| 114 |
+
warmup_steps: 500
|
| 115 |
+
min_lr: 1.0e-6
|
| 116 |
+
# --- resume ---
|
| 117 |
+
resume_from: "" # checkpoint path to resume (empty = fresh start)
|
| 118 |
+
log_every: 20
|
| 119 |
+
ckpt_every: 2500
|
| 120 |
+
ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_prod_20260902 # /data (1.1T free); 40 ckpts x 5 GB model+ema, optimizer sidecars pruned to the last 2
|
| 121 |
+
amp: true
|
| 122 |
+
amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
|
| 123 |
+
seed: 0
|
| 124 |
+
# --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
|
| 125 |
+
backend: ddp # ddp | deepspeed
|
| 126 |
+
deepspeed_config: ""
|
| 127 |
+
# --- validation: held-out flow v-MSE only (fixed RNG). Decode +
|
| 128 |
+
# faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
|
| 129 |
+
val_every: 5000 # optimizer steps between validation passes (0 = never)
|
| 130 |
+
val_max_batches: 8 # val objects per rank per pass
|
| 131 |
+
|
| 132 |
+
# --------------------------------------------------------------------------- #
|
| 133 |
+
# wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
|
| 134 |
+
# val/* per validation pass). enabled: false makes every wandb call a no-op.
|
| 135 |
+
# Extra metric dicts can be pushed from anywhere via
|
| 136 |
+
# from mvsam3d.train.train_slat_flow import log_metrics
|
| 137 |
+
# log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
|
| 138 |
+
# --------------------------------------------------------------------------- #
|
| 139 |
+
wandb:
|
| 140 |
+
enabled: true
|
| 141 |
+
project: "mvsam3d-slatflow"
|
| 142 |
+
entity: "alphabet1"
|
| 143 |
+
name: "slatflow_prod_bs12_lr3e-5_ema0.999_100k"
|
| 144 |
+
dir: "/lp-dev/jonghoon/mv-mesh/wandb"
|
| 145 |
+
|
| 146 |
+
# --------------------------------------------------------------------------- #
|
| 147 |
+
# val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
|
| 148 |
+
# the trained flow through the exact reported protocol (batch_appforce_sam3d
|
| 149 |
+
# sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
|
| 150 |
+
# OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
|
| 151 |
+
# weights, ALL ranks sample their shard then run the reference-env decode +
|
| 152 |
+
# eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
|
| 153 |
+
# Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
|
| 154 |
+
# tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
|
| 155 |
+
# <stage1_cache>/<name>/2v
|
| 156 |
+
# --------------------------------------------------------------------------- #
|
| 157 |
+
val_appforce:
|
| 158 |
+
enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
|
| 159 |
+
every: 5000
|
| 160 |
+
at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
|
| 161 |
+
views: 2
|
| 162 |
+
schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
|
| 163 |
+
seed: 42
|
| 164 |
+
limit: 0 # debug: cap objects per dataset
|
| 165 |
+
batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
|
| 166 |
+
single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
|
| 167 |
+
decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
|
| 168 |
+
ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
|
| 169 |
+
stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
|
| 170 |
+
out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
|
| 171 |
+
ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
|
| 172 |
+
eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
|
| 173 |
+
datasets:
|
| 174 |
+
- name: toys4k100_tex
|
| 175 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
|
| 176 |
+
- name: omni3d
|
| 177 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_2gpu.yaml.bak_predatveiw
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
|
| 2 |
+
# ALL training hyperparameters live here; train_slat_flow.py reads them and
|
| 3 |
+
# nothing training-related is hardcoded in Python. Launch:
|
| 4 |
+
# SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
|
| 5 |
+
# python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
|
| 6 |
+
# (multi-GPU: torchrun --nproc_per_node=N ...)
|
| 7 |
+
# Per-key overrides: --set key.subkey=value (repeatable).
|
| 8 |
+
|
| 9 |
+
# --------------------------------------------------------------------------- #
|
| 10 |
+
# data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
|
| 11 |
+
# Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
|
| 12 |
+
# RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
|
| 13 |
+
# stored views.npz). Production roots get views/rgb + views/bbox.npy +
|
| 14 |
+
# slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
|
| 15 |
+
# --------------------------------------------------------------------------- #
|
| 16 |
+
data:
|
| 17 |
+
roots: # POOL for the 20k->30k resume (user 2026-09-07): objects present at the 20k stop + 5,000 randomly sampled NEW slats (local+HF); lists in .debug/slat_prod/pool_{base,new5k}_shas.txt
|
| 18 |
+
- /data/mv_mesh_data/slat_train/dataset_pool_20k_plus5k
|
| 19 |
+
# Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
|
| 20 |
+
# val = ALL objects under val_roots, train = roots minus those ids.
|
| 21 |
+
# NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
|
| 22 |
+
# set is toys4k-only until the production rgb/bbox precompute lands.
|
| 23 |
+
val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
|
| 24 |
+
# --- view-subset augmentation (the ONLY train-time stochasticity besides
|
| 25 |
+
# the fresh invisible-row noise in x0) ---
|
| 26 |
+
# smoke data has 4 views/object; production renders 24 (paper §3.1.1).
|
| 27 |
+
# TRELLIS-style per-object voxel FILTER (structured_latent.py
|
| 28 |
+
# filter_metadata): objects with more active 64^3 voxels are DROPPED at
|
| 29 |
+
# index-build time (never downsampled). 20000 = the single-object SLAT
|
| 30 |
+
# OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
|
| 31 |
+
# 0 disables the filter.
|
| 32 |
+
max_num_voxels: 20000
|
| 33 |
+
min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
|
| 34 |
+
max_views: 4 # clamped per object to its available view count
|
| 35 |
+
val_num_views: 2 # deterministic view count for validation samples
|
| 36 |
+
aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
|
| 37 |
+
# --- legacy hash split (used only when val_roots is empty) ---
|
| 38 |
+
val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
|
| 39 |
+
split_seed: 0
|
| 40 |
+
# --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
|
| 41 |
+
# p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
|
| 42 |
+
# p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
|
| 43 |
+
# BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
|
| 44 |
+
p_seed_none: 0.0
|
| 45 |
+
p_seed_single: 0.0
|
| 46 |
+
# --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
|
| 47 |
+
# (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
|
| 48 |
+
# metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
|
| 49 |
+
# GPU decode-compat test before training on RecGen's RELEASED latents. ---
|
| 50 |
+
recgen_roots: []
|
| 51 |
+
# encoded_store recgen root with canonical cube cameras (pose_cube.npz /
|
| 52 |
+
# views.npz) looked up by sha; needed for CORRECT recgen voxel
|
| 53 |
+
# visibility (raw view_metadata alone is geometrically approximate).
|
| 54 |
+
recgen_pose_store: null
|
| 55 |
+
|
| 56 |
+
# --------------------------------------------------------------------------- #
|
| 57 |
+
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
| 58 |
+
# SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
|
| 59 |
+
# with the pipeline's effective inference config (steps=25, cfg_strength=1,
|
| 60 |
+
# cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
|
| 61 |
+
# --------------------------------------------------------------------------- #
|
| 62 |
+
model:
|
| 63 |
+
# Warm-start from the deployed SLAT generator (526/526 tensors, verified).
|
| 64 |
+
pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
|
| 65 |
+
# Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
|
| 66 |
+
# PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
|
| 67 |
+
# 3 toys4k objects), so the bias is ENABLED for training. Set false to
|
| 68 |
+
# reproduce the gating configuration.
|
| 69 |
+
enable_soft_mask: true # local -10 soft-mask in cross-attention
|
| 70 |
+
enable_plucker: true # Plücker ray embedding on non-anchor views
|
| 71 |
+
bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
|
| 72 |
+
# Backbone param dtype: false = lossless fp16->fp32 upcast for training
|
| 73 |
+
# (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
|
| 74 |
+
fp16_torso: false
|
| 75 |
+
# Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
|
| 76 |
+
# .use_checkpoint flip, same toggle the batch-size bench used): recompute
|
| 77 |
+
# activations in backward -> big memory cut, ~modest step-time cost.
|
| 78 |
+
# false = current behavior (no checkpointing).
|
| 79 |
+
use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
|
| 80 |
+
# Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
|
| 81 |
+
# towers inside are always frozen). false freezes the whole fuser so only
|
| 82 |
+
# the backbone + Plücker FC train.
|
| 83 |
+
train_condition_embedder: true
|
| 84 |
+
|
| 85 |
+
# --------------------------------------------------------------------------- #
|
| 86 |
+
# optim + train loop
|
| 87 |
+
# --------------------------------------------------------------------------- #
|
| 88 |
+
optim:
|
| 89 |
+
lr: 3.0e-5
|
| 90 |
+
weight_decay: 0.0 # AdamW
|
| 91 |
+
grad_clip: 1.0 # max grad norm
|
| 92 |
+
|
| 93 |
+
# --------------------------------------------------------------------------- #
|
| 94 |
+
# ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
|
| 95 |
+
# updated after every optimizer step; frozen DINO towers excluded).
|
| 96 |
+
# Disabled by default = existing behavior unchanged. State is saved as a
|
| 97 |
+
# per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
|
| 98 |
+
# and restored automatically on resume_from.
|
| 99 |
+
# --------------------------------------------------------------------------- #
|
| 100 |
+
ema:
|
| 101 |
+
enable: true
|
| 102 |
+
rate: 0.999 # ema = rate*ema + (1-rate)*param
|
| 103 |
+
eval_with_ema: true # swap EMA weights in for the in-loop validation
|
| 104 |
+
|
| 105 |
+
train:
|
| 106 |
+
# Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
|
| 107 |
+
# the effective batch is grad_accum_steps x world_size objects per step.
|
| 108 |
+
steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
|
| 109 |
+
num_workers: 16 # CPU work: crops + projections + Plücker per view
|
| 110 |
+
grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
|
| 111 |
+
batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
|
| 112 |
+
# --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
|
| 113 |
+
scheduler: cosine # cosine | constant
|
| 114 |
+
warmup_steps: 500
|
| 115 |
+
min_lr: 1.0e-6
|
| 116 |
+
# --- resume ---
|
| 117 |
+
resume_from: "/data/mv_mesh_data/ckpt/slatflow_prod_20260902/step_0020000.pt" # resume the 20k prod ckpt (+ema/optimizer sidecars)
|
| 118 |
+
log_every: 20
|
| 119 |
+
ckpt_every: 2500
|
| 120 |
+
ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume20k_2gpu
|
| 121 |
+
amp: true
|
| 122 |
+
amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
|
| 123 |
+
seed: 0
|
| 124 |
+
# --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
|
| 125 |
+
backend: ddp # ddp | deepspeed
|
| 126 |
+
deepspeed_config: ""
|
| 127 |
+
# --- validation: held-out flow v-MSE only (fixed RNG). Decode +
|
| 128 |
+
# faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
|
| 129 |
+
val_every: 5000 # optimizer steps between validation passes (0 = never)
|
| 130 |
+
val_max_batches: 8 # val objects per rank per pass
|
| 131 |
+
|
| 132 |
+
# --------------------------------------------------------------------------- #
|
| 133 |
+
# wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
|
| 134 |
+
# val/* per validation pass). enabled: false makes every wandb call a no-op.
|
| 135 |
+
# Extra metric dicts can be pushed from anywhere via
|
| 136 |
+
# from mvsam3d.train.train_slat_flow import log_metrics
|
| 137 |
+
# log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
|
| 138 |
+
# --------------------------------------------------------------------------- #
|
| 139 |
+
wandb:
|
| 140 |
+
enabled: true
|
| 141 |
+
project: "mvsam3d-slatflow"
|
| 142 |
+
entity: "alphabet1"
|
| 143 |
+
name: "slatflow_resume20k_2gpu_bs12_pool55k"
|
| 144 |
+
dir: "/lp-dev/jonghoon/mv-mesh/wandb"
|
| 145 |
+
|
| 146 |
+
# --------------------------------------------------------------------------- #
|
| 147 |
+
# val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
|
| 148 |
+
# the trained flow through the exact reported protocol (batch_appforce_sam3d
|
| 149 |
+
# sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
|
| 150 |
+
# OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
|
| 151 |
+
# weights, ALL ranks sample their shard then run the reference-env decode +
|
| 152 |
+
# eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
|
| 153 |
+
# Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
|
| 154 |
+
# tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
|
| 155 |
+
# <stage1_cache>/<name>/2v
|
| 156 |
+
# --------------------------------------------------------------------------- #
|
| 157 |
+
val_appforce:
|
| 158 |
+
enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
|
| 159 |
+
every: 5000
|
| 160 |
+
at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
|
| 161 |
+
views: 2
|
| 162 |
+
schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
|
| 163 |
+
seed: 42
|
| 164 |
+
limit: 0 # debug: cap objects per dataset
|
| 165 |
+
batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
|
| 166 |
+
single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
|
| 167 |
+
decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
|
| 168 |
+
ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
|
| 169 |
+
stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
|
| 170 |
+
out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
|
| 171 |
+
ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
|
| 172 |
+
eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
|
| 173 |
+
datasets:
|
| 174 |
+
- name: toys4k100_tex
|
| 175 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
|
| 176 |
+
- name: omni3d
|
| 177 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume20k_pslaug_2gpu.yaml.bak_predatveiw
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
|
| 2 |
+
# ALL training hyperparameters live here; train_slat_flow.py reads them and
|
| 3 |
+
# nothing training-related is hardcoded in Python. Launch:
|
| 4 |
+
# SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
|
| 5 |
+
# python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
|
| 6 |
+
# (multi-GPU: torchrun --nproc_per_node=N ...)
|
| 7 |
+
# Per-key overrides: --set key.subkey=value (repeatable).
|
| 8 |
+
|
| 9 |
+
# --------------------------------------------------------------------------- #
|
| 10 |
+
# data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
|
| 11 |
+
# Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
|
| 12 |
+
# RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
|
| 13 |
+
# stored views.npz). Production roots get views/rgb + views/bbox.npy +
|
| 14 |
+
# slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
|
| 15 |
+
# --------------------------------------------------------------------------- #
|
| 16 |
+
data:
|
| 17 |
+
roots: # POOL for the 20k->30k PSL-AUG resume (user 2026-09-07):
|
| 18 |
+
# (a) the 50,621 base-pool objects present at the 20k stop
|
| 19 |
+
# (.debug/slat_prod/pool_base_shas.txt), and
|
| 20 |
+
# (b) ALL new PSL-augmentation objects from
|
| 21 |
+
# /data/mv_mesh_data/psl_aug/objects (psl_aug_shas.txt).
|
| 22 |
+
# The pool root is symlink-only, built by
|
| 23 |
+
# /lp-dev/jonghoon/mv-sam3d-6d-code/build_pool_base_pslaug.sh
|
| 24 |
+
# as <root>/{remote,local,pslaug}/objects/<sha[:2]>/<sha>.
|
| 25 |
+
# psl_aug objects carry NO special weighting: scan_objects
|
| 26 |
+
# flattens all three sources into one uniformly-sampled index.
|
| 27 |
+
- /data/mv_mesh_data/slat_train/dataset_pool_base_pslaug
|
| 28 |
+
# Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
|
| 29 |
+
# val = ALL objects under val_roots, train = roots minus those ids.
|
| 30 |
+
# NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
|
| 31 |
+
# set is toys4k-only until the production rgb/bbox precompute lands.
|
| 32 |
+
val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
|
| 33 |
+
# --- view-subset augmentation (the ONLY train-time stochasticity besides
|
| 34 |
+
# the fresh invisible-row noise in x0) ---
|
| 35 |
+
# smoke data has 4 views/object; production renders 24 (paper §3.1.1).
|
| 36 |
+
# TRELLIS-style per-object voxel FILTER (structured_latent.py
|
| 37 |
+
# filter_metadata): objects with more active 64^3 voxels are DROPPED at
|
| 38 |
+
# index-build time (never downsampled). 20000 = the single-object SLAT
|
| 39 |
+
# OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
|
| 40 |
+
# 0 disables the filter.
|
| 41 |
+
max_num_voxels: 20000
|
| 42 |
+
min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
|
| 43 |
+
max_views: 4 # clamped per object to its available view count
|
| 44 |
+
val_num_views: 2 # deterministic view count for validation samples
|
| 45 |
+
aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
|
| 46 |
+
# --- legacy hash split (used only when val_roots is empty) ---
|
| 47 |
+
val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
|
| 48 |
+
split_seed: 0
|
| 49 |
+
# --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
|
| 50 |
+
# p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
|
| 51 |
+
# p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
|
| 52 |
+
# BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
|
| 53 |
+
p_seed_none: 0.0
|
| 54 |
+
p_seed_single: 0.0
|
| 55 |
+
# --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
|
| 56 |
+
# (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
|
| 57 |
+
# metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
|
| 58 |
+
# GPU decode-compat test before training on RecGen's RELEASED latents. ---
|
| 59 |
+
recgen_roots: []
|
| 60 |
+
# encoded_store recgen root with canonical cube cameras (pose_cube.npz /
|
| 61 |
+
# views.npz) looked up by sha; needed for CORRECT recgen voxel
|
| 62 |
+
# visibility (raw view_metadata alone is geometrically approximate).
|
| 63 |
+
recgen_pose_store: null
|
| 64 |
+
|
| 65 |
+
# --------------------------------------------------------------------------- #
|
| 66 |
+
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
| 67 |
+
# SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
|
| 68 |
+
# with the pipeline's effective inference config (steps=25, cfg_strength=1,
|
| 69 |
+
# cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
|
| 70 |
+
# --------------------------------------------------------------------------- #
|
| 71 |
+
model:
|
| 72 |
+
# Warm-start from the deployed SLAT generator (526/526 tensors, verified).
|
| 73 |
+
pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
|
| 74 |
+
# Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
|
| 75 |
+
# PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
|
| 76 |
+
# 3 toys4k objects), so the bias is ENABLED for training. Set false to
|
| 77 |
+
# reproduce the gating configuration.
|
| 78 |
+
enable_soft_mask: true # local -10 soft-mask in cross-attention
|
| 79 |
+
enable_plucker: true # Plücker ray embedding on non-anchor views
|
| 80 |
+
bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
|
| 81 |
+
# Backbone param dtype: false = lossless fp16->fp32 upcast for training
|
| 82 |
+
# (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
|
| 83 |
+
fp16_torso: false
|
| 84 |
+
# Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
|
| 85 |
+
# .use_checkpoint flip, same toggle the batch-size bench used): recompute
|
| 86 |
+
# activations in backward -> big memory cut, ~modest step-time cost.
|
| 87 |
+
# false = current behavior (no checkpointing).
|
| 88 |
+
use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
|
| 89 |
+
# Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
|
| 90 |
+
# towers inside are always frozen). false freezes the whole fuser so only
|
| 91 |
+
# the backbone + Plücker FC train.
|
| 92 |
+
train_condition_embedder: true
|
| 93 |
+
|
| 94 |
+
# --------------------------------------------------------------------------- #
|
| 95 |
+
# optim + train loop
|
| 96 |
+
# --------------------------------------------------------------------------- #
|
| 97 |
+
optim:
|
| 98 |
+
lr: 3.0e-5
|
| 99 |
+
weight_decay: 0.0 # AdamW
|
| 100 |
+
grad_clip: 1.0 # max grad norm
|
| 101 |
+
|
| 102 |
+
# --------------------------------------------------------------------------- #
|
| 103 |
+
# ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
|
| 104 |
+
# updated after every optimizer step; frozen DINO towers excluded).
|
| 105 |
+
# Disabled by default = existing behavior unchanged. State is saved as a
|
| 106 |
+
# per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
|
| 107 |
+
# and restored automatically on resume_from.
|
| 108 |
+
# --------------------------------------------------------------------------- #
|
| 109 |
+
ema:
|
| 110 |
+
enable: true
|
| 111 |
+
rate: 0.999 # ema = rate*ema + (1-rate)*param
|
| 112 |
+
eval_with_ema: true # swap EMA weights in for the in-loop validation
|
| 113 |
+
|
| 114 |
+
train:
|
| 115 |
+
# Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
|
| 116 |
+
# the effective batch is grad_accum_steps x world_size objects per step.
|
| 117 |
+
steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
|
| 118 |
+
num_workers: 16 # CPU work: crops + projections + Plücker per view
|
| 119 |
+
grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
|
| 120 |
+
batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
|
| 121 |
+
# --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
|
| 122 |
+
scheduler: cosine # cosine | constant
|
| 123 |
+
warmup_steps: 500
|
| 124 |
+
min_lr: 1.0e-6
|
| 125 |
+
# --- resume ---
|
| 126 |
+
resume_from: "/data/mv_mesh_data/ckpt/slatflow_prod_20260902/step_0020000.pt" # resume the 20k prod ckpt (+ema/optimizer sidecars)
|
| 127 |
+
log_every: 20
|
| 128 |
+
ckpt_every: 2500
|
| 129 |
+
ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume20k_pslaug_2gpu
|
| 130 |
+
amp: true
|
| 131 |
+
amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
|
| 132 |
+
seed: 0
|
| 133 |
+
# --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
|
| 134 |
+
backend: ddp # ddp | deepspeed
|
| 135 |
+
deepspeed_config: ""
|
| 136 |
+
# --- validation: held-out flow v-MSE only (fixed RNG). Decode +
|
| 137 |
+
# faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
|
| 138 |
+
val_every: 5000 # optimizer steps between validation passes (0 = never)
|
| 139 |
+
val_max_batches: 8 # val objects per rank per pass
|
| 140 |
+
|
| 141 |
+
# --------------------------------------------------------------------------- #
|
| 142 |
+
# wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
|
| 143 |
+
# val/* per validation pass). enabled: false makes every wandb call a no-op.
|
| 144 |
+
# Extra metric dicts can be pushed from anywhere via
|
| 145 |
+
# from mvsam3d.train.train_slat_flow import log_metrics
|
| 146 |
+
# log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
|
| 147 |
+
# --------------------------------------------------------------------------- #
|
| 148 |
+
wandb:
|
| 149 |
+
enabled: true
|
| 150 |
+
project: "mvsam3d-slatflow"
|
| 151 |
+
entity: "alphabet1"
|
| 152 |
+
name: "slatflow_resume20k_pslaug_bs12"
|
| 153 |
+
dir: "/lp-dev/jonghoon/mv-mesh/wandb"
|
| 154 |
+
|
| 155 |
+
# --------------------------------------------------------------------------- #
|
| 156 |
+
# val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
|
| 157 |
+
# the trained flow through the exact reported protocol (batch_appforce_sam3d
|
| 158 |
+
# sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
|
| 159 |
+
# OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
|
| 160 |
+
# weights, ALL ranks sample their shard then run the reference-env decode +
|
| 161 |
+
# eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
|
| 162 |
+
# Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
|
| 163 |
+
# tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
|
| 164 |
+
# <stage1_cache>/<name>/2v
|
| 165 |
+
# --------------------------------------------------------------------------- #
|
| 166 |
+
val_appforce:
|
| 167 |
+
enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
|
| 168 |
+
every: 5000
|
| 169 |
+
at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
|
| 170 |
+
views: 2
|
| 171 |
+
schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
|
| 172 |
+
seed: 42
|
| 173 |
+
limit: 0 # debug: cap objects per dataset
|
| 174 |
+
batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
|
| 175 |
+
single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
|
| 176 |
+
decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
|
| 177 |
+
ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
|
| 178 |
+
stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
|
| 179 |
+
out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
|
| 180 |
+
ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
|
| 181 |
+
eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
|
| 182 |
+
datasets:
|
| 183 |
+
- name: toys4k100_tex
|
| 184 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
|
| 185 |
+
- name: omni3d
|
| 186 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslfinal_2gpu.yaml.bak_predatveiw
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
|
| 2 |
+
# ALL training hyperparameters live here; train_slat_flow.py reads them and
|
| 3 |
+
# nothing training-related is hardcoded in Python. Launch:
|
| 4 |
+
# SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
|
| 5 |
+
# python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
|
| 6 |
+
# (multi-GPU: torchrun --nproc_per_node=N ...)
|
| 7 |
+
# Per-key overrides: --set key.subkey=value (repeatable).
|
| 8 |
+
|
| 9 |
+
# --------------------------------------------------------------------------- #
|
| 10 |
+
# data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
|
| 11 |
+
# Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
|
| 12 |
+
# RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
|
| 13 |
+
# stored views.npz). Production roots get views/rgb + views/bbox.npy +
|
| 14 |
+
# slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
|
| 15 |
+
# --------------------------------------------------------------------------- #
|
| 16 |
+
data:
|
| 17 |
+
roots: # POOL for the 22.5k->30k PSL-FINAL resume (user 2026-09-08):
|
| 18 |
+
# (a) the 50,621 base-pool objects present at the 20k stop
|
| 19 |
+
# (.debug/slat_prod/pool_base_shas.txt), and
|
| 20 |
+
# (b) the MERGED PSL set listed in
|
| 21 |
+
# /data/mv_mesh_data/psl_aug_occ/psl_final_shas.txt
|
| 22 |
+
# ('<set>\t<sha>': occ -> psl_aug_occ/objects, the
|
| 23 |
+
# occlusion-heavy replacement set; aug -> psl_aug/objects,
|
| 24 |
+
# the earlier whole-view set).
|
| 25 |
+
# Symlink-only root built by
|
| 26 |
+
# /lp-dev/jonghoon/mv-sam3d-6d-code/build_pool_base_pslfinal.sh
|
| 27 |
+
# as <root>/{remote,local,pslocc,pslaug}/objects/<sha[:2]>/<sha>.
|
| 28 |
+
# No special weighting: one uniformly-sampled index.
|
| 29 |
+
- /data/mv_mesh_data/slat_train/dataset_pool_base_pslfinal
|
| 30 |
+
# Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
|
| 31 |
+
# val = ALL objects under val_roots, train = roots minus those ids.
|
| 32 |
+
# NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
|
| 33 |
+
# set is toys4k-only until the production rgb/bbox precompute lands.
|
| 34 |
+
val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
|
| 35 |
+
# --- view-subset augmentation (the ONLY train-time stochasticity besides
|
| 36 |
+
# the fresh invisible-row noise in x0) ---
|
| 37 |
+
# smoke data has 4 views/object; production renders 24 (paper §3.1.1).
|
| 38 |
+
# TRELLIS-style per-object voxel FILTER (structured_latent.py
|
| 39 |
+
# filter_metadata): objects with more active 64^3 voxels are DROPPED at
|
| 40 |
+
# index-build time (never downsampled). 20000 = the single-object SLAT
|
| 41 |
+
# OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
|
| 42 |
+
# 0 disables the filter.
|
| 43 |
+
max_num_voxels: 20000
|
| 44 |
+
min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
|
| 45 |
+
max_views: 4 # clamped per object to its available view count
|
| 46 |
+
val_num_views: 2 # deterministic view count for validation samples
|
| 47 |
+
aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
|
| 48 |
+
# --- legacy hash split (used only when val_roots is empty) ---
|
| 49 |
+
val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
|
| 50 |
+
split_seed: 0
|
| 51 |
+
# --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
|
| 52 |
+
# p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
|
| 53 |
+
# p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
|
| 54 |
+
# BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
|
| 55 |
+
p_seed_none: 0.0
|
| 56 |
+
p_seed_single: 0.0
|
| 57 |
+
# --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
|
| 58 |
+
# (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
|
| 59 |
+
# metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
|
| 60 |
+
# GPU decode-compat test before training on RecGen's RELEASED latents. ---
|
| 61 |
+
recgen_roots: []
|
| 62 |
+
# encoded_store recgen root with canonical cube cameras (pose_cube.npz /
|
| 63 |
+
# views.npz) looked up by sha; needed for CORRECT recgen voxel
|
| 64 |
+
# visibility (raw view_metadata alone is geometrically approximate).
|
| 65 |
+
recgen_pose_store: null
|
| 66 |
+
|
| 67 |
+
# --------------------------------------------------------------------------- #
|
| 68 |
+
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
| 69 |
+
# SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
|
| 70 |
+
# with the pipeline's effective inference config (steps=25, cfg_strength=1,
|
| 71 |
+
# cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
|
| 72 |
+
# --------------------------------------------------------------------------- #
|
| 73 |
+
model:
|
| 74 |
+
# Warm-start from the deployed SLAT generator (526/526 tensors, verified).
|
| 75 |
+
pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
|
| 76 |
+
# Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
|
| 77 |
+
# PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
|
| 78 |
+
# 3 toys4k objects), so the bias is ENABLED for training. Set false to
|
| 79 |
+
# reproduce the gating configuration.
|
| 80 |
+
enable_soft_mask: true # local -10 soft-mask in cross-attention
|
| 81 |
+
enable_plucker: true # Plücker ray embedding on non-anchor views
|
| 82 |
+
bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
|
| 83 |
+
# Backbone param dtype: false = lossless fp16->fp32 upcast for training
|
| 84 |
+
# (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
|
| 85 |
+
fp16_torso: false
|
| 86 |
+
# Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
|
| 87 |
+
# .use_checkpoint flip, same toggle the batch-size bench used): recompute
|
| 88 |
+
# activations in backward -> big memory cut, ~modest step-time cost.
|
| 89 |
+
# false = current behavior (no checkpointing).
|
| 90 |
+
use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
|
| 91 |
+
# Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
|
| 92 |
+
# towers inside are always frozen). false freezes the whole fuser so only
|
| 93 |
+
# the backbone + Plücker FC train.
|
| 94 |
+
train_condition_embedder: true
|
| 95 |
+
|
| 96 |
+
# --------------------------------------------------------------------------- #
|
| 97 |
+
# optim + train loop
|
| 98 |
+
# --------------------------------------------------------------------------- #
|
| 99 |
+
optim:
|
| 100 |
+
lr: 3.0e-5
|
| 101 |
+
weight_decay: 0.0 # AdamW
|
| 102 |
+
grad_clip: 1.0 # max grad norm
|
| 103 |
+
|
| 104 |
+
# --------------------------------------------------------------------------- #
|
| 105 |
+
# ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
|
| 106 |
+
# updated after every optimizer step; frozen DINO towers excluded).
|
| 107 |
+
# Disabled by default = existing behavior unchanged. State is saved as a
|
| 108 |
+
# per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
|
| 109 |
+
# and restored automatically on resume_from.
|
| 110 |
+
# --------------------------------------------------------------------------- #
|
| 111 |
+
ema:
|
| 112 |
+
enable: true
|
| 113 |
+
rate: 0.999 # ema = rate*ema + (1-rate)*param
|
| 114 |
+
eval_with_ema: true # swap EMA weights in for the in-loop validation
|
| 115 |
+
|
| 116 |
+
train:
|
| 117 |
+
# Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
|
| 118 |
+
# the effective batch is grad_accum_steps x world_size objects per step.
|
| 119 |
+
steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
|
| 120 |
+
num_workers: 16 # CPU work: crops + projections + Plücker per view
|
| 121 |
+
grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
|
| 122 |
+
batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
|
| 123 |
+
# --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
|
| 124 |
+
scheduler: cosine # cosine | constant
|
| 125 |
+
warmup_steps: 500
|
| 126 |
+
min_lr: 1.0e-6
|
| 127 |
+
# --- resume ---
|
| 128 |
+
resume_from: "/data/mv_mesh_data/ckpt/slatflow_resume20k_pslaug_2gpu/step_0022500.pt" # resume the 22.5k pslaug ckpt (+ema/optimizer sidecars); pool swapped to base + merged PSL final set
|
| 129 |
+
log_every: 20
|
| 130 |
+
ckpt_every: 2500
|
| 131 |
+
ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume22p5k_pslfinal_2gpu
|
| 132 |
+
amp: true
|
| 133 |
+
amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
|
| 134 |
+
seed: 0
|
| 135 |
+
# --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
|
| 136 |
+
backend: ddp # ddp | deepspeed
|
| 137 |
+
deepspeed_config: ""
|
| 138 |
+
# --- validation: held-out flow v-MSE only (fixed RNG). Decode +
|
| 139 |
+
# faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
|
| 140 |
+
val_every: 5000 # optimizer steps between validation passes (0 = never)
|
| 141 |
+
val_max_batches: 8 # val objects per rank per pass
|
| 142 |
+
|
| 143 |
+
# --------------------------------------------------------------------------- #
|
| 144 |
+
# wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
|
| 145 |
+
# val/* per validation pass). enabled: false makes every wandb call a no-op.
|
| 146 |
+
# Extra metric dicts can be pushed from anywhere via
|
| 147 |
+
# from mvsam3d.train.train_slat_flow import log_metrics
|
| 148 |
+
# log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
|
| 149 |
+
# --------------------------------------------------------------------------- #
|
| 150 |
+
wandb:
|
| 151 |
+
enabled: true
|
| 152 |
+
project: "mvsam3d-slatflow"
|
| 153 |
+
entity: "alphabet1"
|
| 154 |
+
name: "slatflow_resume22p5k_pslfinal_bs12"
|
| 155 |
+
dir: "/lp-dev/jonghoon/mv-mesh/wandb"
|
| 156 |
+
|
| 157 |
+
# --------------------------------------------------------------------------- #
|
| 158 |
+
# val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
|
| 159 |
+
# the trained flow through the exact reported protocol (batch_appforce_sam3d
|
| 160 |
+
# sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
|
| 161 |
+
# OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
|
| 162 |
+
# weights, ALL ranks sample their shard then run the reference-env decode +
|
| 163 |
+
# eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
|
| 164 |
+
# Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
|
| 165 |
+
# tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
|
| 166 |
+
# <stage1_cache>/<name>/2v
|
| 167 |
+
# --------------------------------------------------------------------------- #
|
| 168 |
+
val_appforce:
|
| 169 |
+
enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
|
| 170 |
+
every: 5000
|
| 171 |
+
at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
|
| 172 |
+
views: 2
|
| 173 |
+
schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
|
| 174 |
+
seed: 42
|
| 175 |
+
limit: 0 # debug: cap objects per dataset
|
| 176 |
+
batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
|
| 177 |
+
single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
|
| 178 |
+
decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
|
| 179 |
+
ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
|
| 180 |
+
stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
|
| 181 |
+
out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
|
| 182 |
+
ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
|
| 183 |
+
eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
|
| 184 |
+
datasets:
|
| 185 |
+
- name: toys4k100_tex
|
| 186 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
|
| 187 |
+
- name: omni3d
|
| 188 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/configs/train_slatflow_resume22p5k_pslocc_2gpu.yaml.bak_predatveiw
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training config for the multi-view SLAT flow (SlatFlowModel, SLAT_GEN_PLAN).
|
| 2 |
+
# ALL training hyperparameters live here; train_slat_flow.py reads them and
|
| 3 |
+
# nothing training-related is hardcoded in Python. Launch:
|
| 4 |
+
# SPARSE_ATTN_BACKEND=sdpa PYTHONPATH=. \
|
| 5 |
+
# python mvsam3d/train/train_slat_flow.py --config configs/train_slatflow.yaml
|
| 6 |
+
# (multi-GPU: torchrun --nproc_per_node=N ...)
|
| 7 |
+
# Per-key overrides: --set key.subkey=value (repeatable).
|
| 8 |
+
|
| 9 |
+
# --------------------------------------------------------------------------- #
|
| 10 |
+
# data -> TrellisSlatFlowDataset (DATA_FORMAT.md layout + per-view RGB assets).
|
| 11 |
+
# Smoke: toys4k200 objects with the toys4k1k inputs/renders fallback for
|
| 12 |
+
# RGB+bbox (V=4 views: front/side/oside/back — camera-verified identical to the
|
| 13 |
+
# stored views.npz). Production roots get views/rgb + views/bbox.npy +
|
| 14 |
+
# slat/slat_sam3d.npz per SLAT_GEN_PLAN §8 (precompute jobs pending).
|
| 15 |
+
# --------------------------------------------------------------------------- #
|
| 16 |
+
data:
|
| 17 |
+
roots: # POOL for the 22.5k->30k PSL-OCCLUSION resume (user 2026-09-08):
|
| 18 |
+
# (a) the 50,621 base-pool objects present at the 20k stop
|
| 19 |
+
# (.debug/slat_prod/pool_base_shas.txt), and
|
| 20 |
+
# (b) ALL objects of the REPLACEMENT occlusion set
|
| 21 |
+
# /data/mv_mesh_data/psl_aug_occ/objects (3-4 whole +
|
| 22 |
+
# 4-5 occluded views per sample; psl_aug_occ_shas.txt).
|
| 23 |
+
# The earlier /data/mv_mesh_data/psl_aug set is NOT included.
|
| 24 |
+
# Symlink-only root built by
|
| 25 |
+
# /lp-dev/jonghoon/mv-sam3d-6d-code/build_pool_base_pslocc.sh
|
| 26 |
+
# as <root>/{remote,local,pslocc}/objects/<sha[:2]>/<sha>.
|
| 27 |
+
# No special weighting: one uniformly-sampled index.
|
| 28 |
+
- /data/mv_mesh_data/slat_train/dataset_pool_base_pslocc
|
| 29 |
+
# Explicit validation roots (same policy as train_ssflow.yaml): non-empty ->
|
| 30 |
+
# val = ALL objects under val_roots, train = roots minus those ids.
|
| 31 |
+
# NOTE: val_omni3d has no toys4k1k RGB fallback assets yet, so the smoke val
|
| 32 |
+
# set is toys4k-only until the production rgb/bbox precompute lands.
|
| 33 |
+
val_roots: [] # smoke val set has NO official x1 -> flow-loss val would crash; use a hash split of the production roots instead
|
| 34 |
+
# --- view-subset augmentation (the ONLY train-time stochasticity besides
|
| 35 |
+
# the fresh invisible-row noise in x0) ---
|
| 36 |
+
# smoke data has 4 views/object; production renders 24 (paper §3.1.1).
|
| 37 |
+
# TRELLIS-style per-object voxel FILTER (structured_latent.py
|
| 38 |
+
# filter_metadata): objects with more active 64^3 voxels are DROPPED at
|
| 39 |
+
# index-build time (never downsampled). 20000 = the single-object SLAT
|
| 40 |
+
# OOM-safe bound (PLAN/BATCH_SIZE_BENCH.md; drops ~4.8% of trellis500k).
|
| 41 |
+
# 0 disables the filter.
|
| 42 |
+
max_num_voxels: 20000
|
| 43 |
+
min_views: 1 # |S| ~ Uniform{min_views..max_views} per sample
|
| 44 |
+
max_views: 4 # clamped per object to its available view count
|
| 45 |
+
val_num_views: 2 # deterministic view count for validation samples
|
| 46 |
+
aug_seed: 0 # seeds the per-(epoch,idx) view-subset RNG
|
| 47 |
+
# --- legacy hash split (used only when val_roots is empty) ---
|
| 48 |
+
val_fraction: 0.005 # ~185 held-out production objects for the flow-loss val
|
| 49 |
+
split_seed: 0
|
| 50 |
+
# --- SS-stage seed MIXTURE (unified none/single/multi ratio) ---
|
| 51 |
+
# p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
|
| 52 |
+
# p_seed_single = P(single-view seed -> n_sub == 1); remainder = multi.
|
| 53 |
+
# BOTH 0.0 (default) -> LEGACY n_sub ~ U{min_views..max_views}, unchanged.
|
| 54 |
+
p_seed_none: 0.0
|
| 55 |
+
p_seed_single: 0.0
|
| 56 |
+
# --- RecGen loader (additive / opt-in): dirs of released-SLAT objects
|
| 57 |
+
# (slat_coords.npy + slat_feats.npy + cond_image_*/cond_mask_* + camera
|
| 58 |
+
# metadata), fed to SLAT via matting. Empty = disabled. CAVEAT: run a
|
| 59 |
+
# GPU decode-compat test before training on RecGen's RELEASED latents. ---
|
| 60 |
+
recgen_roots: []
|
| 61 |
+
# encoded_store recgen root with canonical cube cameras (pose_cube.npz /
|
| 62 |
+
# views.npz) looked up by sha; needed for CORRECT recgen voxel
|
| 63 |
+
# visibility (raw view_metadata alone is geometrically approximate).
|
| 64 |
+
recgen_pose_store: null
|
| 65 |
+
|
| 66 |
+
# --------------------------------------------------------------------------- #
|
| 67 |
+
# model -> SlatFlowModel. The generator stack (FlowMatching -> CFG ->
|
| 68 |
+
# SLatFlowModelTdfyWrapper) is hydra-built from the vendored slat_generator.yaml
|
| 69 |
+
# with the pipeline's effective inference config (steps=25, cfg_strength=1,
|
| 70 |
+
# cfg_interval=[0,500], rescale_t=1) baked in by build_slat_generator.
|
| 71 |
+
# --------------------------------------------------------------------------- #
|
| 72 |
+
model:
|
| 73 |
+
# Warm-start from the deployed SLAT generator (526/526 tensors, verified).
|
| 74 |
+
pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/slat_generator.ckpt
|
| 75 |
+
# Local soft-mask cross-attention bias (§4c). Parity gating (§10 step 4)
|
| 76 |
+
# PASSED 2026-09-01 (wrapped/zero-bias bit-identical to the pristine stack on
|
| 77 |
+
# 3 toys4k objects), so the bias is ENABLED for training. Set false to
|
| 78 |
+
# reproduce the gating configuration.
|
| 79 |
+
enable_soft_mask: true # local -10 soft-mask in cross-attention
|
| 80 |
+
enable_plucker: true # Plücker ray embedding on non-anchor views
|
| 81 |
+
bias_value: -10.0 # fixed, fp16-safe; NEVER -inf (§4c / risk #7)
|
| 82 |
+
# Backbone param dtype: false = lossless fp16->fp32 upcast for training
|
| 83 |
+
# (optimizer-friendly); true = yaml-verbatim fp16 torso (the parity config).
|
| 84 |
+
fp16_torso: false
|
| 85 |
+
# Gradient checkpointing on the 24 backbone DiT cross blocks (per-block
|
| 86 |
+
# .use_checkpoint flip, same toggle the batch-size bench used): recompute
|
| 87 |
+
# activations in backward -> big memory cut, ~modest step-time cost.
|
| 88 |
+
# false = current behavior (no checkpointing).
|
| 89 |
+
use_checkpoint: true # 2-GPU resume run (user 2026-09-07): 6 objects/GPU needs activation checkpointing
|
| 90 |
+
# Train the pretrained EmbedderFuser projections + slot embeddings (the DINO
|
| 91 |
+
# towers inside are always frozen). false freezes the whole fuser so only
|
| 92 |
+
# the backbone + Plücker FC train.
|
| 93 |
+
train_condition_embedder: true
|
| 94 |
+
|
| 95 |
+
# --------------------------------------------------------------------------- #
|
| 96 |
+
# optim + train loop
|
| 97 |
+
# --------------------------------------------------------------------------- #
|
| 98 |
+
optim:
|
| 99 |
+
lr: 3.0e-5
|
| 100 |
+
weight_decay: 0.0 # AdamW
|
| 101 |
+
grad_clip: 1.0 # max grad norm
|
| 102 |
+
|
| 103 |
+
# --------------------------------------------------------------------------- #
|
| 104 |
+
# ema — exponential moving average of the TRAINABLE weights (fp32 shadow,
|
| 105 |
+
# updated after every optimizer step; frozen DINO towers excluded).
|
| 106 |
+
# Disabled by default = existing behavior unchanged. State is saved as a
|
| 107 |
+
# per-checkpoint sidecar (step_XXXX_ema.pt, next to the _optimizer.pt sidecar)
|
| 108 |
+
# and restored automatically on resume_from.
|
| 109 |
+
# --------------------------------------------------------------------------- #
|
| 110 |
+
ema:
|
| 111 |
+
enable: true
|
| 112 |
+
rate: 0.999 # ema = rate*ema + (1-rate)*param
|
| 113 |
+
eval_with_ema: true # swap EMA weights in for the in-loop validation
|
| 114 |
+
|
| 115 |
+
train:
|
| 116 |
+
# Loader batch size is PINNED to 1 (per-object coord sets, collate_single);
|
| 117 |
+
# the effective batch is grad_accum_steps x world_size objects per step.
|
| 118 |
+
steps: 100000 # ~32 epochs of 37k objects at 12 objects/step
|
| 119 |
+
num_workers: 16 # CPU work: crops + projections + Plücker per view
|
| 120 |
+
grad_accum_steps: 1 # NO accumulation: effective batch = batch_objects x world_size = 12
|
| 121 |
+
batch_objects: 6 # 6 objects/GPU x 2 GPUs = effective batch 12 (same as the 4-GPU run)
|
| 122 |
+
# --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
|
| 123 |
+
scheduler: cosine # cosine | constant
|
| 124 |
+
warmup_steps: 500
|
| 125 |
+
min_lr: 1.0e-6
|
| 126 |
+
# --- resume ---
|
| 127 |
+
resume_from: "/data/mv_mesh_data/ckpt/slatflow_resume20k_pslaug_2gpu/step_0022500.pt" # resume the 22.5k pslaug ckpt (+ema/optimizer sidecars); pool swapped to base+psl_aug_occ
|
| 128 |
+
log_every: 20
|
| 129 |
+
ckpt_every: 2500
|
| 130 |
+
ckpt_dir: /data/mv_mesh_data/ckpt/slatflow_resume22p5k_pslocc_2gpu
|
| 131 |
+
amp: true
|
| 132 |
+
amp_dtype: bf16 # bf16 | fp16 (fp16 enables GradScaler)
|
| 133 |
+
seed: 0
|
| 134 |
+
# --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
|
| 135 |
+
backend: ddp # ddp | deepspeed
|
| 136 |
+
deepspeed_config: ""
|
| 137 |
+
# --- validation: held-out flow v-MSE only (fixed RNG). Decode +
|
| 138 |
+
# faithfulness eval = the TrainTest phase (frozen-pipeline decode). ---
|
| 139 |
+
val_every: 5000 # optimizer steps between validation passes (0 = never)
|
| 140 |
+
val_max_batches: 8 # val objects per rank per pass
|
| 141 |
+
|
| 142 |
+
# --------------------------------------------------------------------------- #
|
| 143 |
+
# wandb — rank-0 experiment logging (train/loss|lr|grad_norm per log_every step,
|
| 144 |
+
# val/* per validation pass). enabled: false makes every wandb call a no-op.
|
| 145 |
+
# Extra metric dicts can be pushed from anywhere via
|
| 146 |
+
# from mvsam3d.train.train_slat_flow import log_metrics
|
| 147 |
+
# log_metrics({"val/chamfer": ..., "val/psnr": ...}, step)
|
| 148 |
+
# --------------------------------------------------------------------------- #
|
| 149 |
+
wandb:
|
| 150 |
+
enabled: true
|
| 151 |
+
project: "mvsam3d-slatflow"
|
| 152 |
+
entity: "alphabet1"
|
| 153 |
+
name: "slatflow_resume22p5k_pslocc_bs12"
|
| 154 |
+
dir: "/lp-dev/jonghoon/mv-mesh/wandb"
|
| 155 |
+
|
| 156 |
+
# --------------------------------------------------------------------------- #
|
| 157 |
+
# val_appforce — APPEARANCE-FORCING validation (mvsam3d/train/val_appforce.py):
|
| 158 |
+
# the trained flow through the exact reported protocol (batch_appforce_sam3d
|
| 159 |
+
# sam3d_geom_dino 2v -> evaluate_appforce.py) on toys4k-100 (textured) +
|
| 160 |
+
# OmniObject3D-117, every `every` optimizer steps (+ step 0 baseline), EMA
|
| 161 |
+
# weights, ALL ranks sample their shard then run the reference-env decode +
|
| 162 |
+
# eval on their own GPU; rank 0 merges and logs val_af/<ds>/<schedule>/<metric>.
|
| 163 |
+
# Stage-1 (frozen z1fwd geometry forcing) is cached ONCE by
|
| 164 |
+
# tools/val_stage1_ref.py --mode stage1 --exp <exp> --views 2 --out
|
| 165 |
+
# <stage1_cache>/<name>/2v
|
| 166 |
+
# --------------------------------------------------------------------------- #
|
| 167 |
+
val_appforce:
|
| 168 |
+
enabled: false # in-loop val OFF: tools/val_daemon.py validates every 5k ckpt on GPUs 2-3 (training never pauses)
|
| 169 |
+
every: 5000
|
| 170 |
+
at_step0: false # first appearance-forcing val at 5000 (step-0 baseline = the parity run)
|
| 171 |
+
views: 2
|
| 172 |
+
schedules: [seed, reinject] # seed = the training x0 construction; reinject = the reported schedule
|
| 173 |
+
seed: 42
|
| 174 |
+
limit: 0 # debug: cap objects per dataset
|
| 175 |
+
batch_size: 6 # objects per BATCHED 25-step ODE solve (1 = old single path)
|
| 176 |
+
single: true # batched B=6 measured 0.8x slower than single; parity verified but no gain
|
| 177 |
+
decode_procs: 1 # 2 procs x 23 GB OOMed rank 3 at the 10k val
|
| 178 |
+
ref_threads: 8 # OMP/MKL cap for every reference-env subprocess
|
| 179 |
+
stage1_cache: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/stage1
|
| 180 |
+
out_root: /lp-dev/jonghoon/mv-sam3d-6d-code/val_appforce/runs
|
| 181 |
+
ref_python: /lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/bin/python
|
| 182 |
+
eval_dataset_flag: toys4k # evaluate_appforce --dataset (both sets use the toys4k path, as reported)
|
| 183 |
+
datasets:
|
| 184 |
+
- name: toys4k100_tex
|
| 185 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k100_tex
|
| 186 |
+
- name: omni3d
|
| 187 |
+
exp: /lp-dev/jonghoon/mv-mesh/exp_faithfulness/omni3d
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_003.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:14502f014a45dee714dbd919e394b419a72e95419897303bc7386eed8e0a5508
|
| 3 |
+
size 20996178
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/bunny_008.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:92e4ff59ac962b00dc5a5975913c5e952ece70ab3a640d25c0727bd68265ae08
|
| 3 |
+
size 20511442
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/chair_026.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bdf5dc60ce5b61d76bb78762bd5895cdb21ff3a49252f63d9ef8007c73ed3ffd
|
| 3 |
+
size 20690002
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ours_both.log
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
| 2 |
+
import pynvml # type: ignore[import]
|
| 3 |
+
2026-09-01 01:49:09.196 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
|
| 4 |
+
2026-09-01 01:49:12.938 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
|
| 5 |
+
[SPARSE][CONV] spconv algo: native
|
| 6 |
+
[slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
|
| 7 |
+
2026-09-01 01:49:25.245 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 8 |
+
2026-09-01 01:49:29.957 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 9 |
+
2026-09-01 01:49:29.961 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 10 |
+
2026-09-01 01:49:33.784 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 11 |
+
[slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
|
| 12 |
+
[slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
|
| 13 |
+
2026-09-01 01:49:44.041 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 14 |
+
2026-09-01 01:49:47.428 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 15 |
+
2026-09-01 01:49:47.431 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 16 |
+
2026-09-01 01:49:51.389 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 17 |
+
[slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
|
| 18 |
+
[ours] bunny_003 (N=7984, ref steps=25 strength=1.0 rescale_t=1.0 | deployed self-noise floor=0.000e+00 tol=5.000e-02)
|
| 19 |
+
[raw]
|
| 20 |
+
cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
|
| 21 |
+
slat_feats_noise WITHIN-FLOOR max|d|=6.896e-03 (tol 5.000e-02)
|
| 22 |
+
slat_feats_forced WITHIN-FLOOR max|d|=9.228e-03 (tol 5.000e-02)
|
| 23 |
+
[wrapped/zero-bias]
|
| 24 |
+
cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
|
| 25 |
+
slat_feats_noise WITHIN-FLOOR max|d|=6.896e-03 (tol 5.000e-02)
|
| 26 |
+
wrapped==raw (noise) BIT-IDENTICAL shape=(7984, 8) dtype=torch.float32 max|d|=0.000e+00
|
| 27 |
+
slat_feats_forced WITHIN-FLOOR max|d|=9.228e-03 (tol 5.000e-02)
|
| 28 |
+
wrapped==raw (forced) BIT-IDENTICAL shape=(7984, 8) dtype=torch.float32 max|d|=0.000e+00
|
| 29 |
+
[ours] bunny_008 (N=4619, ref steps=25 strength=1.0 rescale_t=1.0 | deployed self-noise floor=0.000e+00 tol=5.000e-02)
|
| 30 |
+
[raw]
|
| 31 |
+
cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
|
| 32 |
+
slat_feats_noise WITHIN-FLOOR max|d|=3.383e-03 (tol 5.000e-02)
|
| 33 |
+
slat_feats_forced WITHIN-FLOOR max|d|=4.269e-03 (tol 5.000e-02)
|
| 34 |
+
[wrapped/zero-bias]
|
| 35 |
+
cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
|
| 36 |
+
slat_feats_noise WITHIN-FLOOR max|d|=3.383e-03 (tol 5.000e-02)
|
| 37 |
+
wrapped==raw (noise) BIT-IDENTICAL shape=(4619, 8) dtype=torch.float32 max|d|=0.000e+00
|
| 38 |
+
slat_feats_forced WITHIN-FLOOR max|d|=4.269e-03 (tol 5.000e-02)
|
| 39 |
+
wrapped==raw (forced) BIT-IDENTICAL shape=(4619, 8) dtype=torch.float32 max|d|=0.000e+00
|
| 40 |
+
[ours] chair_026 (N=5858, ref steps=25 strength=1.0 rescale_t=1.0 | deployed self-noise floor=0.000e+00 tol=5.000e-02)
|
| 41 |
+
[raw]
|
| 42 |
+
cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
|
| 43 |
+
slat_feats_noise WITHIN-FLOOR max|d|=5.830e-03 (tol 5.000e-02)
|
| 44 |
+
slat_feats_forced WITHIN-FLOOR max|d|=1.043e-02 (tol 5.000e-02)
|
| 45 |
+
[wrapped/zero-bias]
|
| 46 |
+
cond BIT-IDENTICAL shape=(1, 5496, 1024) dtype=torch.float16 max|d|=0.000e+00
|
| 47 |
+
slat_feats_noise WITHIN-FLOOR max|d|=5.830e-03 (tol 5.000e-02)
|
| 48 |
+
wrapped==raw (noise) BIT-IDENTICAL shape=(5858, 8) dtype=torch.float32 max|d|=0.000e+00
|
| 49 |
+
slat_feats_forced WITHIN-FLOOR max|d|=1.043e-02 (tol 5.000e-02)
|
| 50 |
+
wrapped==raw (forced) BIT-IDENTICAL shape=(5858, 8) dtype=torch.float32 max|d|=0.000e+00
|
| 51 |
+
[ours] PARITY PASS
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe2_ref.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a9518028c75f9bb983889d290b85f83e15fc8435e2072127aab57e4c7d1a4f53
|
| 3 |
+
size 10685283
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/probe_ref.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:033cd057451f09ac3883647d92a9685d10ca7c0d81f87dc9f1ce94724caf402d
|
| 3 |
+
size 43640209
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen/ref.log
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
| 2 |
+
import pynvml # type: ignore[import]
|
| 3 |
+
2026-08-31 18:47:31.995 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
|
| 4 |
+
2026-08-31 18:47:38.280 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
|
| 5 |
+
[SPARSE][CONV] spconv algo: native
|
| 6 |
+
Warp 1.12.1 initialized:
|
| 7 |
+
CUDA Toolkit 12.9, Driver 13.0
|
| 8 |
+
Devices:
|
| 9 |
+
"cpu" : "x86_64"
|
| 10 |
+
"cuda:0" : "NVIDIA A100-SXM4-80GB" (79 GiB, sm_80, mempool enabled)
|
| 11 |
+
Kernel cache:
|
| 12 |
+
/home/nvidia/.cache/warp/1.12.1
|
| 13 |
+
2026-08-31 18:47:49.271 | INFO | sam3d_objects.pipeline.inference_pipeline:set_attention_backend:15 - GPU name is NVIDIA A100-SXM4-80GB
|
| 14 |
+
2026-08-31 18:47:53.396 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 15 |
+
2026-08-31 18:47:53.397 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 16 |
+
[ref] building deployed pipeline ...
|
| 17 |
+
2026-08-31 18:47:53.453 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 18 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/moge/model/v1.py:172: UserWarning: The following deprecated/invalid arguments are ignored: {'output_mask': True, 'split_head': True}
|
| 19 |
+
warnings.warn(f"The following deprecated/invalid arguments are ignored: {deprecated_kwargs}")
|
| 20 |
+
2026-08-31 18:48:02.212 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 21 |
+
2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:98 - self.device: cuda
|
| 22 |
+
2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:99 - CUDA_VISIBLE_DEVICES: 1
|
| 23 |
+
2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:100 - Actually using GPU: 0
|
| 24 |
+
2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:init_pose_decoder:295 - Using pose decoder: ScaleShiftInvariant
|
| 25 |
+
2026-08-31 18:48:02.214 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:131 - Loading model weights...
|
| 26 |
+
2026-08-31 18:48:02.515 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
|
| 27 |
+
2026-08-31 18:48:13.901 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
|
| 28 |
+
2026-08-31 18:48:19.400 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_decoder.ckpt
|
| 29 |
+
2026-08-31 18:48:19.979 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs.ckpt
|
| 30 |
+
2026-08-31 18:48:20.456 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs_4.ckpt
|
| 31 |
+
2026-08-31 18:48:21.159 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_mesh.ckpt
|
| 32 |
+
2026-08-31 18:48:21.987 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 33 |
+
2026-08-31 18:48:23.321 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 34 |
+
2026-08-31 18:48:23.330 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 35 |
+
2026-08-31 18:48:24.723 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 36 |
+
2026-08-31 18:48:24.776 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
|
| 37 |
+
2026-08-31 18:48:31.503 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 38 |
+
2026-08-31 18:48:32.940 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 39 |
+
2026-08-31 18:48:32.949 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 40 |
+
2026-08-31 18:48:34.490 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 41 |
+
2026-08-31 18:48:34.519 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
|
| 42 |
+
2026-08-31 18:48:40.669 | INFO | sam3d_objects.pipeline.inference_pipeline:override_ss_generator_cfg_config:520 - ss_generator parameters: inference_steps=25, cfg_strength=7, cfg_interval=[0, 500], rescale_t=3, cfg_strength_pm=0.0
|
| 43 |
+
2026-08-31 18:48:40.669 | INFO | sam3d_objects.pipeline.inference_pipeline:override_slat_generator_cfg_config:542 - slat_generator parameters: inference_steps=25, cfg_strength=1, cfg_interval=[0, 500], rescale_t=1
|
| 44 |
+
2026-08-31 18:48:40.669 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:196 - Loading model weights completed!
|
| 45 |
+
[ref] pipeline ready: slat steps=25 strength=1 interval=[0, 500] rescale_t=1
|
| 46 |
+
2026-08-31 18:48:42.723 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 47 |
+
2026-08-31 18:48:43.063 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 48 |
+
2026-08-31 18:48:43.065 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 49 |
+
2026-08-31 18:48:43.065 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 50 |
+
2026-08-31 18:48:43.246 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 51 |
+
2026-08-31 18:48:51.925 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 52 |
+
2026-08-31 18:48:51.925 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 53 |
+
2026-08-31 18:48:52.142 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 54 |
+
[ref] chair_026: N=5858 cond=(1, 5496, 1024)/torch.float16 feats=(5858, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen/chair_026.pt
|
| 55 |
+
2026-08-31 18:49:00.060 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 56 |
+
2026-08-31 18:49:00.226 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 57 |
+
2026-08-31 18:49:00.227 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 58 |
+
2026-08-31 18:49:00.227 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 59 |
+
2026-08-31 18:49:00.389 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 60 |
+
2026-08-31 18:49:09.140 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 61 |
+
2026-08-31 18:49:09.140 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 62 |
+
2026-08-31 18:49:09.316 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 63 |
+
[ref] bunny_003: N=7984 cond=(1, 5496, 1024)/torch.float16 feats=(7984, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen/bunny_003.pt
|
| 64 |
+
2026-08-31 18:49:18.333 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 65 |
+
2026-08-31 18:49:18.496 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 66 |
+
2026-08-31 18:49:18.497 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 67 |
+
2026-08-31 18:49:18.497 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 68 |
+
2026-08-31 18:49:18.660 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 69 |
+
2026-08-31 18:49:26.196 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 70 |
+
2026-08-31 18:49:26.196 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 71 |
+
2026-08-31 18:49:26.401 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 72 |
+
[ref] bunny_008: N=4619 cond=(1, 5496, 1024)/torch.float16 feats=(4619, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen/bunny_008.pt
|
| 73 |
+
[ref] DONE
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_003.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:14502f014a45dee714dbd919e394b419a72e95419897303bc7386eed8e0a5508
|
| 3 |
+
size 20996178
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/bunny_008.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:92e4ff59ac962b00dc5a5975913c5e952ece70ab3a640d25c0727bd68265ae08
|
| 3 |
+
size 20511442
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/chair_026.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bdf5dc60ce5b61d76bb78762bd5895cdb21ff3a49252f63d9ef8007c73ed3ffd
|
| 3 |
+
size 20690002
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/parity_slatgen_b/ref.log
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
| 2 |
+
import pynvml # type: ignore[import]
|
| 3 |
+
2026-08-31 19:04:42.766 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
|
| 4 |
+
2026-08-31 19:04:47.903 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
|
| 5 |
+
[SPARSE][CONV] spconv algo: native
|
| 6 |
+
Warp 1.12.1 initialized:
|
| 7 |
+
CUDA Toolkit 12.9, Driver 13.0
|
| 8 |
+
Devices:
|
| 9 |
+
"cpu" : "x86_64"
|
| 10 |
+
"cuda:0" : "NVIDIA A100-SXM4-80GB" (79 GiB, sm_80, mempool enabled)
|
| 11 |
+
Kernel cache:
|
| 12 |
+
/home/nvidia/.cache/warp/1.12.1
|
| 13 |
+
2026-08-31 19:04:56.352 | INFO | sam3d_objects.pipeline.inference_pipeline:set_attention_backend:15 - GPU name is NVIDIA A100-SXM4-80GB
|
| 14 |
+
2026-08-31 19:04:58.555 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 15 |
+
2026-08-31 19:04:58.555 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 16 |
+
[ref] building deployed pipeline ...
|
| 17 |
+
2026-08-31 19:04:58.594 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 18 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/moge/model/v1.py:172: UserWarning: The following deprecated/invalid arguments are ignored: {'output_mask': True, 'split_head': True}
|
| 19 |
+
warnings.warn(f"The following deprecated/invalid arguments are ignored: {deprecated_kwargs}")
|
| 20 |
+
2026-08-31 19:05:08.207 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 21 |
+
2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:98 - self.device: cuda
|
| 22 |
+
2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:99 - CUDA_VISIBLE_DEVICES: 1
|
| 23 |
+
2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:100 - Actually using GPU: 0
|
| 24 |
+
2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:init_pose_decoder:295 - Using pose decoder: ScaleShiftInvariant
|
| 25 |
+
2026-08-31 19:05:08.209 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:131 - Loading model weights...
|
| 26 |
+
2026-08-31 19:05:08.653 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
|
| 27 |
+
2026-08-31 19:05:20.732 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
|
| 28 |
+
2026-08-31 19:05:25.889 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_decoder.ckpt
|
| 29 |
+
2026-08-31 19:05:26.449 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs.ckpt
|
| 30 |
+
2026-08-31 19:05:26.881 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_gs_4.ckpt
|
| 31 |
+
2026-08-31 19:05:27.467 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_decoder_mesh.ckpt
|
| 32 |
+
2026-08-31 19:05:28.226 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 33 |
+
2026-08-31 19:05:30.056 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 34 |
+
2026-08-31 19:05:30.065 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 35 |
+
2026-08-31 19:05:31.477 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 36 |
+
2026-08-31 19:05:31.522 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/ss_generator.ckpt
|
| 37 |
+
2026-08-31 19:05:39.237 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 38 |
+
2026-08-31 19:05:40.567 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 39 |
+
2026-08-31 19:05:40.576 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:31 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 40 |
+
2026-08-31 19:05:41.857 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:44 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 41 |
+
2026-08-31 19:05:41.882 | INFO | sam3d_objects.model.io:load_model_from_checkpoint:158 - Loading checkpoint from checkpoints/hf/slat_generator.ckpt
|
| 42 |
+
2026-08-31 19:05:48.113 | INFO | sam3d_objects.pipeline.inference_pipeline:override_ss_generator_cfg_config:520 - ss_generator parameters: inference_steps=25, cfg_strength=7, cfg_interval=[0, 500], rescale_t=3, cfg_strength_pm=0.0
|
| 43 |
+
2026-08-31 19:05:48.114 | INFO | sam3d_objects.pipeline.inference_pipeline:override_slat_generator_cfg_config:542 - slat_generator parameters: inference_steps=25, cfg_strength=1, cfg_interval=[0, 500], rescale_t=1
|
| 44 |
+
2026-08-31 19:05:48.114 | INFO | sam3d_objects.pipeline.inference_pipeline:__init__:196 - Loading model weights completed!
|
| 45 |
+
[ref] pipeline ready: slat steps=25 strength=1 interval=[0, 500] rescale_t=1
|
| 46 |
+
2026-08-31 19:05:48.971 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 47 |
+
2026-08-31 19:05:49.265 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 48 |
+
2026-08-31 19:05:49.267 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 49 |
+
2026-08-31 19:05:49.267 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 50 |
+
2026-08-31 19:05:49.431 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 51 |
+
2026-08-31 19:05:57.635 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 52 |
+
2026-08-31 19:05:57.635 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 53 |
+
2026-08-31 19:05:57.803 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 54 |
+
[ref] chair_026: N=5858 cond=(1, 5496, 1024)/torch.float16 feats=(5858, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen_b/chair_026.pt
|
| 55 |
+
2026-08-31 19:06:05.500 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 56 |
+
2026-08-31 19:06:05.675 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 57 |
+
2026-08-31 19:06:05.676 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 58 |
+
2026-08-31 19:06:05.676 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 59 |
+
2026-08-31 19:06:05.827 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 60 |
+
2026-08-31 19:06:14.623 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 61 |
+
2026-08-31 19:06:14.623 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 62 |
+
2026-08-31 19:06:14.843 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 63 |
+
[ref] bunny_003: N=7984 cond=(1, 5496, 1024)/torch.float16 feats=(7984, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen_b/bunny_003.pt
|
| 64 |
+
2026-08-31 19:06:22.977 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 65 |
+
2026-08-31 19:06:23.201 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 66 |
+
2026-08-31 19:06:23.202 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 67 |
+
2026-08-31 19:06:23.202 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 68 |
+
2026-08-31 19:06:23.378 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 69 |
+
2026-08-31 19:06:30.672 | INFO | sam3d_objects.pipeline.inference_pipeline:sample_slat:860 - Sampling sparse latent: inference_steps=25, strength=1, interval=[0, 500], rescale_t=1
|
| 70 |
+
2026-08-31 19:06:30.672 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:748 - Running condition embedder ...
|
| 71 |
+
2026-08-31 19:06:30.837 | INFO | sam3d_objects.pipeline.inference_pipeline:get_condition_input:752 - Condition embedder finishes!
|
| 72 |
+
[ref] bunny_008: N=4619 cond=(1, 5496, 1024)/torch.float16 feats=(4619, 8)/torch.float32 -> /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/migrator/cache/parity_slatgen_b/bunny_008.pt
|
| 73 |
+
[ref] DONE
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/smoke_slatflow.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
| 2 |
+
import pynvml # type: ignore[import]
|
| 3 |
+
2026-09-01 01:55:05.629 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
|
| 4 |
+
2026-09-01 01:55:09.659 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
|
| 5 |
+
2026-09-01 01:55:15.109 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 6 |
+
[SPARSE][CONV] spconv algo: native
|
| 7 |
+
[smoke] object=chair_026 N=5858 n_subset=2 views=[0, 1, 2, 3] crops=(4, 3, 518, 518) uv=(4, 5858, 2) vis=(4, 5858) plucker=(1, 1369, 6) bias_views=1
|
| 8 |
+
[slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
|
| 9 |
+
2026-09-01 01:55:25.930 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 10 |
+
2026-09-01 01:55:31.714 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 11 |
+
2026-09-01 01:55:31.718 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 12 |
+
2026-09-01 01:55:36.823 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 13 |
+
[slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
|
| 14 |
+
[smoke] model built: trainable=617.7M bias_enabled=True
|
| 15 |
+
[smoke] flow_step: loss=0.7099 frac_visible_x0=0.510 frac_target=0.702
|
| 16 |
+
[smoke] grad norms: backbone=6.378e+01 fuser=2.272e+00 plucker_fc=3.835e-02
|
| 17 |
+
[smoke] bias: shape=(1394, 6865) anchor_max|.|=0.0 nonanchor_vals=[-10.0, 0.0] frac_open=0.0066
|
| 18 |
+
[smoke] sample: cond=(1, 6865, 1024) base=(1, 5858, 8) slat feats=(5858, 8) finite=True std=4.557
|
| 19 |
+
[smoke] SELF-CHECK PASS
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/migrator/cache/train_slatflow_smoke.log
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/lp-dev/jonghoon/mv-mesh/envs/mv-sam3d/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
| 2 |
+
import pynvml # type: ignore[import]
|
| 3 |
+
2026-09-01 01:56:18.594 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.sparse:__from_env:39 - [SPARSE] Backend: spconv, Attention: sdpa
|
| 4 |
+
2026-09-01 01:56:22.431 | INFO | sam3d_objects.model.backbone.tdfy_dit.modules.attention:__from_env:30 - [ATTENTION] Using backend: sdpa
|
| 5 |
+
[SPARSE][CONV] spconv algo: native
|
| 6 |
+
[train] train samples: 201
|
| 7 |
+
[train] val samples: 100
|
| 8 |
+
[slat-pretrained] slat_generator: loaded 526/526 own tensors from '_base_models.generator.*' (0 missing, 0 unexpected)
|
| 9 |
+
2026-09-01 01:56:35.604 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 10 |
+
2026-09-01 01:56:40.418 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 11 |
+
2026-09-01 01:56:40.422 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:32 - Loading DINO model: dinov2_vitl14_reg from facebookresearch/dinov2 (source: github)
|
| 12 |
+
2026-09-01 01:56:45.101 | INFO | sam3d_objects.model.backbone.dit.embedder.dino:__init__:51 - Loaded DINO model - type: <class 'dinov2.models.vision_transformer.DinoVisionTransformer'>, embed_dim: 1024, patch_size: (14, 14)
|
| 13 |
+
[slat-pretrained] slat_condition_embedder: loaded 699/699 own tensors from '_base_models.condition_embedder.*' (0 missing, 0 unexpected)
|
| 14 |
+
[setup] trainable params: 617.7M / 1226.5M (enable_bias=True, fp16_torso=False)
|
| 15 |
+
/lp-dev/jonghoon/mv-sam3d-6d-code/migrator/code/mv-sam3d-for-6d-v2/mvsam3d/train/train_slat_flow.py:277: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
|
| 16 |
+
scaler = torch.cuda.amp.GradScaler(enabled=use_scaler)
|
| 17 |
+
[setup] backend=ddp world=1 global_batch=2 (objects/optim-step) steps=2 grad_accum=2 amp=True/bf16 val_every=2
|
| 18 |
+
2026-09-01 01:56:54.846 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 19 |
+
2026-09-01 01:56:54.854 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 20 |
+
2026-09-01 01:56:54.893 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 21 |
+
2026-09-01 01:56:54.945 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 22 |
+
[train] step 1 lr=4.000000e-07 frac_target=0.7416 frac_visible_x0=0.6616 loss=0.9653 loss/slat=0.9653 (0.14 it/s)
|
| 23 |
+
[train] step 2 lr=6.000000e-07 frac_target=0.7212 frac_visible_x0=0.4989 loss=1.9295 loss/slat=1.9295 (0.48 it/s)
|
| 24 |
+
2026-09-01 01:57:04.080 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 25 |
+
2026-09-01 01:57:04.091 | WARNING | sam3d_objects.data.dataset.tdfy.preprocessor:__post_init__:51 - No rgb pointmap normalizer provided, using scale + shift
|
| 26 |
+
[val] step 2 frac_target=0.5975 frac_visible_x0=0.4419 loss=0.9582 loss/slat=0.9582 (n=2)
|
| 27 |
+
[train] saved /lp-dev/jonghoon/mv-sam3d-6d-code/migrator/cache/ckpt_slatflow/step_0000002.pt
|
| 28 |
+
[train] done.
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-310.pyc
ADDED
|
Binary file (673 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (750 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/__init__.cpython-39.pyc
ADDED
|
Binary file (671 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-310.pyc
ADDED
|
Binary file (4.09 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-311.pyc
ADDED
|
Binary file (6.59 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/__pycache__/paths.cpython-39.pyc
ADDED
|
Binary file (4.07 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-310.pyc
ADDED
|
Binary file (184 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (200 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/__init__.cpython-39.pyc
ADDED
|
Binary file (182 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/adapters.cpython-311.pyc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:859d14326540e24088b672bc15940079ab435832247d5921114087cdbcb3a342
|
| 3 |
+
size 118636
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/base_dataset.cpython-311.pyc
ADDED
|
Binary file (16.8 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/canonical.cpython-311.pyc
ADDED
|
Binary file (3.42 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_calib.cpython-311.pyc
ADDED
|
Binary file (5.06 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/dexycb_dataset.cpython-311.pyc
ADDED
|
Binary file (35.6 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/integrated.cpython-311.pyc
ADDED
|
Binary file (8.68 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slat_preprocess.cpython-311.pyc
ADDED
|
Binary file (4.44 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-310.pyc
ADDED
|
Binary file (46.7 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/slatflow_dataset.cpython-311.pyc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:920741be4f6ca85689c3629a6d65f832b0a060dbc701db71a0a7bcfd131e179e
|
| 3 |
+
size 126300
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-310.pyc
ADDED
|
Binary file (17.9 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_imgcond_dataset.cpython-311.pyc
ADDED
|
Binary file (52.9 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ss_preprocess.cpython-311.pyc
ADDED
|
Binary file (12 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-310.pyc
ADDED
|
Binary file (15.9 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-311.pyc
ADDED
|
Binary file (36.9 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/ssflow_dataset.cpython-39.pyc
ADDED
|
Binary file (9.78 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/__pycache__/v_bucket_sampler.cpython-311.pyc
ADDED
|
Binary file (6.04 kB). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py
CHANGED
|
@@ -903,6 +903,53 @@ class TrellisSlatFlowDataset(TrellisSSFlowDataset):
|
|
| 903 |
}
|
| 904 |
|
| 905 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 906 |
DEFAULT_TAR_ROOTS = ("/data/mv_mesh_data/slat_tars/processed",)
|
| 907 |
DEFAULT_TAR_MANIFESTS = "/data/mv_mesh_data/DATASET_INDEX/hubman/processed/manifests"
|
| 908 |
|
|
@@ -960,30 +1007,33 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
|
|
| 960 |
shards: Optional[Sequence[str]] = None,
|
| 961 |
max_objects: Optional[int] = None,
|
| 962 |
use_recgen_images: bool = False,
|
| 963 |
-
p_recgen_image: float = 0.0,
|
| 964 |
cond_cameras_fallback: bool = False):
|
| 965 |
-
# ---- condition-
|
| 966 |
-
#
|
|
|
|
| 967 |
# use_recgen_images : also index ``recgen/<id>/{cameras.json,NN.jpg,
|
| 968 |
# NN_mask.png}`` members (RecGen's own background-baked RGB + its
|
| 969 |
# visible mask + per-view RIGID OpenCV c2w in the SLAT cube) from ANY
|
| 970 |
-
# tar under tar_roots, joined to ``latents/<id>.npz`` by id
|
| 971 |
-
#
|
| 972 |
-
#
|
| 973 |
-
#
|
| 974 |
-
#
|
| 975 |
-
#
|
| 976 |
-
#
|
| 977 |
-
#
|
| 978 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
# DINO seed crops / uv / vis use the mask-matted tight crop exactly
|
| 980 |
-
# like the white path
|
| 981 |
# cond_cameras_fallback : white-cond objects lacking
|
| 982 |
# ``cameras/<id>/cond_cameras.json`` (the 24-view slat5/slat5rs
|
| 983 |
# batches) use ``cond/<id>/transforms.json`` (same transform_matrix;
|
| 984 |
# fx=fy=W/(2 tan(camera_angle_x/2)), cx=cy=W/2 from the PNG width).
|
| 985 |
self.use_recgen_images = bool(use_recgen_images)
|
| 986 |
-
self.p_recgen_image = float(p_recgen_image)
|
| 987 |
self.cond_cameras_fallback = bool(cond_cameras_fallback)
|
| 988 |
# NOTE: deliberately does NOT call TrellisSlatFlowDataset.__init__ (that
|
| 989 |
# scans object DIRS). We set the attributes its shared methods
|
|
@@ -1062,7 +1112,7 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
|
|
| 1062 |
# new index semantics -> distinct cache signature (legacy sig
|
| 1063 |
# unchanged when both knobs are off, so old caches stay valid)
|
| 1064 |
parts.append(f"rg={int(self.use_recgen_images)}"
|
| 1065 |
-
f":fb={int(self.cond_cameras_fallback)}:
|
| 1066 |
return _hashlib.sha1("|".join(parts).encode()).hexdigest()
|
| 1067 |
_sig = _tar_sig(tar_files)
|
| 1068 |
_cache = os.environ.get("TAR_INDEX_CACHE") or os.path.join(
|
|
@@ -1290,9 +1340,27 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
|
|
| 1290 |
views_all.append(dict(depth=None, fx=float(K[0, 0]), fy=float(K[1, 1]),
|
| 1291 |
cx=float(K[0, 2]), cy=float(K[1, 2]),
|
| 1292 |
c2w=np.asarray(c["c2w"], dtype=np.float64),
|
| 1293 |
-
res=int(max(int(c["width"]), int(c["height"])))
|
|
|
|
| 1294 |
return views_all, len(views_all)
|
| 1295 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1296 |
def _load_view_rgba(self, m: Dict, src: str, i: int) -> np.ndarray:
|
| 1297 |
"""FULL-FRAME RGBA uint8 of view i. white: the cond render PNG. recgen:
|
| 1298 |
RecGen's RGB (background kept) + its visible mask as alpha (>127)."""
|
|
@@ -1307,18 +1375,6 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
|
|
| 1307 |
return np.concatenate([rgb, a], axis=2)
|
| 1308 |
return self._load_png_rgba(self._get_tar(m["tar"]), m["cond_pngs"][i])
|
| 1309 |
|
| 1310 |
-
def _cond_source(self, idx: int, m: Dict) -> str:
|
| 1311 |
-
"""'white' | 'recgen' for this (sample, epoch). Own rng stream (9191)."""
|
| 1312 |
-
has_w = bool(m.get("cond_pngs"))
|
| 1313 |
-
has_r = m.get("rg") is not None
|
| 1314 |
-
if has_r and not has_w:
|
| 1315 |
-
return "recgen"
|
| 1316 |
-
if has_r and has_w and self.p_recgen_image > 0.0:
|
| 1317 |
-
r = float(np.random.default_rng(
|
| 1318 |
-
[self.seed, int(self._epoch), int(idx), 9191]).random())
|
| 1319 |
-
return "recgen" if r < self.p_recgen_image else "white"
|
| 1320 |
-
return "white"
|
| 1321 |
-
|
| 1322 |
# ---- OPT-IN bad-view gate (tar variant, keyed by sha) -------------- #
|
| 1323 |
def _tar_valid_views(self, sha: str, tar, views_all, sc, centers, V,
|
| 1324 |
src: str = "white") -> List[int]:
|
|
@@ -1381,11 +1437,17 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
|
|
| 1381 |
tar = self._get_tar(m["tar"])
|
| 1382 |
rng = np.random.default_rng([self.seed, self._epoch, idx])
|
| 1383 |
|
| 1384 |
-
|
| 1385 |
-
|
| 1386 |
-
|
| 1387 |
-
|
| 1388 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1389 |
|
| 1390 |
# x1 = released SLAT: coords 0..63 used DIRECTLY, feats row-aligned.
|
| 1391 |
zl = np.load(io.BytesIO(tar.extractfile(m["latent"]).read()))
|
|
@@ -1398,15 +1460,64 @@ class TarSlatFlowDataset(TrellisSlatFlowDataset):
|
|
| 1398 |
torch.from_numpy(sc).int()], dim=1) # (N,4) int32
|
| 1399 |
centers = (sc.astype(np.float64) + 0.5) / VOX - 0.5
|
| 1400 |
|
| 1401 |
-
|
| 1402 |
-
|
| 1403 |
-
|
| 1404 |
-
|
| 1405 |
-
|
| 1406 |
-
|
| 1407 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1408 |
else:
|
| 1409 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1410 |
order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
|
| 1411 |
|
| 1412 |
crops, uvs, viss, fulls = [], [], [], []
|
|
|
|
| 903 |
}
|
| 904 |
|
| 905 |
|
| 906 |
+
# ---- RecGen camera QC (2026-09-24) ------------------------------------------ #
|
| 907 |
+
# A RecGen view's camera is VALID iff the object's OWN SLAT voxels, projected with
|
| 908 |
+
# that view's camera, land inside RecGen's (modal) mask in proportion to how much
|
| 909 |
+
# of the object RecGen says is visible:
|
| 910 |
+
# p = #voxels projecting inside the mask / #voxels (all voxels)
|
| 911 |
+
# vf = RecGen view_metadata visible_fraction (modal / amodal)
|
| 912 |
+
# valid <=> vf >= RG_VF_MIN and p >= RG_RATIO * vf
|
| 913 |
+
# With a correct pose p ~= vf (occluders remove voxels from the modal mask exactly
|
| 914 |
+
# as they remove visible area). Calibrated on 4,000 random RecGen sets against our
|
| 915 |
+
# latents: keeps 99.0% of views of trusted analytic poses (p/vf p01 = 0.72), 0% of
|
| 916 |
+
# the analytic poses that failed their own seed test. Object guard: if fewer than
|
| 917 |
+
# RG_OBJ_MIN_FRAC of the object's verifiable (vf >= RG_VF_MIN) views pass, the pose
|
| 918 |
+
# is treated as systematically wrong and ALL its RecGen views are dropped (the
|
| 919 |
+
# object falls back to white-only, or is skipped if it has no white cond).
|
| 920 |
+
RG_VF_MIN = 0.10
|
| 921 |
+
RG_RATIO = 0.60
|
| 922 |
+
RG_OBJ_MIN_FRAC = 0.50
|
| 923 |
+
|
| 924 |
+
|
| 925 |
+
def recgen_view_qc(coords3: np.ndarray, views_all: List[Dict], masks: List[np.ndarray],
|
| 926 |
+
vfs: Sequence[float], vf_min: float = RG_VF_MIN,
|
| 927 |
+
ratio: float = RG_RATIO, obj_min_frac: float = RG_OBJ_MIN_FRAC):
|
| 928 |
+
"""-> (valid view indices, per-view p). coords3 (N,3) int voxel coords of the
|
| 929 |
+
object's SLAT latent; views_all[i] has fx,fy,cx,cy,c2w (rigid, SLAT cube);
|
| 930 |
+
masks[i] (H,W) bool; vfs[i] RecGen visible_fraction."""
|
| 931 |
+
pts = (np.asarray(coords3, dtype=np.float64) + 0.5) / VOX - 0.5
|
| 932 |
+
N = max(len(pts), 1)
|
| 933 |
+
ps, ok = [], []
|
| 934 |
+
for v, m, vf in zip(views_all, masks, vfs):
|
| 935 |
+
w2c = np.linalg.inv(v["c2w"])
|
| 936 |
+
xc = pts @ w2c[:3, :3].T + w2c[:3, 3]
|
| 937 |
+
z = xc[:, 2]
|
| 938 |
+
zs = np.where(z == 0, 1e-12, z)
|
| 939 |
+
u = np.round(v["fx"] * xc[:, 0] / zs + v["cx"]).astype(np.int64)
|
| 940 |
+
q = np.round(v["fy"] * xc[:, 1] / zs + v["cy"]).astype(np.int64)
|
| 941 |
+
H, W = m.shape[:2]
|
| 942 |
+
inf = (z > 0) & (u >= 0) & (u < W) & (q >= 0) & (q < H)
|
| 943 |
+
p = float(m[q[inf], u[inf]].sum()) / N
|
| 944 |
+
ps.append(p)
|
| 945 |
+
ok.append(bool(vf >= vf_min and p >= ratio * vf))
|
| 946 |
+
verifiable = sum(1 for vf in vfs if vf >= vf_min)
|
| 947 |
+
valid = [i for i, o in enumerate(ok) if o]
|
| 948 |
+
if not valid or len(valid) < obj_min_frac * verifiable:
|
| 949 |
+
valid = []
|
| 950 |
+
return valid, ps
|
| 951 |
+
|
| 952 |
+
|
| 953 |
DEFAULT_TAR_ROOTS = ("/data/mv_mesh_data/slat_tars/processed",)
|
| 954 |
DEFAULT_TAR_MANIFESTS = "/data/mv_mesh_data/DATASET_INDEX/hubman/processed/manifests"
|
| 955 |
|
|
|
|
| 1007 |
shards: Optional[Sequence[str]] = None,
|
| 1008 |
max_objects: Optional[int] = None,
|
| 1009 |
use_recgen_images: bool = False,
|
|
|
|
| 1010 |
cond_cameras_fallback: bool = False):
|
| 1011 |
+
# ---- condition-source knobs (2026-09-24). Class defaults OFF -> index,
|
| 1012 |
+
# draws and every item byte-identical to the legacy loader. The PROD
|
| 1013 |
+
# config (train_slatflow_prod.yaml) turns BOTH ON.
|
| 1014 |
# use_recgen_images : also index ``recgen/<id>/{cameras.json,NN.jpg,
|
| 1015 |
# NN_mask.png}`` members (RecGen's own background-baked RGB + its
|
| 1016 |
# visible mask + per-view RIGID OpenCV c2w in the SLAT cube) from ANY
|
| 1017 |
+
# tar under tar_roots, joined to ``latents/<id>.npz`` by the SAME id
|
| 1018 |
+
# string. Conditioning then follows the ONE-POOL rule:
|
| 1019 |
+
# * an object with BOTH sources draws its anchor uniformly from the
|
| 1020 |
+
# union of its (valid) white cond views and (valid) RecGen views;
|
| 1021 |
+
# * the anchor's source decides everything: ALL auxiliary views of
|
| 1022 |
+
# the sample come from that same source (and its cameras) — a
|
| 1023 |
+
# sample never mixes white and RecGen views;
|
| 1024 |
+
# * single-source objects draw exactly as the legacy loader does on
|
| 1025 |
+
# the source they have (white-only items are byte-identical to
|
| 1026 |
+
# legacy; latent-only RecGen slats become trainable).
|
| 1027 |
+
# A RecGen anchor goes through the SAME default path: the full-frame
|
| 1028 |
+
# RecGen image (background kept, alpha = its mask) is fed to
|
| 1029 |
+
# preprocess_slat_image -> item["image"] (+ mask/rgb_image/...);
|
| 1030 |
# DINO seed crops / uv / vis use the mask-matted tight crop exactly
|
| 1031 |
+
# like the white path.
|
| 1032 |
# cond_cameras_fallback : white-cond objects lacking
|
| 1033 |
# ``cameras/<id>/cond_cameras.json`` (the 24-view slat5/slat5rs
|
| 1034 |
# batches) use ``cond/<id>/transforms.json`` (same transform_matrix;
|
| 1035 |
# fx=fy=W/(2 tan(camera_angle_x/2)), cx=cy=W/2 from the PNG width).
|
| 1036 |
self.use_recgen_images = bool(use_recgen_images)
|
|
|
|
| 1037 |
self.cond_cameras_fallback = bool(cond_cameras_fallback)
|
| 1038 |
# NOTE: deliberately does NOT call TrellisSlatFlowDataset.__init__ (that
|
| 1039 |
# scans object DIRS). We set the attributes its shared methods
|
|
|
|
| 1112 |
# new index semantics -> distinct cache signature (legacy sig
|
| 1113 |
# unchanged when both knobs are off, so old caches stay valid)
|
| 1114 |
parts.append(f"rg={int(self.use_recgen_images)}"
|
| 1115 |
+
f":fb={int(self.cond_cameras_fallback)}:v2")
|
| 1116 |
return _hashlib.sha1("|".join(parts).encode()).hexdigest()
|
| 1117 |
_sig = _tar_sig(tar_files)
|
| 1118 |
_cache = os.environ.get("TAR_INDEX_CACHE") or os.path.join(
|
|
|
|
| 1340 |
views_all.append(dict(depth=None, fx=float(K[0, 0]), fy=float(K[1, 1]),
|
| 1341 |
cx=float(K[0, 2]), cy=float(K[1, 2]),
|
| 1342 |
c2w=np.asarray(c["c2w"], dtype=np.float64),
|
| 1343 |
+
res=int(max(int(c["width"]), int(c["height"]))),
|
| 1344 |
+
vf=float(c.get("visible_fraction", -1.0))))
|
| 1345 |
return views_all, len(views_all)
|
| 1346 |
|
| 1347 |
+
def _rg_qc_valid(self, sha: str, m: Dict, views_all: List[Dict],
|
| 1348 |
+
sc: np.ndarray) -> List[int]:
|
| 1349 |
+
"""RecGen views passing the camera QC (recgen_view_qc), cached per object."""
|
| 1350 |
+
key = (sha, "rgqc")
|
| 1351 |
+
v = self._valid_cache.get(key)
|
| 1352 |
+
if v is not None:
|
| 1353 |
+
return v
|
| 1354 |
+
r = m["rg"]
|
| 1355 |
+
tar = self._get_tar(r["tar"])
|
| 1356 |
+
masks = [np.array(Image.open(io.BytesIO(tar.extractfile(n).read())).convert("L")) > 127
|
| 1357 |
+
for n in r["masks"]]
|
| 1358 |
+
valid, _ = recgen_view_qc(sc, views_all, masks, [vw["vf"] for vw in views_all])
|
| 1359 |
+
if len(self._valid_cache) > 20000:
|
| 1360 |
+
self._valid_cache.clear()
|
| 1361 |
+
self._valid_cache[key] = valid
|
| 1362 |
+
return valid
|
| 1363 |
+
|
| 1364 |
def _load_view_rgba(self, m: Dict, src: str, i: int) -> np.ndarray:
|
| 1365 |
"""FULL-FRAME RGBA uint8 of view i. white: the cond render PNG. recgen:
|
| 1366 |
RecGen's RGB (background kept) + its visible mask as alpha (>127)."""
|
|
|
|
| 1375 |
return np.concatenate([rgb, a], axis=2)
|
| 1376 |
return self._load_png_rgba(self._get_tar(m["tar"]), m["cond_pngs"][i])
|
| 1377 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1378 |
# ---- OPT-IN bad-view gate (tar variant, keyed by sha) -------------- #
|
| 1379 |
def _tar_valid_views(self, sha: str, tar, views_all, sc, centers, V,
|
| 1380 |
src: str = "white") -> List[int]:
|
|
|
|
| 1437 |
tar = self._get_tar(m["tar"])
|
| 1438 |
rng = np.random.default_rng([self.seed, self._epoch, idx])
|
| 1439 |
|
| 1440 |
+
# ---- condition sources present for this object ------------------- #
|
| 1441 |
+
# white = cond/<id>/NNN.png + its cameras (always, legacy objects)
|
| 1442 |
+
# recgen = recgen/<id>/NN.jpg + NN_mask.png + cameras.json
|
| 1443 |
+
# (only indexed when use_recgen_images=True)
|
| 1444 |
+
has_w = bool(m.get("cond_pngs"))
|
| 1445 |
+
has_r = m.get("rg") is not None
|
| 1446 |
+
cams = {}
|
| 1447 |
+
if has_w:
|
| 1448 |
+
cams["white"] = self._tar_cameras(tar, m)
|
| 1449 |
+
if has_r:
|
| 1450 |
+
cams["recgen"] = self._rg_cameras(m)
|
| 1451 |
|
| 1452 |
# x1 = released SLAT: coords 0..63 used DIRECTLY, feats row-aligned.
|
| 1453 |
zl = np.load(io.BytesIO(tar.extractfile(m["latent"]).read()))
|
|
|
|
| 1460 |
torch.from_numpy(sc).int()], dim=1) # (N,4) int32
|
| 1461 |
centers = (sc.astype(np.float64) + 0.5) / VOX - 0.5
|
| 1462 |
|
| 1463 |
+
# RecGen camera QC (always on for RecGen views): only views whose camera
|
| 1464 |
+
# projects this object's own voxels into RecGen's mask consistently with
|
| 1465 |
+
# RecGen's visible_fraction may enter the pool. None valid -> the
|
| 1466 |
+
# object's RecGen set is dropped (white-only fallback, or skipped).
|
| 1467 |
+
rg_ok = None
|
| 1468 |
+
if has_r:
|
| 1469 |
+
rg_ok = self._rg_qc_valid(sha, m, cams["recgen"][0], sc)
|
| 1470 |
+
if not rg_ok:
|
| 1471 |
+
has_r = False
|
| 1472 |
+
if not (has_w or has_r):
|
| 1473 |
+
raise ValueError(f"{sha}: no condition view passes (white cond absent, "
|
| 1474 |
+
f"all RecGen views fail camera QC)")
|
| 1475 |
+
|
| 1476 |
+
def _valid_for(s):
|
| 1477 |
+
va, Vs = cams[s]
|
| 1478 |
+
fb = (np.asarray(self._tar_valid_views(sha, tar, va, sc, centers, Vs, s),
|
| 1479 |
+
dtype=int) if self.filter_bad_views
|
| 1480 |
+
else np.arange(Vs, dtype=int))
|
| 1481 |
+
if s == "recgen": # QC is the hard rule
|
| 1482 |
+
both = np.asarray([i for i in fb if i in set(rg_ok)], dtype=int)
|
| 1483 |
+
fb = both if len(both) else np.asarray(rg_ok, dtype=int)
|
| 1484 |
+
return fb
|
| 1485 |
+
|
| 1486 |
+
if not (has_w and has_r):
|
| 1487 |
+
# SINGLE source (white-only = every legacy object; recgen-only = the
|
| 1488 |
+
# latent-only new slats): the exact legacy draw on that source.
|
| 1489 |
+
src = "white" if has_w else "recgen"
|
| 1490 |
+
views_all, V = cams[src]
|
| 1491 |
+
n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
|
| 1492 |
+
if src == "recgen": # QC-valid (+ bad-view gate) only
|
| 1493 |
+
valid_arr = _valid_for("recgen")
|
| 1494 |
+
n_sub = min(int(n_sub), len(valid_arr))
|
| 1495 |
+
sub = rng.choice(valid_arr, size=n_sub, replace=False)
|
| 1496 |
+
elif self.filter_bad_views: # OPT-IN: draw only from valid
|
| 1497 |
+
valid_arr = np.asarray(
|
| 1498 |
+
self._tar_valid_views(sha, tar, views_all, sc, centers, V, src),
|
| 1499 |
+
dtype=int)
|
| 1500 |
+
n_sub = min(int(n_sub), len(valid_arr))
|
| 1501 |
+
sub = rng.choice(valid_arr, size=n_sub, replace=False) # anchor in valid
|
| 1502 |
+
else:
|
| 1503 |
+
sub = rng.choice(V, size=n_sub, replace=False)
|
| 1504 |
else:
|
| 1505 |
+
# BOTH sources -> ONE POOL. Candidate anchors = the union of the
|
| 1506 |
+
# object's (valid) white cond views and (QC-valid) RecGen views; the
|
| 1507 |
+
# anchor is drawn uniformly from the pool, and the ANCHOR'S SOURCE
|
| 1508 |
+
# DECIDES EVERYTHING: all auxiliary views are drawn from that same
|
| 1509 |
+
# source only (with that source's cameras) — never mixed.
|
| 1510 |
+
valid = {s: _valid_for(s) for s in ("white", "recgen")}
|
| 1511 |
+
pool = ([("white", int(i)) for i in valid["white"]]
|
| 1512 |
+
+ [("recgen", int(i)) for i in valid["recgen"]])
|
| 1513 |
+
n_sub, no_seed, mode = self._mixture_n_sub(idx, len(pool), rng)
|
| 1514 |
+
src, a_i = pool[int(rng.integers(len(pool)))]
|
| 1515 |
+
views_all, V = cams[src]
|
| 1516 |
+
rest = np.asarray([i for i in valid[src] if i != a_i], dtype=int)
|
| 1517 |
+
n_sub = min(int(n_sub), 1 + len(rest))
|
| 1518 |
+
aux = (rng.choice(rest, size=n_sub - 1, replace=False) if n_sub > 1
|
| 1519 |
+
else np.zeros(0, dtype=int))
|
| 1520 |
+
sub = np.concatenate([[a_i], aux]).astype(int)
|
| 1521 |
order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
|
| 1522 |
|
| 1523 |
crops, uvs, viss, fulls = [], [], [], []
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/data/slatflow_dataset.py.bak_predatveiw
ADDED
|
@@ -0,0 +1,804 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""TrellisSlatFlowDataset — per-view DINO inputs + cameras + SLAT target on top
|
| 2 |
+
of TrellisSSFlowDataset (SLAT_GEN_PLAN §8).
|
| 3 |
+
|
| 4 |
+
Reused verbatim from the SS dataset: object scan / meta.json completion marker /
|
| 5 |
+
split logic, the per-(sample, epoch) view-subset RNG, and the views.npz loader
|
| 6 |
+
(per-view K + c2w_cv — those cameras cover BOTH the Plücker rays and the
|
| 7 |
+
voxel->patch projection; no new camera fields).
|
| 8 |
+
|
| 9 |
+
Per __getitem__ (CPU only — all frozen-GPU work happens in the train step):
|
| 10 |
+
* coords_slat = downsample(prune(voxels.npz coords, dist=1)) — the exact
|
| 11 |
+
inference coordinate ops (pure coordinate transforms, run on CPU torch);
|
| 12 |
+
* a view subset S (|S| ~ U{min_views..max_views}); anchor = S[0]; view order
|
| 13 |
+
of every per-view tensor = [S..., complement...] so the model can slice
|
| 14 |
+
the subset (x0/cond) vs all views;
|
| 15 |
+
* per view: the RGBA crop in the STORED bbox frame -> 518 LANCZOS ->
|
| 16 |
+
premult-alpha-on-black float [3,518,518] (dino_features convention,
|
| 17 |
+
batch_appforce:253-264) — feeds the shared frozen DINO forward;
|
| 18 |
+
* per view: (uv, vis) — voxel-center projections through the ABSOLUTE camera
|
| 19 |
+
(project_visible: nearest-pixel depth test, TOL=0.02) with uv already in
|
| 20 |
+
the crop-frame [-1,1] grid_sample convention (sample_feats :267-275),
|
| 21 |
+
computed in float64 and cast to float32;
|
| 22 |
+
* anchor: the 4 slat condition tensors via the DEPLOYED slat_preprocessor
|
| 23 |
+
recipe (OQ-6) on the raw RGBA input image;
|
| 24 |
+
* non-anchor: per-patch anchor-relative Plücker rays [1369,6] (§4a) and the
|
| 25 |
+
soft-mask camera dicts (§4b);
|
| 26 |
+
* x1: ``slat/slat_official_sam3d.npz`` — the OFFICIAL TRELLIS-toolkit
|
| 27 |
+
latent with SAM3D's slat_encoder (RAW, all active voxels; produced by
|
| 28 |
+
``tools/gen_x1_official.py``). Rows are coord-hash-matched onto
|
| 29 |
+
coords_slat (a prune-subset of the voxel set) and full coverage is
|
| 30 |
+
asserted. There is NO on-the-fly fallback: the former full-view
|
| 31 |
+
visible-mean target (reimplemented, non-official) was deleted — the
|
| 32 |
+
train step errors if the file is absent.
|
| 33 |
+
|
| 34 |
+
RGB SOURCE: the production layout stores ``views/rgb/{i:03d}.webp`` + a
|
| 35 |
+
``views/bbox.npy`` sidecar. The toys4k SMOKE objects pre-date that layout, so
|
| 36 |
+
when those files are missing the loader falls back to the toys4k1k render
|
| 37 |
+
assets (inputs/<obj>_<view>.png = the bbox crop, renders/<obj>_<view>.npz for
|
| 38 |
+
bbox/res) with the fixed front/side/oside/back view order — the exact files
|
| 39 |
+
batch_appforce consumed.
|
| 40 |
+
"""
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
import os
|
| 44 |
+
from typing import Dict, List, Optional, Sequence
|
| 45 |
+
|
| 46 |
+
import numpy as np
|
| 47 |
+
import torch
|
| 48 |
+
from PIL import Image
|
| 49 |
+
|
| 50 |
+
from mvsam3d.data.ssflow_dataset import TrellisSSFlowDataset, load_views_npz
|
| 51 |
+
from mvsam3d.data.slat_preprocess import preprocess_slat_image
|
| 52 |
+
from mvsam3d.model.mv_slat_condition import plucker_rays_for_view
|
| 53 |
+
|
| 54 |
+
TOL = 0.02 # visibility depth tolerance (batch_appforce)
|
| 55 |
+
VOX = 64
|
| 56 |
+
# SS-stage seed-mixture mode -> integer code carried in the item/batch (mirrors
|
| 57 |
+
# ss_imgcond_dataset._MODE_CODE; 'val' shares 'single'==1 as an eval label).
|
| 58 |
+
_SEED_MODE_CODE = {"none": 0, "single": 1, "multi": 2, "legacy": 3, "val": 1}
|
| 59 |
+
TOYS4K1K = "/lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k1k"
|
| 60 |
+
TOYS4K_VIEW_ORDER = ("front", "side", "oside", "back")
|
| 61 |
+
# TRELLIS-style per-object voxel cap (structured_latent.py filter_metadata):
|
| 62 |
+
# objects with MORE active 64^3 voxels than this are DROPPED at index-build
|
| 63 |
+
# time (never downsampled — downsample_sparse_structure is inference-only and
|
| 64 |
+
# corrupts training targets). 20000 = the single-object SLAT OOM-safe bound
|
| 65 |
+
# (PLAN/BATCH_SIZE_BENCH.md; OOM at N~24000). Drops ~4.8% of trellis500k.
|
| 66 |
+
MAX_NUM_VOXELS_DEFAULT = 20000
|
| 67 |
+
# visibility source when a view has no rendered depth (production data):
|
| 68 |
+
# raymarch = exact self-occlusion through the 64^3 grid (default)
|
| 69 |
+
# zbuffer = image-space min-splat (biased ~1 voxel near; see data/vis_check)
|
| 70 |
+
# Calibrated against rendered-depth GT on 301 toys4k objects x 4 views
|
| 71 |
+
# (data/vis_check/vis_check4.py, run4.log; agreement / IoU vs project_visible_np):
|
| 72 |
+
# zbuffer tol .03 .8762 / .6692 raymarch skip 1.5 .8113 / .4355
|
| 73 |
+
# raymarch skip 2.0 .8742 / .6646 raymarch skip 2.5 .8755 / .6724 <- default
|
| 74 |
+
# raymarch skip 3.0 .8519 / .6472 ray2.0 OR zbuf .8757 / .6729
|
| 75 |
+
# skip_vox 2.5 mirrors the GT's own TOL=0.02 slack (~1.3 voxels) plus the
|
| 76 |
+
# half-voxel centre offset; it matches the z-buffer's accuracy without the
|
| 77 |
+
# z-buffer's splat-radius/resolution heuristic. ~40 ms/view (100 ms under a
|
| 78 |
+
# 48-way loaded box) at N=20000.
|
| 79 |
+
VIS_METHOD = os.environ.get("MVSAM3D_VIS_METHOD", "raymarch")
|
| 80 |
+
VIS_SKIP_VOX = float(os.environ.get("MVSAM3D_VIS_SKIP_VOX", "2.5"))
|
| 81 |
+
VIS_ZBUF_TOL = float(os.environ.get("MVSAM3D_VIS_ZBUF_TOL", "0.03"))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def num_active_voxels(voxels_npz: str) -> int:
|
| 85 |
+
"""Active 64^3 voxel count = #rows of voxels.npz 'coords'.
|
| 86 |
+
|
| 87 |
+
Reads only the .npy header inside the zip archive (streaming decompression
|
| 88 |
+
of a few hundred bytes) — cheap enough to run over the whole index at
|
| 89 |
+
dataset-build time. Falls back to a full np.load on any header oddity."""
|
| 90 |
+
import zipfile
|
| 91 |
+
from numpy.lib import format as npfmt
|
| 92 |
+
try:
|
| 93 |
+
with zipfile.ZipFile(voxels_npz) as zf:
|
| 94 |
+
with zf.open("coords.npy") as f:
|
| 95 |
+
version = npfmt.read_magic(f)
|
| 96 |
+
shape, _, _ = npfmt._read_array_header(f, version)
|
| 97 |
+
return int(shape[0])
|
| 98 |
+
except Exception:
|
| 99 |
+
return int(np.load(voxels_npz)["coords"].shape[0])
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# --------------------------------------------------------------------------- #
|
| 103 |
+
def slat_coords_from_voxels(coords3: np.ndarray) -> torch.Tensor:
|
| 104 |
+
"""voxels.npz coords (N,3) -> the SLAT coord set (M,4) int32 (batch col 0):
|
| 105 |
+
prune_sparse_structure(dist=1) + downsample_sparse_structure — the exact
|
| 106 |
+
inference ops (inference_pipeline.py:818-833), on CPU."""
|
| 107 |
+
from sam3d_objects.pipeline.inference_utils import (
|
| 108 |
+
downsample_sparse_structure, prune_sparse_structure)
|
| 109 |
+
c = torch.from_numpy(np.asarray(coords3, dtype=np.int64)).int()
|
| 110 |
+
c4 = torch.cat([torch.zeros(len(c), 1, dtype=torch.int32), c], dim=1)
|
| 111 |
+
c4 = prune_sparse_structure(c4, max_neighbor_axes_dist=1)
|
| 112 |
+
c4, _ = downsample_sparse_structure(c4)
|
| 113 |
+
return c4.int()
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def project_visible_np(centers: np.ndarray, depth: np.ndarray, fx, fy, cx, cy,
|
| 117 |
+
c2w: np.ndarray, res: int):
|
| 118 |
+
"""batch_appforce_sam3d.project_visible (:236-250), verbatim numpy port.
|
| 119 |
+
centers (N,3) float64 canonical -> (u, v, z, visible)."""
|
| 120 |
+
w2c = np.linalg.inv(c2w)
|
| 121 |
+
R, t = w2c[:3, :3], w2c[:3, 3]
|
| 122 |
+
xc = centers @ R.T + t
|
| 123 |
+
z = xc[:, 2]
|
| 124 |
+
u = fx * xc[:, 0] / z + cx
|
| 125 |
+
vv = fy * xc[:, 1] / z + cy
|
| 126 |
+
ui = np.round(u).astype(int)
|
| 127 |
+
vi = np.round(vv).astype(int)
|
| 128 |
+
inframe = (z > 0) & (ui >= 0) & (ui < res) & (vi >= 0) & (vi < res)
|
| 129 |
+
dep = np.zeros(len(centers))
|
| 130 |
+
dep[inframe] = depth[vi[inframe], ui[inframe]]
|
| 131 |
+
visible = inframe & (dep > 0) & (np.abs(z - dep) < TOL)
|
| 132 |
+
return u, vv, z, visible
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def zbuffer_visible(centers: np.ndarray, fx, fy, cx, cy, c2w: np.ndarray,
|
| 136 |
+
res: int, tol: float = TOL):
|
| 137 |
+
"""Per-voxel visibility from the object's OWN voxels (no depth image).
|
| 138 |
+
|
| 139 |
+
Self-occlusion test: every voxel is splatted into a per-view z-buffer over
|
| 140 |
+
its projected footprint (min planar depth per pixel); a voxel is visible
|
| 141 |
+
iff its own depth is within ``tol`` of the buffer at its centre pixel —
|
| 142 |
+
i.e. nothing closer covers it. This is the same decision rule as
|
| 143 |
+
``project_visible_np`` with the rendered depth map replaced by the depth
|
| 144 |
+
of the voxelised surface itself (exact up to the 64^3 voxel size, which is
|
| 145 |
+
below ``tol`` = 0.02 ~ 1.3 voxels). Returns (u, v, z, visible) like
|
| 146 |
+
``project_visible_np``. Production renders (slat50k cond views) ship no
|
| 147 |
+
depth pass, so this is the visibility source for training; the eval sets
|
| 148 |
+
keep their rendered GT depth."""
|
| 149 |
+
w2c = np.linalg.inv(c2w)
|
| 150 |
+
R, t = w2c[:3, :3], w2c[:3, 3]
|
| 151 |
+
xc = centers @ R.T + t
|
| 152 |
+
z = xc[:, 2]
|
| 153 |
+
u = fx * xc[:, 0] / z + cx
|
| 154 |
+
vv = fy * xc[:, 1] / z + cy
|
| 155 |
+
ui = np.round(u).astype(int)
|
| 156 |
+
vi = np.round(vv).astype(int)
|
| 157 |
+
inframe = (z > 0) & (ui >= 0) & (ui < res) & (vi >= 0) & (vi < res)
|
| 158 |
+
visible = np.zeros(len(centers), dtype=bool)
|
| 159 |
+
if not inframe.any():
|
| 160 |
+
return u, vv, z, visible
|
| 161 |
+
# projected voxel footprint (pixels): voxel edge 1/64 at depth z
|
| 162 |
+
zf = z[inframe]
|
| 163 |
+
r = int(np.clip(np.ceil(0.5 * fx / VOX / float(np.median(zf))), 1, 16))
|
| 164 |
+
buf = np.full((res, res), np.inf, dtype=np.float64)
|
| 165 |
+
ui_f, vi_f = ui[inframe], vi[inframe]
|
| 166 |
+
for dy in range(-r, r + 1):
|
| 167 |
+
yy = vi_f + dy
|
| 168 |
+
oky = (yy >= 0) & (yy < res)
|
| 169 |
+
for dx in range(-r, r + 1):
|
| 170 |
+
xx = ui_f + dx
|
| 171 |
+
ok = oky & (xx >= 0) & (xx < res)
|
| 172 |
+
np.minimum.at(buf, (yy[ok], xx[ok]), zf[ok])
|
| 173 |
+
dep = buf[vi_f, ui_f]
|
| 174 |
+
visible[inframe] = np.abs(zf - dep) < tol
|
| 175 |
+
return u, vv, z, visible
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def raymarch_visible(centers: np.ndarray, coords3: np.ndarray, fx, fy, cx, cy,
|
| 179 |
+
c2w: np.ndarray, res: int, skip_vox: float = 2.5,
|
| 180 |
+
step_vox: float = 0.5):
|
| 181 |
+
"""Per-voxel visibility by EXACT ray marching through the 64^3 occupancy
|
| 182 |
+
grid (no image-space splatting, hence none of the z-buffer's near-bias):
|
| 183 |
+
a voxel is visible iff the ray from its centre to the camera centre meets
|
| 184 |
+
no occupied voxel beyond ``skip_vox`` voxels from itself (the skip keeps
|
| 185 |
+
the voxel's own surface neighbours from blocking grazing rays, and matches
|
| 186 |
+
the rendered-depth reference's TOL=0.02 depth slack; see VIS_SKIP_VOX for
|
| 187 |
+
the calibration). Returns (u, v, z, visible) like ``project_visible_np``."""
|
| 188 |
+
occ = np.zeros((VOX, VOX, VOX), dtype=bool)
|
| 189 |
+
c3 = np.asarray(coords3, dtype=np.int64)
|
| 190 |
+
occ[c3[:, 0], c3[:, 1], c3[:, 2]] = True
|
| 191 |
+
w2c = np.linalg.inv(c2w)
|
| 192 |
+
R, t = w2c[:3, :3], w2c[:3, 3]
|
| 193 |
+
xc = centers @ R.T + t
|
| 194 |
+
z = xc[:, 2]
|
| 195 |
+
u = fx * xc[:, 0] / z + cx
|
| 196 |
+
vv = fy * xc[:, 1] / z + cy
|
| 197 |
+
ui = np.round(u).astype(int)
|
| 198 |
+
vi = np.round(vv).astype(int)
|
| 199 |
+
inframe = (z > 0) & (ui >= 0) & (ui < res) & (vi >= 0) & (vi < res)
|
| 200 |
+
cam = c2w[:3, 3]
|
| 201 |
+
d = cam[None, :] - centers # (N,3) towards the camera
|
| 202 |
+
dist = np.linalg.norm(d, axis=1)
|
| 203 |
+
d = d / np.maximum(dist[:, None], 1e-9)
|
| 204 |
+
# march only while still inside the cube: t_max = exit distance of the ray
|
| 205 |
+
# from [-0.5,0.5]^3 (voxels outside are never occupied)
|
| 206 |
+
with np.errstate(divide="ignore", invalid="ignore"):
|
| 207 |
+
t1 = (-0.5 - centers) / d
|
| 208 |
+
t2 = (0.5 - centers) / d
|
| 209 |
+
t_exit = np.nanmin(np.where(d != 0, np.maximum(t1, t2), np.inf), axis=1)
|
| 210 |
+
t_max = np.minimum(dist, t_exit)
|
| 211 |
+
# march the still-unblocked voxels only (active set shrinks fast: most
|
| 212 |
+
# voxels are occluded within a few steps), 16 steps per vectorised chunk
|
| 213 |
+
occf = occ.reshape(-1)
|
| 214 |
+
step = step_vox / VOX
|
| 215 |
+
t = skip_vox / VOX
|
| 216 |
+
blocked = np.zeros(len(centers), dtype=bool)
|
| 217 |
+
act = np.flatnonzero(inframe)
|
| 218 |
+
CH = 16
|
| 219 |
+
while act.size:
|
| 220 |
+
tt = t + step * np.arange(CH) # (CH,)
|
| 221 |
+
ca, da, tm = centers[act], d[act], t_max[act]
|
| 222 |
+
pts = ca[:, None, :] + tt[None, :, None] * da[:, None, :] # (A,CH,3)
|
| 223 |
+
valid = tt[None, :] <= tm[:, None]
|
| 224 |
+
gi = np.floor((pts + 0.5) * VOX).astype(np.int64)
|
| 225 |
+
inside = valid & np.all((gi >= 0) & (gi < VOX), axis=2)
|
| 226 |
+
np.clip(gi, 0, VOX - 1, out=gi)
|
| 227 |
+
flat = (gi[..., 0] * VOX + gi[..., 1]) * VOX + gi[..., 2]
|
| 228 |
+
hit = (occf[flat] & inside).any(axis=1)
|
| 229 |
+
blocked[act[hit]] = True
|
| 230 |
+
t = float(tt[-1]) + step
|
| 231 |
+
act = act[~hit & (tm >= t)]
|
| 232 |
+
visible = inframe & ~blocked
|
| 233 |
+
return u, vv, z, visible
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def crop_uv_norm(u: np.ndarray, v: np.ndarray, bbox) -> np.ndarray:
|
| 237 |
+
"""sample_feats' grid_sample coords (:267-275): render-frame (u,v) -> the
|
| 238 |
+
crop-frame [-1,1] convention, float64 -> float32. (N,2)."""
|
| 239 |
+
y0, y1, x0, x1 = [float(b) for b in bbox]
|
| 240 |
+
# PER-AXIS normalisation: the stored crop is rgba[y0:y1, x0:x1] (no square
|
| 241 |
+
# padding, adapt_slat50k.crop_bbox clamps at the frame edge) and is resized
|
| 242 |
+
# anisotropically to 518x518, so u maps by the crop WIDTH and v by its
|
| 243 |
+
# HEIGHT. Identical to the old single-`side` formula whenever w == h.
|
| 244 |
+
w = x1 - x0
|
| 245 |
+
h = y1 - y0
|
| 246 |
+
un = (u - x0 + 0.5) / w * 2 - 1
|
| 247 |
+
vn = (v - y0 + 0.5) / h * 2 - 1
|
| 248 |
+
return np.stack([un, vn], -1).astype(np.float32)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def rgba_to_crop(rgba_uint8: np.ndarray) -> Dict[str, np.ndarray]:
|
| 252 |
+
"""In-memory RGBA uint8 (H,W,4) -> {'premult' [3,518,518] float32
|
| 253 |
+
(dino_features recipe: 518 LANCZOS + alpha-premult on black),
|
| 254 |
+
'rgba_uint8' (H,W,4)}. Shared by the clean-RGBA file loader and the
|
| 255 |
+
RecGen matting path (which has no on-disk RGBA)."""
|
| 256 |
+
im = Image.fromarray(np.ascontiguousarray(rgba_uint8), mode="RGBA")
|
| 257 |
+
im518 = im.resize((518, 518), Image.Resampling.LANCZOS)
|
| 258 |
+
a = np.array(im518).astype(np.float32) / 255.0
|
| 259 |
+
rgb = a[:, :, :3] * a[:, :, 3:4] # premult alpha on BLACK
|
| 260 |
+
return {"premult": rgb.transpose(2, 0, 1), "rgba_uint8": rgba_uint8}
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def load_crop_rgba(path: str) -> Dict[str, np.ndarray]:
|
| 264 |
+
"""RGBA crop file -> {'premult' [3,518,518] float32 (dino_features recipe:
|
| 265 |
+
518 LANCZOS + alpha-premult on black), 'rgba_uint8' (H,W,4)} ."""
|
| 266 |
+
return rgba_to_crop(np.array(Image.open(path).convert("RGBA")))
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def matte_rgba(rgb_uint8: np.ndarray, mask_uint8: np.ndarray) -> np.ndarray:
|
| 270 |
+
"""RGB (H,W,3) + binary/greyscale mask (H,W) -> RGBA (H,W,4) uint8 CROPPED
|
| 271 |
+
to the mask's bounding box. The RecGen ALPHA the SLAT loader's visibility
|
| 272 |
+
gate needs: cond_image has BACKGROUND baked in and no alpha, so the object
|
| 273 |
+
mask (prefer cond_mask_sam2, fallback cond_mask) becomes the alpha channel.
|
| 274 |
+
Returns the tight-bbox crop so ``crop_uv_norm`` / the alpha gate work exactly
|
| 275 |
+
as they do on the clean transparent-background renders."""
|
| 276 |
+
rgb = np.asarray(rgb_uint8)
|
| 277 |
+
m = np.asarray(mask_uint8)
|
| 278 |
+
if m.ndim == 3:
|
| 279 |
+
m = m[..., 0]
|
| 280 |
+
a = (m > 127).astype(np.uint8) * 255
|
| 281 |
+
ys, xs = np.nonzero(a)
|
| 282 |
+
if len(ys) == 0: # empty mask: whole frame
|
| 283 |
+
y0, y1, x0, x1 = 0, rgb.shape[0], 0, rgb.shape[1]
|
| 284 |
+
else:
|
| 285 |
+
y0, y1 = int(ys.min()), int(ys.max()) + 1
|
| 286 |
+
x0, x1 = int(xs.min()), int(xs.max()) + 1
|
| 287 |
+
rgba = np.concatenate([rgb, a[..., None]], axis=2) # (H,W,4)
|
| 288 |
+
crop = rgba[y0:y1, x0:x1]
|
| 289 |
+
return crop, (float(y0), float(y1), float(x0), float(x1))
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
# --------------------------------------------------------------------------- #
|
| 293 |
+
class TrellisSlatFlowDataset(TrellisSSFlowDataset):
|
| 294 |
+
"""SLAT-flow training samples: SS dataset + per-view RGB/DINO inputs,
|
| 295 |
+
cameras, and the x1 target hookup."""
|
| 296 |
+
|
| 297 |
+
def __init__(self, *args, toys4k1k_root: str = TOYS4K1K,
|
| 298 |
+
max_num_voxels: Optional[int] = MAX_NUM_VOXELS_DEFAULT,
|
| 299 |
+
p_seed_none: float = 0.0, # unified mixture: P(EMPTY seed)
|
| 300 |
+
p_seed_single: float = 0.0, # P(single-view seed); rest=multi
|
| 301 |
+
recgen_roots: Optional[Sequence[str]] = None,
|
| 302 |
+
recgen_pose_store: Optional[str] = None,
|
| 303 |
+
**kwargs):
|
| 304 |
+
super().__init__(*args, **kwargs)
|
| 305 |
+
self.toys4k1k_root = toys4k1k_root
|
| 306 |
+
# encoded_store recgen root holding the canonical cube cameras
|
| 307 |
+
# (pose_cube.npz / views.npz) looked up by sha; None -> analytic fallback.
|
| 308 |
+
self._recgen_pose_store = recgen_pose_store
|
| 309 |
+
# ---- SS-STAGE seed MIXTURE (mirror of ss_imgcond_dataset.seed_mode) --
|
| 310 |
+
# p_seed_none = P(EMPTY appearance seed -> x0 all-noise, no forcing)
|
| 311 |
+
# p_seed_single= P(single-view seed -> n_sub == 1)
|
| 312 |
+
# remainder = multi (n_sub ~ U{max(2,min_views)..max_views})
|
| 313 |
+
# BOTH == 0.0 -> the LEGACY path (n_sub ~ U{min_views..max_views}), which
|
| 314 |
+
# is byte-identical to the pre-mixture behaviour. VAL (fixed_num_views
|
| 315 |
+
# set) is never randomised. Own rng stream (offset 4242 for the mode,
|
| 316 |
+
# 7717 for the multi view-count) so the mixture never perturbs the
|
| 317 |
+
# existing view-subset / crop draws on the main [seed,epoch,idx] stream.
|
| 318 |
+
self.p_seed_none = float(p_seed_none)
|
| 319 |
+
self.p_seed_single = float(p_seed_single)
|
| 320 |
+
assert self.p_seed_none + self.p_seed_single <= 1.0 + 1e-9, (
|
| 321 |
+
self.p_seed_none, self.p_seed_single)
|
| 322 |
+
# TRELLIS-style max_num_voxels FILTER (mirror of datasets/
|
| 323 |
+
# structured_latent.py filter_metadata: metadata['num_voxels'] <= cap):
|
| 324 |
+
# applied at index-build time so oversized objects never reach the
|
| 325 |
+
# loader. None / 0 disables the filter.
|
| 326 |
+
self.max_num_voxels = int(max_num_voxels) if max_num_voxels else None
|
| 327 |
+
if self.max_num_voxels:
|
| 328 |
+
n_before = len(self.dirs)
|
| 329 |
+
self.dirs = [
|
| 330 |
+
d for d in self.dirs
|
| 331 |
+
if num_active_voxels(os.path.join(d, "geometry", "voxels.npz"))
|
| 332 |
+
<= self.max_num_voxels
|
| 333 |
+
]
|
| 334 |
+
n_drop = n_before - len(self.dirs)
|
| 335 |
+
print(f"[slatflow-data] split={self.split} max_num_voxels="
|
| 336 |
+
f"{self.max_num_voxels}: kept {len(self.dirs)}/{n_before} "
|
| 337 |
+
f"objects (dropped {n_drop})", flush=True)
|
| 338 |
+
if not self.dirs:
|
| 339 |
+
raise RuntimeError(
|
| 340 |
+
f"max_num_voxels={self.max_num_voxels} dropped every object "
|
| 341 |
+
f"in split={self.split}")
|
| 342 |
+
|
| 343 |
+
# ---- RecGen roots (ADDITIVE, opt-in) --------------------------------
|
| 344 |
+
# RecGen sample objects (<root>/<id>/ with slat_coords.npy + slat_feats.npy
|
| 345 |
+
# + cond_image_*.jpg + cond_mask*.png + view_metadata.json) do NOT carry a
|
| 346 |
+
# meta.json / geometry/voxels.npz, so they are invisible to scan_objects.
|
| 347 |
+
# Scan them separately and APPEND to self.dirs, tracking which dirs are
|
| 348 |
+
# recgen in self._recgen so __getitem__ can route them to the matting /
|
| 349 |
+
# released-SLAT loader. Non-recgen behaviour is untouched.
|
| 350 |
+
self._recgen = set()
|
| 351 |
+
if recgen_roots:
|
| 352 |
+
import glob as _glob
|
| 353 |
+
if isinstance(recgen_roots, (str, os.PathLike)):
|
| 354 |
+
recgen_roots = [recgen_roots]
|
| 355 |
+
rg = []
|
| 356 |
+
for root in recgen_roots:
|
| 357 |
+
for sc in _glob.iglob(os.path.join(root, "*", "slat_coords.npy")):
|
| 358 |
+
rg.append(os.path.dirname(sc))
|
| 359 |
+
rg = sorted(set(rg))
|
| 360 |
+
self._recgen = set(rg)
|
| 361 |
+
self.dirs = list(self.dirs) + rg
|
| 362 |
+
print(f"[slatflow-data] split={self.split} recgen_roots: added "
|
| 363 |
+
f"{len(rg)} recgen objects (total {len(self.dirs)})", flush=True)
|
| 364 |
+
|
| 365 |
+
# ---- SS-stage seed mixture (mirror of ss_imgcond_dataset.seed_mode) -- #
|
| 366 |
+
def seed_mode(self, idx: int, epoch: int) -> str:
|
| 367 |
+
"""'none' | 'single' | 'multi' | 'legacy' for this (object, epoch).
|
| 368 |
+
|
| 369 |
+
Own rng stream (offset 4242) so the mode decision never perturbs the
|
| 370 |
+
[seed,epoch,idx] view-subset / crop draws. VAL (fixed_num_views set)
|
| 371 |
+
keeps the fixed eval view count (returns 'single' as an unused label —
|
| 372 |
+
__getitem__ pins n_sub to fixed_num_views in that case). With BOTH
|
| 373 |
+
knobs 0.0 the mixture is OFF -> 'legacy' (no draw, main stream
|
| 374 |
+
untouched)."""
|
| 375 |
+
if self.fixed_num_views is not None: # val: unchanged
|
| 376 |
+
return "single"
|
| 377 |
+
if self.p_seed_none <= 0.0 and self.p_seed_single <= 0.0:
|
| 378 |
+
return "legacy"
|
| 379 |
+
r = float(np.random.default_rng(
|
| 380 |
+
[self.seed, int(epoch), int(idx), 4242]).random())
|
| 381 |
+
if r < self.p_seed_none:
|
| 382 |
+
return "none"
|
| 383 |
+
if r < self.p_seed_none + self.p_seed_single:
|
| 384 |
+
return "single"
|
| 385 |
+
return "multi"
|
| 386 |
+
|
| 387 |
+
# ---- per-object RGB/bbox adapter ---------------------------------- #
|
| 388 |
+
def _rgb_assets(self, obj_dir: str, name: str, n_views: int) -> List[Dict]:
|
| 389 |
+
"""-> per-view dict(png=..., bbox=(4,), res=int). Production layout
|
| 390 |
+
first (views/rgb + views/bbox.npy); toys4k1k fallback for smoke."""
|
| 391 |
+
rgb_dir = os.path.join(obj_dir, "views", "rgb")
|
| 392 |
+
bbox_path = os.path.join(obj_dir, "views", "bbox.npy")
|
| 393 |
+
if os.path.isdir(rgb_dir) and os.path.isfile(bbox_path):
|
| 394 |
+
bboxes = np.load(bbox_path)
|
| 395 |
+
out = []
|
| 396 |
+
for i in range(n_views):
|
| 397 |
+
png = os.path.join(rgb_dir, f"{i:03d}.png") # lossless (adapter)
|
| 398 |
+
if not os.path.isfile(png):
|
| 399 |
+
png = os.path.join(rgb_dir, f"{i:03d}.webp")
|
| 400 |
+
out.append(dict(png=png, bbox=bboxes[i].astype(np.float64), res=None))
|
| 401 |
+
return out
|
| 402 |
+
# toys4k smoke fallback (bbox/res from the render npz; png = the crop)
|
| 403 |
+
out = []
|
| 404 |
+
for i in range(n_views):
|
| 405 |
+
tag = TOYS4K_VIEW_ORDER[i]
|
| 406 |
+
png = os.path.join(self.toys4k1k_root, "inputs", f"{name}_{tag}.png")
|
| 407 |
+
rz = np.load(os.path.join(self.toys4k1k_root, "renders",
|
| 408 |
+
f"{name}_{tag}.npz"))
|
| 409 |
+
out.append(dict(png=png, bbox=rz["bbox"].astype(np.float64),
|
| 410 |
+
res=int(rz["res"])))
|
| 411 |
+
return out
|
| 412 |
+
|
| 413 |
+
# ---- SS-stage seed-mixture view-count selection --------------------- #
|
| 414 |
+
def _mixture_n_sub(self, idx: int, V: int, rng) -> tuple:
|
| 415 |
+
"""Return (n_sub, no_seed, mode). With the mixture OFF (both knobs 0.0) this
|
| 416 |
+
is the LEGACY draw — ``rng.integers(min_views, max_views+1)`` off the
|
| 417 |
+
MAIN [seed,epoch,idx] stream — byte-identical to the pre-mixture path.
|
| 418 |
+
With the mixture ON the mode comes from the independent 4242 stream
|
| 419 |
+
(seed_mode), the multi view-count from the independent 7717 stream, so
|
| 420 |
+
the main stream (and thus the ``sub`` view choice) is only advanced by
|
| 421 |
+
the legacy branch — exactly as before."""
|
| 422 |
+
if self.fixed_num_views is not None: # VAL: never randomised
|
| 423 |
+
return min(int(self.fixed_num_views), V), False, "val"
|
| 424 |
+
mode = self.seed_mode(idx, self._epoch)
|
| 425 |
+
if mode == "legacy":
|
| 426 |
+
return (min(int(rng.integers(self.min_views, self.max_views + 1)), V),
|
| 427 |
+
False, "legacy")
|
| 428 |
+
if mode == "none": # EMPTY appearance seed: 1 cond view, x0 all-noise
|
| 429 |
+
return min(1, V), True, "none"
|
| 430 |
+
if mode == "single":
|
| 431 |
+
return min(1, V), False, "single"
|
| 432 |
+
# multi: n_sub ~ U{max(2,min_views) .. max_views}
|
| 433 |
+
lo = max(2, int(self.min_views))
|
| 434 |
+
hi = max(lo, int(self.max_views))
|
| 435 |
+
k = int(np.random.default_rng(
|
| 436 |
+
[self.seed, int(self._epoch), int(idx), 7717]).integers(lo, hi + 1))
|
| 437 |
+
return min(k, V), False, "multi"
|
| 438 |
+
|
| 439 |
+
# ---- item ----------------------------------------------------------- #
|
| 440 |
+
def __getitem__(self, idx: int) -> Dict:
|
| 441 |
+
d = self.dirs[idx]
|
| 442 |
+
if d in self._recgen: # RecGen loader (additive)
|
| 443 |
+
return self._getitem_recgen(idx)
|
| 444 |
+
name = os.path.basename(d)
|
| 445 |
+
views_path = os.path.join(d, "views", "views.npz")
|
| 446 |
+
|
| 447 |
+
# per-(sample, epoch) deterministic RNG — same recipe as SSFlow
|
| 448 |
+
rng = np.random.default_rng([self.seed, self._epoch, idx])
|
| 449 |
+
views_all = load_views_npz(views_path) # K, c2w, depth per view
|
| 450 |
+
V = len(views_all)
|
| 451 |
+
n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
|
| 452 |
+
sub = rng.choice(V, size=n_sub, replace=False)
|
| 453 |
+
order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
|
| 454 |
+
|
| 455 |
+
# SLAT coord set (exact inference coordinate ops)
|
| 456 |
+
vox = np.load(os.path.join(d, "geometry", "voxels.npz"))
|
| 457 |
+
coords = slat_coords_from_voxels(vox["coords"]) # (N,4) int32
|
| 458 |
+
centers = (coords[:, 1:].numpy().astype(np.float64) + 0.5) / VOX - 0.5
|
| 459 |
+
|
| 460 |
+
assets = self._rgb_assets(d, name, V)
|
| 461 |
+
|
| 462 |
+
# Only the SAMPLED subset S feeds the model (cond = anchor + S[1:],
|
| 463 |
+
# x0 seed = visible-mean over S, x1 = the precomputed official target),
|
| 464 |
+
# so crops/uv/vis are built for S ONLY: an 8-view object costs |S| <= 4
|
| 465 |
+
# DINO forwards + projections, not 8. view_order still lists all views.
|
| 466 |
+
crops, uvs, viss = [], [], []
|
| 467 |
+
for i in order[:n_sub]:
|
| 468 |
+
vw = views_all[i]
|
| 469 |
+
a = assets[i]
|
| 470 |
+
if a["res"] is not None:
|
| 471 |
+
res = a["res"]
|
| 472 |
+
elif vw["depth"] is not None:
|
| 473 |
+
res = vw["depth"].shape[0]
|
| 474 |
+
else:
|
| 475 |
+
res = vw["res"]
|
| 476 |
+
if vw["depth"] is not None: # rendered GT depth (eval sets)
|
| 477 |
+
u, vv, z, vis = project_visible_np(
|
| 478 |
+
centers, vw["depth"], vw["fx"], vw["fy"], vw["cx"], vw["cy"],
|
| 479 |
+
vw["c2w"], res)
|
| 480 |
+
elif VIS_METHOD == "zbuffer": # production: from the voxels
|
| 481 |
+
u, vv, z, vis = zbuffer_visible(
|
| 482 |
+
centers, vw["fx"], vw["fy"], vw["cx"], vw["cy"], vw["c2w"], res,
|
| 483 |
+
tol=VIS_ZBUF_TOL)
|
| 484 |
+
else: # production: exact grid ray march
|
| 485 |
+
u, vv, z, vis = raymarch_visible(
|
| 486 |
+
centers, vox["coords"], vw["fx"], vw["fy"], vw["cx"], vw["cy"],
|
| 487 |
+
vw["c2w"], res, skip_vox=VIS_SKIP_VOX)
|
| 488 |
+
crop = load_crop_rgba(a["png"])
|
| 489 |
+
uvn = crop_uv_norm(u, vv, a["bbox"])
|
| 490 |
+
# --- MODAL-MASK (alpha) VISIBILITY GATE ------------------------ #
|
| 491 |
+
# The geometric tests above (rendered depth / z-buffer / ray march)
|
| 492 |
+
# only know about the OBJECT's own surface, so they call a voxel
|
| 493 |
+
# visible even when the stored view actually shows something else
|
| 494 |
+
# in front of it (PSL occlusion crops) or when the projection falls
|
| 495 |
+
# outside the stored crop. The stored RGBA's alpha channel IS the
|
| 496 |
+
# per-view MODAL mask (what is really seen), so gate `vis` by it:
|
| 497 |
+
# nearest-pixel lookup in the crop frame (grid_sample's
|
| 498 |
+
# align_corners=False convention, i.e. px = (un+1)/2*W - 0.5), NO
|
| 499 |
+
# dilation. On occluder-free renders (the base pool) alpha covers
|
| 500 |
+
# the whole silhouette, so this is a near no-op (edge pixels only).
|
| 501 |
+
alpha = crop["rgba_uint8"][..., 3]
|
| 502 |
+
Hc, Wc = alpha.shape[0], alpha.shape[1]
|
| 503 |
+
px = np.rint((uvn[:, 0].astype(np.float64) + 1.0) * 0.5 * Wc - 0.5).astype(np.int64)
|
| 504 |
+
py = np.rint((uvn[:, 1].astype(np.float64) + 1.0) * 0.5 * Hc - 0.5).astype(np.int64)
|
| 505 |
+
inside = (px >= 0) & (px < Wc) & (py >= 0) & (py < Hc)
|
| 506 |
+
vis = np.asarray(vis) & inside
|
| 507 |
+
if inside.any():
|
| 508 |
+
vis[inside] &= alpha[py[inside], px[inside]] > 0
|
| 509 |
+
# --- EMPTY-SEED ('none') gate: contribute ZERO forced x0 rows ---
|
| 510 |
+
# The x0 appearance seed is built model-side as the visible-mean of
|
| 511 |
+
# the rows any view marks visible; an all-False `vis` therefore
|
| 512 |
+
# forces no rows (x0 stays pure noise) while leaving the condition
|
| 513 |
+
# (DINO crops / cameras) untouched. Belt-and-suspenders with the
|
| 514 |
+
# `no_seed` batch flag honoured in slatflow_model.flow_step.
|
| 515 |
+
if no_seed:
|
| 516 |
+
vis = np.zeros_like(np.asarray(vis), dtype=bool)
|
| 517 |
+
# --------------------------------------------------------------- #
|
| 518 |
+
crops.append(crop)
|
| 519 |
+
uvs.append(uvn)
|
| 520 |
+
viss.append(vis)
|
| 521 |
+
|
| 522 |
+
# anchor condition inputs (deployed slat_preprocessor recipe, OQ-6)
|
| 523 |
+
slat_input = preprocess_slat_image(crops[0]["rgba_uint8"])
|
| 524 |
+
|
| 525 |
+
# non-anchor Plücker + soft-mask camera dicts (subset views S[1:])
|
| 526 |
+
anchor_c2w = torch.from_numpy(views_all[order[0]]["c2w"])
|
| 527 |
+
pluckers, bias_views = [], []
|
| 528 |
+
for k in range(1, n_sub):
|
| 529 |
+
i = order[k]
|
| 530 |
+
vw = views_all[i]
|
| 531 |
+
a = assets[i]
|
| 532 |
+
res = (a["res"] if a["res"] is not None else
|
| 533 |
+
(vw["depth"].shape[0] if vw["depth"] is not None else vw["res"]))
|
| 534 |
+
pluckers.append(plucker_rays_for_view(
|
| 535 |
+
anchor_c2w, torch.from_numpy(vw["c2w"]),
|
| 536 |
+
vw["fx"], vw["fy"], vw["cx"], vw["cy"], a["bbox"]))
|
| 537 |
+
bias_views.append(dict(
|
| 538 |
+
w2c=torch.from_numpy(np.linalg.inv(vw["c2w"])).float(),
|
| 539 |
+
fx=vw["fx"], fy=vw["fy"], cx=vw["cx"], cy=vw["cy"],
|
| 540 |
+
bbox=[float(b) for b in a["bbox"]], res=float(res)))
|
| 541 |
+
plucker = (torch.stack(pluckers) if pluckers
|
| 542 |
+
else torch.zeros(0, 1369, 6))
|
| 543 |
+
|
| 544 |
+
item = {
|
| 545 |
+
"name": name,
|
| 546 |
+
"coords": coords, # (N,4) int32
|
| 547 |
+
"slat_input": slat_input, # dict of 4 CPU tensors
|
| 548 |
+
"crops": torch.from_numpy(np.stack([c["premult"] for c in crops])),
|
| 549 |
+
"uv": torch.from_numpy(np.stack(uvs)), # (V,N,2) f32
|
| 550 |
+
"vis": torch.from_numpy(np.stack(viss)), # (V,N) bool
|
| 551 |
+
"plucker": plucker, # (n_sub-1,1369,6)
|
| 552 |
+
"bias_views": bias_views,
|
| 553 |
+
"n_subset": n_sub,
|
| 554 |
+
"view_order": [int(i) for i in order],
|
| 555 |
+
# SS-stage seed-mixture flags (default: legacy/no-op). no_seed=True
|
| 556 |
+
# => EMPTY appearance seed (x0 pure noise, honoured in flow_step).
|
| 557 |
+
"no_seed": torch.tensor(bool(no_seed)),
|
| 558 |
+
"seed_mode": torch.tensor(float(_SEED_MODE_CODE[mode])),
|
| 559 |
+
}
|
| 560 |
+
|
| 561 |
+
# OFFICIAL precomputed x1 target (tools/gen_x1_official.py output):
|
| 562 |
+
# the exact TRELLIS-toolkit latent (SAM3D slat_encoder) on the FULL
|
| 563 |
+
# active-voxel set — coord-hash-match its rows onto coords_slat (a
|
| 564 |
+
# prune-subset of the voxels) and assert every row is covered.
|
| 565 |
+
slat_npz = os.path.join(d, "slat", "slat_official_sam3d.npz")
|
| 566 |
+
if os.path.isfile(slat_npz):
|
| 567 |
+
z = np.load(slat_npz)
|
| 568 |
+
oc = z["coords"].astype(np.int64) # (M,3) all voxels
|
| 569 |
+
ok = (oc[:, 0] * VOX + oc[:, 1]) * VOX + oc[:, 2]
|
| 570 |
+
order = np.argsort(ok)
|
| 571 |
+
want_c = coords[:, 1:].numpy().astype(np.int64)
|
| 572 |
+
want = (want_c[:, 0] * VOX + want_c[:, 1]) * VOX + want_c[:, 2]
|
| 573 |
+
pos = np.searchsorted(ok[order], want)
|
| 574 |
+
sel = order[np.clip(pos, 0, len(order) - 1)]
|
| 575 |
+
assert len(ok) > 0 and np.array_equal(ok[sel], want), \
|
| 576 |
+
(f"{name}: slat_official_sam3d.npz does not cover coords_slat "
|
| 577 |
+
f"(official voxel set mismatch — regenerate with "
|
| 578 |
+
f"tools/gen_x1_official.py)")
|
| 579 |
+
item["x1_feats_raw"] = torch.from_numpy(
|
| 580 |
+
z["feats"][sel].astype(np.float32)) # (N,8) RAW
|
| 581 |
+
return item
|
| 582 |
+
|
| 583 |
+
# ---- RecGen matting loader (additive) ------------------------------- #
|
| 584 |
+
def _recgen_cube_c2w(self, d: str, n: int):
|
| 585 |
+
"""Locate the CORRECT camera->cube poses (c2w in the [-0.5,0.5] voxel
|
| 586 |
+
frame the SLAT loader projects into). RecGen's cube cameras come from
|
| 587 |
+
the ANALYTIC ``pose_cube`` conversion (pose_scale / orientation), NOT a
|
| 588 |
+
simple inv(model2world)@cam2world of the raw view_metadata — that lands
|
| 589 |
+
the object BEHIND the camera (z<0). So prefer, in order:
|
| 590 |
+
1. ``pose_cube.npz`` (c2w_cube) inside the object dir or a views20/;
|
| 591 |
+
2. the encoded_store's pose_cube.npz / views.npz(c2w_cv), looked up by
|
| 592 |
+
sha under ``recgen_pose_store`` when set.
|
| 593 |
+
Returns (c2w array (n,4,4)) or None if no canonical source is found."""
|
| 594 |
+
import glob as _glob
|
| 595 |
+
cands = [os.path.join(d, "pose_cube.npz"),
|
| 596 |
+
os.path.join(d, "views20", "pose_cube.npz"),
|
| 597 |
+
os.path.join(d, "views", "pose_cube.npz")]
|
| 598 |
+
store = getattr(self, "_recgen_pose_store", None)
|
| 599 |
+
if store:
|
| 600 |
+
sha = os.path.basename(d)
|
| 601 |
+
sf = os.path.join(d, "sha256.txt")
|
| 602 |
+
if os.path.isfile(sf):
|
| 603 |
+
sha = open(sf).read().strip() or sha
|
| 604 |
+
base = os.path.join(store, "objects", sha[:2], sha)
|
| 605 |
+
cands += [os.path.join(base, "views20", "pose_cube.npz"),
|
| 606 |
+
os.path.join(base, "views", "pose_cube.npz")]
|
| 607 |
+
for vp in (os.path.join(base, "views20", "views.npz"),
|
| 608 |
+
os.path.join(base, "views", "views.npz")):
|
| 609 |
+
if os.path.isfile(vp):
|
| 610 |
+
z = np.load(vp)
|
| 611 |
+
if "c2w_cv" in z.files:
|
| 612 |
+
c = np.asarray(z["c2w_cv"], dtype=np.float64)
|
| 613 |
+
if c.shape[0] >= n:
|
| 614 |
+
return c[:n]
|
| 615 |
+
for p in cands:
|
| 616 |
+
if os.path.isfile(p):
|
| 617 |
+
z = np.load(p)
|
| 618 |
+
key = "c2w_cube" if "c2w_cube" in z.files else (
|
| 619 |
+
"c2w_cv" if "c2w_cv" in z.files else None)
|
| 620 |
+
if key is not None:
|
| 621 |
+
c = np.asarray(z[key], dtype=np.float64)
|
| 622 |
+
if c.shape[0] >= n:
|
| 623 |
+
return c[:n]
|
| 624 |
+
return None
|
| 625 |
+
|
| 626 |
+
def _recgen_cameras(self, d: str):
|
| 627 |
+
"""RecGen view_metadata.json -> (views_all, V) in the SAME per-view dict
|
| 628 |
+
shape ``load_views_npz`` produces (depth=None, fx/fy/cx/cy, c2w, res).
|
| 629 |
+
|
| 630 |
+
Intrinsics come from view_metadata.json[i]["intrinsics"] (fx=fy=610,
|
| 631 |
+
cx=320, cy=240, 640x480). The camera->cube EXTRINSIC prefers the
|
| 632 |
+
canonical pose_cube (see _recgen_cube_c2w); if none is found it falls
|
| 633 |
+
back to the ANALYTIC inv(model2world)@cam2world, which is format-correct
|
| 634 |
+
but geometrically APPROXIMATE (raw view_metadata alone does not yield the
|
| 635 |
+
cube frame — see report caveat #2). transforms.json is empty ({}) and
|
| 636 |
+
pose_data.json holds the OBJECT 6D pose, not cameras, in these samples."""
|
| 637 |
+
import json
|
| 638 |
+
vm_path = os.path.join(d, "view_metadata.json")
|
| 639 |
+
vm = json.load(open(vm_path))
|
| 640 |
+
vm = [json.loads(s) if isinstance(s, str) else s for s in vm]
|
| 641 |
+
V = len(vm)
|
| 642 |
+
cube = self._recgen_cube_c2w(d, V) # (V,4,4) canonical, or None
|
| 643 |
+
views_all = []
|
| 644 |
+
for i, m in enumerate(vm):
|
| 645 |
+
K = np.asarray(m["intrinsics"], dtype=np.float64)
|
| 646 |
+
H = int(m.get("height", 480)); W = int(m.get("width", 640))
|
| 647 |
+
if cube is not None:
|
| 648 |
+
c2w = cube[i]
|
| 649 |
+
else: # approximate fallback
|
| 650 |
+
c2w = (np.linalg.inv(np.asarray(m["model2world"], dtype=np.float64))
|
| 651 |
+
@ np.asarray(m["cam2world"], dtype=np.float64))
|
| 652 |
+
views_all.append(dict(
|
| 653 |
+
depth=None,
|
| 654 |
+
fx=float(K[0, 0]), fy=float(K[1, 1]),
|
| 655 |
+
cx=float(K[0, 2]), cy=float(K[1, 2]),
|
| 656 |
+
c2w=c2w, res=int(max(H, W))))
|
| 657 |
+
return views_all, V
|
| 658 |
+
|
| 659 |
+
def _recgen_asset(self, d: str, i: int):
|
| 660 |
+
"""View i -> {'rgba': matted RGBA crop, 'bbox': (y0,y1,x0,x1), 'res': int}.
|
| 661 |
+
cond_image_i (RGB, background baked in) x cond_mask (prefer the SAM2 mask,
|
| 662 |
+
fallback the plain mask) -> RGBA so the SLAT alpha-visibility gate works."""
|
| 663 |
+
img_p = os.path.join(d, f"cond_image_{i:02d}.jpg")
|
| 664 |
+
rgb = np.array(Image.open(img_p).convert("RGB"))
|
| 665 |
+
mask_p = os.path.join(d, f"cond_mask_sam2_{i:02d}.png")
|
| 666 |
+
if not os.path.isfile(mask_p):
|
| 667 |
+
mask_p = os.path.join(d, f"cond_mask_{i:02d}.png")
|
| 668 |
+
mask = np.array(Image.open(mask_p).convert("L"))
|
| 669 |
+
crop_rgba, bbox = matte_rgba(rgb, mask)
|
| 670 |
+
return dict(rgba=crop_rgba, bbox=bbox, res=int(max(rgb.shape[0], rgb.shape[1])))
|
| 671 |
+
|
| 672 |
+
def _getitem_recgen(self, idx: int) -> Dict:
|
| 673 |
+
d = self.dirs[idx]
|
| 674 |
+
name = os.path.basename(d)
|
| 675 |
+
|
| 676 |
+
rng = np.random.default_rng([self.seed, self._epoch, idx])
|
| 677 |
+
views_all, V = self._recgen_cameras(d)
|
| 678 |
+
n_sub, no_seed, mode = self._mixture_n_sub(idx, V, rng)
|
| 679 |
+
sub = rng.choice(V, size=n_sub, replace=False)
|
| 680 |
+
order = list(sub) + [i for i in range(V) if i not in set(sub.tolist())]
|
| 681 |
+
|
| 682 |
+
# x1 = RecGen's RELEASED SLAT (slat_coords 0..63 int + slat_feats [N,8]).
|
| 683 |
+
# The released coords ARE the final SLAT coord set (1:1 with slat_feats),
|
| 684 |
+
# so they are used directly as `coords` (no prune/downsample: RecGen ships
|
| 685 |
+
# no voxels.npz) and slat_feats is the row-aligned x1 target. Coverage
|
| 686 |
+
# is 1:1 by construction; assert the row counts match.
|
| 687 |
+
sc = np.load(os.path.join(d, "slat_coords.npy")).astype(np.int64) # (N,3)
|
| 688 |
+
feats = np.load(os.path.join(d, "slat_feats.npy")).astype(np.float32) # (N,8)
|
| 689 |
+
assert sc.ndim == 2 and sc.shape[1] == 3 and feats.shape[0] == sc.shape[0], \
|
| 690 |
+
(f"{name}: recgen slat_coords {sc.shape} / slat_feats {feats.shape} "
|
| 691 |
+
f"mismatch (x1 rows must align 1:1 with coords)")
|
| 692 |
+
assert sc.min() >= 0 and sc.max() < VOX, (name, sc.min(), sc.max())
|
| 693 |
+
coords = torch.cat([torch.zeros(len(sc), 1, dtype=torch.int32),
|
| 694 |
+
torch.from_numpy(sc).int()], dim=1) # (N,4) int32
|
| 695 |
+
centers = (sc.astype(np.float64) + 0.5) / VOX - 0.5
|
| 696 |
+
|
| 697 |
+
crops, uvs, viss = [], [], []
|
| 698 |
+
for i in order[:n_sub]:
|
| 699 |
+
vw = views_all[i]
|
| 700 |
+
a = self._recgen_asset(d, i)
|
| 701 |
+
res = a["res"]
|
| 702 |
+
u, vv, z, vis = raymarch_visible(
|
| 703 |
+
centers, sc, vw["fx"], vw["fy"], vw["cx"], vw["cy"],
|
| 704 |
+
vw["c2w"], res, skip_vox=VIS_SKIP_VOX)
|
| 705 |
+
crop = rgba_to_crop(a["rgba"])
|
| 706 |
+
uvn = crop_uv_norm(u, vv, a["bbox"])
|
| 707 |
+
# modal-mask (alpha) gate — identical to the clean-RGBA path
|
| 708 |
+
alpha = crop["rgba_uint8"][..., 3]
|
| 709 |
+
Hc, Wc = alpha.shape[0], alpha.shape[1]
|
| 710 |
+
px = np.rint((uvn[:, 0].astype(np.float64) + 1.0) * 0.5 * Wc - 0.5).astype(np.int64)
|
| 711 |
+
py = np.rint((uvn[:, 1].astype(np.float64) + 1.0) * 0.5 * Hc - 0.5).astype(np.int64)
|
| 712 |
+
inside = (px >= 0) & (px < Wc) & (py >= 0) & (py < Hc)
|
| 713 |
+
vis = np.asarray(vis) & inside
|
| 714 |
+
if inside.any():
|
| 715 |
+
vis[inside] &= alpha[py[inside], px[inside]] > 0
|
| 716 |
+
if no_seed:
|
| 717 |
+
vis = np.zeros_like(np.asarray(vis), dtype=bool)
|
| 718 |
+
crops.append(crop)
|
| 719 |
+
uvs.append(uvn)
|
| 720 |
+
viss.append(vis)
|
| 721 |
+
a["_bbox"] = a["bbox"] # keep for plucker reuse
|
| 722 |
+
views_all[i]["_asset"] = a
|
| 723 |
+
|
| 724 |
+
slat_input = preprocess_slat_image(crops[0]["rgba_uint8"])
|
| 725 |
+
|
| 726 |
+
anchor_c2w = torch.from_numpy(views_all[order[0]]["c2w"])
|
| 727 |
+
pluckers, bias_views = [], []
|
| 728 |
+
for k in range(1, n_sub):
|
| 729 |
+
i = order[k]
|
| 730 |
+
vw = views_all[i]
|
| 731 |
+
a = vw["_asset"]
|
| 732 |
+
res = a["res"]
|
| 733 |
+
pluckers.append(plucker_rays_for_view(
|
| 734 |
+
anchor_c2w, torch.from_numpy(vw["c2w"]),
|
| 735 |
+
vw["fx"], vw["fy"], vw["cx"], vw["cy"], a["bbox"]))
|
| 736 |
+
bias_views.append(dict(
|
| 737 |
+
w2c=torch.from_numpy(np.linalg.inv(vw["c2w"])).float(),
|
| 738 |
+
fx=vw["fx"], fy=vw["fy"], cx=vw["cx"], cy=vw["cy"],
|
| 739 |
+
bbox=[float(b) for b in a["bbox"]], res=float(res)))
|
| 740 |
+
plucker = (torch.stack(pluckers) if pluckers else torch.zeros(0, 1369, 6))
|
| 741 |
+
|
| 742 |
+
return {
|
| 743 |
+
"name": name,
|
| 744 |
+
"coords": coords,
|
| 745 |
+
"slat_input": slat_input,
|
| 746 |
+
"crops": torch.from_numpy(np.stack([c["premult"] for c in crops])),
|
| 747 |
+
"uv": torch.from_numpy(np.stack(uvs)),
|
| 748 |
+
"vis": torch.from_numpy(np.stack(viss)),
|
| 749 |
+
"plucker": plucker,
|
| 750 |
+
"bias_views": bias_views,
|
| 751 |
+
"n_subset": n_sub,
|
| 752 |
+
"view_order": [int(i) for i in order],
|
| 753 |
+
"no_seed": torch.tensor(bool(no_seed)),
|
| 754 |
+
"seed_mode": torch.tensor(float(_SEED_MODE_CODE[mode])),
|
| 755 |
+
"x1_feats_raw": torch.from_numpy(feats), # (N,8) RAW released SLAT
|
| 756 |
+
}
|
| 757 |
+
|
| 758 |
+
|
| 759 |
+
def collate_single(batch: Sequence[Dict]) -> Dict:
|
| 760 |
+
"""bs=1 collate (variable N per object => per-object steps + grad accum)."""
|
| 761 |
+
assert len(batch) == 1, \
|
| 762 |
+
"SLAT flow trains bs=1 per object (grad accum for larger batches)"
|
| 763 |
+
return batch[0]
|
| 764 |
+
|
| 765 |
+
|
| 766 |
+
def collate_batched(batch: Sequence[Dict]) -> Dict:
|
| 767 |
+
"""Multi-object collate mirroring TRELLIS ``SLat.collate_fn``: the B
|
| 768 |
+
objects' SLAT coords are CONCATENATED into one coordinate set whose batch
|
| 769 |
+
column ([b,x,y,z]) is the object index, with per-object ``layout`` slices
|
| 770 |
+
recorded. Everything with a per-object variable shape (crops, uv, vis,
|
| 771 |
+
plucker, condition inputs, targets) stays a python list — the batched
|
| 772 |
+
model path builds each object's own condition and seed from them.
|
| 773 |
+
|
| 774 |
+
Consumed by ``SlatFlowModel.sample_batch`` (true batched inference:
|
| 775 |
+
one forward per solver step for all B objects)."""
|
| 776 |
+
coords_parts: List[torch.Tensor] = []
|
| 777 |
+
layout: List[slice] = []
|
| 778 |
+
ofs = 0
|
| 779 |
+
for i, it in enumerate(batch):
|
| 780 |
+
c = it["coords"].clone()
|
| 781 |
+
assert int(c[:, 0].max()) == 0, "per-object coords must have batch col 0"
|
| 782 |
+
c[:, 0] = i # batch-index column
|
| 783 |
+
coords_parts.append(c)
|
| 784 |
+
layout.append(slice(ofs, ofs + c.shape[0]))
|
| 785 |
+
ofs += c.shape[0]
|
| 786 |
+
return {
|
| 787 |
+
"coords": torch.cat(coords_parts, dim=0), # (T_total,4) int32
|
| 788 |
+
"layout": layout, # per-object slices
|
| 789 |
+
"names": [it["name"] for it in batch],
|
| 790 |
+
"slat_input": [it["slat_input"] for it in batch],
|
| 791 |
+
"crops": [it["crops"] for it in batch],
|
| 792 |
+
"uv": [it["uv"] for it in batch],
|
| 793 |
+
"vis": [it["vis"] for it in batch],
|
| 794 |
+
"plucker": [it["plucker"] for it in batch],
|
| 795 |
+
"bias_views": [it["bias_views"] for it in batch],
|
| 796 |
+
"n_subset": [int(it["n_subset"]) for it in batch],
|
| 797 |
+
"view_order": [it["view_order"] for it in batch],
|
| 798 |
+
"x1_feats_raw": [it.get("x1_feats_raw") for it in batch],
|
| 799 |
+
# SS-stage seed-mixture flags (per object; default legacy/no-op)
|
| 800 |
+
"no_seed": [bool(it["no_seed"]) if it.get("no_seed") is not None else False
|
| 801 |
+
for it in batch],
|
| 802 |
+
"seed_mode": [float(it["seed_mode"]) if it.get("seed_mode") is not None else 3.0
|
| 803 |
+
for it in batch],
|
| 804 |
+
}
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-310.pyc
ADDED
|
Binary file (185 Bytes). View file
|
|
|
migrator/code/mv-sam3d-for-6d-v2-ssflow/mvsam3d/model/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (201 Bytes). View file
|
|
|