Add joint (unified sparse+dense) Learn2Splat checkpoint + config

joint/checkpoints/epoch_7-step_150000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5defbe956ac66070ea1c055f7e4b2af36751f2091cea9dd761904dc4febfaab6
+size 192637731

joint/config.yaml

@@ -0,0 +1,664 @@
+loss:
+  mse:
+    weight: 1.0
+    l1_loss: true
+    clamp_large_error: 0.0
+  lpips:
+    weight: 0.5
+    apply_after_step: 0
+    perceptual_loss: true
+    half_res: false
+    bf16: true
+  deltas:
+    weight: 2.0
+    exclude_by_norm_grad: true
+    exclude_by_norm_grad_opposite: true
+    eps: 1.0e-08
+    apply_after_step: 100
+  stability:
+    weight: 1.0
+    subset_aware: false
+    targets_only: false
+dataset:
+  name: mixed
+  background_color:
+  - 0.0
+  - 0.0
+  - 0.0
+  eval_source: dense
+  sources:
+    dense:
+      image_shape:
+      - 256
+      - 448
+      background_color:
+      - 0.0
+      - 0.0
+      - 0.0
+      cameras_are_circular: false
+      overfit_to_scene: null
+      opencv_pose_format: false
+      pose_align_middle_view: false
+      test_start_idx: 0
+      view_sampler:
+        name: boundedv2
+        num_target_views: 6
+        num_context_views: 64
+        min_distance_between_context_views: -1
+        max_distance_between_context_views: -1
+        max_distance_to_context_views: 0
+        context_gap_warm_up_steps: 10000
+        target_gap_warm_up_steps: 0
+        initial_min_distance_between_context_views: -1
+        initial_max_distance_between_context_views: -1
+        initial_max_distance_to_context_views: 0
+        extra_views_sampling_strategy: farthest_point
+        target_views_replace_sample: false
+      name: dl3dv
+      roots:
+      - datasets/dl3dv-480p-chunks
+      augment: true
+      max_fov: 100.0
+      skip_bad_shape: true
+      near: 0.01
+      far: 200.0
+      shuffle_val: true
+      test_len: -1
+      test_chunk_interval: 1
+      sort_target_index: true
+      sort_context_index: true
+      train_times_per_scene: 1
+      test_times_per_scene: 1
+      ori_image_shape:
+      - 270
+      - 480
+      overfit_max_views: 148
+      no_mix_test_set: true
+      min_views: 0
+      max_views: 0
+      index_name: index_colmap.json
+    sparse:
+      image_shape:
+      - 256
+      - 448
+      background_color:
+      - 0.0
+      - 0.0
+      - 0.0
+      cameras_are_circular: false
+      overfit_to_scene: null
+      opencv_pose_format: false
+      pose_align_middle_view: false
+      test_start_idx: 0
+      view_sampler:
+        name: boundedv2
+        num_target_views: 6
+        num_context_views: 8
+        min_distance_between_context_views: 24
+        max_distance_between_context_views: 45
+        max_distance_to_context_views: 0
+        context_gap_warm_up_steps: 10000
+        target_gap_warm_up_steps: 0
+        initial_min_distance_between_context_views: 20
+        initial_max_distance_between_context_views: 30
+        initial_max_distance_to_context_views: 0
+        extra_views_sampling_strategy: farthest_point
+        target_views_replace_sample: false
+      name: dl3dv
+      roots:
+      - datasets/dl3dv-480p-chunks
+      augment: true
+      max_fov: 100.0
+      skip_bad_shape: true
+      near: 0.01
+      far: 200.0
+      shuffle_val: true
+      test_len: -1
+      test_chunk_interval: 1
+      sort_target_index: true
+      sort_context_index: true
+      train_times_per_scene: 1
+      test_times_per_scene: 1
+      ori_image_shape:
+      - 270
+      - 480
+      overfit_max_views: 148
+      no_mix_test_set: true
+      min_views: 0
+      max_views: 0
+      index_name: index.json
+  weights:
+    dense: 1.0
+    sparse: 1.0
+scene_trainer:
+  scene_initializer:
+    per_pixel: false
+    per_view: false
+    train_min_gaussians_subsample: 0.1
+    train_max_gaussians_subsample: 1.0
+    eval_min_gaussians_subsample: null
+    eval_max_gaussians_subsample: null
+    train_fixed_gaussians_num: 70000
+    eval_fixed_gaussians_num: null
+    name: colmap
+    path: datasets/dl3dv-colmap-sfm
+    normalize_world_space: false
+    scaling_factor: 1.0
+    init_opacity: 0.1
+    sh_degree: 3
+    dl3dv_settings: true
+    filter_zero_rgb: true
+    points3d_subdir: null
+    points3d_ply_filename: null
+    randomize_opacity: false
+    randomize_opacity_distribution: uniform
+    randomize_opacity_min: 0.0
+    randomize_opacity_std: 0.05
+    override_dataset_poses: true
+  scene_optimizer:
+    refiner:
+      name: none
+      do_densify: false
+      do_prune: false
+      do_opacity_reset: false
+      cap_max: -1
+      pause_refine_after_reset: 0
+      refine_every: 999999999
+      reset_every: 999999999
+      refine_start_iter: 999999999
+      refine_stop_iter: 999999999
+      refine_scale2d_stop_iter: 0
+      grow_grad2d: 0.0
+      grow_scale3d: 0.0
+      prune_scale3d: 0.0
+      prune_scale2d: null
+      grow_scale2d: 0.0
+      min_opacity: 0.0
+      prune_zero_radii: false
+      reduce_opacity: false
+      reduce_factor: 0.0
+      reduce_every: 0
+      fallback_means_lr: 0.0
+      clone_gaussian_state: copy
+      split_gaussian_state: copy
+      relocate_gaussian_state: copy
+      add_gaussian_state: random
+    lr_scheduler:
+      lr_data:
+        _base: 1
+        _means: 1
+        _scales: 1
+        _quats: 1
+        _opacities: 1
+        _sh0: 1
+        _shN: 1
+      apply_scheduler:
+        _base: false
+        _means: false
+        _scales: false
+        _quats: false
+        _opacities: false
+        _sh0: false
+        _shN: false
+      name: none
+    input_gradients_chunk_size: -1
+    input_gradient_with_ssim_loss: true
+    input_gradient_l2_loss: false
+    freeze_mean: false
+    freeze_scale: false
+    freeze_rotation: false
+    freeze_opacity: false
+    freeze_sh0: false
+    freeze_shN: false
+    zero_state_on_densify: false
+    opacity_reg_lambda: 0.0
+    name: l2s
+    no_render_error: false
+    num_basic_blocks: 4
+    num_blocks: 1
+    concat_init_state: false
+    replace_init_state: false
+    state_channels: 256
+    block_rmsnorm: false
+    block_layernorm: false
+    pt_qk_norm: false
+    norm_pt_block: false
+    delta_gaussian_multiple: 1
+    residual_init_state: false
+    clamp_max_scale: 3.0
+    clamp_min_scale: 1.0e-06
+    clamp_min_raw_scales: -8.0
+    clamp_max_raw_scales: 1.0986
+    clamp_min_raw_opacities: -7
+    clamp_max_raw_opacities: 7
+    gaussian_head_multiple: 1
+    clamp_min_sh0: -10000000000.0
+    clamp_max_sh0: 10000000000.0
+    clamp_min_shs: -2.0
+    clamp_max_shs: 2.0
+    clamp_shs_soft: false
+    attn_proj_channels: 64
+    no_knn_attn: false
+    no_tran_block_norm: false
+    tran_block_act: gelu
+    multi_gaussian_scale_smaller: false
+    init_gaussian_multiple: 1
+    condition_pt_feature: true
+    same_num_points: false
+    input_error_rgb_no_shuffle: false
+    input_error_cache_resnet_feature: false
+    init_state_wo_features: false
+    init_state_type: random
+    init_state_scale: 1.0
+    pt_heads: 1
+    with_mv_attn: false
+    with_mv_attn_lowres: false
+    no_mv_attn: false
+    mv_attn_conv_with_norm: false
+    mv_shuffle_attn: false
+    mv_attn_with_pos_enc: false
+    shuffle_attn_no_norm: false
+    mv_unimatch_attn: false
+    knn_samples: 4
+    max_active_gaussians: 100000
+    use_fused_attn: true
+    knn_idx_update_every: 1
+    knn_backend: simple_knn
+    input_alpha: false
+    input_depth: false
+    input_depth_smooth_error: false
+    input_error: false
+    input_error_add_rgb_feature: false
+    input_error_resnet_feature: true
+    input_error_no_freeze_resnet_feature: false
+    input_error_shallow_resnet_feature: false
+    input_error_resnet_feature_layers: 18
+    input_error_additional_cross_attn: false
+    input_error_num_intermediate_views: 8
+    input_error_mv_attn: false
+    input_error_mv_attn_blocks: 2
+    input_error_num_views: 0
+    input_error_remain_context: false
+    input_error_merge_remain_context: false
+    input_error_warp_remain_context: false
+    input_error_random_num_remain_context: false
+    input_error_num_remain_context_test: 0
+    input_gradient: true
+    input_gradient_log: false
+    input_gradient_log_clip_deltas: 0.001
+    input_gradient_scale: 1.0
+    residual_grad_scale: 1.0
+    input_gradient_same_loss: false
+    input_gradient_loss_reduction: mean_pixels_sum_views
+    scale_residual_grads: false
+    train_global_only: false
+    random_step_with_size: false
+    use_amp: true
+    pt_head_amp: true
+    pt_update_amp: true
+    use_checkpointing: false
+    recurrent_use_checkpointing: false
+    input_gradient_normalize: true
+    input_gradient_normalize_type: adam
+    input_normalize_state: false
+    input_normalize_gaussians: false
+    residual_state: false
+    predict_state_scale: true
+    predict_state_scale_norm: false
+    delta_head_concat_img: false
+    delta_head_layer_num: 2
+    delta_head_act: gelu
+    delta_head_final_act: identity
+    delta_head_hidden_dim_matches: input
+    delta_head_scalar_scale: true
+    delta_head_scalar_scale_act: relu
+    delta_head_per_param_heads: false
+    delta_head_per_param_hidden_dim: 48
+    delta_head_per_param_scales: false
+    opt_scales_before_act: true
+    scale_initial_opacities: 1.0
+    sh_d: 16
+    cap_sh_degree: null
+    local_prune_zero_radii: false
+    local_prune_low_weights: false
+    local_prune_low_weights_thresh: -1
+    update_only_nonzero_grad: true
+    log_scale_std: false
+    experimental_run: false
+    experimental_update:
+      _base: true
+      _means: true
+      _scales: true
+      _quats: true
+      _opacities: true
+      _sh0: true
+      _shN: true
+    experimental_use_grads: false
+    experimental_use_norm_grads:
+      _base: false
+      _means: false
+      _scales: false
+      _quats: false
+      _opacities: false
+      _sh0: false
+      _shN: false
+    experimental_lr:
+      _base: 1
+      _means: 0.00016
+      _scales: 0.005
+      _opacities: 0.05
+      _quats: 0.001
+      _sh0: 0.0025
+      _shN: 0.000125
+    use_time_encoding: false
+    time_encoding_max_steps: 2000
+  decoder:
+    name: gsplat
+    use_covariances: false
+    rasterize_mode: classic
+    eps2d: 0.0001
+  use_fsdp: false
+  train_scene_init: false
+  train_scene_opt: true
+  train_min_refine: 1
+  train_max_refine: 6
+  num_update_steps: 2000
+  iter_batch_size: -1
+  opt_batch_size: 8
+  opt_batch_size_min: 0
+  opt_batch_size_max: 0
+  opt_batch_strategy: random
+  sh_degree_interval: 0
+  scene_initializers:
+    sparse:
+      per_pixel: true
+      per_view: true
+      train_min_gaussians_subsample: null
+      train_max_gaussians_subsample: null
+      eval_min_gaussians_subsample: null
+      eval_max_gaussians_subsample: null
+      train_fixed_gaussians_num: null
+      eval_fixed_gaussians_num: null
+      name: resplat_v1
+      num_depth_candidates: 128
+      num_surfaces: 1
+      gaussians_per_pixel: 1
+      gaussian_adapter:
+        gaussian_scale_min: 0.5
+        gaussian_scale_max: 0.3
+        sh_degree: 3
+        exp_scale: false
+        softplus_scale: true
+        clamp_min_scale: 1.0e-06
+        scale_detach_depth: false
+        exp_scale_bias: 4.0
+        no_rotate_sh: true
+        no_sh_mask: true
+        init_rotation_identity: false
+      d_feature: 128
+      visualizer:
+        num_samples: 8
+        min_resolution: 256
+        export_ply: false
+      unimatch_weights_path: pretrained/gmdepth-scale1-resumeflowthings-scannet-5d9d7964.pth
+      multiview_trans_attn_split: 2
+      downscale_factor: 4
+      shim_patch_size: 16
+      local_mv_match: 2
+      monodepth_vit_type: vits
+      return_depth: true
+      num_scales: 1
+      upsample_factor: 8
+      lowest_feature_resolution: 8
+      depth_unet_channels: 128
+      grid_sample_disable_cudnn: false
+      large_gaussian_head: false
+      color_large_unet: false
+      init_sh_input_img: true
+      feature_upsampler_channels: 64
+      gaussian_regressor_channels: 256
+      unet_gaussian_regressor: false
+      resnet_gaussian_regressor: false
+      train_depth_only: false
+      pt_head: true
+      pt_heads: 1
+      init_pt_with_mv_attn: false
+      init_pt_with_mv_attn_lowres: false
+      pt_head_channels: null
+      pt_head_concat_img: false
+      pt_head_conv: false
+      attn_proj_channels: 64
+      knn_samples: 16
+      post_norm: false
+      no_rpe: true
+      no_knn_attn: false
+      num_blocks: 4
+      add_pt_residual: true
+      freeze_depth: false
+      use_gt_depth: false
+      separate_depth_color: false
+      separate_depth_type: small
+      separate_depth_gaussian_scale: false
+      sample_log_depth: true
+      bilinear_upsample_depth: false
+      no_upsample_depth: false
+      return_lowres_depth: false
+      latent_gs: true
+      latent_downsample: 4
+      fixed_latent_size: true
+      latent_gs_img_interp: area
+      dpt_head_depth: false
+      latent_dpt_upsampler: false
+      latent_dpt_upsampler_no_concat: false
+      light_dpt_feature: false
+      avgpool_depth: false
+      nearest_down_depth: false
+      predict_scale: false
+      norm_by_points: false
+      no_pred_depth_range: false
+      point_dist_init_gaussian_scale: false
+      resizeconv_upsampler: false
+      depth_pred_half_res: false
+      use_amp: true
+      pt_head_amp: true
+      use_fsdp: false
+      use_checkpointing: false
+      init_use_checkpointing: false
+      rotate_quat_to_world: false
+      refine_rotate_quat_to_world: false
+      refine_no_use_covariance: false
+      latent_new_reshape: false
+      no_pixel_offset: false
+      init_gaussian_multiple: 1
+      deform_sample_depth: false
+      deform_sample_depth_debug: false
+meta_trainer:
+  test:
+    postprocessing:
+      steps: 0
+      compute_metrics_every: 100
+      lr_data:
+        _base: 1
+        _means: 1
+        _scales: 1
+        _opacities: 1
+        _quats: 1
+        _sh0: 1
+        _shN: 1
+      scheduler: null
+      scheduler_warm_up_ratio: 0.01
+      prior_steps: 0
+      means_lr_final_ratio: 0.0625
+      means_lr_delay_mult: 0.01
+      means_lr_scale_by_scene_extent: true
+      chunk_size: -1
+      adc: null
+      name: none
+    output_path: null
+    compute_scores: true
+    compute_scores_metrics:
+    - psnr
+    - ssim
+    - lpips
+    metrics_batch_size: 32
+    eval_initialization: true
+    save_render_image: false
+    save_render_image_last_only: false
+    save_gt_image: false
+    save_render_depth: false
+    save_gt_depth: false
+    save_init_pred_depth: false
+    save_error_image: false
+    save_video: false
+    save_video_optim: false
+    save_video_view_index: 0
+    save_video_frame_repeat: 1
+    save_video_orbit: false
+    save_video_orbit_steps: null
+    save_video_orbit_with_optim: false
+    save_video_optim_orbit: false
+    save_video_optim_orbit_steps: null
+    save_video_orbit_span: 50
+    save_gaussian: false
+    save_gaussian_last_only: false
+    save_poses: false
+    save_cameras_json: true
+    save_cameras_npz: true
+    render_chunk_size: null
+    dec_chunk_size: 30
+    stablize_camera: false
+    stab_camera_kernel: 50
+    eval_context_views: false
+    inference_window_size: null
+    profile_model: false
+    save_colmap_train_test_views: false
+    ori_colmap_data_path: null
+    adam_optimizer_step: 0
+    save_at_iters:
+    - 0
+    - 1
+    - 5
+    - 10
+    - 50
+    - 100
+    - 200
+    - 300
+    - 400
+    - 500
+    - 1000
+    save_every_freq: null
+    save_every_steps: null
+    skip_if_outputs_exist: false
+    scenes_filter: null
+    experimental_add_noise_to_images: false
+    experimental_add_noise_to_images_std: null
+  train:
+    ckpt_buffer_cfg:
+      capacity: 20
+      sample_batch_size: 1
+      sample_prob: 0.7
+      insert_prob: 0.7
+      return_prob: 0.99
+      rollout: true
+      rollout_min_steps: 1
+      rollout_max_steps: 50
+      rollout_grow: 10000
+      max_t: null
+      push_only_if_not_full: false
+      remove_strategy_when_full: oldest
+    extended_visualization: false
+    print_log_every_n_steps: 100
+    eval_model_every_n_val: 2
+    eval_data_length: 5
+    eval_save_model: true
+    intermediate_loss_weight: 0.9
+    no_viz_video: false
+    eval_depth: false
+    no_log_projections: true
+    no_log_video: true
+    eval_render_depth: false
+    viz_render_depth: false
+    use_gt_depth_range: false
+    loss_on_input_views: true
+    loss_on_target_views: true
+    loss_on_input_views_num: 4
+    loss_on_target_views_num: 6
+    train_window_size: null
+    viz_depth_separate: false
+    scale_l2_loss_weight: 0.0
+    sh_l2_loss_weight: 0.0
+    opacity_l2_loss_weight: 0.0
+    use_ckpt_buffer: true
+  max_steps: 150000
+  val_check_interval: 0.25
+  gradient_clip_val: 0.5
+  num_sanity_val_steps: 1
+  eval_index: null
+  limit_test_batches: 1.0
+  limit_train_batches: 1.0
+  num_nodes: 1
+  meta_optimizer:
+    lr: 0.0001
+    lr_monodepth: 0.0
+    lr_depth: 0.0
+    warm_up_steps: 2000
+    weight_decay: 0.01
+    warm_up_ratio: 0.01
+    adamw_8bit: false
+wandb:
+  project: learn2splat
+  entity: null
+  name: joint
+  mode: online
+  id: null
+  notes: ''
+  tags: []
+mode: train
+data_loader:
+  train:
+    num_workers: 10
+    persistent_workers: true
+    batch_size: 1
+    seed: 1234
+  test:
+    num_workers: 4
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 1
+    persistent_workers: true
+    batch_size: 1
+    seed: 3456
+checkpointing:
+  load: null
+  every_n_train_steps: 1000
+  save_top_k: 5
+  pretrained_model: null
+  pretrained_model_rel_dir: ${checkpoint_rel_dir:${checkpointing.pretrained_model}}
+  pretrained_monodepth: null
+  pretrained_mvdepth: null
+  pretrained_depth: null
+  pretrained_scale_predictor: null
+  pretrained_depth_teacher: null
+  no_strict_load: false
+  resume: true
+  no_resume_upsampler: false
+  partial_load: false
+  freeze_mono_vit: false
+  resume_update_module: null
+  pretrained_initializer: null
+  pretrained_optimizer: null
+  pretrained_source_initializers:
+    sparse: hf://autonomousvision/learn2splat/resplat_init/checkpoints/epoch_20-step_100000.ckpt
+  load_existing_cfg: false
+seed: 111123
+output_dir: !!python/object/apply:optgs.misc.io.CustomPath
+- checkpoints
+- joint
+use_plugins: false
+log_slurm_id: true
+version: 2.5
+profiling:
+  mode: none
+debug_cfg: false