Upload model files from Google Colab

.gitattributes

@@ -1,5 +1,6 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
+*.avro filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
 *.ckpt filter=lfs diff=lfs merge=lfs -text
@@ -8,6 +9,8 @@
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +36,25 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,119 @@
+{
+  "acoustic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "decoder_depths": null,
+    "decoder_n_filters": 32,
+    "decoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0.5,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibevoice_acoustic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "gaussian",
+    "torch_dtype": "float32",
+    "vae_dim": 64,
+    "weight_init_value": 0.01
+  },
+  "acoustic_vae_dim": 64,
+  "architectures": [
+    "VibeVoiceForConditionalGeneration"
+  ],
+  "decoder_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "max_position_embeddings": 65536,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "torch_dtype": "float32",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "diffusion_head_config": {
+    "ddpm_batch_mul": 4,
+    "ddpm_beta_schedule": "cosine",
+    "ddpm_num_inference_steps": 20,
+    "ddpm_num_steps": 1000,
+    "diffusion_type": "ddpm",
+    "head_ffn_ratio": 3.0,
+    "head_layers": 4,
+    "hidden_size": 1536,
+    "latent_size": 64,
+    "model_type": "vibevoice_diffusion_head",
+    "prediction_type": "v_prediction",
+    "rms_norm_eps": 1e-05,
+    "speech_vae_dim": 64,
+    "torch_dtype": "float32"
+  },
+  "model_type": "vibevoice",
+  "semantic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibevoice_semantic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "none",
+    "torch_dtype": "float32",
+    "vae_dim": 128,
+    "weight_init_value": 0.01
+  },
+  "semantic_vae_dim": 128,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true
+}

export_info.json

@@ -0,0 +1,98 @@
+{
+  "base_model": "microsoft/VibeVoice-1.5B",
+  "checkpoint_dir": "/kaggle/working/out_tpu/final",
+  "dtype": "bfloat16",
+  "components_merged": [
+    "language_model",
+    "prediction_head",
+    "acoustic_connector",
+    "semantic_connector"
+  ],
+  "tensors_changed_vs_base": 341,
+  "training_state": {
+    "step": 188,
+    "args": {
+      "model_name_or_path": "microsoft/VibeVoice-1.5B",
+      "data_dir": "/tmp/preprocessed_tpu/train",
+      "eval_data_dir": null,
+      "output_dir": "out_tpu",
+      "lazy_data": false,
+      "max_seq_len": 0,
+      "max_latents": 0,
+      "max_segments": 0,
+      "buckets": null,
+      "auto_buckets": 0,
+      "per_device_batch_size": 4,
+      "gradient_accumulation_steps": 8,
+      "learning_rate": 1e-05,
+      "head_lr_multiplier": 2.0,
+      "weight_decay": 0.01,
+      "adam_beta1": 0.9,
+      "adam_beta2": 0.95,
+      "adam_eps": 1e-08,
+      "max_grad_norm": 1.0,
+      "num_train_epochs": 4.0,
+      "max_steps": 0,
+      "warmup_ratio": 0.08,
+      "warmup_steps": 0,
+      "lr_scheduler": "cosine",
+      "min_lr_ratio": 0.05,
+      "seed": 42,
+      "ce_loss_weight": 1.0,
+      "diffusion_loss_weight": 1.5,
+      "ddpm_batch_mul": 4,
+      "ce_chunk_size": 256,
+      "no_semantic": false,
+      "no_latent_noise": false,
+      "freeze_components": null,
+      "freeze_llm_layers": null,
+      "train_llm_layers": null,
+      "freeze_head_layers": null,
+      "freeze_embeddings": false,
+      "freeze_lm_head": false,
+      "freeze_regex": null,
+      "bf16": true,
+      "gradient_checkpointing": true,
+      "no_spmd": false,
+      "logging_steps": 10,
+      "save_steps": 500,
+      "save_total_limit": 3,
+      "eval_steps": 0,
+      "eval_batches": 20,
+      "resume_from_checkpoint": null,
+      "dataloader_workers": 0,
+      "profile_first_steps": 3
+    },
+    "shapes": {
+      "max_seq_len": 512,
+      "max_latents": 192,
+      "max_segments": 8,
+      "acoustic_dim": 64,
+      "semantic_dim": 128
+    },
+    "preprocess_metadata": {
+      "model_name_or_path": "microsoft/VibeVoice-1.5B",
+      "processor_name_or_path": "vibevoice/processor",
+      "compress_ratio": 3200,
+      "acoustic_dim": 64,
+      "semantic_dim": 128,
+      "fix_std": 0.5,
+      "latent_dtype": "float16",
+      "speech_scaling_factor": 0.1962890625,
+      "speech_bias_factor": -0.04931640625,
+      "scaling_source": "checkpoint",
+      "corpus_latent_mean": 0.01181590482027792,
+      "corpus_latent_var": 31.865126896240167,
+      "shapes": {
+        "max_seq_len": 512,
+        "max_latents": 192,
+        "max_segments": 8,
+        "acoustic_dim": 64,
+        "semantic_dim": 128
+      },
+      "pad_token_id": 151655,
+      "augment_target_silence": true,
+      "normalize_target_audio": false
+    }
+  }
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3cf68b30f87fd63c8754f6cd0ea25e452f61f48f707453ac51c6bcfb5200c38
+size 3991978364

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c7ef0e4ee95c103911bf30ad55578992f4f045b268c3703eeb59ea586d5122a
+size 1416224194