Text-to-Speech
Transformers
Safetensors
moss_tts_local
feature-extraction
podcast
multi-speaker
voice-cloning
expressive-speech
custom-code
research
custom_code
Instructions to use laion/Humanness-Voice-Podcast with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use laion/Humanness-Voice-Podcast with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-to-speech", model="laion/Humanness-Voice-Podcast", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("laion/Humanness-Voice-Podcast", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Commit ·
a1f8946
0
Parent(s):
Public release with neutral metadata
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +48 -0
- CHANGELOG.md +7 -0
- EXPORT_PROOF.json +27 -0
- LICENSE +6 -0
- MANIFEST.json +642 -0
- NOTICE.md +11 -0
- PUBLICATION_ID.json +4 -0
- README.md +229 -0
- TRAINING.md +91 -0
- UPSTREAM_LICENSE +11 -0
- UPSTREAM_NOTICE.md +23 -0
- VALIDATION.md +43 -0
- added_tokens.json +31 -0
- config.json +268 -0
- configuration_moss_tts.py +158 -0
- examples/audio/PROVENANCE.json +26 -0
- examples/audio/de_block_reassembled.mp3 +3 -0
- examples/audio/de_segment_reassembled.mp3 +3 -0
- examples/audio/en_block_reassembled.mp3 +3 -0
- examples/audio/en_segment_reassembled.mp3 +3 -0
- examples/episode_de.json +83 -0
- examples/episode_en.json +83 -0
- examples/references/PROVENANCE.json +30 -0
- examples/references/de_anchors.npz +3 -0
- examples/references/de_s1.mp3 +3 -0
- examples/references/de_s2.mp3 +3 -0
- examples/references/en_anchors.npz +3 -0
- examples/references/en_s1.mp3 +3 -0
- examples/references/en_s2.mp3 +3 -0
- examples/short_de.json +23 -0
- examples/short_en.json +23 -0
- generation_config.json +7 -0
- gpt2_decoder.py +721 -0
- guides/AGENT_GUIDE.md +15 -0
- guides/LEGACY_PIPELINE.md +102 -0
- guides/LEGACY_README.md +173 -0
- guides/MULTI_SPEAKER.md +32 -0
- guides/PIPELINE.md +187 -0
- guides/PODCAST_EXPERIMENT_HISTORY.md +67 -0
- guides/PROMPTING.md +95 -0
- guides/SINGLE_SPEAKER.md +17 -0
- guides/recipe.json +21 -0
- infer.py +12 -0
- infer_legacy.py +3 -0
- model-00001-of-00005.safetensors +3 -0
- model-00002-of-00005.safetensors +3 -0
- model-00003-of-00005.safetensors +3 -0
- model-00004-of-00005.safetensors +3 -0
- model-00005-of-00005.safetensors +3 -0
- model.safetensors.index.json +432 -0
.gitattributes
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
examples/audio/de_block_reassembled.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
examples/audio/de_segment_reassembled.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
examples/audio/en_block_reassembled.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
examples/audio/en_segment_reassembled.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
examples/references/de_s1.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
examples/references/de_s2.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
examples/references/en_s1.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
examples/references/en_s2.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
validation/gpu-smoke/en-turns/episode.wav filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
validation/gpu-smoke/en-pairs/episode.wav filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
validation/gpu-smoke/de-turns/episode.wav filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
validation/gpu-smoke/de-pairs/episode.wav filter=lfs diff=lfs merge=lfs -text
|
CHANGELOG.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Release tooling changes
|
| 2 |
+
|
| 3 |
+
## Pipeline v2
|
| 4 |
+
|
| 5 |
+
The first fresh GPU smoke stopped before generation because a progress-page SQL query used LIMIT4 instead of LIMIT 4. Fixed the query and added empty/populated journal rendering regression checks. Model weights, prompts, candidate rewards and QA thresholds are unchanged. The failed attempt is retained in release validation history. A new GPU smoke uses a fresh output directory and the corrected source snapshot.
|
| 6 |
+
|
| 7 |
+
The portable continued-training loader also uses a real one-example batch sampler with an identity collator. A regression check with the actual Accelerate BatchSamplerShard verifies disjoint rank partitions and preserves the required1×T×13 tensor layout. This does not change the historical training run or inference runtime.
|
EXPORT_PROOF.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"passed": true,
|
| 3 |
+
"source_checkpoint_step": 9874,
|
| 4 |
+
"source_file_sha256": "8c4cd0923553e9522e0b92e5719bfd12ab11f4981b1364c9d134301dfc42b633",
|
| 5 |
+
"source_dtype": "FP32 FSDP full state",
|
| 6 |
+
"deployment_dtype": "BF16; exact per-tensor equality to source cast, as in the listening pilot",
|
| 7 |
+
"parent": "laion/Humaneness-Voice-Large-1.7",
|
| 8 |
+
"tensors": 425,
|
| 9 |
+
"all_tensors_equal_to_evaluated_bf16_cast": true,
|
| 10 |
+
"no_model_generation_or_neural_forward_on_login": true,
|
| 11 |
+
"shard_bytes": {
|
| 12 |
+
"model-00001-of-00005.safetensors": 1997031256,
|
| 13 |
+
"model-00002-of-00005.safetensors": 1997658024,
|
| 14 |
+
"model-00003-of-00005.safetensors": 1997657656,
|
| 15 |
+
"model-00004-of-00005.safetensors": 1963573536,
|
| 16 |
+
"model-00005-of-00005.safetensors": 304110984
|
| 17 |
+
},
|
| 18 |
+
"sha256": {
|
| 19 |
+
"model-00001-of-00005.safetensors": "791c61b2bcad322b58fbac3d0d24174014bb49acad2a52fb581cf96074381675",
|
| 20 |
+
"model-00002-of-00005.safetensors": "1be302af66c882ec7f6ef35954c76b0e55436911596b1602627d53cfbaed1dca",
|
| 21 |
+
"model-00003-of-00005.safetensors": "1d65aa4709289a60e18fc7d7a3fa75c3357755707b167df62e9512c94a3a69c0",
|
| 22 |
+
"model-00004-of-00005.safetensors": "1f8a6db71e26dc79f3f5f0562bf0320f51f1d83fdec0db7b2ca74206f170dde2",
|
| 23 |
+
"model-00005-of-00005.safetensors": "b8033d7393e3d846a6740f041992fa449d2a017613a9130ab5ab0df9d3a15d91"
|
| 24 |
+
},
|
| 25 |
+
"total_parameters": 4129990144,
|
| 26 |
+
"time": 1794075986.055203
|
| 27 |
+
}
|
LICENSE
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Creative Commons Attribution 4.0 International (CC BY 4.0)
|
| 2 |
+
|
| 3 |
+
LAION model weights, new pipeline code and documentation are provided under CC BY 4.0.
|
| 4 |
+
License terms: https://creativecommons.org/licenses/by/4.0/legalcode.en
|
| 5 |
+
Attribution: LAION, Humanness Voice Podcast, 2026.
|
| 6 |
+
Upstream code and dependencies retain their original notices/licenses; example source recordings retain their original terms. See NOTICE.md and UPSTREAM_LICENSE.
|
MANIFEST.json
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model-00005-of-00005.safetensors": {
|
| 3 |
+
"bytes": 304110984,
|
| 4 |
+
"sha256": "b8033d7393e3d846a6740f041992fa449d2a017613a9130ab5ab0df9d3a15d91"
|
| 5 |
+
},
|
| 6 |
+
"config.json": {
|
| 7 |
+
"bytes": 7014,
|
| 8 |
+
"sha256": "34f03ae9ac95c2c6cf0a36a17fd64eec57caa23e344b4e7999015d433a5543b4"
|
| 9 |
+
},
|
| 10 |
+
"LICENSE": {
|
| 11 |
+
"bytes": 432,
|
| 12 |
+
"sha256": "cc980e308d6c61586ea8d4707a3c57a5023f86e24c93ae21aa9dabc43881fe68"
|
| 13 |
+
},
|
| 14 |
+
"processing_moss_tts.py": {
|
| 15 |
+
"bytes": 37501,
|
| 16 |
+
"sha256": "059637b92e7eb9a67df531fe9578e9554d456195828b223eb81f21117c58e8f8"
|
| 17 |
+
},
|
| 18 |
+
"TRAINING.md": {
|
| 19 |
+
"bytes": 8224,
|
| 20 |
+
"sha256": "9d318a9d13304d1c1dd84c204707f3f1a1a1925aa0a486f1bc6141b5495e9af6"
|
| 21 |
+
},
|
| 22 |
+
"model-00004-of-00005.safetensors": {
|
| 23 |
+
"bytes": 1963573536,
|
| 24 |
+
"sha256": "1f8a6db71e26dc79f3f5f0562bf0320f51f1d83fdec0db7b2ca74206f170dde2"
|
| 25 |
+
},
|
| 26 |
+
"voices.json": {
|
| 27 |
+
"bytes": 15113,
|
| 28 |
+
"sha256": "9a80aeafcbfe45247d98170b4139bef99b1d8393588e97cbb93125923a509ea1"
|
| 29 |
+
},
|
| 30 |
+
"requirements.txt": {
|
| 31 |
+
"bytes": 114,
|
| 32 |
+
"sha256": "0a22058ff0d1ff22b7970a9d173b8397cda2fa0209a04ffcaf6ab4653a344002"
|
| 33 |
+
},
|
| 34 |
+
"tokenizer_config.json": {
|
| 35 |
+
"bytes": 5727,
|
| 36 |
+
"sha256": "ec2869dd4c9c7155b3460c291a6fb2b955e5068013ba7efbcaafd1c61e02abce"
|
| 37 |
+
},
|
| 38 |
+
"infer.py": {
|
| 39 |
+
"bytes": 423,
|
| 40 |
+
"sha256": "ebf0e8250a2a5c17c3986c3ab71294c5abc302a0ad6b61f80eac09f01658e37b"
|
| 41 |
+
},
|
| 42 |
+
"model-00002-of-00005.safetensors": {
|
| 43 |
+
"bytes": 1997658024,
|
| 44 |
+
"sha256": "1be302af66c882ec7f6ef35954c76b0e55436911596b1602627d53cfbaed1dca"
|
| 45 |
+
},
|
| 46 |
+
"model.safetensors.index.json": {
|
| 47 |
+
"bytes": 37211,
|
| 48 |
+
"sha256": "cc511c4197d5ae3982936eae3e4b12ef70202dbb163d67526dfbbd7f6a4c6dd7"
|
| 49 |
+
},
|
| 50 |
+
"configuration_moss_tts.py": {
|
| 51 |
+
"bytes": 7160,
|
| 52 |
+
"sha256": "ab6debcb92032cb9dc91ae80aed77dbadd2e59848208baef2b062bd6def3f3be"
|
| 53 |
+
},
|
| 54 |
+
"UPSTREAM_LICENSE": {
|
| 55 |
+
"bytes": 544,
|
| 56 |
+
"sha256": "17241371600bcde523c76ca4f006a1700145535914b24586c5766572640f7cf8"
|
| 57 |
+
},
|
| 58 |
+
"model-00003-of-00005.safetensors": {
|
| 59 |
+
"bytes": 1997657656,
|
| 60 |
+
"sha256": "1d65aa4709289a60e18fc7d7a3fa75c3357755707b167df62e9512c94a3a69c0"
|
| 61 |
+
},
|
| 62 |
+
"gpt2_decoder.py": {
|
| 63 |
+
"bytes": 30896,
|
| 64 |
+
"sha256": "f2e877104669f1e6c7cd34680f0da1a8a159e032123ee56b660b63929b6c8989"
|
| 65 |
+
},
|
| 66 |
+
"UPSTREAM_NOTICE.md": {
|
| 67 |
+
"bytes": 1236,
|
| 68 |
+
"sha256": "a00fd9ba52637b37dce84f9a09596a6747df540a3796b8140e151fb5949d9cf6"
|
| 69 |
+
},
|
| 70 |
+
"model-00001-of-00005.safetensors": {
|
| 71 |
+
"bytes": 1997031256,
|
| 72 |
+
"sha256": "791c61b2bcad322b58fbac3d0d24174014bb49acad2a52fb581cf96074381675"
|
| 73 |
+
},
|
| 74 |
+
"PUBLICATION_ID.json": {
|
| 75 |
+
"bytes": 67,
|
| 76 |
+
"sha256": "89ef756681c83c23a2b778c01c5f90f4e119761e174205b09e5fcc49c89129ea"
|
| 77 |
+
},
|
| 78 |
+
"special_tokens_map.json": {
|
| 79 |
+
"bytes": 394,
|
| 80 |
+
"sha256": "02a153514db64e5378c53ef164c960409882649de6f4aac8f81f66218e7f8797"
|
| 81 |
+
},
|
| 82 |
+
"added_tokens.json": {
|
| 83 |
+
"bytes": 790,
|
| 84 |
+
"sha256": "39bb49db6acfb8e7510b1640f575bdc183f511d85c94ce3ee967f2a6195c4a8a"
|
| 85 |
+
},
|
| 86 |
+
"processor_config.json": {
|
| 87 |
+
"bytes": 209,
|
| 88 |
+
"sha256": "fac7289af2a37d988d708d813c92960e5e91c59d348c17b72c6aa6b86859f2b7"
|
| 89 |
+
},
|
| 90 |
+
"VALIDATION.md": {
|
| 91 |
+
"bytes": 6251,
|
| 92 |
+
"sha256": "ce2472ae62539510a22114427a028c6a4a782d5d3d78a423509dbb82fcd5da84"
|
| 93 |
+
},
|
| 94 |
+
"EXPORT_PROOF.json": {
|
| 95 |
+
"bytes": 1366,
|
| 96 |
+
"sha256": "456c6feffd52f2cd5462bd8575578c81c7c1910885b17858ba3d29390ba8d8b9"
|
| 97 |
+
},
|
| 98 |
+
"qwen3_decoder.py": {
|
| 99 |
+
"bytes": 25473,
|
| 100 |
+
"sha256": "100163bd7ecf31a59bafacc0b032ace9339edc992a3eb4cc80662502e04e46f0"
|
| 101 |
+
},
|
| 102 |
+
"README.md": {
|
| 103 |
+
"bytes": 23905,
|
| 104 |
+
"sha256": "5763750ea138f21907c5104fc2202e0f96d81e9076a494f06c02db2d96359cf8"
|
| 105 |
+
},
|
| 106 |
+
"modeling_moss_tts.py": {
|
| 107 |
+
"bytes": 26379,
|
| 108 |
+
"sha256": "b0a66211943ae580b087f3e71495fea2f455701a4f6c29b6d3562218f7668c5f"
|
| 109 |
+
},
|
| 110 |
+
"CHANGELOG.md": {
|
| 111 |
+
"bytes": 787,
|
| 112 |
+
"sha256": "21038fecbeb284c10ab60eca304b67fd335aedde672c9524cffc351f9760f0f2"
|
| 113 |
+
},
|
| 114 |
+
"tokenizer.json": {
|
| 115 |
+
"bytes": 11423220,
|
| 116 |
+
"sha256": "06902d1fb775216338802205886a24bc715ccc606bd872a892e3d3c83ca1b9e2"
|
| 117 |
+
},
|
| 118 |
+
"generation_config.json": {
|
| 119 |
+
"bytes": 158,
|
| 120 |
+
"sha256": "ec11df43c896a23a66d88eb9281d90bbbca48f3afb047fec0172f7af4d1ccdfc"
|
| 121 |
+
},
|
| 122 |
+
"NOTICE.md": {
|
| 123 |
+
"bytes": 2009,
|
| 124 |
+
"sha256": "63dc5e7668c70550dcb0854dd8349cf22fdc1d1bd4e3e1d0bdd28ae872b64f2e"
|
| 125 |
+
},
|
| 126 |
+
"tests/test_model_contracts.py": {
|
| 127 |
+
"bytes": 4901,
|
| 128 |
+
"sha256": "20ba80d70689b505d827bad32c292c5e1a56f02e39b6622e5d44600bdcc98804"
|
| 129 |
+
},
|
| 130 |
+
"tests/test_pipeline.py": {
|
| 131 |
+
"bytes": 4355,
|
| 132 |
+
"sha256": "e2025aa7fb1e1cd649121451d50ac74c06a10cc2f62455e686c33e0ee1996c15"
|
| 133 |
+
},
|
| 134 |
+
"stats/DONE.json": {
|
| 135 |
+
"bytes": 162,
|
| 136 |
+
"sha256": "09067f0e493d2e2ec4aab55a9d42fd357e068db9fe919cea22741caa68154369"
|
| 137 |
+
},
|
| 138 |
+
"stats/editing_pilot.json": {
|
| 139 |
+
"bytes": 1700,
|
| 140 |
+
"sha256": "3ad9dd4b0640f5639fe093205995b8d27b57d2e646287f3caf35ea52e7b8cb97"
|
| 141 |
+
},
|
| 142 |
+
"stats/release_validation.json": {
|
| 143 |
+
"bytes": 2004,
|
| 144 |
+
"sha256": "4dd00c89df21824943d6d2c1f02dab0c76e4eb3906a97ca950a1ac6358b79976"
|
| 145 |
+
},
|
| 146 |
+
"stats/validation.jsonl": {
|
| 147 |
+
"bytes": 1867,
|
| 148 |
+
"sha256": "90669a2f7a333ec1c6b85115452785d6f730b5d80e95a6dd756078cf42388156"
|
| 149 |
+
},
|
| 150 |
+
"stats/segment_block_pilot.json": {
|
| 151 |
+
"bytes": 36613,
|
| 152 |
+
"sha256": "a11d82e6a83c2d950546be29329730fe901eafbb1602a7085f8fbf343f6f5bf1"
|
| 153 |
+
},
|
| 154 |
+
"stats/training_summary.json": {
|
| 155 |
+
"bytes": 3957,
|
| 156 |
+
"sha256": "33c55870e00f69a0a2dd617e82a5f4fb7a887c5daceec2896ba529e4bbd73686"
|
| 157 |
+
},
|
| 158 |
+
"stats/steps.jsonl": {
|
| 159 |
+
"bytes": 94968,
|
| 160 |
+
"sha256": "dcb081c2a884f0fff80595c848d36a8a2c311e69d28f5ee7c3111cd950731962"
|
| 161 |
+
},
|
| 162 |
+
"guides/recipe.json": {
|
| 163 |
+
"bytes": 608,
|
| 164 |
+
"sha256": "c5cd09a5fb8ddda86d5cc8368cc0906fdb4374351510da2bf8e40458e0255983"
|
| 165 |
+
},
|
| 166 |
+
"guides/MULTI_SPEAKER.md": {
|
| 167 |
+
"bytes": 2767,
|
| 168 |
+
"sha256": "0a45fc63797508a13ee35bdbaced2d3d6fc4f81acdfb3fb7a9230b6b044864d3"
|
| 169 |
+
},
|
| 170 |
+
"guides/SINGLE_SPEAKER.md": {
|
| 171 |
+
"bytes": 2213,
|
| 172 |
+
"sha256": "c9cce422e04a4fe1f950e4fc0014c6bda31356b71d235f57252c8a159cbaf45c"
|
| 173 |
+
},
|
| 174 |
+
"guides/AGENT_GUIDE.md": {
|
| 175 |
+
"bytes": 3513,
|
| 176 |
+
"sha256": "6bb36e793e9112de0377ef9fc95217b0635b7ae86b60ded7d7884fd36d86b109"
|
| 177 |
+
},
|
| 178 |
+
"guides/PIPELINE.md": {
|
| 179 |
+
"bytes": 21725,
|
| 180 |
+
"sha256": "758c0280fff9205b157fd0f0d3d69ab7dc6484a211239c9c79acd5a5e381de08"
|
| 181 |
+
},
|
| 182 |
+
"podcast/engine.py": {
|
| 183 |
+
"bytes": 6370,
|
| 184 |
+
"sha256": "1ddddde10286f779b9d2812da71516ecb06373741daa5ec55c05f4c50731e52c"
|
| 185 |
+
},
|
| 186 |
+
"podcast/pipeline.py": {
|
| 187 |
+
"bytes": 14590,
|
| 188 |
+
"sha256": "a25386e92dd064658d4f36c4886d71d4adc7386bd32b59e1b996dc32915fb5d0"
|
| 189 |
+
},
|
| 190 |
+
"podcast/ranking.py": {
|
| 191 |
+
"bytes": 2372,
|
| 192 |
+
"sha256": "3ba4350a10d98e11cb480a0c7b98ffb7ffec3e680336d26d65b59a078cf2e8b2"
|
| 193 |
+
},
|
| 194 |
+
"podcast/asr.py": {
|
| 195 |
+
"bytes": 1025,
|
| 196 |
+
"sha256": "084a520f2948e3886aecb6ef21c19e5266f46182f362def3dc16ddb3149536f0"
|
| 197 |
+
},
|
| 198 |
+
"podcast/audio.py": {
|
| 199 |
+
"bytes": 5013,
|
| 200 |
+
"sha256": "01155a8975bb9ad09ebedd60c33910370dd84599e5a1bdd0bcd83ae19bc15645"
|
| 201 |
+
},
|
| 202 |
+
"podcast/__init__.py": {
|
| 203 |
+
"bytes": 85,
|
| 204 |
+
"sha256": "c6ef743673c226d84a65d0865559de003b5e34ccd6319ad87b3da7ff38ab7938"
|
| 205 |
+
},
|
| 206 |
+
"podcast/prompting.py": {
|
| 207 |
+
"bytes": 2930,
|
| 208 |
+
"sha256": "391b9fe64002f49846cd88bfea1d8b4d0d57d2982b435e094b686180a0736dfa"
|
| 209 |
+
},
|
| 210 |
+
"podcast/alignment.py": {
|
| 211 |
+
"bytes": 2832,
|
| 212 |
+
"sha256": "ea286efa586e21741e26fba902e9e25b61f79673c03dd12e2271b4905f354784"
|
| 213 |
+
},
|
| 214 |
+
"podcast/conversion.py": {
|
| 215 |
+
"bytes": 869,
|
| 216 |
+
"sha256": "20f497b9c2ab5766a2e619ab3bee04ba914b5b0ccf09faaa5cee4887c3ca72e4"
|
| 217 |
+
},
|
| 218 |
+
"podcast/ears_backend.py": {
|
| 219 |
+
"bytes": 13002,
|
| 220 |
+
"sha256": "3b318831e66fc27e6f1e3d231635b6ab69338631dbdcc37571cb1603468ad0a6"
|
| 221 |
+
},
|
| 222 |
+
"runtime/native_cfg.py": {
|
| 223 |
+
"bytes": 4319,
|
| 224 |
+
"sha256": "d5af8a56d35be6baee5587c019ffbe08d0581ce915e14063194bb2dccbbc9a5f"
|
| 225 |
+
},
|
| 226 |
+
"runtime/base_infer.py": {
|
| 227 |
+
"bytes": 7464,
|
| 228 |
+
"sha256": "4b73145ff3dbeaa6cce4e609973054ea79b1b9ff9f4bb10e3e8a662e7f42b540"
|
| 229 |
+
},
|
| 230 |
+
"examples/short_de.json": {
|
| 231 |
+
"bytes": 844,
|
| 232 |
+
"sha256": "9f8de1d752fc04528b9af635c83d5194fb87421a34b7c88689fcec2af1a4e069"
|
| 233 |
+
},
|
| 234 |
+
"examples/episode_de.json": {
|
| 235 |
+
"bytes": 3421,
|
| 236 |
+
"sha256": "c38c5c59849a6e7226f8189b7dabbd5ea16798e66c5fa2a8adedb927fbd2f700"
|
| 237 |
+
},
|
| 238 |
+
"examples/episode_en.json": {
|
| 239 |
+
"bytes": 3324,
|
| 240 |
+
"sha256": "d5893bf87feab3b8113610ed7100621c2802b35d3517e5726fd02ad7fcedc26c"
|
| 241 |
+
},
|
| 242 |
+
"examples/short_en.json": {
|
| 243 |
+
"bytes": 834,
|
| 244 |
+
"sha256": "7a7084aae9257736d2d262be4afbacf97169bcba8392ff5c4cdfeb9d522c4e15"
|
| 245 |
+
},
|
| 246 |
+
"examples/audio/en_block_reassembled.mp3": {
|
| 247 |
+
"bytes": 911660,
|
| 248 |
+
"sha256": "efdeb8b807a286e5407ac09c4f5c0abc15288774aca734c9e992a0249d077c1b"
|
| 249 |
+
},
|
| 250 |
+
"examples/audio/de_segment_reassembled.mp3": {
|
| 251 |
+
"bytes": 1252268,
|
| 252 |
+
"sha256": "070dfe22bae9fa125022d7ab36346c5d6cc5124fbad41553d0a40ea27d1d4ed5"
|
| 253 |
+
},
|
| 254 |
+
"examples/audio/de_block_reassembled.mp3": {
|
| 255 |
+
"bytes": 947756,
|
| 256 |
+
"sha256": "43462ee94dd0f48ddf3c149ba3450f6b0e08f33e471d124c1b7152f47540d4e2"
|
| 257 |
+
},
|
| 258 |
+
"examples/audio/PROVENANCE.json": {
|
| 259 |
+
"bytes": 1142,
|
| 260 |
+
"sha256": "046c9fadbb7bf2fe11e0517a76fc112c35a5ddcbe8fa4453d40abca5ee231e85"
|
| 261 |
+
},
|
| 262 |
+
"examples/audio/en_segment_reassembled.mp3": {
|
| 263 |
+
"bytes": 1247276,
|
| 264 |
+
"sha256": "1ed0d49e35a7866de27ee71a4112947f37226922d902cd7eeb6352666ec9fec1"
|
| 265 |
+
},
|
| 266 |
+
"examples/references/en_s2.mp3": {
|
| 267 |
+
"bytes": 186284,
|
| 268 |
+
"sha256": "517e5ff6a62e393ac995985a03535ad367322aad14ac5caba3b7b2ee6d2daf92"
|
| 269 |
+
},
|
| 270 |
+
"examples/references/en_s1.mp3": {
|
| 271 |
+
"bytes": 150188,
|
| 272 |
+
"sha256": "b630face220f977dc36e01db02f22063b7f2e4188439e272dc3a02f552082722"
|
| 273 |
+
},
|
| 274 |
+
"examples/references/de_s1.mp3": {
|
| 275 |
+
"bytes": 232364,
|
| 276 |
+
"sha256": "9d5a7823fea82182d6a3ec378e15ae03b0bd61f028c6f50fb6c5b82fb1f1b7dd"
|
| 277 |
+
},
|
| 278 |
+
"examples/references/de_anchors.npz": {
|
| 279 |
+
"bytes": 6799,
|
| 280 |
+
"sha256": "27c051d23d584df782cfcf4abd4051700838169d7bf2a0610b077ec1d30eccdf"
|
| 281 |
+
},
|
| 282 |
+
"examples/references/en_anchors.npz": {
|
| 283 |
+
"bytes": 5199,
|
| 284 |
+
"sha256": "ed071500398ed41980e5ddc81bfc4b928a1469a7e6874792af033b52b92af989"
|
| 285 |
+
},
|
| 286 |
+
"examples/references/de_s2.mp3": {
|
| 287 |
+
"bytes": 209324,
|
| 288 |
+
"sha256": "d2da04db22aeb44206e4f935e95919846e3e5bb4cb116503a7a260c9f48752cd"
|
| 289 |
+
},
|
| 290 |
+
"examples/references/PROVENANCE.json": {
|
| 291 |
+
"bytes": 939,
|
| 292 |
+
"sha256": "a5e08afd717070d35d388ad576a9ad188e05fc667f03231d441b3fce898690b0"
|
| 293 |
+
},
|
| 294 |
+
"training/tokenize_manifest.py": {
|
| 295 |
+
"bytes": 3555,
|
| 296 |
+
"sha256": "381b0938655f62dd23cd66ba4851e35134664a0bf907c9c3d7ccc392d88abd7f"
|
| 297 |
+
},
|
| 298 |
+
"training/va_loss.py": {
|
| 299 |
+
"bytes": 5534,
|
| 300 |
+
"sha256": "2df51c949a72d0ccbad4a1fdb3ce50374cddae34f70c73fd212d7959347095a5"
|
| 301 |
+
},
|
| 302 |
+
"training/train.py": {
|
| 303 |
+
"bytes": 9582,
|
| 304 |
+
"sha256": "234a082f83bdc0022794ea5816c0a0e1b65f5d0ad9bcde9bb21d398f8f7f9f31"
|
| 305 |
+
},
|
| 306 |
+
"training/historical/frozen_fullft_source.tar.gz": {
|
| 307 |
+
"bytes": 39749,
|
| 308 |
+
"sha256": "a232f7f6b5f7ecd7e2def77a4c7e6ec37d8471c73ff3d8caaf9ea1229f93c89a"
|
| 309 |
+
},
|
| 310 |
+
"stats/gpu_smoke_details.json": {
|
| 311 |
+
"bytes": 1802,
|
| 312 |
+
"sha256": "34fb1757ba1b2bf29073c7a694b9c9a2158ddcb8e9fabf1ce4872ddbd37cec42"
|
| 313 |
+
},
|
| 314 |
+
"validation/gpu-smoke/en-turns/episode.wav": {
|
| 315 |
+
"bytes": 1634444,
|
| 316 |
+
"sha256": "7f31f92585479deba672ecdad4521b385f9d12bbe94ad39e58899e3caf846aa4"
|
| 317 |
+
},
|
| 318 |
+
"validation/gpu-smoke/en-turns/episode_timeline.json": {
|
| 319 |
+
"bytes": 213,
|
| 320 |
+
"sha256": "963ef4a873e812f1b042a37af4cdf6b1e8133970b5e2f061d5e703f4dce9b438"
|
| 321 |
+
},
|
| 322 |
+
"validation/gpu-smoke/en-turns/STATUS.json": {
|
| 323 |
+
"bytes": 91,
|
| 324 |
+
"sha256": "31080092245ae8345ac8d9d33051aa6782ec99ad1ab7f9b439407ced8eea06bd"
|
| 325 |
+
},
|
| 326 |
+
"validation/gpu-smoke/de-pairs/episode.wav": {
|
| 327 |
+
"bytes": 2890124,
|
| 328 |
+
"sha256": "5061cdada7a57cfe4ea95730d0aff0bd62d639f9789107b5687922bb428e0fb3"
|
| 329 |
+
},
|
| 330 |
+
"validation/gpu-smoke/de-pairs/episode_timeline.json": {
|
| 331 |
+
"bytes": 216,
|
| 332 |
+
"sha256": "d25c1b878aa98d36f95fe1c4775d2cf3c04a32e035f23e71d193bcf59b361178"
|
| 333 |
+
},
|
| 334 |
+
"validation/gpu-smoke/de-pairs/STATUS.json": {
|
| 335 |
+
"bytes": 91,
|
| 336 |
+
"sha256": "8d24f76ccc148df88f4311724a804c06f88e0d12e764177885a07eaaa4c64931"
|
| 337 |
+
},
|
| 338 |
+
"validation/gpu-smoke/en-pairs/episode.wav": {
|
| 339 |
+
"bytes": 2809484,
|
| 340 |
+
"sha256": "cd95dbfbc64225044092ac43fd951458a74687011a5e948eb8aa3cadbcd5a4b8"
|
| 341 |
+
},
|
| 342 |
+
"validation/gpu-smoke/en-pairs/episode_timeline.json": {
|
| 343 |
+
"bytes": 212,
|
| 344 |
+
"sha256": "cfc9e564da04822707b4cf49c930d9bbcd05785bed94edd39992139f3293f7f7"
|
| 345 |
+
},
|
| 346 |
+
"validation/gpu-smoke/en-pairs/STATUS.json": {
|
| 347 |
+
"bytes": 92,
|
| 348 |
+
"sha256": "085ebcfabbaf4e61a4e7f2d2b5cfe49a7462c69e61cf7ae337a2c5ba4924491a"
|
| 349 |
+
},
|
| 350 |
+
"validation/gpu-smoke/de-turns/episode.wav": {
|
| 351 |
+
"bytes": 1726604,
|
| 352 |
+
"sha256": "ec15965f059af08b805984295ffb08cc85b9d54b5b71172f0a8aa75a198e67cc"
|
| 353 |
+
},
|
| 354 |
+
"validation/gpu-smoke/de-turns/episode_timeline.json": {
|
| 355 |
+
"bytes": 211,
|
| 356 |
+
"sha256": "c4289b2ecf5dee22b15ba98d15add3dc92fa7dfef3b60ab661872c294675c1f5"
|
| 357 |
+
},
|
| 358 |
+
"validation/gpu-smoke/de-turns/STATUS.json": {
|
| 359 |
+
"bytes": 92,
|
| 360 |
+
"sha256": "9afcae48a7133c1d1207e12749fbf4591c8b7e7072512a33818c5615a7f474d1"
|
| 361 |
+
},
|
| 362 |
+
"requirements-legacy.txt": {
|
| 363 |
+
"bytes": 124,
|
| 364 |
+
"sha256": "8bcd23f75f4c83db37dcbd2c7a499edc6887870f5451de9ac85c93656d7aaa88"
|
| 365 |
+
},
|
| 366 |
+
"infer_legacy.py": {
|
| 367 |
+
"bytes": 146,
|
| 368 |
+
"sha256": "4b138e143329b7c50cb1897a65d327936b5bd622a400df92f28b81f4d44ab25d"
|
| 369 |
+
},
|
| 370 |
+
"optimized_pipeline/script-spontaneous-en.json": {
|
| 371 |
+
"bytes": 14275,
|
| 372 |
+
"sha256": "21d1aab0d49ca9e66b56e4c1097ba14b274d251cc1ba3f888383499fdc064a5f"
|
| 373 |
+
},
|
| 374 |
+
"optimized_pipeline/test_spontaneous.py": {
|
| 375 |
+
"bytes": 1480,
|
| 376 |
+
"sha256": "784c283b29b8da7541f4f1a00d54199b3eb6e7e407f25b87e6896719ac8d099f"
|
| 377 |
+
},
|
| 378 |
+
"optimized_pipeline/scoring.py": {
|
| 379 |
+
"bytes": 5106,
|
| 380 |
+
"sha256": "fffc89db6534a326272bb38e1039c46575b911e330fbea393ab49ec550cbce12"
|
| 381 |
+
},
|
| 382 |
+
"optimized_pipeline/.gitignore": {
|
| 383 |
+
"bytes": 108,
|
| 384 |
+
"sha256": "a4f4af8a2429865be28ced3ae59a073e49150ffe039b6e4f8efe871fc9a38781"
|
| 385 |
+
},
|
| 386 |
+
"optimized_pipeline/verify_artifacts.py": {
|
| 387 |
+
"bytes": 3520,
|
| 388 |
+
"sha256": "1c1c7a4b8885b1188a047076c8ce116188905cecee6312b83d6ef708c3a383ca"
|
| 389 |
+
},
|
| 390 |
+
"optimized_pipeline/references.json": {
|
| 391 |
+
"bytes": 488,
|
| 392 |
+
"sha256": "4b21cd86212d5b7e495956a82e6cb630c497e56ad35e8aed8b278cd8e5a54102"
|
| 393 |
+
},
|
| 394 |
+
"optimized_pipeline/test_contracts.py": {
|
| 395 |
+
"bytes": 3875,
|
| 396 |
+
"sha256": "9642836aea9449ba9d294f913ccf5eb785d7bba362b769d01972aa9bb0044aa4"
|
| 397 |
+
},
|
| 398 |
+
"optimized_pipeline/produce.py": {
|
| 399 |
+
"bytes": 1178,
|
| 400 |
+
"sha256": "ea02ca7aeb6053c31b0c211bfbec9fb6ac36706d2b53cb6b4897e5b4ad08aae4"
|
| 401 |
+
},
|
| 402 |
+
"optimized_pipeline/finish.py": {
|
| 403 |
+
"bytes": 13587,
|
| 404 |
+
"sha256": "c9dea792a237d4a90cea9ce1e1b58929c69a329bab2946a84b0419de7025f931"
|
| 405 |
+
},
|
| 406 |
+
"optimized_pipeline/script-de.json": {
|
| 407 |
+
"bytes": 6722,
|
| 408 |
+
"sha256": "49c89ce912de090124ee15645f0fe94acff6f3316343e8c11032f06994b88406"
|
| 409 |
+
},
|
| 410 |
+
"optimized_pipeline/speaker_scoring.py": {
|
| 411 |
+
"bytes": 3880,
|
| 412 |
+
"sha256": "e355d7f3b9571d20564a981cf2028ed74f029ab9e4d0918f81e2140c1526ce9b"
|
| 413 |
+
},
|
| 414 |
+
"optimized_pipeline/script-en.json": {
|
| 415 |
+
"bytes": 6312,
|
| 416 |
+
"sha256": "35bb921b1516483823335a86aaa5afc10a0514dd6739cad60e387b940f95683e"
|
| 417 |
+
},
|
| 418 |
+
"optimized_pipeline/README.md": {
|
| 419 |
+
"bytes": 1282,
|
| 420 |
+
"sha256": "d05e2e462b030afa4bbb1bfe4f9d18a502c959f672e8bd24f3726f90a3c846d9"
|
| 421 |
+
},
|
| 422 |
+
"optimized_pipeline/script-speaker-en.json": {
|
| 423 |
+
"bytes": 16141,
|
| 424 |
+
"sha256": "3dee05ee57db607968a74c27a9ea367187cd7355b95379a1e210d57fc6240bfe"
|
| 425 |
+
},
|
| 426 |
+
"optimized_pipeline/requirements.txt": {
|
| 427 |
+
"bytes": 261,
|
| 428 |
+
"sha256": "c1fb18bc05fe70062f3fed98584f9ba31a327877b43335656e33f0bdacedba8e"
|
| 429 |
+
},
|
| 430 |
+
"optimized_pipeline/script-speaker-de.json": {
|
| 431 |
+
"bytes": 16975,
|
| 432 |
+
"sha256": "8d66a30ee8be0008732f5cfb47365a6fe8ba4ee407d3959608ee0b3fa5b94c77"
|
| 433 |
+
},
|
| 434 |
+
"optimized_pipeline/archive.py": {
|
| 435 |
+
"bytes": 1774,
|
| 436 |
+
"sha256": "a57b71da8081e1cbecddb585490a255a8098872c298c30cf7b77e39f11cbb63b"
|
| 437 |
+
},
|
| 438 |
+
"optimized_pipeline/speaker_finish.py": {
|
| 439 |
+
"bytes": 15682,
|
| 440 |
+
"sha256": "5a9d9a5f64cc8ace567089e5a116579787428745941ce1f3a6bd479e670d158d"
|
| 441 |
+
},
|
| 442 |
+
"optimized_pipeline/generation.py": {
|
| 443 |
+
"bytes": 22018,
|
| 444 |
+
"sha256": "2bcf575383aba32f14e21d9c7087d3e10b0aa6240f88998bce54c4be4c428192"
|
| 445 |
+
},
|
| 446 |
+
"optimized_pipeline/test_inputs.py": {
|
| 447 |
+
"bytes": 2548,
|
| 448 |
+
"sha256": "177acef192ba5edb355fd3a7ff57489e9f0963d0543211f4432c9eb80dcaf748"
|
| 449 |
+
},
|
| 450 |
+
"optimized_pipeline/browser_verify.cjs": {
|
| 451 |
+
"bytes": 2858,
|
| 452 |
+
"sha256": "4b32a23c371e9ab05ee75dd3c3225a150dffd4d37c0d3e63b4d51b6d89965ba6"
|
| 453 |
+
},
|
| 454 |
+
"optimized_pipeline/PROMPTING.md": {
|
| 455 |
+
"bytes": 8521,
|
| 456 |
+
"sha256": "a63ee77066c510f2eae8fc205abbd1c82f61c1fcafccf2e854285d6fd3bc1b9d"
|
| 457 |
+
},
|
| 458 |
+
"optimized_pipeline/prepare.py": {
|
| 459 |
+
"bytes": 1977,
|
| 460 |
+
"sha256": "81752d559ce9901a0e0625efd0de7e8e20243adc531c1e605cc4c9e7a55391e0"
|
| 461 |
+
},
|
| 462 |
+
"optimized_pipeline/run.py": {
|
| 463 |
+
"bytes": 936,
|
| 464 |
+
"sha256": "1773a36cae62fc1de6b3cc59ae893bd54e5cda20065f9a5972b54608eb7dd77e"
|
| 465 |
+
},
|
| 466 |
+
"optimized_pipeline/inputs.py": {
|
| 467 |
+
"bytes": 2061,
|
| 468 |
+
"sha256": "ac95a829fd4b11327a29dde3fe2bc634b22496eab95d909d669001099240af8e"
|
| 469 |
+
},
|
| 470 |
+
"optimized_pipeline/compare_n.py": {
|
| 471 |
+
"bytes": 1907,
|
| 472 |
+
"sha256": "32bada3d5cb534fd7a6a9856d2c4dd16ed61a5b247dae75d900ec62fb72bcc1a"
|
| 473 |
+
},
|
| 474 |
+
"optimized_pipeline/offline_batch.py": {
|
| 475 |
+
"bytes": 8396,
|
| 476 |
+
"sha256": "d5b390625f4351fa1789a74100f0b50abe5f78e5b7b16e645e54b3a0127ee7f3"
|
| 477 |
+
},
|
| 478 |
+
"optimized_pipeline/test_speaker_recipe.py": {
|
| 479 |
+
"bytes": 1258,
|
| 480 |
+
"sha256": "3982784fffe963582ef2b2769cd780953cd8b380538165d2f22eaf357cf71138"
|
| 481 |
+
},
|
| 482 |
+
"optimized_pipeline/speaker_run.py": {
|
| 483 |
+
"bytes": 9641,
|
| 484 |
+
"sha256": "8925320004c85b567571e1d9fd3f66c2a71f253b92fca9a2f4d4760c8ae081bf"
|
| 485 |
+
},
|
| 486 |
+
"optimized_pipeline/script-spontaneous-de.json": {
|
| 487 |
+
"bytes": 15092,
|
| 488 |
+
"sha256": "da7b4050b7aaedd4176f0304441495ec2a216816b4f5a1d160ceb33373defa88"
|
| 489 |
+
},
|
| 490 |
+
"optimized_pipeline/repair_listening.py": {
|
| 491 |
+
"bytes": 4036,
|
| 492 |
+
"sha256": "db2c050be4c429781d35d33648b48a61062d28d549b6801f24c4d9fd42d93272"
|
| 493 |
+
},
|
| 494 |
+
"optimized_pipeline/SPEAKER_RECIPE.md": {
|
| 495 |
+
"bytes": 21735,
|
| 496 |
+
"sha256": "302dd77bfc61098b5e352d24566a77e47f2d9eb4f11b16f296c78d9c832721b2"
|
| 497 |
+
},
|
| 498 |
+
"optimized_pipeline/recipe.py": {
|
| 499 |
+
"bytes": 4794,
|
| 500 |
+
"sha256": "7ed2aa0356b11cc37194c6044cf8dd73bc815348747a686b83a5753b1a86a371"
|
| 501 |
+
},
|
| 502 |
+
"guides/LEGACY_PIPELINE.md": {
|
| 503 |
+
"bytes": 8337,
|
| 504 |
+
"sha256": "140c75a23bafa5d015fdaf51429d9212b95b5cba256a493d581132690e68784a"
|
| 505 |
+
},
|
| 506 |
+
"guides/PROMPTING.md": {
|
| 507 |
+
"bytes": 8521,
|
| 508 |
+
"sha256": "a63ee77066c510f2eae8fc205abbd1c82f61c1fcafccf2e854285d6fd3bc1b9d"
|
| 509 |
+
},
|
| 510 |
+
"guides/LEGACY_README.md": {
|
| 511 |
+
"bytes": 15290,
|
| 512 |
+
"sha256": "bcd4c2388a9c8a5bcadbcb2c955ff3b97cea3172ba5cb2b1f89d00602d8da532"
|
| 513 |
+
},
|
| 514 |
+
"optimized_pipeline/runtime/APACHE-2.0.txt": {
|
| 515 |
+
"bytes": 11358,
|
| 516 |
+
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
|
| 517 |
+
},
|
| 518 |
+
"optimized_pipeline/runtime/published_audio.py": {
|
| 519 |
+
"bytes": 5013,
|
| 520 |
+
"sha256": "01155a8975bb9ad09ebedd60c33910370dd84599e5a1bdd0bcd83ae19bc15645"
|
| 521 |
+
},
|
| 522 |
+
"optimized_pipeline/runtime/semantic_graph.py": {
|
| 523 |
+
"bytes": 6212,
|
| 524 |
+
"sha256": "f5177bd119633417f9704e7967f7d8fe68296429c8e0f6f815350b9da769954d"
|
| 525 |
+
},
|
| 526 |
+
"optimized_pipeline/runtime/generate.py": {
|
| 527 |
+
"bytes": 1112,
|
| 528 |
+
"sha256": "f7fc075e83801574c6e491c6af5e021135882509c28c7965f06d91054c8d9a34"
|
| 529 |
+
},
|
| 530 |
+
"optimized_pipeline/runtime/audio_export.py": {
|
| 531 |
+
"bytes": 3518,
|
| 532 |
+
"sha256": "ee4b770a7fa0125de16800dacd95abea8670cf89844a679683ba852db3110f46"
|
| 533 |
+
},
|
| 534 |
+
"optimized_pipeline/runtime/local_fused.py": {
|
| 535 |
+
"bytes": 23036,
|
| 536 |
+
"sha256": "2a977365935eeffa80f711c61d66b37b7f25db6eef5e1028debc9b2a00652ee9"
|
| 537 |
+
},
|
| 538 |
+
"optimized_pipeline/runtime/codec_fused.py": {
|
| 539 |
+
"bytes": 15807,
|
| 540 |
+
"sha256": "e2deb62ea9344f6073107647ca114c8faef51904d2e6daaa3d1e27b6811f754f"
|
| 541 |
+
},
|
| 542 |
+
"optimized_pipeline/runtime/phased_generate.py": {
|
| 543 |
+
"bytes": 4005,
|
| 544 |
+
"sha256": "57b767cf20e4b7bfaf2e762f25db177677d78d829b6cdcb6c7dddf132dadead6"
|
| 545 |
+
},
|
| 546 |
+
"optimized_pipeline/runtime/performance_core.py": {
|
| 547 |
+
"bytes": 16734,
|
| 548 |
+
"sha256": "645e9d33d866f3113cd132e26c13a1538fa3f497cb550656bd6a3e9628f0d8a2"
|
| 549 |
+
},
|
| 550 |
+
"optimized_pipeline/runtime/PODCAST_UPSTREAM_NOTICE.md": {
|
| 551 |
+
"bytes": 1591,
|
| 552 |
+
"sha256": "256bc9697f7a856f8cf9a2380dc92c1d3845022d861de00c635ac66d56a8bc49"
|
| 553 |
+
},
|
| 554 |
+
"optimized_pipeline/runtime/semantic_layerwise.py": {
|
| 555 |
+
"bytes": 3687,
|
| 556 |
+
"sha256": "59675f59e87450e89e891c9fcc7046a8b128dfd17606f388e294820293e15dcf"
|
| 557 |
+
},
|
| 558 |
+
"optimized_pipeline/runtime/settings.py": {
|
| 559 |
+
"bytes": 273,
|
| 560 |
+
"sha256": "4c6dd861f36eb582ddef65780601a11c513eb4427c32aee461ce283057d9d629"
|
| 561 |
+
},
|
| 562 |
+
"optimized_pipeline/runtime/align_engine.py": {
|
| 563 |
+
"bytes": 24831,
|
| 564 |
+
"sha256": "64d49c130aa7d042fd8f43da1bb5a47a037b0deb408f01e9ae7d2e6ae573cec0"
|
| 565 |
+
},
|
| 566 |
+
"optimized_pipeline/runtime/config.py": {
|
| 567 |
+
"bytes": 180,
|
| 568 |
+
"sha256": "3dec1ab6944c40ee7c44cecd06e676b87620e37c0bd1ca7efdf6203d429d27b3"
|
| 569 |
+
},
|
| 570 |
+
"optimized_pipeline/runtime/PODCAST_UPSTREAM_LICENSE": {
|
| 571 |
+
"bytes": 432,
|
| 572 |
+
"sha256": "cc980e308d6c61586ea8d4707a3c57a5023f86e24c93ae21aa9dabc43881fe68"
|
| 573 |
+
},
|
| 574 |
+
"optimized_pipeline/runtime/ears_reward.py": {
|
| 575 |
+
"bytes": 12273,
|
| 576 |
+
"sha256": "1d607446d9f55f59e62440d98e46519c2489d502e57ee9e14ad8fb6988e47216"
|
| 577 |
+
},
|
| 578 |
+
"optimized_pipeline/references/source.json": {
|
| 579 |
+
"bytes": 141,
|
| 580 |
+
"sha256": "1fa9522adde72ebc5e24b578c946c5c8aa5506a1ea2f5f4c92c14fe6aa64b726"
|
| 581 |
+
},
|
| 582 |
+
"optimized_pipeline/references/PROVENANCE.json": {
|
| 583 |
+
"bytes": 939,
|
| 584 |
+
"sha256": "a5e08afd717070d35d388ad576a9ad188e05fc667f03231d441b3fce898690b0"
|
| 585 |
+
},
|
| 586 |
+
"optimized_pipeline/benchmarks/spontaneous-2026-11-08.json": {
|
| 587 |
+
"bytes": 4021,
|
| 588 |
+
"sha256": "dc7e85fe24ffbbcfe3d7d9d8d8768aab249beb343f154baf7261d33f2391448e"
|
| 589 |
+
},
|
| 590 |
+
"optimized_pipeline/low_memory_compare.py": {
|
| 591 |
+
"bytes": 11528,
|
| 592 |
+
"sha256": "a0d6c1626f1b5f0fd7e667426dd73c6cd3a02f40aeb1c42490b780473e90be33"
|
| 593 |
+
},
|
| 594 |
+
"optimized_pipeline/compare_report.py": {
|
| 595 |
+
"bytes": 7740,
|
| 596 |
+
"sha256": "abd684bfcee6192f357f9457ef8b74795d427204c854f1457cef54d4c453691d"
|
| 597 |
+
},
|
| 598 |
+
"guides/PODCAST_EXPERIMENT_HISTORY.md": {
|
| 599 |
+
"bytes": 8294,
|
| 600 |
+
"sha256": "841e5f4fcd9535beaaf9f5b4373b41dc7aeeb78fcc5e04d0263246e6174f37e7"
|
| 601 |
+
},
|
| 602 |
+
"optimized_pipeline/benchmarks/n-comparison-2026-11-08-turns.json": {
|
| 603 |
+
"bytes": 132495,
|
| 604 |
+
"sha256": "2b3968688232da708aa418f6ab6f2d4d0d1e0b541051f89a25e92f021fad05c4"
|
| 605 |
+
},
|
| 606 |
+
"optimized_pipeline/benchmarks/n-comparison-2026-11-08.json": {
|
| 607 |
+
"bytes": 6269,
|
| 608 |
+
"sha256": "282023c1143d0a6731048c53cbf86ca9a7d2e583fd3591ab4fa80a00f4203553"
|
| 609 |
+
},
|
| 610 |
+
"optimized_pipeline/finish_comparison.py": {
|
| 611 |
+
"bytes": 1399,
|
| 612 |
+
"sha256": "6d295692f56adfbf910858f5f94c7dc0c830c79ef314afdcb7b3a3e727b72616"
|
| 613 |
+
},
|
| 614 |
+
"optimized_pipeline/script-spontaneous-fr.json": {
|
| 615 |
+
"bytes": 15563,
|
| 616 |
+
"sha256": "ba11fb76309337e27e6dd861212047da712e73ef5a24b792db13a15bce9c42c6"
|
| 617 |
+
},
|
| 618 |
+
"optimized_pipeline/audit_alignment.py": {
|
| 619 |
+
"bytes": 3142,
|
| 620 |
+
"sha256": "a729e89d7f17d6d41865dda78086ea8518a957543aa2bfaa04eade1a53be379a"
|
| 621 |
+
},
|
| 622 |
+
"optimized_pipeline/script-spontaneous-es.json": {
|
| 623 |
+
"bytes": 15017,
|
| 624 |
+
"sha256": "e7c4d18468ca6787e6df7440e2b68627e50333b2a25011e01d3d365506879867"
|
| 625 |
+
},
|
| 626 |
+
"optimized_pipeline/languages.py": {
|
| 627 |
+
"bytes": 425,
|
| 628 |
+
"sha256": "f57c3263d9c1e52930f51ecc67cbf674c1a130db953434ea39e1c5ed46314664"
|
| 629 |
+
},
|
| 630 |
+
"optimized_pipeline/script-spontaneous-nl.json": {
|
| 631 |
+
"bytes": 14795,
|
| 632 |
+
"sha256": "85925be8fa1604d0bcfc05b61b5b8fb2ec1d2f6797b8675693285c5612517986"
|
| 633 |
+
},
|
| 634 |
+
"optimized_pipeline/benchmarks/five-language-extension-2026-11-08.json": {
|
| 635 |
+
"bytes": 14316,
|
| 636 |
+
"sha256": "7269325d5cfdce258171fc2636b259c496306aadf30371f653d6f9686c824ce6"
|
| 637 |
+
},
|
| 638 |
+
"optimized_pipeline/benchmarks/five-language-extension-2026-11-08-turns.json": {
|
| 639 |
+
"bytes": 203694,
|
| 640 |
+
"sha256": "6729585516e27882f7e8eaafa868eb19a152fd8ccef7b30a4fe2be91571f94df"
|
| 641 |
+
}
|
| 642 |
+
}
|
NOTICE.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Attribution and scope
|
| 2 |
+
|
| 3 |
+
LAION's Humanness Voice Podcast is a full supervised fine-tune of Humaneness Voice Large 1.7, using the MOSS Local Transformer family implementation and an external MOSS-v2 audio codec. Earlier family provenance and licenses remain in the inherited UPSTREAM files and source copyright headers. The current release's parent, data mixture and checkpoint are documented in TRAINING.md; older inherited notes describe earlier family releases and are not the current model card.
|
| 4 |
+
|
| 5 |
+
New LAION weights, pipeline code and documentation: CC BY 4.0. Third-party implementation files, Python packages, codecs, ASR models, score models and the optional Voice Converter retain their own licenses and notices. The optional converter is separately downloaded and installed; it is not silently embedded in this repository.
|
| 6 |
+
|
| 7 |
+
The German example anchors are short excerpts from recordings supplied by the experiment owner. The English anchors are the previously approved short real-podcast reference excerpts. MP3s and exact cached MOSS codes are included for reproducible conditioning; their provenance is in examples/references/PROVENANCE.json. Source recordings retain original terms. The model's CC BY 4.0 notice does not relicense source recordings or grant rights over someone's identity.
|
| 8 |
+
|
| 9 |
+
Generated example episodes are LAION model outputs, released under CC BY 4.0 with their source-condition records. They demonstrate the earlier script-boundary crop and 400-ms assembly; they are not evidence validating the newly added internal-pause compaction or optional per-turn VC route.
|
| 10 |
+
|
| 11 |
+
The 8 November optimized-pipeline update adds sparse spontaneous prompting, native-normalized genuineness plus ECAPA ranking, bounded archival and measured RTX 3090 results. It does not change model weights or relicense source recordings. The prior native pair workflow remains under infer_legacy.py. Third-party runtime notices, including Apache-2.0 codec optimizations, are retained in optimized_pipeline/runtime/.
|
PUBLICATION_ID.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"project": "humanness-voice-podcast-20261107",
|
| 3 |
+
"step": 9874
|
| 4 |
+
}
|
README.md
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-4.0
|
| 3 |
+
language: [en, de, fr, es, nl]
|
| 4 |
+
pipeline_tag: text-to-speech
|
| 5 |
+
library_name: transformers
|
| 6 |
+
base_model: laion/Humaneness-Voice-Large-1.7
|
| 7 |
+
tags: [podcast, multi-speaker, voice-cloning, expressive-speech, custom-code, research]
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
# Humanness Voice Podcast
|
| 11 |
+
|
| 12 |
+
**Generate natural, reference-conditioned speech and assemble it into a podcast.** This is the complete Podcast full fine-tune of [Humaneness Voice Large 1.7](https://huggingface.co/laion/Humaneness-Voice-Large-1.7), at training update **9,874**. It contains full BF16 weights, not a LoRA. The model speaks a supplied script; an optional API-based or local language model can write the script and its performance directions.
|
| 13 |
+
|
| 14 |
+
## 🎧 Listen: five languages, two reference voices
|
| 15 |
+
|
| 16 |
+
**[Open the public Podcast listening Space →](https://huggingface.co/spaces/laion/Humanness-Voice-Podcast)**
|
| 17 |
+
|
| 18 |
+
Reference audios, an exact prompt example, and **Best-of-4 → Best-of-2 → Single Take** episodes in **German, English, Dutch, French and Spanish**. The same conversation and reference pair make the variants easy to compare; scripts, measured generation times and review notes are included. No installation or paid API is required to listen.
|
| 19 |
+
|
| 20 |
+
The full fine-tune uses real podcasts from these five languages — approximately **300 source hours per language** — with a training mixture of **80% podcast and 20% synthetic voice-acting presentations**. The source-hour estimate is distinct from the packed target-token hours reported in [training details and provenance](TRAINING.md).
|
| 21 |
+
|
| 22 |
+
The current recommended pipeline uses **sparse spontaneous, relaxed and gentle inline directions**, fixed reference voices, and complete-turn Best-of-N. The listener approved this style on 8 November 2026. N=4 is the default; N=2 and N=1 offer cheaper comparisons. Voice conversion, DPO adapters and internal pause compaction are off.
|
| 23 |
+
|
| 24 |
+
**Start here:** [Prompting templates and LLM-writing guide](guides/PROMPTING.md) · [Single-speaker guide](guides/SINGLE_SPEAKER.md) · [Multi-speaker guide](guides/MULTI_SPEAKER.md) · [Pipeline, rewards and speed details](guides/PIPELINE.md) · [Agent quick reference](guides/AGENT_GUIDE.md) · [Machine-readable defaults](guides/recipe.json) · [Runnable optimized code](optimized_pipeline/)
|
| 25 |
+
|
| 26 |
+
[Earlier listening study](https://laion-humaneness-podcast-segment-block-pilot.static.hf.space/editing/index.html) · [Training and provenance](TRAINING.md) · [Validation status](VALIDATION.md) · [GitHub implementation](https://github.com/LAION-AI/Humaneness-Voice/tree/main/podcast_pipeline)
|
| 27 |
+
|
| 28 |
+
## Model and inference at a glance
|
| 29 |
+
|
| 30 |
+
The MOSS Local Transformer architecture combines a Qwen-family semantic backbone with a local acoustic transformer. It generates twelve acoustic codebook tokens per 80-ms frame. The separately downloaded MOSS-v2 codec renders 48-kHz mono audio. The full Podcast policy has approximately 4.13 billion parameters / 8.26 GB of BF16 weights; codec, judges, KV caches and CUDA graphs need additional memory.
|
| 31 |
+
|
| 32 |
+
Supply literal speech, a short delivery direction, and one clean fixed reference per voice. The recommended pipeline generates each speaker turn independently, optionally samples several candidates in parallel, selects a winner, audits its words, and assembles lossless turns before exporting MP3, subtitles, timeline JSON and a listening page.
|
| 33 |
+
|
| 34 |
+
```mermaid
|
| 35 |
+
flowchart LR
|
| 36 |
+
A[Human or LLM writes script and sparse cues] --> B[Fixed reference per speaker]
|
| 37 |
+
B --> C[Podcast policy: N candidates per turn]
|
| 38 |
+
C --> D[Native MOSS-v2 audio decoder]
|
| 39 |
+
D --> E[For N above 1: genuineness + ECAPA selection]
|
| 40 |
+
E --> F[Independent transcript and boundary audit]
|
| 41 |
+
F --> G[400 ms word-clock assembly]
|
| 42 |
+
G --> H[MP3 + HTML + JSON + subtitles]
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
N=1 keeps its first and only take and skips online ranking. Each generation uses only its intended speaker's reference. Several different turns can share a GPU batch without sharing their generated histories.
|
| 46 |
+
|
| 47 |
+
The optimized CLI supports **English, German, Dutch, French and Spanish**. Bundled scripts translate the same twenty-turn conversation; the public Space compares N=4, N=2 and N=1 for each language using the same approved German reference pair. This small fixed-script comparison is not a broad multilingual quality benchmark. Automated content/alignment flags remain visible. The English N4 complete listening edition has one explicitly documented candidate replacement, with the strict original still available.
|
| 48 |
+
|
| 49 |
+
## Quick start: recommended pipeline
|
| 50 |
+
|
| 51 |
+
Use a CUDA/BF16-capable GPU; the measured setup is an **RTX 3090 with 24 GB**, Python 3.12, PyTorch 2.6.0+cu124 and Transformers 5.15.0. Install system `ffmpeg` and `libsndfile`, then use a separate environment. Keep enough disk space for all model assets and candidates. No LLM API key is needed.
|
| 52 |
+
|
| 53 |
+
```bash
|
| 54 |
+
hf download laion/Humanness-Voice-Podcast --local-dir humanness-podcast
|
| 55 |
+
cd humanness-podcast
|
| 56 |
+
python3.12 -m venv .venv
|
| 57 |
+
source .venv/bin/activate
|
| 58 |
+
pip install torch==2.6.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu124
|
| 59 |
+
pip install -r requirements.txt
|
| 60 |
+
|
| 61 |
+
# Fetch the pinned reference/codec/judge assets and record their local paths.
|
| 62 |
+
python optimized_pipeline/prepare.py
|
| 63 |
+
|
| 64 |
+
# One English episode, approved sparse prompting, four takes per speaker turn.
|
| 65 |
+
python infer.py --model . \
|
| 66 |
+
--script optimized_pipeline/script-spontaneous-en.json \
|
| 67 |
+
--n 4 --groups 2 --device cuda:0 --output runs/en-best4
|
| 68 |
+
|
| 69 |
+
# Same German script, two takes per turn; N=1 is a true single take.
|
| 70 |
+
python infer.py --model . \
|
| 71 |
+
--script optimized_pipeline/script-spontaneous-de.json \
|
| 72 |
+
--n 2 --groups 4 --device cuda:0 --output runs/de-best2
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
Open `runs/en-best4/index.html`. Without `--script`, the bundled English **and** German episodes are generated. Use `--n 1 --groups 8` for single takes, or omit `--groups` to run the small automatic batch-size pilot. Groups × N must not exceed 16; reduce groups if VRAM is insufficient. The example group settings keep eight simultaneous candidates.
|
| 76 |
+
|
| 77 |
+
For Dutch, French or Spanish, pass `optimized_pipeline/script-spontaneous-nl.json`, `script-spontaneous-fr.json` or `script-spontaneous-es.json` respectively. Repeat `--script` to generate several languages. The [five-language reproduction commands](guides/PIPELINE.md#reproduce-the-additional-dutch-french-and-spanish-episodes) also show the phased token/codec runner and shared final audit for N=4/2/1 on a GPU with limited free VRAM.
|
| 78 |
+
|
| 79 |
+
`infer.py` now runs the optimized generation → audit → export route. The former native single/pair driver remains available explicitly as **`infer_legacy.py`**, with its own [legacy guide](guides/LEGACY_PIPELINE.md) and `requirements-legacy.txt` environment. Its identity×inverse-WER reward and Photon ASR differ from the current defaults; the two workflows are deliberately labeled. No old model weights, training code or examples were removed.
|
| 80 |
+
|
| 81 |
+
## Single-speaker and multi-speaker prompting
|
| 82 |
+
|
| 83 |
+
Use `GENERAL` for consistent delivery and parenthesized directions for local expression. Keep the literal text separate from acting cues. A single-turn generation gets only that turn's reference. For a multi-speaker episode, keep a stable speaker-to-reference mapping and alternate independently generated turns; see the [full JSON contract](guides/PROMPTING.md).
|
| 84 |
+
|
| 85 |
+
```text
|
| 86 |
+
GENERAL:
|
| 87 |
+
A warm, emotionally thoughtful two-host podcast conversation. Very authentic, very genuine.
|
| 88 |
+
Natural, intimate, spontaneous spoken delivery with clear words, subtle emotion,
|
| 89 |
+
relaxed conversational pacing and brief meaningful pauses.
|
| 90 |
+
Be sincere, without an exaggerated announcer voice or forced laughter.
|
| 91 |
+
Perform only the current speaker turn and speak exactly the SCRIPT.
|
| 92 |
+
Keep the same speaker identity as the reference audio.
|
| 93 |
+
|
| 94 |
+
SCRIPT:
|
| 95 |
+
(warm, curious; relaxed, spontaneous, gently spoken) What if meaning is something we practise?
|
| 96 |
+
(gently, as a spontaneous thought) Maybe it begins with noticing who needs us today.
|
| 97 |
+
```
|
| 98 |
+
|
| 99 |
+
Do not include the parentheses in the literal `text` passed to the processor. Do not add speaker labels to the words to be spoken. No “very pleasant/nice to listen to” instruction is used in this approved recipe. Use **one short cue per turn and at most one extra cue at an existing sentence boundary**; do not repeatedly demand strong emotion before every sentence.
|
| 100 |
+
|
| 101 |
+
The preserved native two-speaker path is different: `infer_legacy.py --mode pairs` uses `(speaker=S1, ...)` and `(speaker=S2, ...)` plus two labeled reference slots in a single call. Its wrapper, prior-context limits and caveats are documented in the [multi-speaker guide](guides/MULTI_SPEAKER.md). Do not mix that wrapper with the optimized one-speaker-per-call route.
|
| 102 |
+
|
| 103 |
+
## Have a language model write the directions
|
| 104 |
+
|
| 105 |
+
We recommend using an **API-based or local LLM** to write the dialogue and its sparse cues. Manually formatting every turn and direction is unnecessarily time-consuming. The [prompting guide](guides/PROMPTING.md#use-an-llm-as-the-script-and-direction-writer) includes a reusable writing instruction, JSON templates and validation rules.
|
| 106 |
+
|
| 107 |
+
Tell the writing model the topic, audience, speaker roles and intended length. Ask for natural conversation, subtle contextual emotion, exact preservation of any supplied transcript, and no more than two delivery chunks per turn. Keep each turn comfortably below 25.6 seconds; split longer explanations. The TTS pipeline does not call an LLM itself or bind you to a particular provider.
|
| 108 |
+
|
| 109 |
+
For custom scripts use `--script my-script.json`; for your own voices use `--references my-references.json`. Reference paths are relative to the mapping JSON. Keep clean single-speaker anchors of roughly **5–15 seconds**. The approved S1/S2 clips supply 181/163 codec frames (14.48/13.04 s of conditioning), with no old three-second truncation. The [reference provenance](examples/references/PROVENANCE.json) and original source terms remain applicable. Remove or update cached reference codes when replacing a recording.
|
| 110 |
+
|
| 111 |
+
## Model assets and reproducibility
|
| 112 |
+
|
| 113 |
+
| Component | Role | Source |
|
| 114 |
+
|---|---|---|
|
| 115 |
+
| Humanness Voice Podcast, full FT step 9874 | Speech policy | [This repository](https://huggingface.co/laion/Humanness-Voice-Podcast), weight revision `91f8110cd815f2233301a8b1dd16dbbf3fae8e05` |
|
| 116 |
+
| MOSS Audio Tokenizer v2 | Reference encoding and waveform decoding | [OpenMOSS-Team/MOSS-Audio-Tokenizer-v2](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2) |
|
| 117 |
+
| Humaneness Ears Medium | Genuineness, additional diagnostic scores | [laion/humaneness-ears-base-medium](https://huggingface.co/laion/humaneness-ears-base-medium) |
|
| 118 |
+
| SpeechBrain ECAPA, 192D | Similarity to the intended reference | [speechbrain/spkrec-ecapa-voxceleb](https://huggingface.co/speechbrain/spkrec-ecapa-voxceleb) |
|
| 119 |
+
| Parakeet TDT 0.6B v3 | Final transcript diagnostic | [nvidia/parakeet-tdt-0.6b-v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) |
|
| 120 |
+
| Qwen3 ForcedAligner | Approximate script-word boundaries for EN/DE/FR/ES; Dutch uses conservative Parakeet TDT estimates | [Qwen/Qwen3-ForcedAligner-0.6B-hf](https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B-hf) |
|
| 121 |
+
|
| 122 |
+
All asset revisions and commands are in [the pipeline guide](guides/PIPELINE.md). Use fixed model snapshots, script/reference hashes and separate output folders for different recipes. Preserve batch grouping and seeds. Fixed seeds do not imply identical audio across different batch sizes, hardware, precision or software versions. Model weight hashes remain in `EXPORT_PROOF.json`; no model weights changed in this pipeline update.
|
| 123 |
+
|
| 124 |
+
## Detailed defaults and selection
|
| 125 |
+
|
| 126 |
+
| Setting | Current value |
|
| 127 |
+
|---|---|
|
| 128 |
+
| Prompt style | `spontaneous`: restrained, relaxed, gentle, sparse inline cues |
|
| 129 |
+
| Precision | BF16 |
|
| 130 |
+
| Temperature / CFG | 1.0 / 2.5 |
|
| 131 |
+
| Top-p / top-k | 0.95 / 50 |
|
| 132 |
+
| Timing | Free timing: `tokens=None`, no inline duration request |
|
| 133 |
+
| Maximum per turn | 320 semantic frames = 25.6 seconds |
|
| 134 |
+
| Default selection | Best-of-4 complete raw turns |
|
| 135 |
+
| Voice conversion / added adapters | Off / none |
|
| 136 |
+
| Internal pause compaction | Off |
|
| 137 |
+
|
| 138 |
+
For N>1:
|
| 139 |
+
|
| 140 |
+
```text
|
| 141 |
+
G = clip(native Humaneness Ears genuineness / 6, 0, 1)
|
| 142 |
+
S = clip(cosine(ECAPA(candidate), ECAPA(correct reference)), 0, 1)
|
| 143 |
+
reward = 0.5 * G + 0.5 * S
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
Use fixed scales, not per-batch min/max. Compute the correct reference embedding once and reuse it. Select the maximum separately for each turn; exact ties choose the lowest candidate index. **Content Enjoyment and WER do not enter this reward.** N=1 keeps its sole take and does not run online ranking.
|
| 147 |
+
|
| 148 |
+
Every selected output gets an independent final transcript/alignment audit. The boundary edit preserves up to 80 ms before the first word and 150 ms after the last, with small fades outside those words. Uncertain alignment remains untrimmed and flagged. Reassembly uses a **400-ms word-to-word gap**, including the retained guards, then exports mono 48-kHz / 192-kbps MP3 with a lossless master and audit records. Internal timing is preserved.
|
| 149 |
+
|
| 150 |
+
A high genuineness/identity score can still select an incomplete sentence. In the approved Best-of-4 sample set, English turn 6's strict winner stopped after its first sentence. The experiment preserves that strict output. An additional full listening copy replaces only that turn with an existing complete alternative, explicitly adding a content eligibility check. It must not be described as unchanged strict reward argmax. See [the exact failure and repair record](optimized_pipeline/benchmarks/spontaneous-2026-11-08.json).
|
| 151 |
+
|
| 152 |
+
## Speed on RTX 3090 and practical optimizations
|
| 153 |
+
|
| 154 |
+
These are measured **single-GPU** results for the current sparse-prompt Best-of-4 run. The English row describes the strict experiment, including the flagged incomplete turn; its separately repaired listening copy lasts 201.22 s.
|
| 155 |
+
|
| 156 |
+
| Language | Final audio | Generation + ranking | Useful RTF | Genuineness / 6 | ECAPA cosine | Mean turn WER |
|
| 157 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 158 |
+
| German, N=4 | 219.38 s | 179.98 s | 0.821 | 3.943 | 0.640 | 1.98% |
|
| 159 |
+
| English, N=4, strict | 193.30 s | 195.83 s | 1.013 | 3.880 | 0.407 | 5.50% |
|
| 160 |
+
|
| 161 |
+
RTF = generation/ranking time divided by final edited audio duration; lower is faster. It excludes model loading, correctness checks, final audit and MP3 encoding and is not browser first-audio latency. Both languages together required 380.10 s of production including archival/cleanup, or 645.59 s from process start to exports with an existing compiler cache. The extra listening repair and live-service restoration are excluded.
|
| 162 |
+
|
| 163 |
+
Implemented optimizations:
|
| 164 |
+
|
| 165 |
+
- BF16 policy, layerwise semantic CUDA graphs, reusable local-decoder graphs, fast sampling and fused codec kernels. Real-model tests verify the checked codec/token/PCM paths.
|
| 166 |
+
- Batch different turns together while preserving independent candidate histories. Two turns × four candidates produced eight simultaneous takes in this run.
|
| 167 |
+
- Reuse identical candidate-prompt prefill only after exact valid-KV/hidden-state checks pass. A two-group check measured 2.009 s normally versus 0.518 s with reuse.
|
| 168 |
+
- Keep the unused reference codec encoder in CPU RAM after reference codes are ready; keep the decoder on the GPU.
|
| 169 |
+
- Cache reference embeddings, batch candidate judges, and reserve ASR for the final audit.
|
| 170 |
+
- Use one bounded CPU writer alongside the next GPU batch. Archival consumed 24.87 CPU seconds but blocked production for only 1.08 s; all takes remain available for later reranking.
|
| 171 |
+
- Reuse a known group size to skip repeated timing sweeps, and reuse a warmed model/compilation cache when generating several episodes. `optimized_pipeline/compare_n.py` shares one model load across N=2 and N=1.
|
| 172 |
+
|
| 173 |
+
Policy token prediction remains the main cost: 324.85 s across both episodes, versus 24.85 s codec decoding and 21.25 s scoring including preprocessing. These stages total 375.80 s with remaining generation overhead. Of 160 takes, 37 reached the 25.6-s ceiling; 19 of 20 batches waited for at least one capped take. Safely recognizing complete lagging takes, grouping similar lengths or compacting finished rows are future opportunities. Simply shortening the ceiling risks losing actual speech.
|
| 174 |
+
|
| 175 |
+
Timing depends on prompts, batch grouping, graph bucket settings, cache warmth and other GPU workloads. Overall production time was about the same as the preceding run; the archival overlap is measured, but this is not evidence of a controlled aggregate speedup. Fresh N=2/N=1 results are documented below. Full receipts and limitations: [pipeline guide](guides/PIPELINE.md), [benchmark JSON](optimized_pipeline/benchmarks/spontaneous-2026-11-08.json).
|
| 176 |
+
|
| 177 |
+
## Training, earlier evidence and limitations
|
| 178 |
+
|
| 179 |
+
Training used two epochs, approximately 80% real-podcast / 20% original-format acting replay by presentation count, 32 GPUs and 9,874 optimizer updates. Peak learning rates were 2e-5 for the semantic backbone and 4e-5 for the local/acoustic parameters; 5% warm-up followed by cosine decay to 10% of peak. Development loss was 4.4653 versus 4.5745 before training. This is not a human quality rating. See [TRAINING.md](TRAINING.md) for counts, manifests and continued-training instructions. The public weights are deployment weights, not the original 32-rank optimizer resume state.
|
| 180 |
+
|
| 181 |
+
The earlier segment/native-block experiment and its original metrics remain accessible in the [archived model card](guides/LEGACY_README.md), [study summary](stats/segment_block_pilot.json) and [listening grid](https://laion-humaneness-podcast-segment-block-pilot.static.hf.space/index.html). Those Ears-identity/Parakeet-Redux measurements must not be compared as identical metrics to the new ECAPA/Parakeet-v3 pipeline. Legacy optional VC and internal-pause compaction retain their original experimental status.
|
| 182 |
+
|
| 183 |
+
Speaker identities can drift; words can be omitted or repeated; ASR, alignment and quality judges can be wrong. The listener-approved prompt is a useful recipe, not a universal guarantee. Inspect flags and listen before publishing an episode. Reference recordings, third-party implementations and external models retain their own licenses. LAION model weights and newly authored release materials are **CC BY 4.0**; see [NOTICE.md](NOTICE.md). Use voices with appropriate permission and disclose synthetic speech where appropriate.
|
| 184 |
+
|
| 185 |
+
## Fresh N=1 and N=2 comparison with the approved prompts
|
| 186 |
+
|
| 187 |
+
Both episodes were freshly generated at N=2 and N=1 after publishing the new defaults. Same literal scripts, cues, reference pair, temperature 1.0 and CFG 2.5. N1 keeps its sole take and skips online ranking. N2 uses the same 50/50 reward as N4; there is no WER gate or hidden retry.
|
| 188 |
+
|
| 189 |
+
| Language | N | Final audio | Generate + rank | Useful RTF ↓ | Genuineness / 6 ↑ | ECAPA cosine ↑ | Mean turn WER ↓ |
|
| 190 |
+
|---|---:|---:|---:|---:|---:|---:|---:|
|
| 191 |
+
| DE | 1 | 220.75 s | 55.24 s | 0.250 | 3.916 | 0.553 | 1.21% |
|
| 192 |
+
| DE | 2 | 222.26 s | 115.80 s | 0.521 | 3.897 | 0.630 | 2.56% |
|
| 193 |
+
| DE | 4 | 219.38 s | 179.98 s | 0.821 | 3.943 | 0.640 | 1.98% |
|
| 194 |
+
| EN | 1 | 199.30 s | 68.56 s | 0.344 | 3.863 | 0.352 | 2.12% |
|
| 195 |
+
| EN | 2 | 199.54 s | 122.77 s | 0.615 | 3.904 | 0.383 | 2.93% |
|
| 196 |
+
| EN | 4 | 193.30 s | 195.83 s | 1.013 | 3.880 | 0.407 | 5.50% |
|
| 197 |
+
|
| 198 |
+
In this sample, **N=2 is a useful speed/identity compromise**: German speaker cosine rises from 0.553 at N1 to 0.630 at N2, close to N4's 0.640. English rises from 0.352 to 0.383; N4 is 0.407. Genuineness stays roughly 3.9/6. N1 is faster but less consistent in speaker identity. These are twenty turns per language, not evidence of a universal optimum. The old English N4 row includes its documented incomplete turn; the repaired listening copy is separate.
|
| 199 |
+
|
| 200 |
+
All 80 new selected turns passed the configured review checks (per-turn WER threshold 15% and usable forced alignment); none was regenerated or replaced. This is not a guarantee of perfect speech. Both candidate counts passed artifact validation, exact prompt/reference routing, strict selection and PCM equality checks. The token/full-prefill checks and optional fused decoding were bit-identical on their tested inputs. Fourteen CPU tests pass.
|
| 201 |
+
|
| 202 |
+
**Timing scope:** N2 production across both languages totals 238.57 s; N1 totals 123.80 s; old N4 totals 375.80 s. Token prediction accounts for 209.57 / 113.22 / 324.85 s respectively. N2 codec/ranking take 18.18/10.07 s; N1 codec takes 9.91 s and online ranking zero. Loading, correctness probes, final audit and export are excluded from these production sums. The complete successful N2+N1 experiment, from process start to all four MP3s, took **660.70 s**. The failed earlier OOM attempt and live-service restoration are excluded.
|
| 203 |
+
|
| 204 |
+
N2/N1 used phased execution under additional unrelated GPU training; N4 used integrated execution with a different background load. The other training ended before final auditing. Do not claim these are controlled percentage speedups from N alone. N2 uses four turns per batch and N1 eight (final partial batches smaller), maintaining up to eight simultaneous candidates. Different groups/seeds mean these are fresh runs, not nested subsets of the N4 pool.
|
| 205 |
+
|
| 206 |
+
See [aggregate results](optimized_pipeline/benchmarks/n-comparison-2026-11-08.json) and [all per-turn candidate scores, seeds and audits](optimized_pipeline/benchmarks/n-comparison-2026-11-08-turns.json). Recreate the listening HTML with `compare_report.py`; the six strict episodes and additional labeled N4 English listening copy are independently selectable.
|
| 207 |
+
|
| 208 |
+
<!-- FIVE_LANGUAGE_RESULTS -->
|
| 209 |
+
## Five-language listening release: additional NL / FR / ES results
|
| 210 |
+
|
| 211 |
+
[Open the public listening Space](https://huggingface.co/spaces/laion/Humanness-Voice-Podcast). The nine additional episodes use the same fixed reference pair, approved sparse prompting and translated twenty-turn story as the DE/EN examples. All N2/N4 reward winners and all N1 sole takes are retained; no retries, content re-selection or voice conversion.
|
| 212 |
+
|
| 213 |
+
| Language | N | Final audio | Production | Useful RTF | Genuineness / 6 | ECAPA cosine | Mean turn WER | Transcript / boundary flags, of 20 |
|
| 214 |
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 215 |
+
| NL | 4 | 303.00 s | 261.95 s | 0.865 | 3.969 | 0.388 | 14.4% | 8 / 10 |
|
| 216 |
+
| NL | 2 | 298.90 s | 134.25 s | 0.449 | 3.884 | 0.352 | 13.5% | 5 / 8 |
|
| 217 |
+
| NL | 1 | 280.82 s | 73.28 s | 0.261 | 3.897 | 0.333 | 15.1% | 6 / 9 |
|
| 218 |
+
| FR | 4 | 238.82 s | 242.20 s | 1.014 | 4.021 | 0.420 | 21.9% | 8 / 4 |
|
| 219 |
+
| FR | 2 | 231.62 s | 131.62 s | 0.568 | 4.032 | 0.385 | 14.2% | 6 / 3 |
|
| 220 |
+
| FR | 1 | 238.10 s | 65.74 s | 0.276 | 3.906 | 0.362 | 14.5% | 5 / 4 |
|
| 221 |
+
| ES | 4 | 234.02 s | 250.40 s | 1.070 | 4.008 | 0.450 | 2.2% | 0 / 0 |
|
| 222 |
+
| ES | 2 | 233.78 s | 134.51 s | 0.575 | 4.018 | 0.385 | 2.6% | 0 / 0 |
|
| 223 |
+
| ES | 1 | 232.66 s | 57.90 s | 0.249 | 4.020 | 0.392 | 2.1% | 1 / 0 |
|
| 224 |
+
|
| 225 |
+
Flags are materially more frequent in Dutch and French than Spanish in this fixed-reference run. Some recognized transcripts omit words or drift across languages. ASR itself is fallible, and a boundary warning alone does not imply missing speech. Dutch uses conservative Parakeet TDT boundaries because the Qwen aligner does not support Dutch. Uncertain clips remain untrimmed, so some episodes include longer tails. These examples are not endorsed as error-free multilingual outputs. Scripts and detailed diagnostics are available next to every player.
|
| 226 |
+
|
| 227 |
+
Production is the sum of policy, codec and online selection time, excluding loading, correctness probes, final audit and encoding. Phased inference used one RTX 3090 alongside unrelated jobs; cross-run GPU loads differ. N4/N2/N1 use separate draws and grouping, not nested subsets. Do not interpret these measurements as a controlled speed or quality ablation.
|
| 228 |
+
|
| 229 |
+
[Aggregate measurements](optimized_pipeline/benchmarks/five-language-extension-2026-11-08.json) · [All candidate scores and selected-turn audits](optimized_pipeline/benchmarks/five-language-extension-2026-11-08-turns.json).
|
TRAINING.md
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training and checkpoint provenance
|
| 2 |
+
|
| 3 |
+
## Released checkpoint
|
| 4 |
+
|
| 5 |
+
- Parent: [Humaneness Voice Large 1.7](https://huggingface.co/laion/Humaneness-Voice-Large-1.7), the merged supervised acting release selected at update 44,250 in its earlier run.
|
| 6 |
+
- This release: a subsequent **full supervised fine-tune**, final update **9,874**. It is the real-podcast checkpoint used in the linked listening experiment, not the earlier edition and not an additional LoRA.
|
| 7 |
+
- Architecture: MOSS Local Transformer family; semantic backbone plus local acoustic transformer, twelve MOSS-v2 channels per 80-ms frame, 48-kHz mono output.
|
| 8 |
+
- Distribution: 32 GPUs, effective global batch 32, accumulation 1, FSDP FULL_SHARD, FP32 optimizer parameters/reductions and BF16 compute.
|
| 9 |
+
- Optimizer: AdamW, betas (.9,.95), epsilon 1e-8, weight decay .01, gradient norm limit 1.
|
| 10 |
+
- Schedule: two epochs; semantic peak 2e-5, local/acoustic peak 4e-5, 5% linear warm-up, cosine decay to 10% of each peak. No learning-rate restart.
|
| 11 |
+
- Actual nonpadding presentations: 315,932, with 36 distributed-padding presentations across the complete run.
|
| 12 |
+
- Per epoch: 126,373 podcast training targets and 31,593 acting-replay targets; approximately 80/20 by presentations.
|
| 13 |
+
- Full checkpoints included optimizer, per-rank RNG, scheduler and sampler state. The historical checkpoint states remain at their original storage; this public model package contains deployment weights and training code.
|
| 14 |
+
- Final development loss: 4.4653007 on 100 balanced audit cases, versus 4.5745 at initialization. The final update had the lowest recorded development loss. This does not establish the best human listening score.
|
| 15 |
+
|
| 16 |
+
[Exact numerical training summary](stats/training_summary.json), [development history](stats/validation.jsonl), [logged optimizer-step measurements](stats/steps.jsonl), and [BF16 export proof](EXPORT_PROOF.json) are included. Every exported tensor was checked against the source full checkpoint cast to BF16, matching the precision used in the listening pilot.
|
| 17 |
+
|
| 18 |
+
## Data and prompting
|
| 19 |
+
|
| 20 |
+
The prepared corpus uses real podcast speech in en/de/fr/es/nl, plus an original-format expressive-acting replay subset. Targets, source splits, literal scripts, word timestamps and speaker labels were retained. The replay subset was language/speaker-balanced and preserved the earlier acting training membership; held-out acting items were excluded from replay.
|
| 21 |
+
|
| 22 |
+
Podcast examples contain two clean episode-local S1/S2 anchors and, where available, a continuously encoded earlier-conversation block. Prefer one to three previous turns per speaker, allowing unequal histories and explicitly documented anchors-only or single-speaker-history cases. Combined anchor+history budget: 750 frames / 60 seconds. Each anchor remains 63–187 frames. Only the final assistant target audio is supervised; references and past dialogue must never enter the target labels.
|
| 23 |
+
|
| 24 |
+
The frozen mixture and actual token-hour breakdown are in stats/training_summary.json. Token hours are sums of selected codec-frame durations; they should not be treated as a count of unique original podcast episode hours.
|
| 25 |
+
|
| 26 |
+
The prepared source corpus was published to the access-controlled [TTS-AGI/Humaneness-Voice-Real-Podcasts-HQ-Multilingual](https://huggingface.co/datasets/TTS-AGI/Humaneness-Voice-Real-Podcasts-HQ-Multilingual), including the continuous-context sidecar in the preparation chain. Access to those training data is separate from downloading this public model. Original-format acting material is described by the parent model's provenance and source preparation. This model repository does not make a private training corpus public.
|
| 27 |
+
|
| 28 |
+
## Original code versus portable continued training
|
| 29 |
+
|
| 30 |
+
[The frozen historical launch-source archive](training/historical/frozen_fullft_source.tar.gz) records the actual Jupiter trainer, data/packing contracts and immediate parent dependencies. It retains original paths for audit. It is not a claim that a 32-rank Jupiter job can be copied verbatim onto another machine with different filesystem paths.
|
| 31 |
+
|
| 32 |
+
The portable entry point training/train.py implements the same MOSS hidden-state supervised loss, twelve-book target masking, semantic/local learning-rate groups and warm-up/cosine recipe. It starts a **new fine-tune** from a downloaded model. Use a canonical local JSONL plus NPZ token arrays, optionally packed inside TARs. This loader avoids the historical machine's SQLite paths and supports your own authorized data.
|
| 33 |
+
|
| 34 |
+
### Prepare token data
|
| 35 |
+
|
| 36 |
+
A source JSONL record can be:
|
| 37 |
+
|
| 38 |
+
~~~json
|
| 39 |
+
{
|
| 40 |
+
"language": "en",
|
| 41 |
+
"mode": "pairs",
|
| 42 |
+
"general": "A warm, genuine, realistic conversation.",
|
| 43 |
+
"timed": true,
|
| 44 |
+
"references": {"S1": "speaker1.wav", "S2": "speaker2.wav"},
|
| 45 |
+
"target_audio": "dialogue.wav",
|
| 46 |
+
"turns": [
|
| 47 |
+
{"speaker": "S1", "text": "I was hoping you would ask.", "direction": "warm, curious", "seconds": 2.1},
|
| 48 |
+
{"speaker": "S2", "text": "Then tell me what surprised you.", "direction": "amused, attentive", "seconds": 2.4}
|
| 49 |
+
]
|
| 50 |
+
}
|
| 51 |
+
~~~
|
| 52 |
+
|
| 53 |
+
Optional context_audio is a single earlier-conversation waveform, already assembled chronologically; context_description must describe that exact waveform. Do not concatenate independent context code arrays or use future target audio as history. Optional instruction can supply the authoritative original GENERAL/SCRIPT prompt, preserving detailed acting directions rather than replacing them with automatic captions.
|
| 54 |
+
|
| 55 |
+
~~~bash
|
| 56 |
+
python training/tokenize_manifest.py \
|
| 57 |
+
--model . --input /data/authorized-records.jsonl \
|
| 58 |
+
--output /data/tokenized-podcasts --device cuda:0
|
| 59 |
+
~~~
|
| 60 |
+
|
| 61 |
+
The tokenizer writes TAR containers with compressed NPZ records and a train.jsonl. NPZ keys are target, S1, S2 and optional context, each an integer [frames,12] array. A native single-speaker record uses mode turns, one turn and the matching speaker reference; no-reference acting examples may omit reference arrays. Keep train/development splits separate before tokenization.
|
| 62 |
+
|
| 63 |
+
### Start continued full SFT on eight GPUs
|
| 64 |
+
|
| 65 |
+
~~~bash
|
| 66 |
+
torchrun --standalone --nproc_per_node=8 training/train.py \
|
| 67 |
+
--model . --data /data/tokenized-podcasts/train.jsonl \
|
| 68 |
+
--output /runs/podcast-sft --epochs 2 \
|
| 69 |
+
--semantic-lr 2e-5 --local-lr 4e-5 --warmup 0.05 \
|
| 70 |
+
--save-every 200
|
| 71 |
+
~~~
|
| 72 |
+
|
| 73 |
+
The portable trainer shuffles deterministically per epoch. It does not silently rebalance your input JSONL; construct the desired podcast/replay mixture explicitly. Default accumulation is 1; an eight-GPU run therefore has a smaller effective batch than the original 32-GPU training. Match the global batch if making a controlled scientific comparison.
|
| 74 |
+
|
| 75 |
+
### Continue an interrupted portable run
|
| 76 |
+
|
| 77 |
+
Use the same arguments and the same world/data/recipe, adding:
|
| 78 |
+
|
| 79 |
+
~~~bash
|
| 80 |
+
--resume /runs/podcast-sft/checkpoints/step-00000200
|
| 81 |
+
~~~
|
| 82 |
+
|
| 83 |
+
A READY.json is written only after the Accelerate checkpoint is complete. Model/optimizer/RNG states are saved by Accelerate; the ready metadata binds the data manifest, world and schedule, plus the deterministic next epoch/local batch. Keep token archives immutable. A different dataset, schedule or world is a new experiment, not an exact optimizer resume.
|
| 84 |
+
|
| 85 |
+
The released BF16 deployment weights cannot reconstruct the original FP32 optimizer moments or the original 32 per-rank RNG states. Do not describe a new run initialized from this repository as that historical exact resume.
|
| 86 |
+
|
| 87 |
+
## Loss details
|
| 88 |
+
|
| 89 |
+
Target supervision uses the native binary continue/stop head and all twelve codec channels. Channel weights are 1 for the stop head and 32/12 per audio channel, normalized by the total active weights. Loss is computed from global hidden states with a teacher-forced local transformer in chunks of 256 positions. References, instructions and prior context are masked from target labels. The training prefix is constructed through the same single/pair reference builder as inference, with the actual target frame count used for the teacher-forced global timing request.
|
| 90 |
+
|
| 91 |
+
The new portable trainer/tokenizer and the new inference orchestrator are release tooling. Their CPU contract checks and any allocated GPU smoke results are recorded separately in [VALIDATION.md](VALIDATION.md); they are not retroactively labelled as the original historical run.
|
UPSTREAM_LICENSE
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Humaneness Voice Large 1.3 Alpha: model weights and newly authored release materials.
|
| 2 |
+
SPDX-License-Identifier: CC-BY-4.0
|
| 3 |
+
Copyright 2026 LAION contributors.
|
| 4 |
+
|
| 5 |
+
Creative Commons Attribution 4.0 International:
|
| 6 |
+
https://creativecommons.org/licenses/by/4.0/legalcode.en
|
| 7 |
+
|
| 8 |
+
Attribute LAION contributors, Humaneness Voice Large 1.3 Alpha, 2026,
|
| 9 |
+
https://huggingface.co/laion/Humaneness-Voice-Large-1.3-Alpha
|
| 10 |
+
and indicate changes. Third-party code/model/codec/data terms and personality
|
| 11 |
+
rights remain applicable. See NOTICE.md; this does not relicense them.
|
UPSTREAM_NOTICE.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Upstream provenance
|
| 2 |
+
|
| 3 |
+
Immediate parent weights:
|
| 4 |
+
https://huggingface.co/laion/Humaneness-Voice-Large-1.2-Alpha
|
| 5 |
+
(CC BY 4.0), final step 7,851. The 1.3 continuation is a full supervised
|
| 6 |
+
fine-tune, not a LoRA merge and not reinforcement learning.
|
| 7 |
+
|
| 8 |
+
Earlier model lineage:
|
| 9 |
+
https://huggingface.co/laion/moss-tts-local-transformer-4.55b-voice-acting-v2-sft3
|
| 10 |
+
(CC BY 4.0 as stated by that parent repository).
|
| 11 |
+
Architecture: MOSS Local Transformer 1.5 family, Qwen3-family semantic backbone.
|
| 12 |
+
External codec: https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2
|
| 13 |
+
Synthetic sources: https://huggingface.co/datasets/laion/german-english-voice-acting-challenges
|
| 14 |
+
|
| 15 |
+
Original implementation files retain existing attribution. Dependencies,
|
| 16 |
+
tokenizer assets, the external codec and source recordings retain their
|
| 17 |
+
own upstream terms. The CC BY 4.0 declaration does not relicense third-party
|
| 18 |
+
software or grant rights over another person's identity. Source voice
|
| 19 |
+
aliases are fictional catalog labels, not real-person endorsements.
|
| 20 |
+
|
| 21 |
+
The demo reference crops originate in the public synthetic acting corpus;
|
| 22 |
+
their UIDs and attribution remain in the linked listening-audio datasets.
|
| 23 |
+
The CLI canary reference uses the same public synthetic reference pool.
|
VALIDATION.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Current optimized-pipeline validation
|
| 2 |
+
|
| 3 |
+
## Five-language listening release
|
| 4 |
+
|
| 5 |
+
The [public Podcast listening Space](https://huggingface.co/spaces/laion/Humanness-Voice-Podcast) combines the existing six EN/DE examples with nine newly generated NL/FR/ES episodes. The extension produced **420 candidates and 180 selected turns** at N=4/2/1 on one shared RTX 3090, using the same reference pair and sparse prompting. Strict ranking, correct reference routing, exact candidate-to-selected PCM, transcript/prompt preservation, MP3 hashes and decoding, estimated-boundary joins, and timing accounting passed artifact validation for all three modes. Sixteen CPU tests pass.
|
| 6 |
+
|
| 7 |
+
All **17 players** (15 episodes and two references) progressed in Chromium both locally and on the public HTTPS Space; candidate order, language navigation, transcripts, prompt-copy feedback, exclusive playback, and 1440/768/390-pixel layouts passed. These playback checks do not imply perceptual correctness of every spoken word.
|
| 8 |
+
|
| 9 |
+
The extension has **49 configured review flags among 180 selected turns**: 39 transcript checks exceed 15% ASR WER, and 38 have uncertain boundaries, with overlap between those sets. Dutch contributes many boundary flags because it uses conservative Parakeet TDT estimates rather than unsupported Qwen forced alignment. Uncertain clips stay untrimmed. French and Dutch have possible omissions or language drift; ASR can also be wrong. Spanish has no review flags at N4/N2 and one at N1. Nothing was silently regenerated or reselected. See [aggregate evidence](optimized_pipeline/benchmarks/five-language-extension-2026-11-08.json) and [all per-turn records](optimized_pipeline/benchmarks/five-language-extension-2026-11-08-turns.json).
|
| 10 |
+
|
| 11 |
+
## Earlier EN/DE runs
|
| 12 |
+
|
| 13 |
+
The new default was executed end to end on one RTX3090:40turns,160candidates, both DE/EN, exact prefill and codec/token/PCM checks, independent audit and MP3 export. All seven players in the listening comparison progressed in Chromium on desktop/mobile. English turn6 had a real incomplete top-ranked take; strict output is retained and a separately documented complete copy uses another candidate. See [measurements and limits](guides/PIPELINE.md) and [the benchmark receipt](optimized_pipeline/benchmarks/spontaneous-2026-11-08.json).
|
| 14 |
+
|
| 15 |
+
Fourteen CPU tests cover the original cache/reward contracts, sparse cue word preservation, failed archival receipts, new default selection and custom script validation. Run `cd optimized_pipeline && python -m unittest test_contracts test_speaker_recipe test_spontaneous test_inputs`.
|
| 16 |
+
|
| 17 |
+
The listener has approved the spontaneous samples; that is not a blinded study or a quality guarantee. The fresh N=2/N=1 comparison is complete: 120 candidates, 80 selected turns, four MP3 episodes; zero configured content/alignment review flags. All seven listening-comparison players actually played in Chromium; desktop/mobile checks passed. Exact tested prefill/token and fused-decoder checks passed. See [aggregate comparison](optimized_pipeline/benchmarks/n-comparison-2026-11-08.json) and [per-turn records](optimized_pipeline/benchmarks/n-comparison-2026-11-08-turns.json). Native pairs, optional VC and internal pause compaction belong to the preserved legacy route; the prior evidence follows.
|
| 18 |
+
|
| 19 |
+
---
|
| 20 |
+
|
| 21 |
+
# Validation status
|
| 22 |
+
|
| 23 |
+
This page separates the trained model evidence from newly packaged tooling.
|
| 24 |
+
|
| 25 |
+
- The source full checkpoint at update 9,874 completed training successfully and was used for 1,536 generated/scored candidates in the controlled EN/DE segment/block experiment. All 160 original episode conditions assembled, and 271 selected pause variants received full-dialogue evaluation.
|
| 26 |
+
- The BF16 deployment export was checked tensor by tensor against the source FP32 checkpoint cast to BF16, matching the evaluated loading precision. Shard checksums are in EXPORT_PROOF.json.
|
| 27 |
+
- The prior CPU-only script-boundary edit produced 1,416 trimmed previews and 123 complete montage conditions. 120 candidate boundaries required review; 37 montage conditions therefore had no safe strict-endpoint edited version. All original audio remained available.
|
| 28 |
+
- The prior local embedded MP3 and public players passed real Chromium playback, exclusive-player, trimmed/raw-toggle and responsive-layout checks.
|
| 29 |
+
- New portable package CPU tests cover script-vs-extra-ASR boundaries, internal hallucination rejection, waveform-word preservation, exact180-ms quiet-gap compaction, non-quiet-gap preservation, no constant-background crossfade gain bump, exact400-ms joins, global timestamp handling, identity-collapse penalty, prompt routing/timing, heterogeneous batch routing with tiny CPU fakes, CFG1 behavior, and target-only twelve-codebook training labels.
|
| 30 |
+
- The new internal-pause compaction remains **UNVALIDATED for listening quality** despite its signal-level tests.
|
| 31 |
+
- The optional split/convert/re-ASR/reassemble workflow is **experimental and not validated end to end** in the published listening grid.
|
| 32 |
+
- The fresh allocated four-GPU functional smoke **passed** for EN/DE, turns/pairs, two candidates, identity scoring, ASR, assembly and the quiet-pause option. It does not establish perceptual quality or validate optional VC.
|
| 33 |
+
- The historical trainer completed on the original 32-GPU environment. The new portable continued-training entry point is separate release tooling; its packer has CPU contract tests, not a completed fresh multi-GPU retraining result.
|
| 34 |
+
|
| 35 |
+
Reproduce the CPU checks:
|
| 36 |
+
|
| 37 |
+
~~~bash
|
| 38 |
+
python -m unittest discover -s tests -v
|
| 39 |
+
~~~
|
| 40 |
+
|
| 41 |
+
The exact release-time test report is provided in stats/release_validation.json. It must be interpreted together with the unvalidated-feature labels above. Automatic scores do not replace a listening study.
|
| 42 |
+
|
| 43 |
+
The successful v2 smoke generated 16 candidates;10 passed the configured content checks and8 selected units formed four complete episodes (EN turns11.35s,EN pairs19.51s,DE turns11.99s,DE pairs20.07s). The quiet-pause option was enabled, but no real pause met its conservative trigger in these short examples; actual splice behavior is covered only by synthetic CPU tests and remains perceptually unvalidated. [Detailed result](stats/gpu_smoke_details.json) · [The four new audio outputs](validation/gpu-smoke/).
|
added_tokens.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"</think>": 151668,
|
| 3 |
+
"</tool_call>": 151658,
|
| 4 |
+
"</tool_response>": 151666,
|
| 5 |
+
"<think>": 151667,
|
| 6 |
+
"<tool_call>": 151657,
|
| 7 |
+
"<tool_response>": 151665,
|
| 8 |
+
"<|audio_end|>": 151670,
|
| 9 |
+
"<|audio_pad|>": 151671,
|
| 10 |
+
"<|audio_start|>": 151669,
|
| 11 |
+
"<|box_end|>": 151649,
|
| 12 |
+
"<|box_start|>": 151648,
|
| 13 |
+
"<|endoftext|>": 151643,
|
| 14 |
+
"<|file_sep|>": 151664,
|
| 15 |
+
"<|fim_middle|>": 151660,
|
| 16 |
+
"<|fim_pad|>": 151662,
|
| 17 |
+
"<|fim_prefix|>": 151659,
|
| 18 |
+
"<|fim_suffix|>": 151661,
|
| 19 |
+
"<|im_end|>": 151645,
|
| 20 |
+
"<|im_start|>": 151644,
|
| 21 |
+
"<|image_pad|>": 151655,
|
| 22 |
+
"<|object_ref_end|>": 151647,
|
| 23 |
+
"<|object_ref_start|>": 151646,
|
| 24 |
+
"<|quad_end|>": 151651,
|
| 25 |
+
"<|quad_start|>": 151650,
|
| 26 |
+
"<|repo_name|>": 151663,
|
| 27 |
+
"<|video_pad|>": 151656,
|
| 28 |
+
"<|vision_end|>": 151653,
|
| 29 |
+
"<|vision_pad|>": 151654,
|
| 30 |
+
"<|vision_start|>": 151652
|
| 31 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"MossTTSLocalModel"
|
| 4 |
+
],
|
| 5 |
+
"attn_implementation": "sdpa",
|
| 6 |
+
"audio_assistant_gen_slot_token_id": 151656,
|
| 7 |
+
"audio_assistant_slot_token_id": 151656,
|
| 8 |
+
"audio_codebook_sizes": [
|
| 9 |
+
1024,
|
| 10 |
+
1024,
|
| 11 |
+
1024,
|
| 12 |
+
1024,
|
| 13 |
+
1024,
|
| 14 |
+
1024,
|
| 15 |
+
1024,
|
| 16 |
+
1024,
|
| 17 |
+
1024,
|
| 18 |
+
1024,
|
| 19 |
+
1024,
|
| 20 |
+
1024
|
| 21 |
+
],
|
| 22 |
+
"audio_end_token_id": 151670,
|
| 23 |
+
"audio_pad_code": 1024,
|
| 24 |
+
"audio_pad_token_id": 1024,
|
| 25 |
+
"audio_start_token_id": 151669,
|
| 26 |
+
"audio_tokenizer_name_or_path": "OpenMOSS-Team/MOSS-Audio-Tokenizer-v2",
|
| 27 |
+
"audio_user_slot_token_id": 151654,
|
| 28 |
+
"audio_vocab_size": 1024,
|
| 29 |
+
"auto_map": {
|
| 30 |
+
"AutoConfig": "configuration_moss_tts.MossTTSLocalConfig",
|
| 31 |
+
"AutoModel": "modeling_moss_tts.MossTTSLocalModel",
|
| 32 |
+
"AutoProcessor": "processing_moss_tts.MossTTSLocalProcessor"
|
| 33 |
+
},
|
| 34 |
+
"dtype": "bfloat16",
|
| 35 |
+
"gpt2_config": {
|
| 36 |
+
"activation_function": "silu",
|
| 37 |
+
"add_cross_attention": false,
|
| 38 |
+
"attn_pdrop": 0.0,
|
| 39 |
+
"bos_token_id": null,
|
| 40 |
+
"cross_attention_hidden_size": null,
|
| 41 |
+
"decoder_start_token_id": null,
|
| 42 |
+
"embd_pdrop": 0.0,
|
| 43 |
+
"eos_token_id": 151645,
|
| 44 |
+
"finetuning_task": null,
|
| 45 |
+
"initializer_range": 0.02,
|
| 46 |
+
"is_decoder": false,
|
| 47 |
+
"layer_norm_epsilon": 1e-06,
|
| 48 |
+
"model_type": "gpt2",
|
| 49 |
+
"n_ctx": 10240,
|
| 50 |
+
"n_embd": 2560,
|
| 51 |
+
"n_head": 32,
|
| 52 |
+
"n_inner": 9728,
|
| 53 |
+
"n_layer": 1,
|
| 54 |
+
"n_positions": 10240,
|
| 55 |
+
"pad_token_id": null,
|
| 56 |
+
"position_embedding_type": "rope",
|
| 57 |
+
"prefix": null,
|
| 58 |
+
"pruned_heads": {},
|
| 59 |
+
"reorder_and_upcast_attn": false,
|
| 60 |
+
"resid_pdrop": 0.0,
|
| 61 |
+
"rope_base": 1000000.0,
|
| 62 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 63 |
+
"scale_attn_weights": true,
|
| 64 |
+
"sep_token_id": null,
|
| 65 |
+
"summary_activation": null,
|
| 66 |
+
"summary_first_dropout": 0.1,
|
| 67 |
+
"summary_proj_to_labels": true,
|
| 68 |
+
"summary_type": "cls_index",
|
| 69 |
+
"summary_use_proj": true,
|
| 70 |
+
"task_specific_params": null,
|
| 71 |
+
"tf_legacy_loss": false,
|
| 72 |
+
"tie_encoder_decoder": false,
|
| 73 |
+
"tie_word_embeddings": true,
|
| 74 |
+
"tokenizer_class": null,
|
| 75 |
+
"torchscript": false,
|
| 76 |
+
"use_bfloat16": false,
|
| 77 |
+
"use_cache": true,
|
| 78 |
+
"vocab_size": 151936
|
| 79 |
+
},
|
| 80 |
+
"hidden_size": 2560,
|
| 81 |
+
"im_end_token_id": 151645,
|
| 82 |
+
"im_start_token_id": 151644,
|
| 83 |
+
"initializer_range": 0.02,
|
| 84 |
+
"language_config": {
|
| 85 |
+
"add_cross_attention": false,
|
| 86 |
+
"architectures": [
|
| 87 |
+
"Qwen3ForCausalLM"
|
| 88 |
+
],
|
| 89 |
+
"attention_bias": false,
|
| 90 |
+
"attention_dropout": 0.0,
|
| 91 |
+
"bos_token_id": 151643,
|
| 92 |
+
"cross_attention_hidden_size": null,
|
| 93 |
+
"decoder_start_token_id": null,
|
| 94 |
+
"dtype": "bfloat16",
|
| 95 |
+
"eos_token_id": 151643,
|
| 96 |
+
"finetuning_task": null,
|
| 97 |
+
"gradient_checkpointing_use_reentrant": false,
|
| 98 |
+
"head_dim": 128,
|
| 99 |
+
"hidden_act": "silu",
|
| 100 |
+
"hidden_size": 2560,
|
| 101 |
+
"initializer_range": 0.02,
|
| 102 |
+
"intermediate_size": 9728,
|
| 103 |
+
"is_decoder": false,
|
| 104 |
+
"layer_types": [
|
| 105 |
+
"full_attention",
|
| 106 |
+
"full_attention",
|
| 107 |
+
"full_attention",
|
| 108 |
+
"full_attention",
|
| 109 |
+
"full_attention",
|
| 110 |
+
"full_attention",
|
| 111 |
+
"full_attention",
|
| 112 |
+
"full_attention",
|
| 113 |
+
"full_attention",
|
| 114 |
+
"full_attention",
|
| 115 |
+
"full_attention",
|
| 116 |
+
"full_attention",
|
| 117 |
+
"full_attention",
|
| 118 |
+
"full_attention",
|
| 119 |
+
"full_attention",
|
| 120 |
+
"full_attention",
|
| 121 |
+
"full_attention",
|
| 122 |
+
"full_attention",
|
| 123 |
+
"full_attention",
|
| 124 |
+
"full_attention",
|
| 125 |
+
"full_attention",
|
| 126 |
+
"full_attention",
|
| 127 |
+
"full_attention",
|
| 128 |
+
"full_attention",
|
| 129 |
+
"full_attention",
|
| 130 |
+
"full_attention",
|
| 131 |
+
"full_attention",
|
| 132 |
+
"full_attention",
|
| 133 |
+
"full_attention",
|
| 134 |
+
"full_attention",
|
| 135 |
+
"full_attention",
|
| 136 |
+
"full_attention",
|
| 137 |
+
"full_attention",
|
| 138 |
+
"full_attention",
|
| 139 |
+
"full_attention",
|
| 140 |
+
"full_attention"
|
| 141 |
+
],
|
| 142 |
+
"max_position_embeddings": 32768,
|
| 143 |
+
"max_window_layers": 36,
|
| 144 |
+
"model_type": "qwen3",
|
| 145 |
+
"num_attention_heads": 32,
|
| 146 |
+
"num_hidden_layers": 36,
|
| 147 |
+
"num_key_value_heads": 8,
|
| 148 |
+
"pad_token_id": 151643,
|
| 149 |
+
"prefix": null,
|
| 150 |
+
"pruned_heads": {},
|
| 151 |
+
"rms_norm_eps": 1e-06,
|
| 152 |
+
"rope_parameters": {
|
| 153 |
+
"rope_theta": 1000000,
|
| 154 |
+
"rope_type": "default"
|
| 155 |
+
},
|
| 156 |
+
"sep_token_id": null,
|
| 157 |
+
"sliding_window": null,
|
| 158 |
+
"task_specific_params": null,
|
| 159 |
+
"tf_legacy_loss": false,
|
| 160 |
+
"tie_encoder_decoder": false,
|
| 161 |
+
"tie_word_embeddings": true,
|
| 162 |
+
"tokenizer_class": null,
|
| 163 |
+
"torchscript": false,
|
| 164 |
+
"use_bfloat16": false,
|
| 165 |
+
"use_cache": false,
|
| 166 |
+
"use_sliding_window": false,
|
| 167 |
+
"vocab_size": 151936
|
| 168 |
+
},
|
| 169 |
+
"local_hidden_size": 2560,
|
| 170 |
+
"local_text_head_mode": "binary",
|
| 171 |
+
"local_transformer_attn_implementation": "sdpa",
|
| 172 |
+
"local_transformer_layers": 1,
|
| 173 |
+
"model_type": "moss_tts_local",
|
| 174 |
+
"n_vq": 12,
|
| 175 |
+
"pad_token_id": 151643,
|
| 176 |
+
"processor_class": "MossTTSLocalProcessor",
|
| 177 |
+
"qwen3_config": {
|
| 178 |
+
"add_cross_attention": false,
|
| 179 |
+
"architectures": [
|
| 180 |
+
"Qwen3ForCausalLM"
|
| 181 |
+
],
|
| 182 |
+
"attention_bias": false,
|
| 183 |
+
"attention_dropout": 0.0,
|
| 184 |
+
"bos_token_id": 151643,
|
| 185 |
+
"cross_attention_hidden_size": null,
|
| 186 |
+
"decoder_start_token_id": null,
|
| 187 |
+
"dtype": "bfloat16",
|
| 188 |
+
"eos_token_id": 151643,
|
| 189 |
+
"finetuning_task": null,
|
| 190 |
+
"gradient_checkpointing_use_reentrant": false,
|
| 191 |
+
"head_dim": 128,
|
| 192 |
+
"hidden_act": "silu",
|
| 193 |
+
"hidden_size": 2560,
|
| 194 |
+
"initializer_range": 0.02,
|
| 195 |
+
"intermediate_size": 9728,
|
| 196 |
+
"is_decoder": false,
|
| 197 |
+
"layer_types": [
|
| 198 |
+
"full_attention",
|
| 199 |
+
"full_attention",
|
| 200 |
+
"full_attention",
|
| 201 |
+
"full_attention",
|
| 202 |
+
"full_attention",
|
| 203 |
+
"full_attention",
|
| 204 |
+
"full_attention",
|
| 205 |
+
"full_attention",
|
| 206 |
+
"full_attention",
|
| 207 |
+
"full_attention",
|
| 208 |
+
"full_attention",
|
| 209 |
+
"full_attention",
|
| 210 |
+
"full_attention",
|
| 211 |
+
"full_attention",
|
| 212 |
+
"full_attention",
|
| 213 |
+
"full_attention",
|
| 214 |
+
"full_attention",
|
| 215 |
+
"full_attention",
|
| 216 |
+
"full_attention",
|
| 217 |
+
"full_attention",
|
| 218 |
+
"full_attention",
|
| 219 |
+
"full_attention",
|
| 220 |
+
"full_attention",
|
| 221 |
+
"full_attention",
|
| 222 |
+
"full_attention",
|
| 223 |
+
"full_attention",
|
| 224 |
+
"full_attention",
|
| 225 |
+
"full_attention",
|
| 226 |
+
"full_attention",
|
| 227 |
+
"full_attention",
|
| 228 |
+
"full_attention",
|
| 229 |
+
"full_attention",
|
| 230 |
+
"full_attention",
|
| 231 |
+
"full_attention",
|
| 232 |
+
"full_attention",
|
| 233 |
+
"full_attention"
|
| 234 |
+
],
|
| 235 |
+
"max_position_embeddings": 32768,
|
| 236 |
+
"max_window_layers": 36,
|
| 237 |
+
"model_type": "qwen3",
|
| 238 |
+
"num_attention_heads": 32,
|
| 239 |
+
"num_hidden_layers": 36,
|
| 240 |
+
"num_key_value_heads": 8,
|
| 241 |
+
"pad_token_id": 151643,
|
| 242 |
+
"prefix": null,
|
| 243 |
+
"pruned_heads": {},
|
| 244 |
+
"rms_norm_eps": 1e-06,
|
| 245 |
+
"rope_parameters": {
|
| 246 |
+
"rope_theta": 1000000,
|
| 247 |
+
"rope_type": "default"
|
| 248 |
+
},
|
| 249 |
+
"sep_token_id": null,
|
| 250 |
+
"sliding_window": null,
|
| 251 |
+
"task_specific_params": null,
|
| 252 |
+
"tf_legacy_loss": false,
|
| 253 |
+
"tie_encoder_decoder": false,
|
| 254 |
+
"tie_word_embeddings": true,
|
| 255 |
+
"tokenizer_class": null,
|
| 256 |
+
"torchscript": false,
|
| 257 |
+
"use_bfloat16": false,
|
| 258 |
+
"use_cache": false,
|
| 259 |
+
"use_sliding_window": false,
|
| 260 |
+
"vocab_size": 151936
|
| 261 |
+
},
|
| 262 |
+
"sampling_rate": 48000,
|
| 263 |
+
"tie_word_embeddings": true,
|
| 264 |
+
"transformers_version": "5.14.1",
|
| 265 |
+
"use_static_local_kv_cache": true,
|
| 266 |
+
"vocab_size": 151936,
|
| 267 |
+
"torch_dtype": "bfloat16"
|
| 268 |
+
}
|
configuration_moss_tts.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
"""Configuration for the MOSS-TTS-Local-Transformer-v1.5 release."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Any, Dict, Optional, Union
|
| 7 |
+
|
| 8 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 9 |
+
from transformers.models.gpt2.configuration_gpt2 import GPT2Config
|
| 10 |
+
from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
SUPPORTED_ATTENTION_IMPLEMENTATIONS = {"flash_attention_2", "sdpa", "eager"}
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _normalize_attention_implementation(value: Optional[str], default: str = "flash_attention_2") -> str:
|
| 17 |
+
normalized = str(value or default).strip().lower()
|
| 18 |
+
if normalized in {"flash", "flash_attn", "flash-attn", "flash_attention"}:
|
| 19 |
+
normalized = "flash_attention_2"
|
| 20 |
+
if normalized not in SUPPORTED_ATTENTION_IMPLEMENTATIONS:
|
| 21 |
+
raise ValueError(
|
| 22 |
+
"attn_implementation must be one of "
|
| 23 |
+
f"{sorted(SUPPORTED_ATTENTION_IMPLEMENTATIONS)}, got {value!r}."
|
| 24 |
+
)
|
| 25 |
+
return normalized
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class MossTTSLocalConfig(PretrainedConfig):
|
| 29 |
+
model_type = "moss_tts_local"
|
| 30 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
qwen3_config: Optional[Union[Qwen3Config, Dict[str, Any]]] = None,
|
| 35 |
+
gpt2_config: Optional[Union[GPT2Config, Dict[str, Any]]] = None,
|
| 36 |
+
language_config: Optional[Union[Qwen3Config, Dict[str, Any]]] = None,
|
| 37 |
+
n_vq: int = 12,
|
| 38 |
+
audio_vocab_size: int = 1024,
|
| 39 |
+
audio_codebook_sizes: Optional[list[int]] = None,
|
| 40 |
+
audio_pad_token_id: int = 1024,
|
| 41 |
+
audio_pad_code: Optional[int] = None,
|
| 42 |
+
pad_token_id: int = 151643,
|
| 43 |
+
im_start_token_id: int = 151644,
|
| 44 |
+
im_end_token_id: int = 151645,
|
| 45 |
+
audio_start_token_id: int = 151669,
|
| 46 |
+
audio_end_token_id: int = 151670,
|
| 47 |
+
audio_user_slot_token_id: int = 151654,
|
| 48 |
+
audio_assistant_slot_token_id: int = 151656,
|
| 49 |
+
audio_assistant_gen_slot_token_id: Optional[int] = None,
|
| 50 |
+
sampling_rate: int = 48000,
|
| 51 |
+
audio_tokenizer_name_or_path: Optional[str] = None,
|
| 52 |
+
attn_implementation: str = "flash_attention_2",
|
| 53 |
+
local_transformer_attn_implementation: Optional[str] = None,
|
| 54 |
+
local_text_head_mode: str = "binary",
|
| 55 |
+
initializer_range: float = 0.02,
|
| 56 |
+
**kwargs: Any,
|
| 57 |
+
) -> None:
|
| 58 |
+
if qwen3_config is None and language_config is not None:
|
| 59 |
+
qwen3_config = language_config
|
| 60 |
+
if isinstance(qwen3_config, dict):
|
| 61 |
+
self.qwen3_config = Qwen3Config(**qwen3_config)
|
| 62 |
+
elif qwen3_config is None:
|
| 63 |
+
self.qwen3_config = Qwen3Config()
|
| 64 |
+
else:
|
| 65 |
+
self.qwen3_config = qwen3_config
|
| 66 |
+
|
| 67 |
+
if isinstance(gpt2_config, dict):
|
| 68 |
+
self.gpt2_config = GPT2Config(**gpt2_config)
|
| 69 |
+
elif gpt2_config is None:
|
| 70 |
+
self.gpt2_config = GPT2Config(
|
| 71 |
+
vocab_size=int(self.qwen3_config.vocab_size),
|
| 72 |
+
n_embd=int(self.qwen3_config.hidden_size),
|
| 73 |
+
n_layer=1,
|
| 74 |
+
n_head=max(1, int(self.qwen3_config.hidden_size) // 80),
|
| 75 |
+
n_positions=int(n_vq) + 1,
|
| 76 |
+
n_ctx=int(n_vq) + 1,
|
| 77 |
+
activation_function="silu",
|
| 78 |
+
layer_norm_epsilon=1e-6,
|
| 79 |
+
resid_pdrop=0.0,
|
| 80 |
+
embd_pdrop=0.0,
|
| 81 |
+
attn_pdrop=0.0,
|
| 82 |
+
)
|
| 83 |
+
else:
|
| 84 |
+
self.gpt2_config = gpt2_config
|
| 85 |
+
|
| 86 |
+
self.n_vq = int(n_vq)
|
| 87 |
+
if self.n_vq <= 0:
|
| 88 |
+
raise ValueError("n_vq must be positive.")
|
| 89 |
+
if audio_codebook_sizes is None:
|
| 90 |
+
self.audio_codebook_sizes = [int(audio_vocab_size)] * self.n_vq
|
| 91 |
+
else:
|
| 92 |
+
self.audio_codebook_sizes = [int(size) for size in audio_codebook_sizes]
|
| 93 |
+
if len(self.audio_codebook_sizes) != self.n_vq:
|
| 94 |
+
raise ValueError(
|
| 95 |
+
f"audio_codebook_sizes must have length n_vq={self.n_vq}, "
|
| 96 |
+
f"got {len(self.audio_codebook_sizes)}."
|
| 97 |
+
)
|
| 98 |
+
if any(size <= 0 for size in self.audio_codebook_sizes):
|
| 99 |
+
raise ValueError("audio_codebook_sizes must contain positive integers.")
|
| 100 |
+
self.audio_vocab_size = int(max(int(audio_vocab_size), max(self.audio_codebook_sizes)))
|
| 101 |
+
self.audio_pad_token_id = int(audio_pad_code if audio_pad_code is not None else audio_pad_token_id)
|
| 102 |
+
self.audio_pad_code = self.audio_pad_token_id
|
| 103 |
+
if self.audio_pad_token_id < self.audio_vocab_size:
|
| 104 |
+
raise ValueError("audio_pad_token_id/audio_pad_code must be outside the audio vocab.")
|
| 105 |
+
|
| 106 |
+
self.pad_token_id = int(pad_token_id)
|
| 107 |
+
self.im_start_token_id = int(im_start_token_id)
|
| 108 |
+
self.im_end_token_id = int(im_end_token_id)
|
| 109 |
+
self.audio_start_token_id = int(audio_start_token_id)
|
| 110 |
+
self.audio_end_token_id = int(audio_end_token_id)
|
| 111 |
+
self.audio_user_slot_token_id = int(audio_user_slot_token_id)
|
| 112 |
+
self.audio_assistant_slot_token_id = int(
|
| 113 |
+
audio_assistant_slot_token_id
|
| 114 |
+
if audio_assistant_gen_slot_token_id is None
|
| 115 |
+
else audio_assistant_gen_slot_token_id
|
| 116 |
+
)
|
| 117 |
+
self.audio_assistant_gen_slot_token_id = self.audio_assistant_slot_token_id
|
| 118 |
+
|
| 119 |
+
self.sampling_rate = int(sampling_rate)
|
| 120 |
+
self.audio_tokenizer_name_or_path = audio_tokenizer_name_or_path
|
| 121 |
+
self.attn_implementation = _normalize_attention_implementation(attn_implementation)
|
| 122 |
+
self.local_transformer_attn_implementation = _normalize_attention_implementation(
|
| 123 |
+
local_transformer_attn_implementation,
|
| 124 |
+
default=self.attn_implementation,
|
| 125 |
+
)
|
| 126 |
+
self.initializer_range = float(initializer_range)
|
| 127 |
+
|
| 128 |
+
self.hidden_size = int(self.qwen3_config.hidden_size)
|
| 129 |
+
self.vocab_size = int(self.qwen3_config.vocab_size)
|
| 130 |
+
self.local_hidden_size = int(self.gpt2_config.hidden_size)
|
| 131 |
+
if self.local_hidden_size != self.hidden_size:
|
| 132 |
+
raise ValueError(
|
| 133 |
+
"This MOSS-TTS-Local-Transformer-v1.5 release expects local hidden size to "
|
| 134 |
+
"match Qwen3 hidden size so audio embeddings and heads are tied."
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
normalized_text_head_mode = str(local_text_head_mode or "full_vocab").strip().lower()
|
| 138 |
+
if normalized_text_head_mode in {"full", "full-vocab", "vocab"}:
|
| 139 |
+
normalized_text_head_mode = "full_vocab"
|
| 140 |
+
if normalized_text_head_mode not in {"full_vocab", "binary"}:
|
| 141 |
+
raise ValueError("local_text_head_mode must be 'full_vocab' or 'binary'.")
|
| 142 |
+
self.local_text_head_mode = normalized_text_head_mode
|
| 143 |
+
|
| 144 |
+
kwargs.setdefault("tie_word_embeddings", True)
|
| 145 |
+
super().__init__(pad_token_id=self.pad_token_id, **kwargs)
|
| 146 |
+
|
| 147 |
+
@property
|
| 148 |
+
def language_config(self) -> Qwen3Config:
|
| 149 |
+
return self.qwen3_config
|
| 150 |
+
|
| 151 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 152 |
+
output = super().to_dict()
|
| 153 |
+
output["qwen3_config"] = self.qwen3_config.to_dict()
|
| 154 |
+
output["language_config"] = self.qwen3_config.to_dict()
|
| 155 |
+
output["gpt2_config"] = self.gpt2_config.to_dict()
|
| 156 |
+
output["audio_pad_code"] = self.audio_pad_token_id
|
| 157 |
+
output["audio_assistant_gen_slot_token_id"] = self.audio_assistant_slot_token_id
|
| 158 |
+
return output
|
examples/audio/PROVENANCE.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"file": "en_segment_reassembled.mp3",
|
| 4 |
+
"source_condition": "en-conversation-02-podcast-segment-free-n1",
|
| 5 |
+
"duration_s": 77.91,
|
| 6 |
+
"editing": "Verified script boundaries,150-ms tails,400-ms word-clock gaps. Internal pause compaction and optional VC were NOT applied."
|
| 7 |
+
},
|
| 8 |
+
{
|
| 9 |
+
"file": "en_block_reassembled.mp3",
|
| 10 |
+
"source_condition": "en-conversation-01-podcast-block-free-n8",
|
| 11 |
+
"duration_s": 56.95,
|
| 12 |
+
"editing": "Verified script boundaries,150-ms tails,400-ms word-clock gaps. Internal pause compaction and optional VC were NOT applied."
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"file": "de_segment_reassembled.mp3",
|
| 16 |
+
"source_condition": "de-conversation-02-podcast-segment-free-n1",
|
| 17 |
+
"duration_s": 78.23,
|
| 18 |
+
"editing": "Verified script boundaries,150-ms tails,400-ms word-clock gaps. Internal pause compaction and optional VC were NOT applied."
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"file": "de_block_reassembled.mp3",
|
| 22 |
+
"source_condition": "de-conversation-01-podcast-block-free-n8",
|
| 23 |
+
"duration_s": 59.19,
|
| 24 |
+
"editing": "Verified script boundaries,150-ms tails,400-ms word-clock gaps. Internal pause compaction and optional VC were NOT applied."
|
| 25 |
+
}
|
| 26 |
+
]
|
examples/audio/de_block_reassembled.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:43462ee94dd0f48ddf3c149ba3450f6b0e08f33e471d124c1b7152f47540d4e2
|
| 3 |
+
size 947756
|
examples/audio/de_segment_reassembled.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:070dfe22bae9fa125022d7ab36346c5d6cc5124fbad41553d0a40ea27d1d4ed5
|
| 3 |
+
size 1252268
|
examples/audio/en_block_reassembled.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:efdeb8b807a286e5407ac09c4f5c0abc15288774aca734c9e992a0249d077c1b
|
| 3 |
+
size 911660
|
examples/audio/en_segment_reassembled.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1ed0d49e35a7866de27ee71a4112947f37226922d902cd7eeb6352666ec9fec1
|
| 3 |
+
size 1247276
|
examples/episode_de.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"language": "de",
|
| 3 |
+
"references": {
|
| 4 |
+
"S1": "references/de_s1.mp3",
|
| 5 |
+
"S2": "references/de_s2.mp3"
|
| 6 |
+
},
|
| 7 |
+
"reference_codes": "references/de_anchors.npz",
|
| 8 |
+
"general": "A very genuine, warm, realistic conversation, pleasant and nice to listen to. Respond naturally to the other host; speak only the scripted words.",
|
| 9 |
+
"turns": [
|
| 10 |
+
{
|
| 11 |
+
"speaker": "S1",
|
| 12 |
+
"text": "Ich habe Humaneness Podcast gehört, eine Feinabstimmung von Humaneness Voice Large.",
|
| 13 |
+
"direction": "warm, curious, quietly excited, genuine and pleasant to listen to",
|
| 14 |
+
"seconds": 3.93
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"speaker": "S2",
|
| 18 |
+
"text": "Du klingst verdächtig begeistert. Hat dich ein Sprachmodell wirklich überrascht?",
|
| 19 |
+
"direction": "skeptical, teasing, genuinely surprised, genuine and pleasant to listen to",
|
| 20 |
+
"seconds": 3.57
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"speaker": "S1",
|
| 24 |
+
"text": "Es liest ein Skript mit mehreren Stimmen und eigenen Spielanweisungen pro Satz.",
|
| 25 |
+
"direction": "engaged, confident, conversational, genuine and pleasant to listen to",
|
| 26 |
+
"seconds": 4.29
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"speaker": "S2",
|
| 30 |
+
"text": "Moment, eine Stimme darf dir misstrauen, während die andere schon jubelt?",
|
| 31 |
+
"direction": "incredulous, playful doubt, genuine and pleasant to listen to",
|
| 32 |
+
"seconds": 3.93
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"speaker": "S1",
|
| 36 |
+
"text": "Genau. Wir testen vierzig Emotionsklassen und behaupten nicht, dass jede perfekt gelingt.",
|
| 37 |
+
"direction": "thoughtful, candid, reassuring, genuine and pleasant to listen to",
|
| 38 |
+
"seconds": 4.29
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"speaker": "S2",
|
| 42 |
+
"text": "Diese Einschränkung macht die Vorführung irgendwie glaubwürdiger. Ärgerlich vernünftig von dir.",
|
| 43 |
+
"direction": "amused, slightly vulnerable, genuine and pleasant to listen to",
|
| 44 |
+
"seconds": 3.93
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"speaker": "S1",
|
| 48 |
+
"text": "Im Training kommen echte Gespräche und Schauspielbeispiele zusammen, damit beide Formate erhalten bleiben.",
|
| 49 |
+
"direction": "warm, matter-of-fact, genuine, genuine and pleasant to listen to",
|
| 50 |
+
"seconds": 4.64
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"speaker": "S2",
|
| 54 |
+
"text": "Na gut. Zeig mir diese kleine Pause, bevor jemand widerwillig zugibt, beeindruckt zu sein.",
|
| 55 |
+
"direction": "wonder, affectionate teasing, relaxed amusement, genuine and pleasant to listen to",
|
| 56 |
+
"seconds": 5.0
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"speaker": "S1",
|
| 60 |
+
"text": "Ich mag Gespräche, in denen noch Platz für eine echte, unerwartete Reaktion bleibt.",
|
| 61 |
+
"direction": "warm, curious, quietly excited, genuine and pleasant to listen to",
|
| 62 |
+
"seconds": 4.64
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"speaker": "S2",
|
| 66 |
+
"text": "Ja, manchmal erzählt dieses kleine Zögern mehr als der nächste völlig glatt formulierte Satz.",
|
| 67 |
+
"direction": "skeptical, teasing, genuinely surprised, genuine and pleasant to listen to",
|
| 68 |
+
"seconds": 5.0
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"speaker": "S1",
|
| 72 |
+
"text": "Dann hören wir genau hin, auch an den Stellen, an denen die schöne Illusion kurz zerbricht.",
|
| 73 |
+
"direction": "engaged, confident, conversational, genuine and pleasant to listen to",
|
| 74 |
+
"seconds": 5.71
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"speaker": "S2",
|
| 78 |
+
"text": "Einverstanden. Ich möchte Wärme und Persönlichkeit hören und trotzdem erkennen, wer gerade spricht.",
|
| 79 |
+
"direction": "incredulous, playful doubt, genuine and pleasant to listen to",
|
| 80 |
+
"seconds": 4.64
|
| 81 |
+
}
|
| 82 |
+
]
|
| 83 |
+
}
|
examples/episode_en.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"language": "en",
|
| 3 |
+
"references": {
|
| 4 |
+
"S1": "references/de_s1.mp3",
|
| 5 |
+
"S2": "references/de_s2.mp3"
|
| 6 |
+
},
|
| 7 |
+
"reference_codes": "references/de_anchors.npz",
|
| 8 |
+
"general": "A very genuine, warm, realistic conversation, pleasant and nice to listen to. Respond naturally to the other host; speak only the scripted words.",
|
| 9 |
+
"turns": [
|
| 10 |
+
{
|
| 11 |
+
"speaker": "S1",
|
| 12 |
+
"text": "I have been listening to Humaneness Podcast, a fine-tune of Humaneness Voice Large.",
|
| 13 |
+
"direction": "warm, curious, quietly excited, genuine and pleasant to listen to",
|
| 14 |
+
"seconds": 4.33
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"speaker": "S2",
|
| 18 |
+
"text": "You sound suspiciously excited. Did a speech model actually surprise you?",
|
| 19 |
+
"direction": "skeptical, teasing, genuinely surprised, genuine and pleasant to listen to",
|
| 20 |
+
"seconds": 3.67
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"speaker": "S1",
|
| 24 |
+
"text": "It can read a script with different speakers and sentence-level acting directions.",
|
| 25 |
+
"direction": "engaged, confident, conversational, genuine and pleasant to listen to",
|
| 26 |
+
"seconds": 4.0
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"speaker": "S2",
|
| 30 |
+
"text": "Wait, so I can ask one voice to doubt you while the other celebrates?",
|
| 31 |
+
"direction": "incredulous, playful doubt, genuine and pleasant to listen to",
|
| 32 |
+
"seconds": 4.67
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"speaker": "S1",
|
| 36 |
+
"text": "Exactly. We are testing forty emotion labels, not claiming forty perfect performances.",
|
| 37 |
+
"direction": "thoughtful, candid, reassuring, genuine and pleasant to listen to",
|
| 38 |
+
"seconds": 4.0
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"speaker": "S2",
|
| 42 |
+
"text": "That distinction makes me trust the demonstration a little more. Annoyingly sensible of you.",
|
| 43 |
+
"direction": "amused, slightly vulnerable, genuine and pleasant to listen to",
|
| 44 |
+
"seconds": 4.67
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"speaker": "S1",
|
| 48 |
+
"text": "The new training mixes real conversations with acting examples to retain both formats.",
|
| 49 |
+
"direction": "warm, matter-of-fact, genuine, genuine and pleasant to listen to",
|
| 50 |
+
"seconds": 4.33
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"speaker": "S2",
|
| 54 |
+
"text": "All right, show me the awkward pause before someone finally admits they are impressed.",
|
| 55 |
+
"direction": "wonder, affectionate teasing, relaxed amusement, genuine and pleasant to listen to",
|
| 56 |
+
"seconds": 4.67
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"speaker": "S1",
|
| 60 |
+
"text": "There is something lovely about a conversation that leaves room for a real reaction.",
|
| 61 |
+
"direction": "warm, curious, quietly excited, genuine and pleasant to listen to",
|
| 62 |
+
"seconds": 4.67
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"speaker": "S2",
|
| 66 |
+
"text": "Yes, a tiny hesitation can say more than another perfectly polished sentence.",
|
| 67 |
+
"direction": "skeptical, teasing, genuinely surprised, genuine and pleasant to listen to",
|
| 68 |
+
"seconds": 4.0
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"speaker": "S1",
|
| 72 |
+
"text": "So let us compare the voices carefully, including the moments where the illusion breaks.",
|
| 73 |
+
"direction": "engaged, confident, conversational, genuine and pleasant to listen to",
|
| 74 |
+
"seconds": 4.67
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"speaker": "S2",
|
| 78 |
+
"text": "Agreed. I want warmth and character, and I still want to recognize who is talking.",
|
| 79 |
+
"direction": "incredulous, playful doubt, genuine and pleasant to listen to",
|
| 80 |
+
"seconds": 5.0
|
| 81 |
+
}
|
| 82 |
+
]
|
| 83 |
+
}
|
examples/references/PROVENANCE.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"pack": "de-synthetic-reference",
|
| 4 |
+
"language": "de",
|
| 5 |
+
"speaker_files": [
|
| 6 |
+
"de_s1.mp3",
|
| 7 |
+
"de_s2.mp3"
|
| 8 |
+
],
|
| 9 |
+
"codes": "de_anchors.npz",
|
| 10 |
+
"conditioning_frames": {
|
| 11 |
+
"S1": 181,
|
| 12 |
+
"S2": 163
|
| 13 |
+
},
|
| 14 |
+
"source_terms": "Original source terms; this model release does not relicense source recordings. German anchors were supplied by the experiment owner; English anchors are the previously approved short real-podcast reference excerpts."
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"pack": "en-real-2",
|
| 18 |
+
"language": "en",
|
| 19 |
+
"speaker_files": [
|
| 20 |
+
"en_s1.mp3",
|
| 21 |
+
"en_s2.mp3"
|
| 22 |
+
],
|
| 23 |
+
"codes": "en_anchors.npz",
|
| 24 |
+
"conditioning_frames": {
|
| 25 |
+
"S1": 117,
|
| 26 |
+
"S2": 145
|
| 27 |
+
},
|
| 28 |
+
"source_terms": "Original source terms; this model release does not relicense source recordings. German anchors were supplied by the experiment owner; English anchors are the previously approved short real-podcast reference excerpts."
|
| 29 |
+
}
|
| 30 |
+
]
|
examples/references/de_anchors.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:27c051d23d584df782cfcf4abd4051700838169d7bf2a0610b077ec1d30eccdf
|
| 3 |
+
size 6799
|
examples/references/de_s1.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9d5a7823fea82182d6a3ec378e15ae03b0bd61f028c6f50fb6c5b82fb1f1b7dd
|
| 3 |
+
size 232364
|
examples/references/de_s2.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d2da04db22aeb44206e4f935e95919846e3e5bb4cb116503a7a260c9f48752cd
|
| 3 |
+
size 209324
|
examples/references/en_anchors.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed071500398ed41980e5ddc81bfc4b928a1469a7e6874792af033b52b92af989
|
| 3 |
+
size 5199
|
examples/references/en_s1.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b630face220f977dc36e01db02f22063b7f2e4188439e272dc3a02f552082722
|
| 3 |
+
size 150188
|
examples/references/en_s2.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:517e5ff6a62e393ac995985a03535ad367322aad14ac5caba3b7b2ee6d2daf92
|
| 3 |
+
size 186284
|
examples/short_de.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"language": "de",
|
| 3 |
+
"references": {
|
| 4 |
+
"S1": "references/de_s1.mp3",
|
| 5 |
+
"S2": "references/de_s2.mp3"
|
| 6 |
+
},
|
| 7 |
+
"reference_codes": "references/de_anchors.npz",
|
| 8 |
+
"general": "A very genuine, warm, realistic conversation, pleasant and nice to listen to. Respond naturally to the other host; speak only the scripted words.",
|
| 9 |
+
"turns": [
|
| 10 |
+
{
|
| 11 |
+
"speaker": "S1",
|
| 12 |
+
"text": "Ich habe Humaneness Podcast gehört, eine Feinabstimmung von Humaneness Voice Large.",
|
| 13 |
+
"direction": "warm, curious, quietly excited, genuine and pleasant to listen to",
|
| 14 |
+
"seconds": 3.93
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"speaker": "S2",
|
| 18 |
+
"text": "Du klingst verdächtig begeistert. Hat dich ein Sprachmodell wirklich überrascht?",
|
| 19 |
+
"direction": "skeptical, teasing, genuinely surprised, genuine and pleasant to listen to",
|
| 20 |
+
"seconds": 3.57
|
| 21 |
+
}
|
| 22 |
+
]
|
| 23 |
+
}
|
examples/short_en.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"language": "en",
|
| 3 |
+
"references": {
|
| 4 |
+
"S1": "references/de_s1.mp3",
|
| 5 |
+
"S2": "references/de_s2.mp3"
|
| 6 |
+
},
|
| 7 |
+
"reference_codes": "references/de_anchors.npz",
|
| 8 |
+
"general": "A very genuine, warm, realistic conversation, pleasant and nice to listen to. Respond naturally to the other host; speak only the scripted words.",
|
| 9 |
+
"turns": [
|
| 10 |
+
{
|
| 11 |
+
"speaker": "S1",
|
| 12 |
+
"text": "I have been listening to Humaneness Podcast, a fine-tune of Humaneness Voice Large.",
|
| 13 |
+
"direction": "warm, curious, quietly excited, genuine and pleasant to listen to",
|
| 14 |
+
"seconds": 4.33
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"speaker": "S2",
|
| 18 |
+
"text": "You sound suspiciously excited. Did a speech model actually surprise you?",
|
| 19 |
+
"direction": "skeptical, teasing, genuinely surprised, genuine and pleasant to listen to",
|
| 20 |
+
"seconds": 3.67
|
| 21 |
+
}
|
| 22 |
+
]
|
| 23 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"output_attentions": false,
|
| 4 |
+
"output_hidden_states": false,
|
| 5 |
+
"pad_token_id": 151643,
|
| 6 |
+
"transformers_version": "5.14.1"
|
| 7 |
+
}
|
gpt2_decoder.py
ADDED
|
@@ -0,0 +1,721 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
import torch.utils.checkpoint
|
| 10 |
+
from transformers.activations import ACT2FN
|
| 11 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast
|
| 12 |
+
from transformers.models.gpt2.configuration_gpt2 import GPT2Config
|
| 13 |
+
|
| 14 |
+
try:
|
| 15 |
+
from flash_attn import flash_attn_func, flash_attn_varlen_func
|
| 16 |
+
from flash_attn.bert_padding import pad_input, unpad_input
|
| 17 |
+
|
| 18 |
+
_FLASH_ATTN_AVAILABLE = True
|
| 19 |
+
except Exception:
|
| 20 |
+
flash_attn_func = None
|
| 21 |
+
flash_attn_varlen_func = None
|
| 22 |
+
pad_input = None
|
| 23 |
+
unpad_input = None
|
| 24 |
+
_FLASH_ATTN_AVAILABLE = False
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@dataclass
|
| 28 |
+
class PackedSequenceMetadata:
|
| 29 |
+
cu_seqlens: torch.Tensor
|
| 30 |
+
max_seqlen: int
|
| 31 |
+
indices: Optional[torch.Tensor] = None
|
| 32 |
+
batch_size: Optional[int] = None
|
| 33 |
+
seq_len: Optional[int] = None
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _is_static_kv_cache_layer(layer_past: object) -> bool:
|
| 37 |
+
return isinstance(layer_past, dict) and bool(layer_past.get("static_kv_cache", False))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class MossTTSNanoGPT2RotaryEmbedding(nn.Module):
|
| 41 |
+
def __init__(self, dim: int, base: float = 10000.0) -> None:
|
| 42 |
+
super().__init__()
|
| 43 |
+
if dim % 2 != 0:
|
| 44 |
+
raise ValueError(f"RoPE head_dim must be even, got {dim}")
|
| 45 |
+
self.dim = int(dim)
|
| 46 |
+
self.base = float(base)
|
| 47 |
+
self.register_buffer("inv_freq", self._compute_inv_freq(), persistent=False)
|
| 48 |
+
|
| 49 |
+
def _compute_inv_freq(self, device: Optional[torch.device] = None) -> torch.Tensor:
|
| 50 |
+
return 1.0 / (
|
| 51 |
+
self.base ** (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim)
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
def forward(
|
| 55 |
+
self,
|
| 56 |
+
position_ids: torch.LongTensor,
|
| 57 |
+
*,
|
| 58 |
+
device: torch.device,
|
| 59 |
+
dtype: torch.dtype,
|
| 60 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 61 |
+
if position_ids.ndim == 1:
|
| 62 |
+
position_ids = position_ids.unsqueeze(0)
|
| 63 |
+
inv_freq = self._compute_inv_freq(device=device)
|
| 64 |
+
freqs = torch.einsum("bs,d->bsd", position_ids.to(device=device, dtype=inv_freq.dtype), inv_freq)
|
| 65 |
+
cos = freqs.cos().repeat_interleave(2, dim=-1).unsqueeze(2).to(dtype=dtype)
|
| 66 |
+
sin = freqs.sin().repeat_interleave(2, dim=-1).unsqueeze(2).to(dtype=dtype)
|
| 67 |
+
return cos, sin
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def rotate_half(hidden_states: torch.Tensor) -> torch.Tensor:
|
| 71 |
+
even = hidden_states[..., ::2]
|
| 72 |
+
odd = hidden_states[..., 1::2]
|
| 73 |
+
return torch.stack((-odd, even), dim=-1).reshape_as(hidden_states)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def apply_rotary_pos_emb(
|
| 77 |
+
hidden_states: torch.Tensor,
|
| 78 |
+
cos: torch.Tensor,
|
| 79 |
+
sin: torch.Tensor,
|
| 80 |
+
) -> torch.Tensor:
|
| 81 |
+
return (hidden_states * cos) + (rotate_half(hidden_states) * sin)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class MossTTSNanoGPT2MLP(nn.Module):
|
| 85 |
+
def __init__(self, config: GPT2Config) -> None:
|
| 86 |
+
super().__init__()
|
| 87 |
+
hidden_size = int(config.hidden_size)
|
| 88 |
+
inner_size = int(config.n_inner or 4 * hidden_size)
|
| 89 |
+
self.fc_in = nn.Linear(hidden_size, inner_size)
|
| 90 |
+
self.fc_out = nn.Linear(inner_size, hidden_size)
|
| 91 |
+
self.act = ACT2FN[config.activation_function]
|
| 92 |
+
self.dropout = nn.Dropout(config.resid_pdrop)
|
| 93 |
+
|
| 94 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 95 |
+
hidden_states = self.fc_in(hidden_states)
|
| 96 |
+
hidden_states = self.act(hidden_states)
|
| 97 |
+
hidden_states = self.fc_out(hidden_states)
|
| 98 |
+
return self.dropout(hidden_states)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class MossTTSNanoGPT2Attention(nn.Module):
|
| 102 |
+
def __init__(self, config: GPT2Config, layer_idx: int, attn_implementation: str) -> None:
|
| 103 |
+
super().__init__()
|
| 104 |
+
hidden_size = int(config.hidden_size)
|
| 105 |
+
num_heads = int(config.num_attention_heads)
|
| 106 |
+
if hidden_size % num_heads != 0:
|
| 107 |
+
raise ValueError(f"hidden_size={hidden_size} must be divisible by num_attention_heads={num_heads}")
|
| 108 |
+
|
| 109 |
+
self.num_heads = num_heads
|
| 110 |
+
self.head_dim = hidden_size // num_heads
|
| 111 |
+
self.embed_dim = hidden_size
|
| 112 |
+
self.layer_idx = layer_idx
|
| 113 |
+
self.attn_implementation = attn_implementation
|
| 114 |
+
self.attn_dropout = float(config.attn_pdrop)
|
| 115 |
+
self.resid_dropout = nn.Dropout(config.resid_pdrop)
|
| 116 |
+
self.scale_attn_weights = bool(getattr(config, "scale_attn_weights", True))
|
| 117 |
+
self.scale_attn_by_inverse_layer_idx = bool(getattr(config, "scale_attn_by_inverse_layer_idx", False))
|
| 118 |
+
self.position_embedding_type = str(getattr(config, "position_embedding_type", "absolute")).lower()
|
| 119 |
+
if self.position_embedding_type not in {"absolute", "rope"}:
|
| 120 |
+
raise ValueError(f"Unsupported position_embedding_type={self.position_embedding_type!r}")
|
| 121 |
+
|
| 122 |
+
self.c_attn = nn.Linear(hidden_size, 3 * hidden_size)
|
| 123 |
+
self.c_proj = nn.Linear(hidden_size, hidden_size)
|
| 124 |
+
self.rotary_emb = None
|
| 125 |
+
if self.position_embedding_type == "rope":
|
| 126 |
+
self.rotary_emb = MossTTSNanoGPT2RotaryEmbedding(
|
| 127 |
+
self.head_dim,
|
| 128 |
+
base=float(getattr(config, "rope_base", 10000.0)),
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
def _split_heads(self, tensor: torch.Tensor) -> torch.Tensor:
|
| 132 |
+
if tensor.ndim == 3:
|
| 133 |
+
batch_size, seq_len, _ = tensor.shape
|
| 134 |
+
return tensor.view(batch_size, seq_len, self.num_heads, self.head_dim)
|
| 135 |
+
if tensor.ndim == 2:
|
| 136 |
+
total_tokens, _ = tensor.shape
|
| 137 |
+
return tensor.view(total_tokens, self.num_heads, self.head_dim)
|
| 138 |
+
raise ValueError(f"Unsupported tensor rank for attention split: {tensor.ndim}")
|
| 139 |
+
|
| 140 |
+
def _merge_heads(self, tensor: torch.Tensor) -> torch.Tensor:
|
| 141 |
+
if tensor.ndim == 4:
|
| 142 |
+
batch_size, seq_len, _, _ = tensor.shape
|
| 143 |
+
return tensor.reshape(batch_size, seq_len, self.embed_dim)
|
| 144 |
+
if tensor.ndim == 3:
|
| 145 |
+
total_tokens, _, _ = tensor.shape
|
| 146 |
+
return tensor.reshape(total_tokens, self.embed_dim)
|
| 147 |
+
raise ValueError(f"Unsupported tensor rank for attention merge: {tensor.ndim}")
|
| 148 |
+
|
| 149 |
+
def _causal_attention_mask(
|
| 150 |
+
self,
|
| 151 |
+
attention_mask: Optional[torch.Tensor],
|
| 152 |
+
query_length: int,
|
| 153 |
+
key_length: int,
|
| 154 |
+
device: torch.device,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
query_positions = torch.arange(query_length, device=device, dtype=torch.long)
|
| 157 |
+
query_positions = query_positions + max(key_length - query_length, 0)
|
| 158 |
+
key_positions = torch.arange(key_length, device=device, dtype=torch.long)
|
| 159 |
+
causal = key_positions.unsqueeze(0) <= query_positions.unsqueeze(1)
|
| 160 |
+
causal = causal.unsqueeze(0).unsqueeze(0)
|
| 161 |
+
if attention_mask is None:
|
| 162 |
+
return causal
|
| 163 |
+
key_mask = attention_mask[:, None, None, :].to(dtype=torch.bool)
|
| 164 |
+
return causal & key_mask
|
| 165 |
+
|
| 166 |
+
def _eager_attention(
|
| 167 |
+
self,
|
| 168 |
+
query: torch.Tensor,
|
| 169 |
+
key: torch.Tensor,
|
| 170 |
+
value: torch.Tensor,
|
| 171 |
+
attention_mask: Optional[torch.Tensor],
|
| 172 |
+
) -> torch.Tensor:
|
| 173 |
+
query = query.transpose(1, 2)
|
| 174 |
+
key = key.transpose(1, 2)
|
| 175 |
+
value = value.transpose(1, 2)
|
| 176 |
+
|
| 177 |
+
scale = 1.0
|
| 178 |
+
if self.scale_attn_weights:
|
| 179 |
+
scale /= self.head_dim ** 0.5
|
| 180 |
+
if self.scale_attn_by_inverse_layer_idx:
|
| 181 |
+
scale /= float(self.layer_idx + 1)
|
| 182 |
+
|
| 183 |
+
scores = torch.matmul(query, key.transpose(-1, -2)) * scale
|
| 184 |
+
causal_mask = self._causal_attention_mask(
|
| 185 |
+
attention_mask=attention_mask,
|
| 186 |
+
query_length=query.shape[-2],
|
| 187 |
+
key_length=key.shape[-2],
|
| 188 |
+
device=query.device,
|
| 189 |
+
)
|
| 190 |
+
scores = scores.masked_fill(~causal_mask, torch.finfo(scores.dtype).min)
|
| 191 |
+
probs = torch.softmax(scores, dim=-1)
|
| 192 |
+
if self.training and self.attn_dropout > 0:
|
| 193 |
+
probs = torch.dropout(probs, self.attn_dropout, train=True)
|
| 194 |
+
output = torch.matmul(probs, value)
|
| 195 |
+
return output.transpose(1, 2).contiguous()
|
| 196 |
+
|
| 197 |
+
def _sdpa_attention(
|
| 198 |
+
self,
|
| 199 |
+
query: torch.Tensor,
|
| 200 |
+
key: torch.Tensor,
|
| 201 |
+
value: torch.Tensor,
|
| 202 |
+
attention_mask: Optional[torch.Tensor],
|
| 203 |
+
) -> torch.Tensor:
|
| 204 |
+
query = query.transpose(1, 2)
|
| 205 |
+
key = key.transpose(1, 2)
|
| 206 |
+
value = value.transpose(1, 2)
|
| 207 |
+
mask = None
|
| 208 |
+
if attention_mask is not None or query.shape[-2] != key.shape[-2]:
|
| 209 |
+
mask = self._causal_attention_mask(
|
| 210 |
+
attention_mask=attention_mask,
|
| 211 |
+
query_length=query.shape[-2],
|
| 212 |
+
key_length=key.shape[-2],
|
| 213 |
+
device=query.device,
|
| 214 |
+
)
|
| 215 |
+
output = torch.nn.functional.scaled_dot_product_attention(
|
| 216 |
+
query,
|
| 217 |
+
key,
|
| 218 |
+
value,
|
| 219 |
+
attn_mask=mask,
|
| 220 |
+
dropout_p=self.attn_dropout if self.training else 0.0,
|
| 221 |
+
is_causal=mask is None,
|
| 222 |
+
)
|
| 223 |
+
return output.transpose(1, 2).contiguous()
|
| 224 |
+
|
| 225 |
+
def _flash_attention(
|
| 226 |
+
self,
|
| 227 |
+
query: torch.Tensor,
|
| 228 |
+
key: torch.Tensor,
|
| 229 |
+
value: torch.Tensor,
|
| 230 |
+
attention_mask: Optional[torch.Tensor],
|
| 231 |
+
packed_metadata: Optional[PackedSequenceMetadata],
|
| 232 |
+
) -> torch.Tensor:
|
| 233 |
+
if not _FLASH_ATTN_AVAILABLE:
|
| 234 |
+
raise ImportError("flash_attn is not installed, but attn_implementation='flash_attention_2' was requested.")
|
| 235 |
+
if query.device.type != "cuda":
|
| 236 |
+
raise ValueError("flash_attention_2 requires CUDA tensors.")
|
| 237 |
+
if query.dtype not in (torch.float16, torch.bfloat16):
|
| 238 |
+
raise ValueError(
|
| 239 |
+
f"flash_attention_2 requires fp16/bf16 tensors, but received dtype={query.dtype}."
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
dropout_p = self.attn_dropout if self.training else 0.0
|
| 243 |
+
if packed_metadata is not None:
|
| 244 |
+
if packed_metadata.indices is not None:
|
| 245 |
+
query = query.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices)
|
| 246 |
+
key = key.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices)
|
| 247 |
+
value = value.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices)
|
| 248 |
+
output = flash_attn_varlen_func(
|
| 249 |
+
query,
|
| 250 |
+
key,
|
| 251 |
+
value,
|
| 252 |
+
packed_metadata.cu_seqlens,
|
| 253 |
+
packed_metadata.cu_seqlens,
|
| 254 |
+
packed_metadata.max_seqlen,
|
| 255 |
+
packed_metadata.max_seqlen,
|
| 256 |
+
dropout_p=dropout_p,
|
| 257 |
+
causal=True,
|
| 258 |
+
)
|
| 259 |
+
if packed_metadata.indices is None:
|
| 260 |
+
return output
|
| 261 |
+
return pad_input(
|
| 262 |
+
output,
|
| 263 |
+
packed_metadata.indices,
|
| 264 |
+
packed_metadata.batch_size,
|
| 265 |
+
packed_metadata.seq_len,
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
if attention_mask is None or bool(attention_mask.all()):
|
| 269 |
+
return flash_attn_func(
|
| 270 |
+
query,
|
| 271 |
+
key,
|
| 272 |
+
value,
|
| 273 |
+
dropout_p=dropout_p,
|
| 274 |
+
causal=True,
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
if query.shape[1] != key.shape[1]:
|
| 278 |
+
query_attention_mask = attention_mask[:, -query.shape[1] :]
|
| 279 |
+
unpadded_query, query_indices, cu_seqlens_q, max_seqlen_q, _ = unpad_input(
|
| 280 |
+
query,
|
| 281 |
+
query_attention_mask,
|
| 282 |
+
)
|
| 283 |
+
unpadded_key, _, cu_seqlens_k, max_seqlen_k, _ = unpad_input(key, attention_mask)
|
| 284 |
+
unpadded_value, _, _, _, _ = unpad_input(value, attention_mask)
|
| 285 |
+
output = flash_attn_varlen_func(
|
| 286 |
+
unpadded_query,
|
| 287 |
+
unpadded_key,
|
| 288 |
+
unpadded_value,
|
| 289 |
+
cu_seqlens_q,
|
| 290 |
+
cu_seqlens_k,
|
| 291 |
+
max_seqlen_q,
|
| 292 |
+
max_seqlen_k,
|
| 293 |
+
dropout_p=dropout_p,
|
| 294 |
+
causal=True,
|
| 295 |
+
)
|
| 296 |
+
return pad_input(output, query_indices, query.shape[0], query.shape[1])
|
| 297 |
+
|
| 298 |
+
unpadded_query, indices, cu_seqlens, max_seqlen, _ = unpad_input(query, attention_mask)
|
| 299 |
+
unpadded_key, _, _, _, _ = unpad_input(key, attention_mask)
|
| 300 |
+
unpadded_value, _, _, _, _ = unpad_input(value, attention_mask)
|
| 301 |
+
output = flash_attn_varlen_func(
|
| 302 |
+
unpadded_query,
|
| 303 |
+
unpadded_key,
|
| 304 |
+
unpadded_value,
|
| 305 |
+
cu_seqlens,
|
| 306 |
+
cu_seqlens,
|
| 307 |
+
max_seqlen,
|
| 308 |
+
max_seqlen,
|
| 309 |
+
dropout_p=dropout_p,
|
| 310 |
+
causal=True,
|
| 311 |
+
)
|
| 312 |
+
return pad_input(output, indices, query.shape[0], query.shape[1])
|
| 313 |
+
|
| 314 |
+
def forward(
|
| 315 |
+
self,
|
| 316 |
+
hidden_states: torch.Tensor,
|
| 317 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 318 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 319 |
+
packed_metadata: Optional[PackedSequenceMetadata] = None,
|
| 320 |
+
layer_past: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
|
| 321 |
+
use_cache: bool = False,
|
| 322 |
+
) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, torch.Tensor]]]:
|
| 323 |
+
qkv = self.c_attn(hidden_states)
|
| 324 |
+
query, key, value = qkv.split(self.embed_dim, dim=-1)
|
| 325 |
+
query = self._split_heads(query)
|
| 326 |
+
key = self._split_heads(key)
|
| 327 |
+
value = self._split_heads(value)
|
| 328 |
+
|
| 329 |
+
if self.rotary_emb is not None:
|
| 330 |
+
if position_ids is None:
|
| 331 |
+
raise ValueError("position_ids must be provided when position_embedding_type='rope'.")
|
| 332 |
+
cos, sin = self.rotary_emb(
|
| 333 |
+
position_ids.to(device=query.device),
|
| 334 |
+
device=query.device,
|
| 335 |
+
dtype=query.dtype,
|
| 336 |
+
)
|
| 337 |
+
query = apply_rotary_pos_emb(query, cos, sin)
|
| 338 |
+
key = apply_rotary_pos_emb(key, cos, sin)
|
| 339 |
+
|
| 340 |
+
static_layer_past = layer_past is not None and _is_static_kv_cache_layer(layer_past)
|
| 341 |
+
if static_layer_past:
|
| 342 |
+
past_length = int(layer_past.get("length", 0))
|
| 343 |
+
new_length = past_length + int(key.shape[1])
|
| 344 |
+
key_cache = layer_past["key"]
|
| 345 |
+
value_cache = layer_past["value"]
|
| 346 |
+
if new_length > int(key_cache.shape[1]):
|
| 347 |
+
raise ValueError(
|
| 348 |
+
f"Static KV cache is too short: need {new_length}, capacity={int(key_cache.shape[1])}."
|
| 349 |
+
)
|
| 350 |
+
key_cache[:, past_length:new_length].copy_(key)
|
| 351 |
+
value_cache[:, past_length:new_length].copy_(value)
|
| 352 |
+
key = key_cache[:, :new_length]
|
| 353 |
+
value = value_cache[:, :new_length]
|
| 354 |
+
layer_past["length"] = new_length
|
| 355 |
+
elif layer_past is not None:
|
| 356 |
+
past_key, past_value = layer_past
|
| 357 |
+
key = torch.cat([past_key.to(device=key.device, dtype=key.dtype), key], dim=1)
|
| 358 |
+
value = torch.cat([past_value.to(device=value.device, dtype=value.dtype), value], dim=1)
|
| 359 |
+
|
| 360 |
+
present = layer_past if (use_cache and static_layer_past) else ((key, value) if use_cache else None)
|
| 361 |
+
|
| 362 |
+
if self.attn_implementation == "flash_attention_2":
|
| 363 |
+
attn_output = self._flash_attention(
|
| 364 |
+
query=query,
|
| 365 |
+
key=key,
|
| 366 |
+
value=value,
|
| 367 |
+
attention_mask=attention_mask,
|
| 368 |
+
packed_metadata=packed_metadata,
|
| 369 |
+
)
|
| 370 |
+
elif self.attn_implementation == "sdpa":
|
| 371 |
+
attn_output = self._sdpa_attention(
|
| 372 |
+
query=query,
|
| 373 |
+
key=key,
|
| 374 |
+
value=value,
|
| 375 |
+
attention_mask=attention_mask,
|
| 376 |
+
)
|
| 377 |
+
else:
|
| 378 |
+
attn_output = self._eager_attention(
|
| 379 |
+
query=query,
|
| 380 |
+
key=key,
|
| 381 |
+
value=value,
|
| 382 |
+
attention_mask=attention_mask,
|
| 383 |
+
)
|
| 384 |
+
|
| 385 |
+
attn_output = self._merge_heads(attn_output)
|
| 386 |
+
attn_output = self.c_proj(attn_output)
|
| 387 |
+
return self.resid_dropout(attn_output), present
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
class MossTTSNanoGPT2Block(nn.Module):
|
| 391 |
+
def __init__(self, config: GPT2Config, layer_idx: int, attn_implementation: str) -> None:
|
| 392 |
+
super().__init__()
|
| 393 |
+
hidden_size = int(config.hidden_size)
|
| 394 |
+
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
| 395 |
+
self.attn = MossTTSNanoGPT2Attention(config, layer_idx=layer_idx, attn_implementation=attn_implementation)
|
| 396 |
+
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
| 397 |
+
self.mlp = MossTTSNanoGPT2MLP(config)
|
| 398 |
+
|
| 399 |
+
def forward(
|
| 400 |
+
self,
|
| 401 |
+
hidden_states: torch.Tensor,
|
| 402 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 403 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 404 |
+
packed_metadata: Optional[PackedSequenceMetadata] = None,
|
| 405 |
+
layer_past: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
|
| 406 |
+
use_cache: bool = False,
|
| 407 |
+
) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, torch.Tensor]]]:
|
| 408 |
+
attn_output, present = self.attn(
|
| 409 |
+
self.ln_1(hidden_states),
|
| 410 |
+
attention_mask=attention_mask,
|
| 411 |
+
position_ids=position_ids,
|
| 412 |
+
packed_metadata=packed_metadata,
|
| 413 |
+
layer_past=layer_past,
|
| 414 |
+
use_cache=use_cache,
|
| 415 |
+
)
|
| 416 |
+
hidden_states = hidden_states + attn_output
|
| 417 |
+
hidden_states = hidden_states + self.mlp(self.ln_2(hidden_states))
|
| 418 |
+
return hidden_states, present
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
class MossTTSNanoGPT2Model(nn.Module):
|
| 422 |
+
def __init__(self, config: GPT2Config, attn_implementation: str = "eager") -> None:
|
| 423 |
+
super().__init__()
|
| 424 |
+
self.config = config
|
| 425 |
+
self.attn_implementation = attn_implementation
|
| 426 |
+
self.position_embedding_type = str(getattr(config, "position_embedding_type", "absolute")).lower()
|
| 427 |
+
if self.position_embedding_type not in {"absolute", "rope"}:
|
| 428 |
+
raise ValueError(f"Unsupported position_embedding_type={self.position_embedding_type!r}")
|
| 429 |
+
hidden_size = int(config.hidden_size)
|
| 430 |
+
self.wte = nn.Embedding(config.vocab_size, hidden_size)
|
| 431 |
+
self.wpe = nn.Embedding(config.n_positions, hidden_size) if self.position_embedding_type == "absolute" else nn.Identity()
|
| 432 |
+
self.drop = nn.Dropout(config.embd_pdrop)
|
| 433 |
+
self.h = nn.ModuleList(
|
| 434 |
+
[MossTTSNanoGPT2Block(config, layer_idx=index, attn_implementation=attn_implementation) for index in range(config.n_layer)]
|
| 435 |
+
)
|
| 436 |
+
self.ln_f = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
| 437 |
+
self.gradient_checkpointing = False
|
| 438 |
+
self._reset_parameters()
|
| 439 |
+
|
| 440 |
+
def _reset_parameters(self) -> None:
|
| 441 |
+
init_std = float(self.config.initializer_range)
|
| 442 |
+
for module in self.modules():
|
| 443 |
+
if isinstance(module, nn.Linear):
|
| 444 |
+
nn.init.normal_(module.weight, mean=0.0, std=init_std)
|
| 445 |
+
if module.bias is not None:
|
| 446 |
+
nn.init.zeros_(module.bias)
|
| 447 |
+
elif isinstance(module, nn.Embedding):
|
| 448 |
+
nn.init.normal_(module.weight, mean=0.0, std=init_std)
|
| 449 |
+
elif isinstance(module, nn.LayerNorm):
|
| 450 |
+
nn.init.ones_(module.weight)
|
| 451 |
+
nn.init.zeros_(module.bias)
|
| 452 |
+
|
| 453 |
+
@staticmethod
|
| 454 |
+
def _normalize_num_sequences(
|
| 455 |
+
cu_seqlens: torch.Tensor,
|
| 456 |
+
num_sequences: Optional[torch.Tensor],
|
| 457 |
+
device: torch.device,
|
| 458 |
+
) -> torch.Tensor:
|
| 459 |
+
if cu_seqlens.ndim == 1:
|
| 460 |
+
cu_seqlens = cu_seqlens.unsqueeze(0)
|
| 461 |
+
if num_sequences is None:
|
| 462 |
+
diffs = cu_seqlens[:, 1:] - cu_seqlens[:, :-1]
|
| 463 |
+
return diffs.gt(0).sum(dim=-1).to(device=device, dtype=torch.long)
|
| 464 |
+
if num_sequences.ndim == 0:
|
| 465 |
+
num_sequences = num_sequences.unsqueeze(0)
|
| 466 |
+
return num_sequences.to(device=device, dtype=torch.long)
|
| 467 |
+
|
| 468 |
+
@staticmethod
|
| 469 |
+
def _packed_segments_from_cu_seqlens(
|
| 470 |
+
cu_seqlens: torch.Tensor,
|
| 471 |
+
num_sequences: Optional[torch.Tensor],
|
| 472 |
+
device: torch.device,
|
| 473 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 474 |
+
if cu_seqlens.ndim == 1:
|
| 475 |
+
cu_seqlens = cu_seqlens.unsqueeze(0)
|
| 476 |
+
cu_seqlens = cu_seqlens.to(device=device)
|
| 477 |
+
batch_size, boundary_count = cu_seqlens.shape
|
| 478 |
+
segment_slots = boundary_count - 1
|
| 479 |
+
if segment_slots <= 0:
|
| 480 |
+
empty = torch.empty(0, dtype=torch.long, device=device)
|
| 481 |
+
return empty, empty, empty
|
| 482 |
+
|
| 483 |
+
counts = MossTTSNanoGPT2Model._normalize_num_sequences(cu_seqlens, num_sequences, device=device)
|
| 484 |
+
counts = counts.clamp(min=0, max=segment_slots)
|
| 485 |
+
segment_slots = int(counts.max().item()) if counts.numel() > 0 else 0
|
| 486 |
+
if segment_slots <= 0:
|
| 487 |
+
empty = torch.empty(0, dtype=torch.long, device=device)
|
| 488 |
+
return empty, empty, empty
|
| 489 |
+
cu_seqlens = cu_seqlens[:, : segment_slots + 1]
|
| 490 |
+
slot_ids = torch.arange(segment_slots, device=device).unsqueeze(0)
|
| 491 |
+
valid_slots = slot_ids < counts.unsqueeze(1)
|
| 492 |
+
|
| 493 |
+
starts = cu_seqlens[:, :-1].to(dtype=torch.long)
|
| 494 |
+
ends = cu_seqlens[:, 1:].to(dtype=torch.long)
|
| 495 |
+
lengths = (ends - starts).clamp_min(0)
|
| 496 |
+
lengths = torch.where(valid_slots, lengths, torch.zeros((), dtype=torch.long, device=device))
|
| 497 |
+
|
| 498 |
+
batch_ids = torch.arange(batch_size, device=device, dtype=torch.long).unsqueeze(1).expand(batch_size, segment_slots)
|
| 499 |
+
batch_ids = batch_ids.reshape(-1)
|
| 500 |
+
starts = starts.reshape(-1)
|
| 501 |
+
lengths = lengths.reshape(-1)
|
| 502 |
+
|
| 503 |
+
valid_segments = lengths.gt(0)
|
| 504 |
+
valid_count = int(valid_segments.to(dtype=torch.long).sum().item())
|
| 505 |
+
if valid_count <= 0:
|
| 506 |
+
empty = torch.empty(0, dtype=torch.long, device=device)
|
| 507 |
+
return empty, empty, empty
|
| 508 |
+
if valid_count == lengths.numel():
|
| 509 |
+
return batch_ids, starts, lengths
|
| 510 |
+
|
| 511 |
+
valid_order = torch.argsort(valid_segments.to(dtype=torch.long), descending=True, stable=True)[:valid_count]
|
| 512 |
+
return (
|
| 513 |
+
batch_ids.index_select(0, valid_order),
|
| 514 |
+
starts.index_select(0, valid_order),
|
| 515 |
+
lengths.index_select(0, valid_order),
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
@staticmethod
|
| 519 |
+
def _packed_token_indices(
|
| 520 |
+
batch_ids: torch.Tensor,
|
| 521 |
+
starts: torch.Tensor,
|
| 522 |
+
lengths: torch.Tensor,
|
| 523 |
+
seq_len: int,
|
| 524 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 525 |
+
total_tokens = int(lengths.sum().item())
|
| 526 |
+
if total_tokens <= 0:
|
| 527 |
+
empty = torch.empty(0, dtype=torch.long, device=lengths.device)
|
| 528 |
+
return empty, empty
|
| 529 |
+
|
| 530 |
+
segment_ids = torch.repeat_interleave(
|
| 531 |
+
torch.arange(lengths.numel(), device=lengths.device, dtype=torch.long),
|
| 532 |
+
lengths,
|
| 533 |
+
output_size=total_tokens,
|
| 534 |
+
)
|
| 535 |
+
segment_starts = torch.cumsum(lengths, dim=0) - lengths
|
| 536 |
+
positions = torch.arange(total_tokens, device=lengths.device, dtype=torch.long) - segment_starts[segment_ids]
|
| 537 |
+
indices = batch_ids[segment_ids] * seq_len + starts[segment_ids] + positions
|
| 538 |
+
return indices, positions
|
| 539 |
+
|
| 540 |
+
@staticmethod
|
| 541 |
+
def build_packed_position_ids(
|
| 542 |
+
attention_mask: Optional[torch.Tensor],
|
| 543 |
+
cu_seqlens: torch.Tensor,
|
| 544 |
+
num_sequences: Optional[torch.Tensor],
|
| 545 |
+
sequence_length: Optional[int] = None,
|
| 546 |
+
) -> torch.Tensor:
|
| 547 |
+
if cu_seqlens.ndim == 1:
|
| 548 |
+
cu_seqlens = cu_seqlens.unsqueeze(0)
|
| 549 |
+
batch_size = cu_seqlens.shape[0]
|
| 550 |
+
seq_len = int(sequence_length or (cu_seqlens.shape[1] - 1))
|
| 551 |
+
device = cu_seqlens.device
|
| 552 |
+
position_ids = torch.zeros((batch_size, seq_len), dtype=torch.long, device=device)
|
| 553 |
+
batch_ids, starts, lengths = MossTTSNanoGPT2Model._packed_segments_from_cu_seqlens(
|
| 554 |
+
cu_seqlens,
|
| 555 |
+
num_sequences,
|
| 556 |
+
device,
|
| 557 |
+
)
|
| 558 |
+
if lengths.numel() > 0:
|
| 559 |
+
indices, positions = MossTTSNanoGPT2Model._packed_token_indices(batch_ids, starts, lengths, seq_len)
|
| 560 |
+
position_ids.view(-1).scatter_(0, indices, positions)
|
| 561 |
+
if attention_mask is not None:
|
| 562 |
+
position_ids = position_ids * attention_mask.to(dtype=position_ids.dtype)
|
| 563 |
+
return position_ids
|
| 564 |
+
|
| 565 |
+
@staticmethod
|
| 566 |
+
def build_packed_metadata(
|
| 567 |
+
hidden_states: torch.Tensor,
|
| 568 |
+
cu_seqlens: torch.Tensor,
|
| 569 |
+
num_sequences: Optional[torch.Tensor],
|
| 570 |
+
) -> PackedSequenceMetadata:
|
| 571 |
+
if cu_seqlens.ndim == 1:
|
| 572 |
+
cu_seqlens = cu_seqlens.unsqueeze(0)
|
| 573 |
+
device = hidden_states.device
|
| 574 |
+
seq_len = hidden_states.shape[1]
|
| 575 |
+
batch_ids, starts, lengths = MossTTSNanoGPT2Model._packed_segments_from_cu_seqlens(
|
| 576 |
+
cu_seqlens,
|
| 577 |
+
num_sequences,
|
| 578 |
+
device,
|
| 579 |
+
)
|
| 580 |
+
if lengths.numel() == 0:
|
| 581 |
+
raise ValueError("cu_seqlens did not describe any non-empty packed sequences.")
|
| 582 |
+
|
| 583 |
+
indices, _ = MossTTSNanoGPT2Model._packed_token_indices(batch_ids, starts, lengths, seq_len)
|
| 584 |
+
cumulative = torch.empty(lengths.numel() + 1, dtype=torch.int32, device=device)
|
| 585 |
+
cumulative[0] = 0
|
| 586 |
+
cumulative[1:] = lengths.to(dtype=torch.int32).cumsum(dim=0)
|
| 587 |
+
return PackedSequenceMetadata(
|
| 588 |
+
cu_seqlens=cumulative,
|
| 589 |
+
max_seqlen=int(lengths.max().item()),
|
| 590 |
+
indices=indices,
|
| 591 |
+
batch_size=hidden_states.shape[0],
|
| 592 |
+
seq_len=hidden_states.shape[1],
|
| 593 |
+
)
|
| 594 |
+
|
| 595 |
+
def forward(
|
| 596 |
+
self,
|
| 597 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 598 |
+
past_key_values: Optional[tuple[tuple[torch.Tensor, torch.Tensor], ...]] = None,
|
| 599 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 600 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 601 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 602 |
+
use_cache: Optional[bool] = None,
|
| 603 |
+
output_attentions: Optional[bool] = None,
|
| 604 |
+
output_hidden_states: Optional[bool] = None,
|
| 605 |
+
return_dict: bool = True,
|
| 606 |
+
cu_seqlens: Optional[torch.Tensor] = None,
|
| 607 |
+
num_sequences: Optional[torch.Tensor] = None,
|
| 608 |
+
) -> BaseModelOutputWithPast:
|
| 609 |
+
del input_ids, output_attentions
|
| 610 |
+
|
| 611 |
+
if inputs_embeds is None:
|
| 612 |
+
raise ValueError("inputs_embeds must be provided.")
|
| 613 |
+
|
| 614 |
+
use_cache = bool(use_cache)
|
| 615 |
+
if use_cache and cu_seqlens is not None:
|
| 616 |
+
raise ValueError("use_cache=True is not supported together with cu_seqlens packing.")
|
| 617 |
+
|
| 618 |
+
hidden_states = inputs_embeds
|
| 619 |
+
query_attention_mask = None
|
| 620 |
+
if attention_mask is not None:
|
| 621 |
+
attention_mask = attention_mask.to(dtype=torch.bool, device=hidden_states.device)
|
| 622 |
+
query_attention_mask = attention_mask[:, -hidden_states.shape[1] :]
|
| 623 |
+
|
| 624 |
+
packed_metadata = None
|
| 625 |
+
if position_ids is None:
|
| 626 |
+
if cu_seqlens is not None:
|
| 627 |
+
if attention_mask is None:
|
| 628 |
+
raise ValueError("attention_mask must be provided with cu_seqlens packing.")
|
| 629 |
+
position_ids = self.build_packed_position_ids(
|
| 630 |
+
attention_mask=attention_mask,
|
| 631 |
+
cu_seqlens=cu_seqlens.to(device=hidden_states.device),
|
| 632 |
+
num_sequences=num_sequences.to(device=hidden_states.device) if num_sequences is not None else None,
|
| 633 |
+
sequence_length=hidden_states.shape[1],
|
| 634 |
+
)
|
| 635 |
+
elif attention_mask is not None:
|
| 636 |
+
position_ids = attention_mask.long().cumsum(dim=-1) - 1
|
| 637 |
+
position_ids = position_ids.masked_fill(~attention_mask, 0)
|
| 638 |
+
position_ids = position_ids[:, -hidden_states.shape[1] :]
|
| 639 |
+
else:
|
| 640 |
+
past_length = 0
|
| 641 |
+
if past_key_values is not None and len(past_key_values) > 0:
|
| 642 |
+
first_layer_past = past_key_values[0]
|
| 643 |
+
if _is_static_kv_cache_layer(first_layer_past):
|
| 644 |
+
past_length = int(first_layer_past.get("length", 0))
|
| 645 |
+
else:
|
| 646 |
+
past_length = first_layer_past[0].shape[1]
|
| 647 |
+
position_ids = torch.arange(hidden_states.shape[1], device=hidden_states.device, dtype=torch.long)
|
| 648 |
+
position_ids = position_ids + past_length
|
| 649 |
+
position_ids = position_ids.unsqueeze(0).expand(hidden_states.shape[0], -1)
|
| 650 |
+
|
| 651 |
+
if cu_seqlens is not None and self.attn_implementation == "flash_attention_2":
|
| 652 |
+
packed_metadata = self.build_packed_metadata(
|
| 653 |
+
hidden_states=hidden_states,
|
| 654 |
+
cu_seqlens=cu_seqlens.to(device=hidden_states.device),
|
| 655 |
+
num_sequences=num_sequences.to(device=hidden_states.device) if num_sequences is not None else None,
|
| 656 |
+
)
|
| 657 |
+
|
| 658 |
+
if self.position_embedding_type == "absolute":
|
| 659 |
+
hidden_states = hidden_states + self.wpe(position_ids)
|
| 660 |
+
hidden_states = self.drop(hidden_states)
|
| 661 |
+
if query_attention_mask is not None:
|
| 662 |
+
hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype)
|
| 663 |
+
|
| 664 |
+
all_hidden_states = () if output_hidden_states else None
|
| 665 |
+
presents = [] if use_cache else None
|
| 666 |
+
for layer_index, block in enumerate(self.h):
|
| 667 |
+
if output_hidden_states:
|
| 668 |
+
all_hidden_states = all_hidden_states + (hidden_states,)
|
| 669 |
+
|
| 670 |
+
if self.gradient_checkpointing and self.training:
|
| 671 |
+
if use_cache:
|
| 672 |
+
raise ValueError("use_cache=True is not supported when gradient checkpointing is enabled during training.")
|
| 673 |
+
|
| 674 |
+
def custom_forward(*inputs):
|
| 675 |
+
output, _ = block(
|
| 676 |
+
inputs[0],
|
| 677 |
+
attention_mask=inputs[1],
|
| 678 |
+
position_ids=inputs[2],
|
| 679 |
+
packed_metadata=packed_metadata,
|
| 680 |
+
layer_past=None,
|
| 681 |
+
use_cache=False,
|
| 682 |
+
)
|
| 683 |
+
return output
|
| 684 |
+
|
| 685 |
+
hidden_states = torch.utils.checkpoint.checkpoint(
|
| 686 |
+
custom_forward,
|
| 687 |
+
hidden_states,
|
| 688 |
+
attention_mask,
|
| 689 |
+
position_ids,
|
| 690 |
+
use_reentrant=False,
|
| 691 |
+
)
|
| 692 |
+
present = None
|
| 693 |
+
else:
|
| 694 |
+
hidden_states, present = block(
|
| 695 |
+
hidden_states,
|
| 696 |
+
attention_mask=attention_mask,
|
| 697 |
+
position_ids=position_ids,
|
| 698 |
+
packed_metadata=packed_metadata,
|
| 699 |
+
layer_past=None if past_key_values is None else past_key_values[layer_index],
|
| 700 |
+
use_cache=use_cache,
|
| 701 |
+
)
|
| 702 |
+
if query_attention_mask is not None:
|
| 703 |
+
hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype)
|
| 704 |
+
if presents is not None:
|
| 705 |
+
presents.append(present)
|
| 706 |
+
|
| 707 |
+
hidden_states = self.ln_f(hidden_states)
|
| 708 |
+
if query_attention_mask is not None:
|
| 709 |
+
hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype)
|
| 710 |
+
if output_hidden_states:
|
| 711 |
+
all_hidden_states = all_hidden_states + (hidden_states,)
|
| 712 |
+
|
| 713 |
+
if not return_dict:
|
| 714 |
+
return (hidden_states, tuple(presents) if presents is not None else None, all_hidden_states, None)
|
| 715 |
+
|
| 716 |
+
return BaseModelOutputWithPast(
|
| 717 |
+
last_hidden_state=hidden_states,
|
| 718 |
+
past_key_values=tuple(presents) if presents is not None else None,
|
| 719 |
+
hidden_states=all_hidden_states,
|
| 720 |
+
attentions=None,
|
| 721 |
+
)
|
guides/AGENT_GUIDE.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Agent reference — current Podcast default
|
| 2 |
+
|
| 3 |
+
1. Read the main README, [PROMPTING.md](PROMPTING.md) and [PIPELINE.md](PIPELINE.md). Current default is `spontaneous`, approved by the listener on 8 November 2026. Do not confuse it with the earlier exaggerated sentence-emotion variant.
|
| 4 |
+
2. Use the full `laion/Humanness-Voice-Podcast` checkpoint, pinned weight revision 91f8110cd815f2233301a8b1dd16dbbf3fae8e05, not the parent Large 1.7 weights and not a LoRA. Download the separately linked codec and judges.
|
| 5 |
+
3. Have an API or local LLM write literal dialogue and sparse cues as validated JSON. It is much less laborious than hand-authoring every cue. TTS itself needs no LLM API key. Preserve supplied literal text exactly.
|
| 6 |
+
4. Recommended multi-speaker episodes consist of independently conditioned turns. Keep one fixed reference per speaker. Use `--script` and `--references` for custom input. Update/remove cached codes when changing reference audio.
|
| 7 |
+
5. Temperature 1.0, CFG 2.5, top-p 0.95, top-k 50, BF16, free timing. At most 25.6 s per turn; split long text. No VC, DPO or added LoRA.
|
| 8 |
+
6. Default N4 ranks each raw candidate with .5*clip(native Ears genuineness/6)+.5*clip(ECAPA reference cosine), both clipped to [0,1]. No CE or WER factor/gate. N2 uses the same formula. N1 keeps the sole take and skips online scoring.
|
| 9 |
+
7. Audit selected output independently with Parakeet v3 and Qwen forced alignment. Qwen does not support Dutch: that language, or a failed Qwen alignment, uses explicitly labeled Parakeet TDT boundary estimates only with <=15% transcript WER and matching first/last two words. Keep 80 ms lead / 150 ms tail and fades outside words; uncertain alignment remains untrimmed and flagged. Assemble 400 ms gaps between estimated word boundaries (unverified clips fall back to clip edges). Do not silently repair strict benchmark output or turn N1 into retries.
|
| 10 |
+
8. Best-of4 English turn 6 exposed a real content omission. The public documentation distinguishes strict selection from an extra listening copy repaired with a content check. A high reward is not a complete-script guarantee.
|
| 11 |
+
9. Use `infer.py`/`optimized_pipeline/produce.py` for the current route; `infer_legacy.py` explicitly preserves native pairs, old identity×inverse-WER selection and optional VC. Use separate environments for their recorded Torch stacks.
|
| 12 |
+
10. Save every candidate, seeds, source/reference hashes and timing receipts. Do not include warmup or export in a generation-only claim, confuse candidate throughput with delivered audio, or call model scores human preference evidence. Record batch grouping and compiler-cache state.
|
| 13 |
+
|
| 14 |
+
11. Fresh EN/DE N1/N2 measurements and per-turn scores are linked from the README. N2/N1 ran in separate policy/codec phases because another GPU user started training; no unrelated process was stopped. Compare timing scope and GPU load. None of those 80 selected turns crossed the configured review threshold, but model metrics are not a guarantee.
|
| 15 |
+
12. The [public listening Space](https://huggingface.co/spaces/laion/Humanness-Voice-Podcast) adds Dutch, French and Spanish translations at N4/N2/N1. Scripts are `script-spontaneous-{en,de,nl,fr,es}.json`; pass repeated `--script` arguments, preserving explicit language codes. The same fixed reference pair is used in all five languages. Review flags are shown on the relevant sample cards. `low_memory_compare.py --n-values 4 2 1 --candidates 8` plus `finish_comparison.py` reproduces the additional run; see the pipeline guide.
|
guides/LEGACY_PIPELINE.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> Archived pre-8-November pipeline. Its rewards and defaults differ from the current recommended pipeline. See the main README and current guides.
|
| 2 |
+
|
| 3 |
+
# Episode-generation pipeline
|
| 4 |
+
|
| 5 |
+
## The complete sequence
|
| 6 |
+
|
| 7 |
+
1. Read an episode JSON with language, S1/S2 reference files and ordered literal turns. Optional cached anchor codes avoid repeated reference encoding.
|
| 8 |
+
2. Divide into one-turn or two-turn units. Keep the same voice-to-reference assignment for the complete episode.
|
| 9 |
+
3. Generate one, two, four or eight seeded candidates per unit. Microbatches can contain different units and prompts. Stop/candidate histories are separate for every row.
|
| 10 |
+
4. Commit raw generated audio to SQLite before ASR or scoring. Interrupted generated-but-unscored candidates are evaluated on resume.
|
| 11 |
+
5. Run word-timestamp ASR, match the requested literal script, require both boundaries and sufficient coverage, and reject excessive internal WER.
|
| 12 |
+
6. Rank passing candidates with identity contrast × inverse WER, or the documented alternative rewards.
|
| 13 |
+
7. Optionally split and voice-convert each intended turn, then rerun ASR and content checks on the converted output.
|
| 14 |
+
8. Optionally shorten long near-silent word gaps. This is **UNVALIDATED** and off by default.
|
| 15 |
+
9. Keep 80-ms pre-word / 150-ms post-word guards and gentle outside-word fades.
|
| 16 |
+
10. Assemble lossless waveforms with an exact configurable word-clock gap. Write episode.wav, its timeline, the SQLite audit journal and a continuously updated listening.html.
|
| 17 |
+
|
| 18 |
+
## Commands
|
| 19 |
+
|
| 20 |
+
~~~bash
|
| 21 |
+
# Fastest: one take per individual speaker turn, content-only selection.
|
| 22 |
+
python infer.py --model . --script examples/episode_en.json \
|
| 23 |
+
--output runs/one-take --mode turns --candidates 1 --batch-size 2 \
|
| 24 |
+
--reward asr
|
| 25 |
+
|
| 26 |
+
# Two candidates per unit, default EARS identity contrast and inverse WER.
|
| 27 |
+
python infer.py --model . --script examples/episode_de.json \
|
| 28 |
+
--output runs/two-takes --mode pairs --candidates 2 --batch-size 2
|
| 29 |
+
|
| 30 |
+
# Extra pleasantness preference. This combined recipe is proposed/unvalidated.
|
| 31 |
+
python infer.py --model . --script examples/episode_en.json \
|
| 32 |
+
--output runs/pleasant --mode turns --candidates 2 --batch-size 2 \
|
| 33 |
+
--reward pleasant --compact-pauses --pause-keep 0.18
|
| 34 |
+
~~~
|
| 35 |
+
|
| 36 |
+
Use a separate output directory for a different recipe, prompt, reference or seed. The contract binds script contents, reference bytes/cached codes, runtime code and model snapshot. Resume the **same command** to reuse existing raw candidates and selected units. A different contract fails explicitly. Partial ASR/scoring interruption does not turn a runtime error into a content rejection.
|
| 37 |
+
|
| 38 |
+
## ASR interface
|
| 39 |
+
|
| 40 |
+
The built-in default is moondream/parakeet-redux, revision 2bf128600aac4b16946f7ed8372e56117fe5e23b in the source evaluation. The package uses Photon word timestamps. For a downloaded model, pass --asr-model-path /models/parakeet-redux. CPU is the portable default; --asr-device cuda uses GPU Photon.
|
| 41 |
+
|
| 42 |
+
No official multilingual Parakeet V4 ID was verified. To use your own ASR process, put its argument vector in a JSON file:
|
| 43 |
+
|
| 44 |
+
~~~json
|
| 45 |
+
["/opt/asr-env/bin/python", "/opt/asr/transcribe_json.py"]
|
| 46 |
+
~~~
|
| 47 |
+
|
| 48 |
+
Then pass --asr-command asr_command.json. The process receives JSON on stdin:
|
| 49 |
+
|
| 50 |
+
~~~json
|
| 51 |
+
{"audio": "/absolute/current.wav", "timestamps": "word"}
|
| 52 |
+
~~~
|
| 53 |
+
|
| 54 |
+
It must return one JSON object on stdout:
|
| 55 |
+
|
| 56 |
+
~~~json
|
| 57 |
+
{
|
| 58 |
+
"text": "Hello world",
|
| 59 |
+
"segments": [
|
| 60 |
+
{"words": [
|
| 61 |
+
{"word": "Hello", "start": 0.12, "end": 0.51},
|
| 62 |
+
{"word": "world", "start": 0.55, "end": 0.94}
|
| 63 |
+
]}
|
| 64 |
+
]
|
| 65 |
+
}
|
| 66 |
+
~~~
|
| 67 |
+
|
| 68 |
+
Times must already be global seconds in that waveform. Write process logs to stderr, not stdout. Untimed, nonfinite, backwards or out-of-range words cause failure rather than invented timestamps. The command is executed as an argument vector, without a shell.
|
| 69 |
+
|
| 70 |
+
## Content QA and rewards
|
| 71 |
+
|
| 72 |
+
Default maximum WER is .30 and per-turn token coverage is .80, with both outer script endpoints required. Strict QA also needs at least two matched tokens per turn; one-word backchannels are currently rejected. Adjust these only as an explicit new recipe; inspect their effect on your languages and ASR. A boundary check can fail because ASR missed a real word. Conversely, low WER cannot establish good acting or correct identity.
|
| 73 |
+
|
| 74 |
+
Identity reward uses mean intended-reference cosine C and a positive own-versus-other contrast M, normalized by the separation of the two references. Its speaker-balanced definition prevents a long speaker dominating a short one. Missing short/unscorable identity windows contribute zero. Windows below 1.2 seconds cannot be reliably scored here; longer identity windows are limited to twelve seconds.
|
| 75 |
+
|
| 76 |
+
The optional pleasant recipe gives .4 C, .2 M, .2 normalized Genuineness and .2 actual Audiobox CE, multiplied by 1/(1+WER). It uses fixed EARS training normalization, not a min/max fitted independently to each candidate group. Candidate scoring precedes waveform edits and optional VC; it is not a reported score for the final episode.
|
| 77 |
+
|
| 78 |
+
## Internal quiet-pause compaction
|
| 79 |
+
|
| 80 |
+
--compact-pauses is **UNVALIDATED**. Eligible gaps are at least --pause-trigger seconds, normally .8. Each gap is tested against both --quiet-dbfs, normally -45 dBFS, and a threshold 30 dB below the median recognized-speech RMS. The more conservative threshold wins. A gap is shortened only if every tested 10-ms frame of the removal region is quiet.
|
| 81 |
+
|
| 82 |
+
Retained word-to-word pause is --pause-keep, between .15 and .20 seconds. A 10-ms cosine-squared/sine-squared crossfade is placed within the quiet material. Its gains sum to one, avoiding a correlated-background gain bump. Splices stay inside the quiet region, away from word boundaries. Recognized-word samples are checked for equality after the splice; timestamps are updated.
|
| 83 |
+
|
| 84 |
+
This is not a speech/nonverbal classifier. Faint missed speech, breaths, laughter or a deliberately dramatic pause can still be harmed. Disable compaction to keep those timings. The outer tail cut can also remove intentional wordless sounds after the final scripted word.
|
| 85 |
+
|
| 86 |
+
## Optional per-turn conversion
|
| 87 |
+
|
| 88 |
+
Install the complete Humaneness Voice Converter in a separate environment following its own README, then pass --vc-model-dir and --vc-python. The converter receives generated speech as source and the matching S1/S2 anchor as target. It uses source emotion/style and target speaker/timbre conditioning internally.
|
| 89 |
+
|
| 90 |
+
For two-turn blocks, buffer each aligned turn by up to 100 ms and cap the buffers at neighboring-turn midpoints. Overlapping or insufficiently aligned speech is not silently assigned a speaker. Re-transcribe every converted output and apply the same content QA. A converted voice is a proposed output, not a certified identity match.
|
| 91 |
+
|
| 92 |
+
Conversion can change duration. The pipeline rebuilds a chronological timeline rather than blindly reusing old offsets, then applies edge fades and the chosen final gap. The conversion route is experimental and was not included in the earlier editing-grid scores.
|
| 93 |
+
|
| 94 |
+
## Batching and throughput
|
| 95 |
+
|
| 96 |
+
Use BF16 with the provided SDPA path. --batch-size counts independent generation requests, not entire episodes. Different speaker turns, texts and candidate seeds can share a batch. Each row retains its own RNG; fixed seeds still need fixed hardware, software, precision and batch grouping for reproducibility.
|
| 97 |
+
|
| 98 |
+
One candidate is cheapest. Two candidates roughly double acoustic generation work; batching may amortize it. Larger pools additionally require more ASR/EARS work. This package makes no fresh cross-hardware speed claim. Lower the batch size if model + codec + judge memory is too large. Optional VC is a separate phase, after releasing the TTS model from GPU memory; it can run in another environment.
|
| 99 |
+
|
| 100 |
+
Use the same prepared reference codes and loaded model for the complete run. The default does not repeatedly download or reload the TTS model per turn. The raw SQLite journal avoids thousands of separate candidate files; selected unit WAVs and four rolling raw previews support the listening page.
|
| 101 |
+
|
| 102 |
+
The optional VC adapter currently invokes the converter CLI once per turn, so its models are reloaded for each conversion. Budget additional latency for this prototype path; the heterogeneous batching described above applies to TTS generation. A persistent converter service is a future optimization, not a measured feature of this release.
|
guides/LEGACY_README.md
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> Archived pre-8-November pipeline. Its rewards and defaults differ from the current recommended pipeline. See the main README and current guides.
|
| 2 |
+
|
| 3 |
+
---
|
| 4 |
+
license: cc-by-4.0
|
| 5 |
+
language: [en, de, fr, es, nl]
|
| 6 |
+
pipeline_tag: text-to-speech
|
| 7 |
+
library_name: transformers
|
| 8 |
+
base_model: laion/Humaneness-Voice-Large-1.7
|
| 9 |
+
tags: [podcast, multi-speaker, voice-cloning, expressive-speech, custom-code, research]
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Humanness Voice Podcast
|
| 13 |
+
|
| 14 |
+
Generate a scripted podcast using two reference voices. This is the complete real-podcast full fine-tune of **Humaneness Voice Large 1.7 at update 9,874**, heard in our segment/block listening experiment. It includes the full model weights, portable inference and training code, example reference voices, and a pipeline for turning separately generated speech into longer episodes.
|
| 15 |
+
|
| 16 |
+
**[Listen: raw, trimmed and reassembled conversations](https://laion-humaneness-podcast-segment-block-pilot.static.hf.space/editing/index.html)** · **[Original measured comparison](https://laion-humaneness-podcast-segment-block-pilot.static.hf.space/index.html)** · **[Single-speaker prompting guide](../guides/SINGLE_SPEAKER.md)** · **[Multi-speaker prompting guide](../guides/MULTI_SPEAKER.md)** · **[Pipeline guide](../guides/PIPELINE.md)** · **[Training and provenance](../TRAINING.md)** · **[Agent reference](../guides/AGENT_GUIDE.md)**
|
| 17 |
+
|
| 18 |
+
Model weights and newly authored LAION release materials: **CC BY 4.0**. This is a full checkpoint; no LoRA needs to be merged. The external MOSS audio codec, upstream software and reference recordings retain their own terms. See [NOTICE.md](../NOTICE.md).
|
| 19 |
+
|
| 20 |
+
## What it does
|
| 21 |
+
|
| 22 |
+
You supply the dialogue, its speaker labels and emotion directions, plus one clean reference recording for each voice. The model produces speech; the surrounding pipeline checks the spoken words, trims unused audio and assembles the episode. It does not write the podcast text.
|
| 23 |
+
|
| 24 |
+
Two generation paths are included:
|
| 25 |
+
|
| 26 |
+
| Path | Generation unit | Why use it? | Main limitation |
|
| 27 |
+
|---|---|---|---|
|
| 28 |
+
| **turns** | One speaker's turn per call, with only that person's anchor | More direct control over the intended identity; easy to replace an individual turn | Separately generated turns may feel less acoustically connected |
|
| 29 |
+
| **pairs** | Two consecutive turns together, normally S1 then S2, with both anchors | Preserves the local interaction and one speaker change within a generation | The two identities can still merge or switch incorrectly |
|
| 30 |
+
|
| 31 |
+
Both paths can batch **different texts, prompts, speakers and candidate seeds**. Use one candidate for speed or two candidates for a small selection pool; four and eight are also supported. A run keeps every raw candidate and its acceptance/rejection record in a SQLite journal. Prefer a downloaded local model directory or pin --revision when resuming: a later repository documentation commit can otherwise resolve to a different snapshot.
|
| 32 |
+
|
| 33 |
+
The model was trained on English, German, French, Spanish and Dutch material. The current detailed reference/segment comparison covers **English and German**, using the same listener-approved German voices for both. Do not assume equally established quality in all five languages.
|
| 34 |
+
|
| 35 |
+
## Quick start
|
| 36 |
+
|
| 37 |
+
Use a CUDA-capable machine. The recorded software stack is Python 3.13.5, PyTorch 2.9.1 and Transformers 5.14.1; install matching CUDA Torch and torchaudio 2.9.1 builds appropriate for your machine first. The new portable driver has CPU contract tests; consult [VALIDATION.md](../VALIDATION.md) for the exact GPU test status and known limits.
|
| 38 |
+
|
| 39 |
+
The quick-start commands below enable **unvalidated automatic quiet-pause trimming** as an experiment. Omit --compact-pauses to retain internal timing.
|
| 40 |
+
|
| 41 |
+
~~~bash
|
| 42 |
+
hf download laion/Humanness-Voice-Podcast --local-dir humanness-podcast
|
| 43 |
+
cd humanness-podcast
|
| 44 |
+
python -m pip install -r requirements.txt
|
| 45 |
+
|
| 46 |
+
# One referenced speaker per generation; two seeded candidates per turn.
|
| 47 |
+
python infer.py --model . \
|
| 48 |
+
--script examples/short_en.json --output runs/en-turns \
|
| 49 |
+
--mode turns --candidates 2 --batch-size 2 --compact-pauses
|
| 50 |
+
|
| 51 |
+
# Generate two turns at once, then assemble the chosen blocks.
|
| 52 |
+
python infer.py --model . \
|
| 53 |
+
--script examples/episode_de.json --output runs/de-pairs \
|
| 54 |
+
--mode pairs --candidates 2 --batch-size 2 --compact-pauses
|
| 55 |
+
~~~
|
| 56 |
+
|
| 57 |
+
Open runs/en-turns/listening.html or runs/de-pairs/listening.html. It updates as candidates and selected units appear. The completed episode is episode.wav; episode_timeline.json records the assembly clock.
|
| 58 |
+
|
| 59 |
+
The script examples use the same two German voices as the listening experiment. [Four short input references and exact cached MOSS anchor codes](../examples/references/) are included: a German pair and the earlier English pair. The English and German example scripts deliberately share the German anchors. Edit their references and reference_codes fields to use the English pair, or remove reference_codes when supplying new audio. Use clean, single-speaker references of approximately **5–15 seconds**; the codec limit is 63–187 frames, or 5.04–14.96 seconds. The old three-second truncation is not used.
|
| 60 |
+
|
| 61 |
+
The speech decoder uses the separately downloaded [MOSS Audio Tokenizer v2](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2). The package downloads it when needed. Pass --codec /local/codec to reuse an existing copy. The repository is self-contained for the TTS weights and code; codec and optional scoring/conversion dependencies are separate downloads.
|
| 62 |
+
|
| 63 |
+
## Automatic trimming and episode assembly
|
| 64 |
+
|
| 65 |
+
The default speech cut keeps up to **80 ms before the first verified script word and 150 ms after the last**. Gentle fades of up to 20 ms at the start and 50 ms at the end stay outside those words. The first/last **requested script words** define the cut, rather than any later ASR words from mumbling.
|
| 66 |
+
|
| 67 |
+
Selected units are assembled from lossless 48-kHz masters and encoded/written as one episode. The default **400-ms word-to-word gap** includes retained tail/head padding. It does not add another 400 ms on top of those guards.
|
| 68 |
+
|
| 69 |
+
**Additional long-pause trimming is experimental and UNVALIDATED.** Enable it explicitly:
|
| 70 |
+
|
| 71 |
+
~~~bash
|
| 72 |
+
python infer.py --model . \
|
| 73 |
+
--script examples/episode_en.json --output runs/en-quiet-trim \
|
| 74 |
+
--mode turns --candidates 2 --batch-size 2 \
|
| 75 |
+
--tail-ms 150 --compact-pauses --pause-trigger 0.8 --pause-keep 0.18
|
| 76 |
+
~~~
|
| 77 |
+
|
| 78 |
+
This option tests gaps between recognized words, preserves the words themselves, and shortens only long gaps that are quiet under both an absolute and a relative RMS threshold. Eligible pauses become approximately **180 ms**, with a 10-ms constant-gain crossfade to avoid a background-level bump. It can still remove faint speech or intentional nonverbal sounds that ASR misses. It has signal-level tests but no listening validation. Keep it off when preserving those details is more important than shortening pauses.
|
| 79 |
+
|
| 80 |
+
The listening grid demonstrates the earlier script-boundary editing workflow, **not this new internal-pause compaction**. Its diagnostic examples compare 100/150/200-ms tails. Source quality and WER values are not copied onto newly edited outputs.
|
| 81 |
+
|
| 82 |
+
## Candidate selection and content checks
|
| 83 |
+
|
| 84 |
+
These rewards select fixed-model candidates; the episode CLI does not update model weights. The default reward is:
|
| 85 |
+
|
| 86 |
+
~~~text
|
| 87 |
+
R = (0.5 × C + 0.5 × M) / (1 + WER)
|
| 88 |
+
|
| 89 |
+
C: speaker-balanced mean cosine to the intended EARS identity embedding.
|
| 90 |
+
M: positive own-reference versus other-reference contrast, normalized by
|
| 91 |
+
max(1 - reference_cross_cosine, 0.05), then clipped to [0,1].
|
| 92 |
+
~~~
|
| 93 |
+
|
| 94 |
+
Missing/scant aligned speech contributes zero to identity ranking. Speaker labels are never swapped to improve the score. The inverse-WER factor is bounded at zero WER.
|
| 95 |
+
|
| 96 |
+
Before ranking, candidates must have verified script endpoints, sufficient literal-word coverage for every intended turn, and WER below the configured threshold inside the kept script span. Extra post-script material is removed; hallucinated words **inside** the kept span still count as errors. If no candidate passes, the pipeline stops with its journal and listening page intact. It does not silently publish an incomplete episode or loosen QA.
|
| 97 |
+
|
| 98 |
+
--reward pleasant adds a **proposed, unvalidated** pleasantness preference:
|
| 99 |
+
|
| 100 |
+
~~~text
|
| 101 |
+
R = (0.4 × C + 0.2 × M + 0.2 × Genuine_CDF + 0.2 × CE_CDF) / (1 + WER)
|
| 102 |
+
~~~
|
| 103 |
+
|
| 104 |
+
Genuine and actual Audiobox CE use the fixed EARS training-distribution Gaussian-CDF normalization. --reward asr skips identity scoring and ranks by 1/(1+WER); it is a faster content-only option, not an identity check.
|
| 105 |
+
|
| 106 |
+
The default judge is [Humaneness EARS Medium](https://huggingface.co/laion/humaneness-ears-base-medium). Identity ranking uses its 250-dimensional identity embedding; the separate 128-dimensional timbre embedding should not be confused with it.
|
| 107 |
+
|
| 108 |
+
## ASR: exact implementation
|
| 109 |
+
|
| 110 |
+
The verified default is **[Parakeet Redux](https://huggingface.co/moondream/parakeet-redux)** through Photon, with actual word timestamps. Its timestamps are already global; the code does not add segment offsets twice.
|
| 111 |
+
|
| 112 |
+
An official multilingual “Parakeet V4” checkpoint could not be verified when this package was prepared. We do not invent a model ID or call Redux “V4.” Use --asr-model for a compatible Photon model or --asr-command asr_command.json for a separately installed ASR service. The command receives JSON on stdin and must return text plus globally timestamped words under segments[].words[]. See [the pipeline guide](../guides/PIPELINE.md). --asr-model-path supports an already downloaded Photon checkpoint.
|
| 113 |
+
|
| 114 |
+
## Optional identity conversion
|
| 115 |
+
|
| 116 |
+
The [Humaneness Voice Converter](https://huggingface.co/laion/Humaneness-Voice-Converter) can be used as an **experimental optional post-processing stage**:
|
| 117 |
+
|
| 118 |
+
~~~bash
|
| 119 |
+
# Install the converter separately, following its own README.
|
| 120 |
+
hf download laion/Humaneness-Voice-Converter --local-dir ../humaneness-vc
|
| 121 |
+
|
| 122 |
+
python infer.py --model . \
|
| 123 |
+
--script examples/short_en.json --output runs/en-pairs-vc \
|
| 124 |
+
--mode pairs --candidates 2 --batch-size 2 \
|
| 125 |
+
--vc-model-dir ../humaneness-vc \
|
| 126 |
+
--vc-python /path/to/converter-venv/bin/python \
|
| 127 |
+
--vc-buffer-ms 100 --tail-ms 150
|
| 128 |
+
~~~
|
| 129 |
+
|
| 130 |
+
For a paired block, ASR locates each intended turn. Up to 100 ms of source buffer is kept on either side, limited by the midpoint to neighboring turns. Each speaker is converted separately to **their own** reference. Converted speech is resampled, transcribed again, checked against its intended text, trimmed and reassembled in chronological order. Source turns must fit the converter's tested 1–30-second range. An unsafe split or failed post-conversion check stops the run.
|
| 131 |
+
|
| 132 |
+
Conversion can change duration and expression; this route does not pretend that converted audio can always be pasted back at the old sample offsets. It rebuilds the timeline with controlled word-clock gaps and edge fades. It has not been validated end to end as a podcast production workflow and does not guarantee correct identity.
|
| 133 |
+
|
| 134 |
+
## Training, architecture and reproducibility
|
| 135 |
+
|
| 136 |
+
This is a full supervised fine-tune of [Humaneness Voice Large 1.7](https://huggingface.co/laion/Humaneness-Voice-Large-1.7). The MOSS Local Transformer architecture uses a Qwen-family semantic backbone and a local acoustic transformer to generate twelve codec channels per 80-ms frame. The external MOSS-v2 decoder renders 48-kHz mono speech.
|
| 137 |
+
|
| 138 |
+
Training used two epochs, approximately 80% real-podcast / 20% original-format acting replay by presentation count, 32 GPUs and 9,874 optimizer updates. Peak learning rates were 2e-5 for the semantic backbone and 4e-5 for the local/acoustic parameters; 5% warm-up followed by cosine decay to 10% of the peak. The final measured development loss was **4.4653**, versus **4.5745** before training. This alone is not a human listening-quality result.
|
| 139 |
+
|
| 140 |
+
See [TRAINING.md](../TRAINING.md) for the actual counts, data contract, original frozen launch archive, portable continued-SFT entry point and checkpoint/resume instructions. This repository contains deployment weights, not the 32-rank historical optimizer state. Starting a new fine-tune from these BF16 weights is not an exact optimizer resume of the completed training run.
|
| 141 |
+
|
| 142 |
+
## Earlier measured segment/block comparison
|
| 143 |
+
|
| 144 |
+
These values describe the earlier fixed-400-ms, natural-timing montages with ten scripts per condition. They do not score the new internal-pause compaction or optional VC. The parent1.7 block wrapper was exploratory. All automatic metrics come from EARS Medium and Parakeet Redux, not a blinded listening study.
|
| 145 |
+
|
| 146 |
+
| Model | Generation | N | CE ↑ | Genuine ↑ | Intended identity cosine ↑ | WER ↓ |
|
| 147 |
+
|---|---|---:|---:|---:|---:|---:|
|
| 148 |
+
|Parent Large 1.7|segment|1|5.872|3.614|0.477|34.7%|
|
| 149 |
+
|Parent Large 1.7|block|8|6.103|3.719|0.429|10.2%|
|
| 150 |
+
|Podcast step 9874|segment|1|6.051|3.532|0.560|37.4%|
|
| 151 |
+
|Podcast step 9874|block|8|6.328|3.373|0.730|2.2%|
|
| 152 |
+
|
| 153 |
+
Speaker cosine averages available aligned turns. Missing-coverage effects and per-metric denominators remain in the full [summary](../stats/segment_block_pilot.json). A higher cosine is not a percentage of correct identity.
|
| 154 |
+
|
| 155 |
+
## Examples and prompting references
|
| 156 |
+
|
| 157 |
+
- [English episode example](../examples/episode_en.json), [German episode example](../examples/episode_de.json), and shorter two-turn examples.
|
| 158 |
+
- [Four generated, reassembled listening examples](../examples/audio/) with exact source-condition provenance.
|
| 159 |
+
- [Single-speaker GENERAL/SCRIPT guide](../guides/SINGLE_SPEAKER.md).
|
| 160 |
+
- [Multi-speaker labels, two anchors and optional past context](../guides/MULTI_SPEAKER.md).
|
| 161 |
+
- [Batching, rewards, pause editing and optional VC](../guides/PIPELINE.md).
|
| 162 |
+
- [Machine-readable recipe](../guides/recipe.json) and [compact agent guide](../guides/AGENT_GUIDE.md).
|
| 163 |
+
- [Underlying Large 1.7 manual](https://laion-humaneness-voice-large-1-7-prompting-manual.static.hf.space/) and [Large 1.7 model repository](https://huggingface.co/laion/Humaneness-Voice-Large-1.7).
|
| 164 |
+
|
| 165 |
+
## Fresh functional smoke
|
| 166 |
+
|
| 167 |
+
The corrected portable driver passed an allocated four-GPU smoke: English/German, one-turn/two-turn paths, different-text batches, two candidates per unit, ASR checks, identity ranking and assembly. [Listen to the four fresh outputs](../validation/gpu-smoke/) and read [the exact acceptance counts](../stats/gpu_smoke_details.json). This is a functional check, not a listening study; optional VC remains untested and the quiet-pause shortening remains unvalidated.
|
| 168 |
+
|
| 169 |
+
## Known limitations and intended use
|
| 170 |
+
|
| 171 |
+
Speaker identities can collapse, switches can be late, and generation can continue into silence or mumbling. Duration requests are approximate, not millisecond-accuracy guarantees. ASR-based checks and automatic identity/quality scores can be wrong. Batched generation changes floating-point arithmetic; fixed seeds do not promise bit-identical results across hardware, precision or batch size.
|
| 172 |
+
|
| 173 |
+
This is a research and dataset-augmentation release. Test your own languages, voices and content before deployment. Use reference voices with appropriate permission, disclose synthetic speech where appropriate, and do not use it for deceptive impersonation. LAION does not endorse misuse. This research guidance does not replace or amend the CC BY 4.0 license or applicable third-party rights.
|
guides/MULTI_SPEAKER.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Multi-speaker episodes and native two-speaker blocks
|
| 2 |
+
|
| 3 |
+
## Recommended: independently generated turns, assembled into a conversation
|
| 4 |
+
|
| 5 |
+
Write ordered JSON turns with stable `speaker` IDs and separate literal text/directions. The optimized pipeline routes each turn to its matching fixed reference. Different speakers and scripts can share a GPU batch; each candidate keeps its own history. Winners are assembled with a 400-ms word-to-word gap after conservative boundary alignment/fades. This is the listener-approved, measured route.
|
| 6 |
+
|
| 7 |
+
Use the [prompting/JSON guide](PROMPTING.md) and [pipeline guide](PIPELINE.md). The supplied German and English episodes share the same S1/S2 references. The labels are routing IDs, not spoken words. N counts candidates per turn, while groups counts independent turns per policy batch: N4×groups2 gives eight candidate streams plus their CFG branches.
|
| 8 |
+
|
| 9 |
+
## Preserved alternative: native S1/S2 two-turn generation
|
| 10 |
+
|
| 11 |
+
`infer_legacy.py --mode pairs` uses the trained native two-speaker wrapper. It is an explicit alternative with different rewards, ASR checks and software requirements; its timings are not the optimized table. See [the legacy pipeline](LEGACY_PIPELINE.md). Recommended sparse style cues also work as instructions, but the new quality comparison evaluates independent turns.
|
| 12 |
+
|
| 13 |
+
```text
|
| 14 |
+
GENERAL: A warm, authentic and genuine conversation. Keep S1 and S2 distinct, each matching its own reference. Speak only the scripted words, naturally, gently and spontaneously.
|
| 15 |
+
SCRIPT:
|
| 16 |
+
(speaker=S1, curious; relaxed and spontaneous) Could meaning begin with a small act of care?
|
| 17 |
+
(speaker=S2, reflective; gently spoken) Yes. It need not look impressive to matter.
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
The literal text passed separately joins the two sentences without labels/directions. `podcast.prompting.build_user` attaches two correctly labeled codec anchors:
|
| 21 |
+
|
| 22 |
+
```text
|
| 23 |
+
- Reference(s):
|
| 24 |
+
Speaker S1 identity reference:
|
| 25 |
+
<|audio|>
|
| 26 |
+
Speaker S2 identity reference:
|
| 27 |
+
<|audio|>
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
Each placeholder contains the matching MOSS codes. Do not simply prepend a label to an audio filename. Native pair anchors are 63–187 frames each; optional chronologically prior conversation context plus anchors has a 750-frame/60-second combined limit. History must be one continuously encoded assembled waveform, not concatenated independently encoded token arrays, and must never contain the target. Automatic generated-history feedback is not evaluated. The optimized route does not use it.
|
| 31 |
+
|
| 32 |
+
Three-speaker native joint generation is outside the trained wrapper’s demonstrated contract. A separately assembled episode can route more IDs, but the benchmark evaluates two voices. Speaker collapse and late switches remain possible; inspect identity and transcript diagnostics and listen.
|
guides/PIPELINE.md
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Podcast generation pipeline — reproducible Best-of-N
|
| 2 |
+
|
| 3 |
+
This recipe uses the **Podcast full fine-tune**, not the general Large 1.7 checkpoint. It generates complete independent speaker turns, samples N takes per turn in parallel, ranks them, and assembles the winners into a two-host episode. Voice conversion is disabled. English, German, Dutch, French and Spanish use the same two reference recordings.
|
| 4 |
+
|
| 5 |
+
**[Public listening room: references and Best-of-4 / Best-of-2 / Single Take in five languages](https://huggingface.co/spaces/laion/Humanness-Voice-Podcast).** The page includes the exact scripts, prompt example, per-turn audit and measured generation times. The English N4 complete listening copy has one explicitly documented candidate replacement; its strict original remains downloadable.
|
| 6 |
+
|
| 7 |
+
## Current best practice: sparse spontaneous, relaxed and gentle delivery
|
| 8 |
+
|
| 9 |
+
On 8 November 2026 the listener approved the **sparse spontaneous/relaxed/gentle inline variant** and requested it as the standard. `--prompt-style spontaneous` is now the default. The earlier restrained `legacy` prompt remains available, while the exaggerated per-sentence emotion experiment is retained only for reproduction. This is a listener preference, not a universal or blinded-study optimum.
|
| 10 |
+
|
| 11 |
+
Read the [exact prompting guide and LLM-writing template](../optimized_pipeline/PROMPTING.md). An API-based or local LLM is recommended for writing the script and sparse cues: hand-authoring every turn and direction is unnecessarily laborious. The TTS itself does not write the dialogue and needs no LLM API key.
|
| 12 |
+
|
| 13 |
+
Exact GENERAL:
|
| 14 |
+
|
| 15 |
+
> A warm, emotionally thoughtful two-host podcast conversation. Very authentic, very genuine. Natural, intimate, spontaneous spoken delivery with clear words, subtle emotion, relaxed conversational pacing and brief meaningful pauses. Be sincere, without an exaggerated announcer voice or forced laughter. Perform only the current speaker turn and speak exactly the SCRIPT. Keep the same speaker identity as the reference audio.
|
| 16 |
+
|
| 17 |
+
Use **one moderate parenthesized direction plus a short spontaneous/gentle cue per turn**, and at most one extra cue at an existing sentence boundary, e.g. `(warm, intimate welcome; gently curious)`, followed by the literal transcript. Do not repeatedly demand strong emotions, forced laughter, or exaggerated acting. “Very pleasant to listen to” is deliberately absent. The experimental per-sentence style remains available only for reproducing the rejected comparison.
|
| 18 |
+
|
| 19 |
+
Temperature **1.0**, CFG **2.5**, top-p **0.95**, top-k **50**, BF16, free timing (`tokens=None`), 320 semantic frames / 25.6 s maximum per turn. Split longer scripts into shorter turns. No DPO, additional LoRAs, or voice conversion. These settings apply to this Podcast checkpoint, not all Humaneness models.
|
| 20 |
+
|
| 21 |
+
## Exact reward and quality audit
|
| 22 |
+
|
| 23 |
+
```
|
| 24 |
+
G = clip(native Humaneness Ears genuineness / 6, 0, 1)
|
| 25 |
+
S = clip(cosine(ECAPA(candidate), ECAPA(correct reference)), 0, 1)
|
| 26 |
+
reward = 0.5 * G + 0.5 * S
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
There is **no Content Enjoyment term and no WER multiplier or WER selection gate**. Fixed scales avoid unstable per-batch min/max normalization. Exact ties select the lowest candidate index. Score complete raw candidates; audit the edited winner separately. ECAPA is an approximate identity cue, not a guarantee of successful cloning. A high reward does not guarantee correct words.
|
| 30 |
+
|
| 31 |
+
After selection, Qwen forced alignment supplies approximate word boundaries for its supported languages (EN/DE/FR/ES here); keep 80 ms before the first word and 150 ms after the last, with small fades outside the aligned words. Dutch is **not** a supported Qwen alignment language. For Dutch, or unusable Qwen boundaries, Parakeet TDT estimates boundaries only when its transcript has at most 15% WER and matches the script's first and last two normalized words. These are explicitly labeled **ASR token boundary estimates**, not forced alignment. Uncertain cases remain untrimmed and flagged. Reassembly targets a 400 ms **word-to-word** gap using those estimates; for an unverified clip it falls back to clip edges. Final Parakeet WER is diagnostic only; never silently replace the reward winner. Export mono 48 kHz / 192 kbps MP3 plus lossless source, subtitles, per-turn scores, review flags, and listening HTML. No internal pause compaction.
|
| 32 |
+
|
| 33 |
+
## Weights and references
|
| 34 |
+
|
| 35 |
+
| Component | Published source | Pinned revision |
|
| 36 |
+
|---|---|---|
|
| 37 |
+
| Podcast policy, full FT step 9874, 4.13B | [laion/Humanness-Voice-Podcast](https://huggingface.co/laion/Humanness-Voice-Podcast) | `91f8110cd815f2233301a8b1dd16dbbf3fae8e05` |
|
| 38 |
+
| MOSS v2 codec | [OpenMOSS-Team/MOSS-Audio-Tokenizer-v2](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2) | `f6e20e543b33d2c252a7ef71bdf8aa71e5ff9169` |
|
| 39 |
+
| Genuineness judge | [Humaneness Ears Medium](https://huggingface.co/laion/humaneness-ears-base-medium) | `818506970d93c809a3295a0a02c40dee8ff2bfc3` |
|
| 40 |
+
| Speaker similarity, 192D | [SpeechBrain ECAPA VoxCeleb](https://huggingface.co/speechbrain/spkrec-ecapa-voxceleb) | `0f99f2d0ebe89ac095bcc5903c4dd8f72b367286` |
|
| 41 |
+
| Final transcript diagnostic | [Parakeet TDT 0.6B v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) | `541d1f99c6b0c3cd0b11a95167540bb8edefd82b` |
|
| 42 |
+
| Final alignment | [Qwen3 ForcedAligner](https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B-hf) | `c07281df297b9905d24a508279258cccf987a064` |
|
| 43 |
+
|
| 44 |
+
All five languages condition S1 on `de_s1.mp3` and S2 on `de_s2.mp3`, from the Podcast repository's [reference release](https://huggingface.co/laion/Humanness-Voice-Podcast/tree/0947ffdcedf85705683b256ccc310c90ccf3b6a9/examples/references). Recordings are approximately 14.52 / 13.08 s; cached codec conditioning contains 181 / 163 frames (14.48 / 13.04 s). `references.json` contains SHA-256 checks. Do not confuse fictional host labels with real identities. Source recordings retain original terms; this code package downloads them rather than relicensing them. The listening Space includes byte-identical copies with provenance.
|
| 45 |
+
|
| 46 |
+
## Run on another machine
|
| 47 |
+
|
| 48 |
+
The [standalone code](../optimized_pipeline/) includes the optimized policy/codec runtime, scoring, example scripts, alignment and reassembly. It does not import the Robby server, depend on private absolute paths, or require API keys. NVIDIA CUDA with BF16 support is required for this optimized path. Tested environment: RTX 3090 24 GB, Python 3.12, PyTorch 2.6.0+cu124, Transformers 5.15.0. Install system `ffmpeg` and `libsndfile` first. Use an empty GPU; unrelated workloads can cause OOM or change timings. The policy alone occupies about 8.26 GB; codec, judges, KV caches and graph capture need additional memory.
|
| 49 |
+
|
| 50 |
+
```bash
|
| 51 |
+
cd optimized_pipeline
|
| 52 |
+
python3.12 -m venv .venv
|
| 53 |
+
source .venv/bin/activate
|
| 54 |
+
pip install torch==2.6.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu124
|
| 55 |
+
pip install -r requirements.txt
|
| 56 |
+
# Optionally set HF_HOME and PODCAST_CACHE on a drive with sufficient free space.
|
| 57 |
+
python prepare.py
|
| 58 |
+
MODEL=$(python -c 'import json; print(json.load(open("assets.local.json"))["model"])')
|
| 59 |
+
python -m unittest test_contracts test_speaker_recipe
|
| 60 |
+
python speaker_run.py --model "$MODEL" --device cuda:0 --n 4 \
|
| 61 |
+
--prompt-style spontaneous --output outputs/approved/generation
|
| 62 |
+
python speaker_finish.py --runs outputs/approved/generation \
|
| 63 |
+
--output outputs/approved --device cuda:0
|
| 64 |
+
# Open outputs/approved/index.html.
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
`--n 1` keeps its only take and skips online scoring; final audit still measures its quality. `--n 2` uses the same 50/50 selection as N=4. N is takes **per segment**; the automatic pilot chooses one or two independent segments per policy batch, at most 16 simultaneous candidates. With two segments and N=4 the policy handles eight candidate histories, plus their CFG branches. The sample scripts have 20 turns per language. Supply custom JSON with `--script` (repeatable) and `--references`; see the prompting guide. `produce.py` runs generation plus audit/export in one command. `--limit 2` is a useful first smoke test. Re-run exactly the same configuration to resume completed turns; use a fresh directory when prompts/references/settings change. A live local Parakeet service is optional via the finisher's `--asr-url`; the default audit loads it locally after generation releases the policy.
|
| 68 |
+
|
| 69 |
+
Code includes checks that reused prompt prefill has identical hidden states and valid KV entries, that batched generation reproduces the prior one-segment runner, and that codec encoder offloading preserves decoding. If prefill equivalence fails, reuse is disabled. CUDA/Triton compilation is cold-start work and can take minutes. Same seeds do not promise identical outputs across hardware, libraries or batch grouping. Preserve all recorded scripts, source hashes, seeds and candidate scores.
|
| 70 |
+
|
| 71 |
+
The archived causal chunk search lives in `generation.py`; the default command above is **offline complete-turn selection**, not real-time Best-of-N streaming. Its legacy search reward differs; do not present it as the 50/50 experiment.
|
| 72 |
+
|
| 73 |
+
## Measured approved Best-of-4 baseline
|
| 74 |
+
|
| 75 |
+
Hardware: one RTX 3090 24 GB on a shared host; BF16, two turns × four takes per batch, 512-position semantic buckets, 26-frame internal waveform blocks, existing compiler cache. Both languages use the same fixed reference pair and approved sparse spontaneous directions.
|
| 76 |
+
|
| 77 |
+
| Language | Final audio | Generation + ranking | Useful RTF ↓ | Genuineness / 6 ↑ | ECAPA cosine ↑ | Mean turn WER ↓ |
|
| 78 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 79 |
+
| German | 219.38 s | 179.98 s | 0.821 | 3.943 | 0.640 | 1.98% |
|
| 80 |
+
| English, strict selection | 193.30 s | 195.83 s | 1.013 | 3.880 | 0.407 | 5.50% |
|
| 81 |
+
|
| 82 |
+
Useful RTF divides generation/ranking time by **final edited audio duration**, not total candidate audio. It excludes loading, correctness probes, final audit and export. Combined production including archival/cleanup took 380.10 s; process start to MP3 exports took 645.59 s, excluding service restoration and the additional listening repair.
|
| 83 |
+
|
| 84 |
+
Across both episodes: policy prediction 324.85 s, waveform decoding 24.85 s, ranking/preprocessing 21.25 s, remaining generation overhead 4.86 s. Ears forward work 4.35 s and ECAPA 3.08 s are included in ranking. Archival used 24.87 CPU seconds but blocked GPU production for just 1.08 s. These are measured stages, not promises for another machine.
|
| 85 |
+
|
| 86 |
+
**Content failure retained:** strict English turn 6 stopped after the first sentence. A separate complete listening copy replaces only that turn with existing candidate 1 (reward 0.499669; diagnostic WER 3.33%) and lasts 201.22 s. No new synthesis was used. This explicitly adds a content eligibility check; it is not the original strict 50/50 result. The default pipeline keeps strict winners and flags problems instead of silently repairing them. The listener approved the sparse style including that complete listening copy.
|
| 87 |
+
|
| 88 |
+
See [baseline machine-readable results](../optimized_pipeline/benchmarks/spontaneous-2026-11-08.json) and [historical experiments](PODCAST_EXPERIMENT_HISTORY.md). Metrics are fallible model estimates, not blinded human ratings.
|
| 89 |
+
|
| 90 |
+
## Implemented speed optimizations
|
| 91 |
+
|
| 92 |
+
- BF16 policy, reusable semantic and local-decoder CUDA graphs, fast token sampling and tested fused codec operations.
|
| 93 |
+
- Multiple independent turns in one batch. Histories remain separate; candidate selection is local to each turn.
|
| 94 |
+
- Identical candidate prompts reuse prefill only after hidden-state and valid-KV equality checks. A two-group check measured 2.009 s normally versus 0.518 s with reuse.
|
| 95 |
+
- Cache reference acoustic codes and speaker embeddings. Keep the unused codec encoder in CPU RAM after reference preparation.
|
| 96 |
+
- Batch the Ears/ECAPA judges. Skip online ranking for N=1; reserve Parakeet for final audit.
|
| 97 |
+
- Archive every candidate with one bounded CPU writer while the next GPU batch runs. Propagate write failures before publishing a completed-turn receipt.
|
| 98 |
+
- Reuse an explicitly selected `--groups` value, a warmed worker and compiler caches instead of repeating setup and timing sweeps for every episode.
|
| 99 |
+
|
| 100 |
+
`--groups 2 --n 4`, `--groups 4 --n 2`, and `--groups 8 --n 1` each maintain eight candidates per policy batch, plus CFG branches. These layouts change RNG trajectories; N1/N2 are not necessarily subsets of an N4 run. Resume requires unchanged settings and stable batch grouping.
|
| 101 |
+
|
| 102 |
+
### When VRAM is shared or limited
|
| 103 |
+
|
| 104 |
+
`low_memory_compare.py` separates **policy token generation → release policy → waveform decoding and ranking**. It saves acoustic tokens for every candidate, retains the same token sampling loop, checks shared/full prefill equality and tests optional fused decoding. It requires prepared reference codes. This is an offline comparison runner, not a streaming service and not a general low-memory implementation for arbitrary recordings.
|
| 105 |
+
|
| 106 |
+
```bash
|
| 107 |
+
python low_memory_compare.py --model "$MODEL" --device cuda:0 --output outputs/n-comparison
|
| 108 |
+
for N in 2 1; do
|
| 109 |
+
python speaker_finish.py --runs "outputs/n-comparison/n$N/generation" \
|
| 110 |
+
--output "outputs/n-comparison/n$N" --device cuda:0
|
| 111 |
+
python verify_artifacts.py "outputs/n-comparison/n$N"
|
| 112 |
+
done
|
| 113 |
+
python compare_report.py --n4 outputs/spontaneous --n2 outputs/n-comparison/n2 \
|
| 114 |
+
--n1 outputs/n-comparison/n1 --output outputs/n-comparison
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
Stage sums exclude shared loading and inter-phase waiting; the parent receipt records total generation-job time. Do not label their sum request latency. An integrated attempt ran out of memory when another user's training job appeared; no unrelated job was stopped. The phased comparison uses the same shared GPU under that extra load. A fair speed ablation should repeat N=1/2/4 with identical warm-up and an otherwise idle GPU.
|
| 118 |
+
|
| 119 |
+
### Reproduce the additional Dutch, French and Spanish episodes
|
| 120 |
+
|
| 121 |
+
The bundled `script-spontaneous-{en,de,nl,fr,es}.json` files contain the same twenty-turn conversation, translated by a language model and preserving the sparse directions. Language routing is explicit throughout the processor and alignment stages. Default scripts remain EN/DE for compatibility; supply the other files explicitly. The example below maintains up to eight candidates per policy batch: two turns at N4, four at N2, eight at N1. It needs a fresh output directory and prepared reference codes.
|
| 122 |
+
|
| 123 |
+
```bash
|
| 124 |
+
python low_memory_compare.py --model "$MODEL" --device cuda:0 \
|
| 125 |
+
--script script-spontaneous-nl.json \
|
| 126 |
+
--script script-spontaneous-fr.json \
|
| 127 |
+
--script script-spontaneous-es.json \
|
| 128 |
+
--n-values 4 2 1 --candidates 8 --output outputs/five-language-extension
|
| 129 |
+
python finish_comparison.py --runs outputs/five-language-extension --device cuda:0
|
| 130 |
+
for N in 4 2 1; do
|
| 131 |
+
python verify_artifacts.py "outputs/five-language-extension/n$N"
|
| 132 |
+
done
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
`finish_comparison.py` shares frozen scoring/alignment models across the three modes and writes its parent completion receipt only after all exports finish. This run produces 420 candidates and nine episodes (180 selected turns). It is not a nested re-ranking of one N4 pool: N2/N1 are separate runs with their own recorded seeds and batch grouping. Publication retains strict reward winners and exposes review flags, including uncertain boundaries and possible omitted words. Machine ASR is an audit aid, not a native-speaker listening verdict. The same fixed German references deliberately test cross-language voice conditioning.
|
| 136 |
+
|
| 137 |
+
## Next speed improvements to measure
|
| 138 |
+
|
| 139 |
+
Policy prediction dominates. In N4, 37/160 candidates hit the 25.6-s cap and 19/20 batches waited for a capped candidate. Length-aware grouping and safe finished-row compaction may reduce wasted work, but require verified CFG/cache remapping and complete-script checks. Do not simply reduce the ceiling or remove words to improve RTF. A persistent dedicated worker is the lowest-risk way to avoid loading and compilation latency.
|
| 140 |
+
|
| 141 |
+
Reassembly and scoring are much smaller costs. Optimizing the codec alone cannot remove the policy bottleneck. Keep truthful per-stage, per-episode and end-to-end timings; preserve all candidate files so selection experiments need no fresh synthesis.
|
| 142 |
+
|
| 143 |
+
## Fresh N=1 and N=2 comparison with the approved prompts
|
| 144 |
+
|
| 145 |
+
Both episodes were freshly generated at N=2 and N=1 after publishing the new defaults. Same literal scripts, cues, reference pair, temperature 1.0 and CFG 2.5. N1 keeps its sole take and skips online ranking. N2 uses the same 50/50 reward as N4; there is no WER gate or hidden retry.
|
| 146 |
+
|
| 147 |
+
| Language | N | Final audio | Generate + rank | Useful RTF ↓ | Genuineness / 6 ↑ | ECAPA cosine ↑ | Mean turn WER ↓ |
|
| 148 |
+
|---|---:|---:|---:|---:|---:|---:|---:|
|
| 149 |
+
| DE | 1 | 220.75 s | 55.24 s | 0.250 | 3.916 | 0.553 | 1.21% |
|
| 150 |
+
| DE | 2 | 222.26 s | 115.80 s | 0.521 | 3.897 | 0.630 | 2.56% |
|
| 151 |
+
| DE | 4 | 219.38 s | 179.98 s | 0.821 | 3.943 | 0.640 | 1.98% |
|
| 152 |
+
| EN | 1 | 199.30 s | 68.56 s | 0.344 | 3.863 | 0.352 | 2.12% |
|
| 153 |
+
| EN | 2 | 199.54 s | 122.77 s | 0.615 | 3.904 | 0.383 | 2.93% |
|
| 154 |
+
| EN | 4 | 193.30 s | 195.83 s | 1.013 | 3.880 | 0.407 | 5.50% |
|
| 155 |
+
|
| 156 |
+
In this sample, **N=2 is a useful speed/identity compromise**: German speaker cosine rises from 0.553 at N1 to 0.630 at N2, close to N4's 0.640. English rises from 0.352 to 0.383; N4 is 0.407. Genuineness stays roughly 3.9/6. N1 is faster but less consistent in speaker identity. These are twenty turns per language, not evidence of a universal optimum. The old English N4 row includes its documented incomplete turn; the repaired listening copy is separate.
|
| 157 |
+
|
| 158 |
+
All 80 new selected turns passed the configured review checks (per-turn WER threshold 15% and usable forced alignment); none was regenerated or replaced. This is not a guarantee of perfect speech. Both candidate counts passed artifact validation, exact prompt/reference routing, strict selection and PCM equality checks. The token/full-prefill checks and optional fused decoding were bit-identical on their tested inputs. Fourteen CPU tests pass.
|
| 159 |
+
|
| 160 |
+
**Timing scope:** N2 production across both languages totals 238.57 s; N1 totals 123.80 s; old N4 totals 375.80 s. Token prediction accounts for 209.57 / 113.22 / 324.85 s respectively. N2 codec/ranking take 18.18/10.07 s; N1 codec takes 9.91 s and online ranking zero. Loading, correctness probes, final audit and export are excluded from these production sums. The complete successful N2+N1 experiment, from process start to all four MP3s, took **660.70 s**. The failed earlier OOM attempt and live-service restoration are excluded.
|
| 161 |
+
|
| 162 |
+
N2/N1 used phased execution under additional unrelated GPU training; N4 used integrated execution with a different background load. The other training ended before final auditing. Do not claim these are controlled percentage speedups from N alone. N2 uses four turns per batch and N1 eight (final partial batches smaller), maintaining up to eight simultaneous candidates. Different groups/seeds mean these are fresh runs, not nested subsets of the N4 pool.
|
| 163 |
+
|
| 164 |
+
See [aggregate results](../optimized_pipeline/benchmarks/n-comparison-2026-11-08.json) and [all per-turn candidate scores, seeds and audits](../optimized_pipeline/benchmarks/n-comparison-2026-11-08-turns.json). Recreate the listening HTML with `compare_report.py`; the six strict episodes and additional labeled N4 English listening copy are independently selectable.
|
| 165 |
+
|
| 166 |
+
<!-- FIVE_LANGUAGE_RESULTS -->
|
| 167 |
+
## Five-language listening release: additional NL / FR / ES results
|
| 168 |
+
|
| 169 |
+
[Open the public listening Space](https://huggingface.co/spaces/laion/Humanness-Voice-Podcast). The nine additional episodes use the same fixed reference pair, approved sparse prompting and translated twenty-turn story as the DE/EN examples. All N2/N4 reward winners and all N1 sole takes are retained; no retries, content re-selection or voice conversion.
|
| 170 |
+
|
| 171 |
+
| Language | N | Final audio | Production | Useful RTF | Genuineness / 6 | ECAPA cosine | Mean turn WER | Transcript / boundary flags, of 20 |
|
| 172 |
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 173 |
+
| NL | 4 | 303.00 s | 261.95 s | 0.865 | 3.969 | 0.388 | 14.4% | 8 / 10 |
|
| 174 |
+
| NL | 2 | 298.90 s | 134.25 s | 0.449 | 3.884 | 0.352 | 13.5% | 5 / 8 |
|
| 175 |
+
| NL | 1 | 280.82 s | 73.28 s | 0.261 | 3.897 | 0.333 | 15.1% | 6 / 9 |
|
| 176 |
+
| FR | 4 | 238.82 s | 242.20 s | 1.014 | 4.021 | 0.420 | 21.9% | 8 / 4 |
|
| 177 |
+
| FR | 2 | 231.62 s | 131.62 s | 0.568 | 4.032 | 0.385 | 14.2% | 6 / 3 |
|
| 178 |
+
| FR | 1 | 238.10 s | 65.74 s | 0.276 | 3.906 | 0.362 | 14.5% | 5 / 4 |
|
| 179 |
+
| ES | 4 | 234.02 s | 250.40 s | 1.070 | 4.008 | 0.450 | 2.2% | 0 / 0 |
|
| 180 |
+
| ES | 2 | 233.78 s | 134.51 s | 0.575 | 4.018 | 0.385 | 2.6% | 0 / 0 |
|
| 181 |
+
| ES | 1 | 232.66 s | 57.90 s | 0.249 | 4.020 | 0.392 | 2.1% | 1 / 0 |
|
| 182 |
+
|
| 183 |
+
Flags are materially more frequent in Dutch and French than Spanish in this fixed-reference run. Some recognized transcripts omit words or drift across languages. ASR itself is fallible, and a boundary warning alone does not imply missing speech. Dutch uses conservative Parakeet TDT boundaries because the Qwen aligner does not support Dutch. Uncertain clips remain untrimmed, so some episodes include longer tails. These examples are not endorsed as error-free multilingual outputs. Scripts and detailed diagnostics are available next to every player.
|
| 184 |
+
|
| 185 |
+
Production is the sum of policy, codec and online selection time, excluding loading, correctness probes, final audit and encoding. Phased inference used one RTX 3090 alongside unrelated jobs; cross-run GPU loads differ. N4/N2/N1 use separate draws and grouping, not nested subsets. Do not interpret these measurements as a controlled speed or quality ablation.
|
| 186 |
+
|
| 187 |
+
[Aggregate measurements](../optimized_pipeline/benchmarks/five-language-extension-2026-11-08.json) · [All candidate scores and selected-turn audits](../optimized_pipeline/benchmarks/five-language-extension-2026-11-08-turns.json).
|
guides/PODCAST_EXPERIMENT_HISTORY.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Earlier Podcast experiments and measurements
|
| 2 |
+
|
| 3 |
+
Historical record. For current defaults use [Podcast Generation Pipeline](PIPELINE.md). Earlier styles, rewards and hardware conditions differ; labels below preserve those distinctions.
|
| 4 |
+
|
| 5 |
+
## Measurements and interpretation
|
| 6 |
+
|
| 7 |
+
Single RTX 3090, BF16. The earlier fresh 50/50 experiment used the subsequently rejected per-sentence emotion prompts; these figures measure throughput, not approval of that style.
|
| 8 |
+
|
| 9 |
+
| Fresh Best-of-4 run | Final audio | Generation + ranking | Useful RTF |
|
| 10 |
+
|---|---:|---:|---:|
|
| 11 |
+
| German | 236 s | 179.13 s | 0.760 |
|
| 12 |
+
| English | 212 s | 174.23 s | 0.820 |
|
| 13 |
+
|
| 14 |
+
RTF = generation/ranking / final edited duration, not all candidate audio. Combined production including archival and cleanup was 380.65 s; generation/ranking accounted for 353.36 s. Cold setup, tests and timing probes took 356.52 s. Final audit load 30.10 s, audit 45.10 s, parallel CPU export 49.01 s; full experiment to exports 894.12 s. Do not call these request-to-first-audio latencies.
|
| 15 |
+
|
| 16 |
+
Shared prefill measured 0.273 vs 1.125 s for one group and 0.544 vs 2.110 s for two groups, with exact equality. A warm 104-frame probe took 3.572 s for one segment and 5.238 s for two (2.619 s/segment). The latter is an approximate workload comparison: two different scripts and different RNG batch shapes. An earlier run shared the card with a training job; speed differences cannot all be assigned to code.
|
| 17 |
+
|
| 18 |
+
The **earlier approved restrained rerank** reused 160 saved candidates, generated no new TTS, and changed 15/20 German and 12/20 English winners. Final German / English means: genuineness 3.967 / 3.822 out of 6; ECAPA cosine 0.655 / 0.401; turn WER 1.01% / 1.79%. Reranking took 32.18 s; do not report that as synthesis throughput. These are model metrics, not human quality ratings.
|
| 19 |
+
|
| 20 |
+
## Next efficiency experiments
|
| 21 |
+
|
| 22 |
+
1. Already implemented: overlap bounded CPU candidate archival with the next GPU batch; preserve all takes and propagate write errors before declaring completion. The earlier run spent roughly 27 s outside measured generation/ranking, including archival and cleanup; that is a ceiling, not a promised speedup.
|
| 23 |
+
2. Reuse a warmed worker, cached reference embeddings and tested graph shapes across episodes. Report cold start separately.
|
| 24 |
+
3. Explicitly reuse a previously measured batch size instead of repeating timing sweeps every episode, while retaining correctness checks after runtime changes.
|
| 25 |
+
4. Bucket comparable turn lengths to reduce completed-row padding, but record batch assignment because it changes RNG histories.
|
| 26 |
+
5. Benchmark EOS/continuation behavior before shortening frame caps. Never accelerate by silently clipping words or dropping candidates.
|
| 27 |
+
|
| 28 |
+
## Current standard: sparse spontaneous delivery
|
| 29 |
+
|
| 30 |
+
`--prompt-style spontaneous` retains every spoken word and original moderate turn direction, adding a short relaxed/gentle/spontaneous cue. Alternating turns get at most one additional cue at an existing sentence boundary. It does not repeat a list of emotions before every sentence. The listener-approved `spontaneous` style is now the default; `legacy` reproduces the earlier restrained variant.
|
| 31 |
+
|
| 32 |
+
```bash
|
| 33 |
+
python speaker_run.py --model "$MODEL" --device cuda:0 --n 4 --groups 2 \
|
| 34 |
+
--prompt-style spontaneous --output outputs/spontaneous/generation
|
| 35 |
+
python speaker_finish.py --runs outputs/spontaneous/generation \
|
| 36 |
+
--output outputs/spontaneous --device cuda:0
|
| 37 |
+
python verify_artifacts.py outputs/spontaneous
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
The explicit `--groups 2` reuses the previously measured batch size and skips repeated timing sweeps. It retains the real-model codec, generation-equivalence and prefill checks. First production batches can include cold graph/compilation costs; those are not hidden. If comparing throughput, keep this warm-up policy identical.
|
| 41 |
+
|
| 42 |
+
Candidate FLAC archival now runs on one bounded CPU thread alongside the next GPU batch. No candidates are discarded; write failures propagate, and a `result.json` is published only after its files exist. `archive-timing.json` distinguishes CPU work from the remaining blocking wait. Resume keeps stable batch grouping and replaces the same batch timing record rather than double-counting it.
|
| 43 |
+
|
| 44 |
+
The listener approved these samples and requested this prompt style as the default. Model-score changes alone still do not establish general superiority.
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
## New spontaneous variant: measured outcome, 8 November 2026
|
| 48 |
+
|
| 49 |
+
The portable package generated and scored all **160 candidates** (40 turns × 4), passed exact GPU codec/token/PCM and prefill checks, and exported both episodes. The eleven CPU tests cover prefix selection, CFG state copying, prompt word preservation, exact reward and incomplete archival receipts.
|
| 50 |
+
|
| 51 |
+
| Strict 50/50 selection | Final duration | Generation + ranking | Useful RTF | Genuineness / 6 | ECAPA cosine | Mean turn WER |
|
| 52 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 53 |
+
| German | 219.38 s | 179.98 s | 0.821 | 3.943 | 0.640 | 1.98% |
|
| 54 |
+
| English | 193.30 s | 195.83 s | 1.013 | 3.880 | 0.407 | 5.50% |
|
| 55 |
+
|
| 56 |
+
**Important observed failure:** the highest-reward take for English turn 6 ends after its first sentence (4 s instead of the complete turn). Its reward is 0.500111; the next candidate scores 0.499803. The strict experiment is kept unchanged and explicitly flagged. A separately labeled content-repaired listening copy can use a complete alternative, but that adds a content eligibility check and must not be reported as unconstrained 50/50 argmax. This illustrates why genuineness and speaker similarity alone cannot certify a complete script. The sparse spontaneous style is now the listener-approved default. The earlier restrained variant remains available for comparison.
|
| 57 |
+
|
| 58 |
+
Across both episodes, generation/ranking took 375.80 s: policy token prediction 324.85 s, codec decoding 24.85 s, scoring including preprocessing 21.25 s, and approximately 4.86 s in remaining generation overhead. Ears forward work (4.35 s) and ECAPA work (3.08 s) are **included** in scoring, not extra. Archived audio writing consumed 24.87 CPU seconds but blocked production for only 1.08 s; almost all writing overlapped GPU work. Total production including final archival/cleanup was 380.10 s. Process start to both MP3 exports was 645.59 s, excluding service restoration and the additional listening repair.
|
| 59 |
+
|
| 60 |
+
Do not attribute the entire startup reduction from the earlier 894.12 s experiment to code: this run reused an existing compiler cache, skipped repeated speed probes, used different prompts and inherited 512-position semantic buckets rather than the earlier 256. Overall production time was about the same as the earlier 380.65 s run; the archival overlap itself is measured, but a controlled aggregate speedup has not been established.
|
| 61 |
+
|
| 62 |
+
The largest remaining opportunity is **long-tail candidate computation**: 37 of 160 takes reached the 25.6 s cap and 19 of 20 batches waited for at least one capped take. Only 2,726.64 s of candidate audio were produced versus 4,096 s if all rows ran to the cap, but a batched policy can still spend time on already finished rows. Next, test conservative text-completion checks for lagging candidates, length-aware grouping, or active-row compaction with verified CFG/codec cache copying. Require complete-script checks and keep the original outputs for comparison; simply shortening the cap risks losing real speech. Keeping a warmed dedicated worker is the lowest-risk way to avoid repeated loading and compilation.
|
| 63 |
+
|
| 64 |
+
Machine-readable measurements: [spontaneous-2026-11-08.json](../optimized_pipeline/benchmarks/spontaneous-2026-11-08.json).
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
The additional English listening copy is **201.22 s** long. It replaces only turn 6 with existing candidate 1 (raw reward 0.499669, ASR WER 3.33%, plausible forced alignment). Candidate 3 was also checked first in reward order. Nothing was regenerated; the strict episode and its scores remain unchanged. `repair_listening.py OUTPUT --turn 6 --language en --asr-url http://127.0.0.1:8801/transcribe/parakeet_v3` reproduces this explicit repair with the recorded loopback ASR service; the default generation/audit commands never apply this content gate. Its HTTP ASR contract is mono 16 kHz float32 bytes in, JSON containing `text` out. CPU alignment uses FP32 to avoid slow BF16 emulation on older CPUs.
|
guides/PROMPTING.md
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Prompting guide: spontaneous, relaxed and gentle
|
| 2 |
+
|
| 3 |
+
This is the listener-approved default as of 8 November 2026 (`--prompt-style spontaneous`). It keeps the original restrained delivery and adds sparse inline cues for spontaneous, easy, gently spoken conversation. The listener preferred these samples; that does not establish a universal optimum for all voices or scripts. The older `legacy` turn-level style and exaggerated `sentence` experiment remain explicit reproduction options.
|
| 4 |
+
|
| 5 |
+
## Use an LLM as the script and direction writer
|
| 6 |
+
|
| 7 |
+
Writing every literal turn and every carefully placed acting cue by hand is unnecessarily laborious. We recommend asking an API-based or local language model to produce the JSON script. The LLM writes the dialogue and sparse directions; this Podcast model speaks them. No particular LLM vendor, API key, or network service is required by inference. Review factual content before synthesis.
|
| 8 |
+
|
| 9 |
+
Give the writing model this instruction:
|
| 10 |
+
|
| 11 |
+
> Write a natural, thoughtful spoken conversation in the requested language. Use the requested speakers, with stable IDs. Separate literal spoken text from performance cues. Keep each turn short enough to fit comfortably below 25.6 seconds; split long explanations into turns. Make the exchange spontaneous, relaxed, gentle and genuine, with modest contextual emotion rather than theatrical intensity. Use one short cue at the start of a turn and at most one additional cue at a meaningful existing sentence boundary. Do not label every sentence with a strong emotion. Do not request forced laughter, excessive breathiness, exaggerated slowness, or “very pleasant/nice to listen to.” Do not insert speaker labels, directions or duration requests into the spoken text. Output valid JSON with `language` and an ordered `turns` array. Each turn has `index`, `speaker`, `text`, `direction`, and optionally one or two `delivery_chunks`, each containing `text` and `cue`. Joining chunk texts with one space must exactly reproduce the turn's literal `text`. Do not change a supplied transcript. Use no generated audio as a reference; references are selected separately.
|
| 12 |
+
|
| 13 |
+
Include the topic, factual constraints, speaker roles, intended audience and desired approximate episode length in the writing request. For the evaluated example, two hosts discuss meaning, Viktor Frankl, everyday care and connection. A calm question, gentle curiosity, understated warmth or mild amusement usually suffice. Strong emotions should arise from actual content and an explicit request.
|
| 14 |
+
|
| 15 |
+
## Exact approved GENERAL
|
| 16 |
+
|
| 17 |
+
```text
|
| 18 |
+
A warm, emotionally thoughtful two-host podcast conversation. Very authentic, very genuine. Natural, intimate, spontaneous spoken delivery with clear words, subtle emotion, relaxed conversational pacing and brief meaningful pauses. Be sincere, without an exaggerated announcer voice or forced laughter. Perform only the current speaker turn and speak exactly the SCRIPT. Keep the same speaker identity as the reference audio.
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
GENERAL remains constant across the tested episodes. It does not include a voice gender, age, timbre redesign, or “very pleasant to listen to.” Identity comes from the fixed reference audio. For a turn within a podcast, use this exact text even though the individual call contains only one speaker.
|
| 22 |
+
|
| 23 |
+
## Single-speaker call template
|
| 24 |
+
|
| 25 |
+
```text
|
| 26 |
+
GENERAL: [exact GENERAL above]
|
| 27 |
+
SCRIPT:
|
| 28 |
+
(warm, gently curious; relaxed, spontaneous, gently spoken) What if meaning is something we practise?
|
| 29 |
+
(gently, as a spontaneous thought) Maybe it begins with noticing who needs us today.
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
The processor receives the separate literal text `What if meaning is something we practise? Maybe it begins with noticing who needs us today.` and **only this speaker's** reference codes. Inline directions belong in `instruction`, not in the literal transcript. Use `tokens=None` and no inline duration brackets: the 320-frame generation ceiling is a limit, not a requested duration.
|
| 33 |
+
|
| 34 |
+
Prefer cues such as:
|
| 35 |
+
|
| 36 |
+
- `easy, spontaneous, gently spoken`
|
| 37 |
+
- `relaxed, unforced, like talking to a friend`
|
| 38 |
+
- `soft natural delivery, thinking aloud`
|
| 39 |
+
- `casual and sincere, unhurried`
|
| 40 |
+
- `gentle conversational flow, in the moment`
|
| 41 |
+
|
| 42 |
+
Do not stack all of them onto one sentence. The approved example files rotate a single short cue and add one extra cue inside alternating turns. Most emotion remains subtle. The second cue is optional.
|
| 43 |
+
|
| 44 |
+
## JSON accepted by the optimized pipeline
|
| 45 |
+
|
| 46 |
+
```json
|
| 47 |
+
{
|
| 48 |
+
"language": "en",
|
| 49 |
+
"turns": [
|
| 50 |
+
{
|
| 51 |
+
"index": 0,
|
| 52 |
+
"speaker": "S1",
|
| 53 |
+
"text": "What if meaning is something we practise? Maybe it begins with noticing who needs us today.",
|
| 54 |
+
"direction": "warm, gently curious",
|
| 55 |
+
"delivery_chunks": [
|
| 56 |
+
{"text": "What if meaning is something we practise?", "cue": "warm, gently curious; relaxed, spontaneous, gently spoken"},
|
| 57 |
+
{"text": "Maybe it begins with noticing who needs us today.", "cue": "gently, as a spontaneous thought"}
|
| 58 |
+
]
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"index": 1,
|
| 62 |
+
"speaker": "S2",
|
| 63 |
+
"text": "Yes. Perhaps it begins with a small gesture that makes a real difference to someone.",
|
| 64 |
+
"direction": "warm and reflective"
|
| 65 |
+
}
|
| 66 |
+
]
|
| 67 |
+
}
|
| 68 |
+
```
|
| 69 |
+
|
| 70 |
+
Use one consistent language per file. Supported codes are `en`, `de`, `nl`, `fr` and `es`; the bundled spontaneous scripts cover all five. Set `language` to `de` and write all turns in German for a German episode. A simpler turn with only `speaker`, `text` and `direction` is valid: the default adds one sparse spontaneous cue without changing the literal words. `index` defaults to its position. Pass one `--script` argument per language in a job. Explicit routing preserves the requested language instead of treating non-German text as English. The CLI validates the exact chunk-to-text reconstruction and rejects ambiguous duplicate language/index pairs.
|
| 71 |
+
|
| 72 |
+
Pass a separate reference mapping with `--references` when changing voices:
|
| 73 |
+
|
| 74 |
+
```json
|
| 75 |
+
{
|
| 76 |
+
"S1": {"path": "audio/voice-one.wav"},
|
| 77 |
+
"S2": {"path": "audio/voice-two.wav"}
|
| 78 |
+
}
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
Paths are relative to this reference JSON. Audio SHA-256 values are recorded automatically when absent, and verified when supplied. The bundled mapping additionally provides pinned cached MOSS codes. Remove or regenerate `codes` and `code_key` when replacing audio: otherwise the old codes remain the actual conditioning input. Use clean single-speaker references around 5–15 seconds and reuse them for the entire episode.
|
| 82 |
+
|
| 83 |
+
## Multi-speaker episodes versus native two-speaker blocks
|
| 84 |
+
|
| 85 |
+
The recommended optimized route generates each speaker turn separately, always with its correct reference, and then assembles the turns chronologically. A GPU batch can contain different speakers and texts, but their generated histories and rewards stay separate. This is a multi-speaker **episode**, not two speakers mixed into one model call. Any supported speaker ID in the reference mapping can route an independently generated turn; only the two-voice example has been evaluated here.
|
| 86 |
+
|
| 87 |
+
The preserved native pair route (`infer_legacy.py --mode pairs`) instead uses the trained S1/S2 reference wrapper and `(speaker=S1, ...)` / `(speaker=S2, ...)` instructions within one call. Labels must not appear in the literal transcript. This route has different content checks and rewards, and is not covered by the new optimized Best-of-N timing table. See the model repository's multi-speaker guide before using it. Do not pass two references to the optimized single-turn call and assume it will switch identities correctly.
|
| 88 |
+
|
| 89 |
+
## Settings and selection
|
| 90 |
+
|
| 91 |
+
BF16 policy, temperature **1.0**, CFG **2.5**, top-p **0.95**, top-k **50**, free timing, up to **320 × 80 ms = 25.6 s per turn**. No voice conversion, DPO or extra LoRA. Default N=4; use `--n 2` or `--n 1` to trade selection opportunity for less compute.
|
| 92 |
+
|
| 93 |
+
For N>1, choose the maximum of `0.5*clip(Ears native genuineness/6,0,1) + 0.5*clip(ECAPA reference cosine,0,1)` per complete raw turn. This is fixed normalization, not candidate-wise min/max. WER and Content Enjoyment are not selection terms. N=1 retains its sole take and skips online scoring. All modes receive the same independent final audit.
|
| 94 |
+
|
| 95 |
+
Quality checks are essential: a highly genuine, similar-sounding take can still omit words. The experiment keeps strict ranking results and flags such failures. If a complete listening copy is repaired using another candidate, document that additional content eligibility check explicitly; never present it as unconstrained reward argmax or as an unchanged Single Take.
|
guides/SINGLE_SPEAKER.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Single-speaker inference and prompting
|
| 2 |
+
|
| 3 |
+
The default optimized pipeline generates one speaker per turn. Supply only that speaker’s reference for the call; the code selects it from the fixed reference map. A one-turn script produces one utterance; several same-speaker turns can form a monologue.
|
| 4 |
+
|
| 5 |
+
Use the [exact GENERAL/SCRIPT template and JSON contract](PROMPTING.md). Literal words belong in `text`; acting directions belong in `direction` or `delivery_chunks[].cue`. Joining chunk texts with one space must equal `text` exactly. The default adds a short spontaneous/gentle cue when none is supplied. Keep one cue per turn, optionally one more at a meaningful existing sentence boundary. Do not repeatedly exaggerate emotion or add speaker labels to the literal words.
|
| 6 |
+
|
| 7 |
+
```text
|
| 8 |
+
GENERAL: A warm, emotionally thoughtful two-host podcast conversation. Very authentic, very genuine. Natural, intimate, spontaneous spoken delivery with clear words, subtle emotion, relaxed conversational pacing and brief meaningful pauses. Be sincere, without an exaggerated announcer voice or forced laughter. Perform only the current speaker turn and speak exactly the SCRIPT. Keep the same speaker identity as the reference audio.
|
| 9 |
+
SCRIPT:
|
| 10 |
+
(warm, curious; relaxed, spontaneous, gently spoken) What if meaning begins with paying attention?
|
| 11 |
+
```
|
| 12 |
+
|
| 13 |
+
For a pure monologue, changing the first GENERAL sentence to describe a monologue is a reasonable adaptation, but the published benchmark uses the exact podcast GENERAL above.
|
| 14 |
+
|
| 15 |
+
Settings: BF16, temperature 1.0, CFG 2.5, top-p 0.95, top-k 50, `tokens=None`, no inline duration brackets. Generation stops at EOS or the 320-frame/25.6-second ceiling. Split longer text. Use clean 5–15-second references; never silently truncate them to three seconds. The approved examples use 181/163 codec frames.
|
| 16 |
+
|
| 17 |
+
Run `python infer.py --model . --script my-script.json --references my-references.json --n 1 --output runs/single` after the README setup. N=1 keeps the first and only take; use N=2/4 for candidate selection. An LLM can produce the script JSON using [the writing instruction](PROMPTING.md#use-an-llm-as-the-script-and-direction-writer), but the inference runtime does not require one.
|
guides/recipe.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": 3,
|
| 3 |
+
"default_prompt_style": "spontaneous",
|
| 4 |
+
"listener_approved": true,
|
| 5 |
+
"model": "laion/Humanness-Voice-Podcast",
|
| 6 |
+
"revision": "91f8110cd815f2233301a8b1dd16dbbf3fae8e05",
|
| 7 |
+
"temperature": 1.0,
|
| 8 |
+
"cfg": 2.5,
|
| 9 |
+
"top_p": 0.95,
|
| 10 |
+
"top_k": 50,
|
| 11 |
+
"n": 4,
|
| 12 |
+
"mode": "independent turns",
|
| 13 |
+
"max_frames": 320,
|
| 14 |
+
"free_timing": true,
|
| 15 |
+
"voice_conversion": false,
|
| 16 |
+
"reward": "0.5*clip(genuineness/6,0,1)+0.5*clip(ecapa_cosine,0,1)",
|
| 17 |
+
"wer_role": "audit only",
|
| 18 |
+
"n1": "first and only take; online ranking skipped",
|
| 19 |
+
"prompt_guide": "guides/PROMPTING.md",
|
| 20 |
+
"pipeline_guide": "guides/PIPELINE.md"
|
| 21 |
+
}
|
infer.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Current default: optimized spontaneous Podcast generation, audit and export.
|
| 2 |
+
For the preserved native pair/legacy workflow use infer_legacy.py.
|
| 3 |
+
"""
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import sys
|
| 6 |
+
ROOT=Path(__file__).resolve().parent
|
| 7 |
+
sys.path.insert(0,str(ROOT/'optimized_pipeline'))
|
| 8 |
+
from produce import main
|
| 9 |
+
if __name__=='__main__':
|
| 10 |
+
args=sys.argv[1:]
|
| 11 |
+
if '--model' not in args:args=['--model',str(ROOT),*args]
|
| 12 |
+
main(args)
|
infer_legacy.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Entry point: python infer.py --script examples/short_en.json --output run."""
|
| 2 |
+
from podcast.pipeline import main
|
| 3 |
+
if __name__=='__main__':main()
|
model-00001-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:791c61b2bcad322b58fbac3d0d24174014bb49acad2a52fb581cf96074381675
|
| 3 |
+
size 1997031256
|
model-00002-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1be302af66c882ec7f6ef35954c76b0e55436911596b1602627d53cfbaed1dca
|
| 3 |
+
size 1997658024
|
model-00003-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d65aa4709289a60e18fc7d7a3fa75c3357755707b167df62e9512c94a3a69c0
|
| 3 |
+
size 1997657656
|
model-00004-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1f8a6db71e26dc79f3f5f0562bf0320f51f1d83fdec0db7b2ca74206f170dde2
|
| 3 |
+
size 1963573536
|
model-00005-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b8033d7393e3d846a6740f041992fa449d2a017613a9130ab5ab0df9d3a15d91
|
| 3 |
+
size 304110984
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,432 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_size": 8259980288
|
| 4 |
+
},
|
| 5 |
+
"weight_map": {
|
| 6 |
+
"audio_embeddings.0.weight": "model-00001-of-00005.safetensors",
|
| 7 |
+
"audio_embeddings.1.weight": "model-00001-of-00005.safetensors",
|
| 8 |
+
"audio_embeddings.10.weight": "model-00001-of-00005.safetensors",
|
| 9 |
+
"audio_embeddings.11.weight": "model-00001-of-00005.safetensors",
|
| 10 |
+
"audio_embeddings.2.weight": "model-00001-of-00005.safetensors",
|
| 11 |
+
"audio_embeddings.3.weight": "model-00001-of-00005.safetensors",
|
| 12 |
+
"audio_embeddings.4.weight": "model-00001-of-00005.safetensors",
|
| 13 |
+
"audio_embeddings.5.weight": "model-00001-of-00005.safetensors",
|
| 14 |
+
"audio_embeddings.6.weight": "model-00001-of-00005.safetensors",
|
| 15 |
+
"audio_embeddings.7.weight": "model-00001-of-00005.safetensors",
|
| 16 |
+
"audio_embeddings.8.weight": "model-00001-of-00005.safetensors",
|
| 17 |
+
"audio_embeddings.9.weight": "model-00001-of-00005.safetensors",
|
| 18 |
+
"local_text_lm_head.weight": "model-00001-of-00005.safetensors",
|
| 19 |
+
"local_transformer.h.0.attn.c_attn.bias": "model-00001-of-00005.safetensors",
|
| 20 |
+
"local_transformer.h.0.attn.c_attn.weight": "model-00001-of-00005.safetensors",
|
| 21 |
+
"local_transformer.h.0.attn.c_proj.bias": "model-00001-of-00005.safetensors",
|
| 22 |
+
"local_transformer.h.0.attn.c_proj.weight": "model-00001-of-00005.safetensors",
|
| 23 |
+
"local_transformer.h.0.ln_1.bias": "model-00001-of-00005.safetensors",
|
| 24 |
+
"local_transformer.h.0.ln_1.weight": "model-00001-of-00005.safetensors",
|
| 25 |
+
"local_transformer.h.0.ln_2.bias": "model-00001-of-00005.safetensors",
|
| 26 |
+
"local_transformer.h.0.ln_2.weight": "model-00001-of-00005.safetensors",
|
| 27 |
+
"local_transformer.h.0.mlp.fc_in.bias": "model-00001-of-00005.safetensors",
|
| 28 |
+
"local_transformer.h.0.mlp.fc_in.weight": "model-00001-of-00005.safetensors",
|
| 29 |
+
"local_transformer.h.0.mlp.fc_out.bias": "model-00001-of-00005.safetensors",
|
| 30 |
+
"local_transformer.h.0.mlp.fc_out.weight": "model-00001-of-00005.safetensors",
|
| 31 |
+
"local_transformer.ln_f.bias": "model-00001-of-00005.safetensors",
|
| 32 |
+
"local_transformer.ln_f.weight": "model-00001-of-00005.safetensors",
|
| 33 |
+
"transformer.embed_tokens.weight": "model-00001-of-00005.safetensors",
|
| 34 |
+
"transformer.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 35 |
+
"transformer.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 36 |
+
"transformer.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 37 |
+
"transformer.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 38 |
+
"transformer.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 39 |
+
"transformer.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
|
| 40 |
+
"transformer.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
|
| 41 |
+
"transformer.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
|
| 42 |
+
"transformer.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
|
| 43 |
+
"transformer.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
|
| 44 |
+
"transformer.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
|
| 45 |
+
"transformer.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 46 |
+
"transformer.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 47 |
+
"transformer.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 48 |
+
"transformer.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 49 |
+
"transformer.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 50 |
+
"transformer.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
|
| 51 |
+
"transformer.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
|
| 52 |
+
"transformer.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
|
| 53 |
+
"transformer.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
|
| 54 |
+
"transformer.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
|
| 55 |
+
"transformer.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
|
| 56 |
+
"transformer.layers.10.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 57 |
+
"transformer.layers.10.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 58 |
+
"transformer.layers.10.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 59 |
+
"transformer.layers.10.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 60 |
+
"transformer.layers.10.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 61 |
+
"transformer.layers.10.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
|
| 62 |
+
"transformer.layers.10.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
|
| 63 |
+
"transformer.layers.10.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
|
| 64 |
+
"transformer.layers.10.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
|
| 65 |
+
"transformer.layers.10.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
|
| 66 |
+
"transformer.layers.10.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
|
| 67 |
+
"transformer.layers.11.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 68 |
+
"transformer.layers.11.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 69 |
+
"transformer.layers.11.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 70 |
+
"transformer.layers.11.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 71 |
+
"transformer.layers.11.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 72 |
+
"transformer.layers.11.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
|
| 73 |
+
"transformer.layers.11.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
|
| 74 |
+
"transformer.layers.11.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
|
| 75 |
+
"transformer.layers.11.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
|
| 76 |
+
"transformer.layers.11.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
|
| 77 |
+
"transformer.layers.11.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
|
| 78 |
+
"transformer.layers.12.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 79 |
+
"transformer.layers.12.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 80 |
+
"transformer.layers.12.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 81 |
+
"transformer.layers.12.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 82 |
+
"transformer.layers.12.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 83 |
+
"transformer.layers.12.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
|
| 84 |
+
"transformer.layers.12.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
|
| 85 |
+
"transformer.layers.12.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
|
| 86 |
+
"transformer.layers.12.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
|
| 87 |
+
"transformer.layers.12.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
|
| 88 |
+
"transformer.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 89 |
+
"transformer.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 90 |
+
"transformer.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 91 |
+
"transformer.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 92 |
+
"transformer.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 93 |
+
"transformer.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 94 |
+
"transformer.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 95 |
+
"transformer.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 96 |
+
"transformer.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 97 |
+
"transformer.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 98 |
+
"transformer.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 99 |
+
"transformer.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 100 |
+
"transformer.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 101 |
+
"transformer.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 102 |
+
"transformer.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 103 |
+
"transformer.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 104 |
+
"transformer.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 105 |
+
"transformer.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 106 |
+
"transformer.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 107 |
+
"transformer.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 108 |
+
"transformer.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 109 |
+
"transformer.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 110 |
+
"transformer.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 111 |
+
"transformer.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 112 |
+
"transformer.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 113 |
+
"transformer.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 114 |
+
"transformer.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 115 |
+
"transformer.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 116 |
+
"transformer.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 117 |
+
"transformer.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 118 |
+
"transformer.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 119 |
+
"transformer.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 120 |
+
"transformer.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 121 |
+
"transformer.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 122 |
+
"transformer.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 123 |
+
"transformer.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 124 |
+
"transformer.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 125 |
+
"transformer.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 126 |
+
"transformer.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 127 |
+
"transformer.layers.16.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 128 |
+
"transformer.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 129 |
+
"transformer.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 130 |
+
"transformer.layers.16.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 131 |
+
"transformer.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 132 |
+
"transformer.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 133 |
+
"transformer.layers.17.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 134 |
+
"transformer.layers.17.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 135 |
+
"transformer.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 136 |
+
"transformer.layers.17.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 137 |
+
"transformer.layers.17.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 138 |
+
"transformer.layers.17.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 139 |
+
"transformer.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 140 |
+
"transformer.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 141 |
+
"transformer.layers.17.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 142 |
+
"transformer.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 143 |
+
"transformer.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 144 |
+
"transformer.layers.18.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 145 |
+
"transformer.layers.18.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 146 |
+
"transformer.layers.18.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 147 |
+
"transformer.layers.18.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 148 |
+
"transformer.layers.18.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 149 |
+
"transformer.layers.18.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 150 |
+
"transformer.layers.18.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 151 |
+
"transformer.layers.18.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 152 |
+
"transformer.layers.18.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 153 |
+
"transformer.layers.18.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 154 |
+
"transformer.layers.18.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 155 |
+
"transformer.layers.19.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 156 |
+
"transformer.layers.19.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 157 |
+
"transformer.layers.19.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 158 |
+
"transformer.layers.19.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 159 |
+
"transformer.layers.19.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 160 |
+
"transformer.layers.19.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 161 |
+
"transformer.layers.19.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 162 |
+
"transformer.layers.19.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 163 |
+
"transformer.layers.19.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 164 |
+
"transformer.layers.19.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 165 |
+
"transformer.layers.19.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 166 |
+
"transformer.layers.2.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 167 |
+
"transformer.layers.2.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 168 |
+
"transformer.layers.2.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 169 |
+
"transformer.layers.2.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 170 |
+
"transformer.layers.2.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 171 |
+
"transformer.layers.2.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 172 |
+
"transformer.layers.2.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 173 |
+
"transformer.layers.2.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 174 |
+
"transformer.layers.2.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 175 |
+
"transformer.layers.2.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 176 |
+
"transformer.layers.2.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 177 |
+
"transformer.layers.20.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 178 |
+
"transformer.layers.20.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 179 |
+
"transformer.layers.20.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 180 |
+
"transformer.layers.20.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 181 |
+
"transformer.layers.20.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 182 |
+
"transformer.layers.20.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 183 |
+
"transformer.layers.20.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 184 |
+
"transformer.layers.20.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 185 |
+
"transformer.layers.20.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 186 |
+
"transformer.layers.20.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 187 |
+
"transformer.layers.20.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 188 |
+
"transformer.layers.21.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 189 |
+
"transformer.layers.21.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 190 |
+
"transformer.layers.21.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 191 |
+
"transformer.layers.21.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 192 |
+
"transformer.layers.21.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 193 |
+
"transformer.layers.21.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 194 |
+
"transformer.layers.21.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 195 |
+
"transformer.layers.21.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 196 |
+
"transformer.layers.21.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 197 |
+
"transformer.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 198 |
+
"transformer.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 199 |
+
"transformer.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 200 |
+
"transformer.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 201 |
+
"transformer.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 202 |
+
"transformer.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 203 |
+
"transformer.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 204 |
+
"transformer.layers.22.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 205 |
+
"transformer.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 206 |
+
"transformer.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 207 |
+
"transformer.layers.22.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 208 |
+
"transformer.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 209 |
+
"transformer.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 210 |
+
"transformer.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 211 |
+
"transformer.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 212 |
+
"transformer.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 213 |
+
"transformer.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 214 |
+
"transformer.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 215 |
+
"transformer.layers.23.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 216 |
+
"transformer.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 217 |
+
"transformer.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 218 |
+
"transformer.layers.23.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 219 |
+
"transformer.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 220 |
+
"transformer.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 221 |
+
"transformer.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 222 |
+
"transformer.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 223 |
+
"transformer.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 224 |
+
"transformer.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 225 |
+
"transformer.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 226 |
+
"transformer.layers.24.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 227 |
+
"transformer.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 228 |
+
"transformer.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 229 |
+
"transformer.layers.24.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 230 |
+
"transformer.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 231 |
+
"transformer.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 232 |
+
"transformer.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 233 |
+
"transformer.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 234 |
+
"transformer.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 235 |
+
"transformer.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 236 |
+
"transformer.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 237 |
+
"transformer.layers.25.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 238 |
+
"transformer.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 239 |
+
"transformer.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 240 |
+
"transformer.layers.25.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 241 |
+
"transformer.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 242 |
+
"transformer.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 243 |
+
"transformer.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 244 |
+
"transformer.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 245 |
+
"transformer.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 246 |
+
"transformer.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 247 |
+
"transformer.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 248 |
+
"transformer.layers.26.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 249 |
+
"transformer.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 250 |
+
"transformer.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 251 |
+
"transformer.layers.26.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 252 |
+
"transformer.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 253 |
+
"transformer.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 254 |
+
"transformer.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 255 |
+
"transformer.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 256 |
+
"transformer.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 257 |
+
"transformer.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 258 |
+
"transformer.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 259 |
+
"transformer.layers.27.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 260 |
+
"transformer.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 261 |
+
"transformer.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 262 |
+
"transformer.layers.27.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 263 |
+
"transformer.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 264 |
+
"transformer.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 265 |
+
"transformer.layers.28.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 266 |
+
"transformer.layers.28.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 267 |
+
"transformer.layers.28.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 268 |
+
"transformer.layers.28.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 269 |
+
"transformer.layers.28.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 270 |
+
"transformer.layers.28.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 271 |
+
"transformer.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 272 |
+
"transformer.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 273 |
+
"transformer.layers.28.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 274 |
+
"transformer.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 275 |
+
"transformer.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 276 |
+
"transformer.layers.29.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 277 |
+
"transformer.layers.29.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 278 |
+
"transformer.layers.29.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 279 |
+
"transformer.layers.29.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 280 |
+
"transformer.layers.29.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 281 |
+
"transformer.layers.29.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 282 |
+
"transformer.layers.29.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 283 |
+
"transformer.layers.29.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 284 |
+
"transformer.layers.29.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 285 |
+
"transformer.layers.29.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 286 |
+
"transformer.layers.29.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 287 |
+
"transformer.layers.3.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 288 |
+
"transformer.layers.3.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 289 |
+
"transformer.layers.3.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 290 |
+
"transformer.layers.3.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 291 |
+
"transformer.layers.3.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 292 |
+
"transformer.layers.3.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 293 |
+
"transformer.layers.3.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 294 |
+
"transformer.layers.3.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 295 |
+
"transformer.layers.3.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 296 |
+
"transformer.layers.3.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 297 |
+
"transformer.layers.3.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 298 |
+
"transformer.layers.30.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 299 |
+
"transformer.layers.30.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 300 |
+
"transformer.layers.30.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 301 |
+
"transformer.layers.30.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 302 |
+
"transformer.layers.30.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 303 |
+
"transformer.layers.30.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 304 |
+
"transformer.layers.30.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 305 |
+
"transformer.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 306 |
+
"transformer.layers.30.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 307 |
+
"transformer.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 308 |
+
"transformer.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 309 |
+
"transformer.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 310 |
+
"transformer.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 311 |
+
"transformer.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 312 |
+
"transformer.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 313 |
+
"transformer.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 314 |
+
"transformer.layers.31.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 315 |
+
"transformer.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 316 |
+
"transformer.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 317 |
+
"transformer.layers.31.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 318 |
+
"transformer.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 319 |
+
"transformer.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 320 |
+
"transformer.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 321 |
+
"transformer.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 322 |
+
"transformer.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 323 |
+
"transformer.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 324 |
+
"transformer.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 325 |
+
"transformer.layers.32.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 326 |
+
"transformer.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 327 |
+
"transformer.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 328 |
+
"transformer.layers.32.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 329 |
+
"transformer.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 330 |
+
"transformer.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 331 |
+
"transformer.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 332 |
+
"transformer.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 333 |
+
"transformer.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 334 |
+
"transformer.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 335 |
+
"transformer.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 336 |
+
"transformer.layers.33.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 337 |
+
"transformer.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 338 |
+
"transformer.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 339 |
+
"transformer.layers.33.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 340 |
+
"transformer.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 341 |
+
"transformer.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 342 |
+
"transformer.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 343 |
+
"transformer.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 344 |
+
"transformer.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 345 |
+
"transformer.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 346 |
+
"transformer.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 347 |
+
"transformer.layers.34.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 348 |
+
"transformer.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 349 |
+
"transformer.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 350 |
+
"transformer.layers.34.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 351 |
+
"transformer.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 352 |
+
"transformer.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 353 |
+
"transformer.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 354 |
+
"transformer.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 355 |
+
"transformer.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 356 |
+
"transformer.layers.35.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 357 |
+
"transformer.layers.35.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 358 |
+
"transformer.layers.35.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 359 |
+
"transformer.layers.35.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 360 |
+
"transformer.layers.35.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 361 |
+
"transformer.layers.35.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 362 |
+
"transformer.layers.35.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 363 |
+
"transformer.layers.35.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 364 |
+
"transformer.layers.4.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 365 |
+
"transformer.layers.4.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 366 |
+
"transformer.layers.4.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 367 |
+
"transformer.layers.4.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 368 |
+
"transformer.layers.4.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 369 |
+
"transformer.layers.4.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 370 |
+
"transformer.layers.4.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 371 |
+
"transformer.layers.4.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 372 |
+
"transformer.layers.4.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 373 |
+
"transformer.layers.4.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 374 |
+
"transformer.layers.4.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 375 |
+
"transformer.layers.5.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 376 |
+
"transformer.layers.5.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 377 |
+
"transformer.layers.5.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 378 |
+
"transformer.layers.5.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 379 |
+
"transformer.layers.5.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 380 |
+
"transformer.layers.5.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 381 |
+
"transformer.layers.5.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 382 |
+
"transformer.layers.5.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 383 |
+
"transformer.layers.5.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 384 |
+
"transformer.layers.5.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 385 |
+
"transformer.layers.5.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 386 |
+
"transformer.layers.6.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 387 |
+
"transformer.layers.6.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 388 |
+
"transformer.layers.6.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 389 |
+
"transformer.layers.6.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 390 |
+
"transformer.layers.6.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 391 |
+
"transformer.layers.6.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 392 |
+
"transformer.layers.6.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 393 |
+
"transformer.layers.6.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 394 |
+
"transformer.layers.6.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 395 |
+
"transformer.layers.6.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 396 |
+
"transformer.layers.6.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 397 |
+
"transformer.layers.7.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 398 |
+
"transformer.layers.7.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 399 |
+
"transformer.layers.7.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 400 |
+
"transformer.layers.7.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 401 |
+
"transformer.layers.7.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 402 |
+
"transformer.layers.7.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 403 |
+
"transformer.layers.7.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 404 |
+
"transformer.layers.7.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 405 |
+
"transformer.layers.7.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 406 |
+
"transformer.layers.7.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 407 |
+
"transformer.layers.7.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 408 |
+
"transformer.layers.8.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 409 |
+
"transformer.layers.8.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 410 |
+
"transformer.layers.8.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 411 |
+
"transformer.layers.8.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
|
| 412 |
+
"transformer.layers.8.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 413 |
+
"transformer.layers.8.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
|
| 414 |
+
"transformer.layers.8.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
|
| 415 |
+
"transformer.layers.8.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
|
| 416 |
+
"transformer.layers.8.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
|
| 417 |
+
"transformer.layers.8.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
|
| 418 |
+
"transformer.layers.8.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
|
| 419 |
+
"transformer.layers.9.input_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 420 |
+
"transformer.layers.9.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
|
| 421 |
+
"transformer.layers.9.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
|
| 422 |
+
"transformer.layers.9.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
|
| 423 |
+
"transformer.layers.9.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 424 |
+
"transformer.layers.9.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
|
| 425 |
+
"transformer.layers.9.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
|
| 426 |
+
"transformer.layers.9.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
|
| 427 |
+
"transformer.layers.9.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
|
| 428 |
+
"transformer.layers.9.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
|
| 429 |
+
"transformer.layers.9.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
|
| 430 |
+
"transformer.norm.weight": "model-00005-of-00005.safetensors"
|
| 431 |
+
}
|
| 432 |
+
}
|