diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..35198d13b0028d82a60273a5363f393580083570 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +examples/de-S1.wav filter=lfs diff=lfs merge=lfs -text +examples/de-S2.wav filter=lfs diff=lfs merge=lfs -text +examples/en-S1.wav filter=lfs diff=lfs merge=lfs -text +examples/en-S2.wav filter=lfs diff=lfs merge=lfs -text +examples/es-S1.wav filter=lfs diff=lfs merge=lfs -text +examples/es-S2.wav filter=lfs diff=lfs merge=lfs -text +examples/fr-S1.wav filter=lfs diff=lfs merge=lfs -text +examples/fr-S2.wav filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/CLI_INFERENCE_VERIFIED.json b/CLI_INFERENCE_VERIFIED.json new file mode 100644 index 0000000000000000000000000000000000000000..e06d9a1be651012acd75b4df19cea28d08e3165c --- /dev/null +++ b/CLI_INFERENCE_VERIFIED.json @@ -0,0 +1,8 @@ +{ + "passed": true, + "input": "actual original-media WAV references encoded by portable CLI", + "output": "/e/scratch/laionize/schuhmann1/hv_notebooklm_edition_20261103/release/cli-canary.wav", + "duration_s": 18.0, + "sample_rate": 48000, + "job": "2658385" +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..3e82e6c892abe0fa3a95fe794445d7dffb4dd8e9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +Creative Commons Attribution 4.0 International (CC BY 4.0) +Copyright 2026 LAION and contributors. +https://creativecommons.org/licenses/by/4.0/legalcode +Fine-tuned weights are released under CC BY 4.0. Third-party components retain their own licenses. diff --git a/MERGE_VERIFICATION.json b/MERGE_VERIFICATION.json new file mode 100644 index 0000000000000000000000000000000000000000..1611fa9dcad298e2ba237ea0ee052270427c5305 --- /dev/null +++ b/MERGE_VERIFICATION.json @@ -0,0 +1,28 @@ +{ + "passed": true, + "parent_step": 5613, + "adapter_step": 652, + "rank": 64, + "alpha": 64, + "scale": 1.0, + "modules": 146, + "merged": true, + "linear_max_relative_error": 0.007352941176470588, + "export_reload_semantic_prefix_exact": true, + "weights": { + "model-00001-of-00003.safetensors": { + "bytes": 3994689160, + "sha256": "3c8b9d4e555370073a298b34d65927b7956752de969aade484f05544c9d4acaa" + }, + "model-00003-of-00003.safetensors": { + "bytes": 304110936, + "sha256": "b87a011a7f1e37373e5a4489e9dbbffbb298c4760505e94474bfc63f37d1660c" + }, + "model-00002-of-00003.safetensors": { + "bytes": 3961231200, + "sha256": "a9d4328d7b8a2f7e677f3e9aa35568f14de8f4e0fb7df615fc36adc5cfb8c2a0" + } + }, + "adapter_sha256": "844cf1707ab8e3980e9aab09131132de955be68fc0eab62dd933d24a810744e0", + "time": 1793662358.5163062 +} diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000000000000000000000000000000000000..f0644403ab5695b8019bd17ebbd2d3d63a62d6b9 --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,6 @@ +Parent: https://huggingface.co/laion/Humaneness-Voice-Large-1.3-Alpha +Architecture lineage: https://huggingface.co/laion/moss-tts-local-transformer-4.55b-voice-acting-v2-sft3 +External codec: https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2 +Podcast source: https://huggingface.co/datasets/TTS-AGI/NotebookLM-Podcasts +Prepared data: https://huggingface.co/datasets/laion/NotebookLM-Podcasts-Humaneness-Prepared +Original code attribution is retained in implementation files. This release does not relicense third-party software or grant identity rights. diff --git a/PODCAST_PROMPTING.md b/PODCAST_PROMPTING.md new file mode 100644 index 0000000000000000000000000000000000000000..48e0342f229aaedc27d7421566f5ef907ee9c417 --- /dev/null +++ b/PODCAST_PROMPTING.md @@ -0,0 +1,41 @@ +# Podcast prompting and input contract + +Use podcast_infer.py with a dialogue JSON. Directions remain English; the words spoken stay in their original language. The CLI supplies the two voice recordings separately. Never insert a filename, audio placeholder or speaker identity into the literal spoken text. + +## Minimal dialogue + + { + "language": "en", + "general": "Two friends chatting naturally in a quiet studio.", + "pause_s": 0.24, + "turns": [ + {"speaker": "S1", "text": "Did you try the new walking route?", "seconds": 3.0, "direction": "warm, curious, conversational"}, + {"speaker": "S2", "text": "I did... and I found a lovely little bookshop.", "seconds": 4.4, "direction": "friendly, amused, spontaneous"} + ] + } + +## The rendered performance prompt + + GENERAL: Two friends chatting naturally in a quiet studio. + SCRIPT: + (speaker=S1, warm, curious, conversational) [3.00 seconds duration] Did you try the new walking route? + [0.24 seconds pause] + (speaker=S2, friendly, amused, spontaneous) [4.40 seconds duration] I did... and I found a lovely little bookshop. + +GENERAL may be empty. SCRIPT's speaker= syntax, parentheses, square timing cues and ordering are part of the trained input. Directions are not literal words. Keep the transcript itself free of speaker labels. Ellipses can suggest a hesitation, but do not guarantee a precise pause; square pause cues are clearer. Requested turn durations include the speech in that turn; separate pause cues add to the total. Do not count the same pause twice. + +## Reference audio + +Supply one clean voice per file, preferably 6–12 seconds, with no music, clipping or competing speaker. The allowed range after codec quantization is 63–187 frames (5.04–14.96 seconds). The old 2.96-second reference limit is not used. S1 maps to --reference-s1, and S2 maps to --reference-s2. The two files may be independent authorized voices. The new demo uses clean original HQ training-source crops; do not judge the older codec-reconstructed players as untouched reference recordings. + +## Optional conversation history + +Use --context previous.wav --context-description "S1: 0.0–5.0 seconds; S2: 5.2–11.0 seconds" only for genuinely earlier audio. Keep chronological context and a truthful speaker timeline; maximum 60 seconds. Identity references and dialogue history are distinct inputs. Never include the desired future target in previous.wav. The new HQ demonstration deliberately tests a fresh dialogue with two identity references and no earlier conversation. + +## Longer conversations + +The test matrix requests 2 turns at 18 seconds, 6 turns at 38.4 seconds and 8 turns at 59.2 seconds. These are requested durations; the measured generated durations are shown separately. The model generates a whole continuation jointly. Do not claim that requested labels prove it actually changed speakers; listen to each transition. For longer sessions, retain recent chronological output as context and request the next part, keeping identity labels consistent. + +## Reproducibility + +The examples use seed 20261103 plus topic index, temperature 1, top-p .95, top-k 50 and CFG 1. Prompt files, sampling and source-reference provenance are included. A fixed seed does not promise bit-identical outputs across different GPU hardware, software versions, precision or kernel choices. diff --git a/PROMPT_PREFIX_VERIFIED.json b/PROMPT_PREFIX_VERIFIED.json new file mode 100644 index 0000000000000000000000000000000000000000..612aaf2b8a2ab57ad31fe175396744af2f8ed775 --- /dev/null +++ b/PROMPT_PREFIX_VERIFIED.json @@ -0,0 +1,5 @@ +{ + "passed": true, + "tasks": 30, + "method": "Actual tokenizer/processor; portable helper input_ids and attention_mask match frozen podcast training packer exactly for every new dialogue. No neural inference in this CPU-only format test." +} diff --git a/PUBLICATION_ID.json b/PUBLICATION_ID.json new file mode 100644 index 0000000000000000000000000000000000000000..011f298db32962bf152ce984095b26361125ac1b --- /dev/null +++ b/PUBLICATION_ID.json @@ -0,0 +1,5 @@ +{ + "project": "notebooklm-edition-20261103", + "parent_step": 5613, + "adapter_step": 652 +} diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c2210590c1002028e8b0559def579d378c417b46 --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +license: cc-by-4.0 +language: +- en +- de +- fr +- es +pipeline_tag: text-to-speech +base_model: laion/Humaneness-Voice-Large-1.3-Alpha +tags: +- podcast +- multi-speaker +- moss-tts +- notebooklm +- merged-lora +--- + +# Humaneness Voice Large — NotebookLM Edition + +Generate a two-person podcast conversation from a written dialogue and two clean voice references. This is the user's listening-selected **NotebookLM + rank 64 · final 652**, with its adapter already merged at scale 1. Download the root weights and run inference directly; do not apply the included adapter a second time. + +The model begins with [Humaneness Voice Large 1.3 Alpha](https://huggingface.co/laion/Humaneness-Voice-Large-1.3-Alpha), receives a three-epoch full fine-tune on filtered two-speaker podcasts, and then a small rank-64 quality-ladder fine-tune. It produces the entire requested exchange as one acoustic sequence. It is not a pipeline that separately synthesizes each turn and joins files. + +**Start here:** [Podcast prompting guide](PODCAST_PROMPTING.md) · [Inference script](podcast_infer.py) · [Training and lineage](TRAINING.md) · [Existing held-out comparison](https://laion-humaneness-voice-large-podcast-listening.static.hf.space/lora-comparison.html) + +## Install and generate + +Use a CUDA GPU with sufficient memory, a recent compatible PyTorch installation, and Python 3.11 or newer. The tested JUPITER environment used PyTorch 2.9.1 and Transformers 5.14.1. CPU-only inference and other precision modes are not validated here. + +```bash +hf download laion/Humaneness-Voice-Large-NotebookLM-Edition --local-dir notebooklm-edition +cd notebooklm-edition +pip install -r requirements.txt +python podcast_infer.py --model . --dialogue examples/dialogue.json \ + --reference-s1 examples/en-S1.wav --reference-s2 examples/en-S2.wav \ + --seed 20261103 --output conversation.wav +``` + +The first invocation also downloads the separately licensed [MOSS Audio Tokenizer v2](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2). The model's custom implementation requires trust_remote_code; inspect the supplied code before running it. Supply your own authorized clean single-speaker WAV references to change the voices. The provided reference files are original-media crops with source provenance in examples/reference-provenance.json. + +## What the model receives + +Two distinct S1/S2 identity-reference blocks precede the prompt. Each reference must cover 5.04–14.96 seconds; longer input files are cropped before encoding. Optional strictly earlier conversation may supply up to 60 seconds of context. The GENERAL block describes the overall performance. SCRIPT contains alternating speaker labels, English delivery directions, literal words in the requested language, and duration/pause cues. The CLI builds the exact trained ChatML serialization and preserves all audio placeholders. The generic base-model single-speaker helper does not implement this podcast contract. + +S1 and S2 are local routing labels for the two recordings, not global identities, registered personal names or spoken words. English, German, French and Spanish are the focus of the HQ demonstrations. Dutch was present in the first full fine-tune but excluded from the quality adapter because publisher labels were unreliable; Dutch performance is not advertised. + +## Architecture and sampling + +The architecture is the Large MOSS Local Transformer family: a 36-layer Qwen3-family semantic transformer with 2,560-dimensional states and a one-layer local acoustic transformer using the same width. Twelve codebooks are generated at each 80-ms acoustic frame, or 150 codec indices per second. MOSS-v2 decodes the result to 48-kHz mono audio. The tokenizer and network architecture were not expanded for speaker routing. + +The reference command uses audio temperature 1, top-p 0.95, top-k 50, repetition penalty 1, and CFG 1. Whole-dialogue generation is stochastic. Requested durations and speaker switches are learned instructions, not hard constraints or forced waveform edits. Start with two short turns, then try longer dialogues. + +## Evidence and limits + +The user selected this checkpoint after listening. The earlier frozen 50-task comparison gave this adapter CE 6.079, genuineness 3.959 and mean per-clip ASR WER 11.44%; its NotebookLM full-fine-tuned parent gave 6.084, 3.943 and 7.76%. Those metrics do not establish an overall automatic improvement. They also do not measure speaker-switch correctness. The HQ reference study uses new text and training-domain identities; it is a demonstration, not a held-out-speaker benchmark. All raw model-predicted scores and prompts are preserved. + +## License and intended use + +The released fine-tuned weights are CC BY 4.0; see LICENSE and NOTICE.md. Upstream code, tokenizer assets, the external codec, and source recordings retain their own terms. Research and authorized dataset augmentation are recommended uses. Use only voices for which you have appropriate permission; do not use the model for deceptive impersonation. Test quality, language accuracy and speaker stability before any production use. LAION does not endorse generated statements or speaker identities. + +The new 30-dialogue HQ-reference study is running. Its page will be linked here automatically after public HTML and audio publication. diff --git a/TRAINING.md b/TRAINING.md new file mode 100644 index 0000000000000000000000000000000000000000..1299d645916d03e2e002d1390530fbc7adf299e1 --- /dev/null +++ b/TRAINING.md @@ -0,0 +1,11 @@ +# Training lineage and reproducibility + +1. Start from Humaneness Voice Large 1.3 Alpha, final step 9,799. This is already an extensively supervised Large model, not random initialization or the original SFT3 checkpoint. +2. Full fine-tune all parameters on the prepared two-speaker podcast corpus for three shuffled epochs. Starting and middle 30-second windows each require predicted DNSMOS background >=4.0. Five publisher-language pools were prepared. Episode-disjoint heldouts and duplicate-audio checks precede training. Use 64 GPUs, accumulation 2, peak semantic LR 2e-5 and Talker LR 4e-5, 5% warmup and one cosine schedule to a 10% floor. Final checkpoint: step 5,613. Audio-only target labels exclude reference and conversation-prefix audio. +3. Attach rank-64, alpha-64 adapters to 146 attention projections across semantic and local transformers. Use the same two-speaker prefix and strict-past-context contract. The clean quality ladder contains 732 unique examples in EN/DE/FR/ES; Dutch is excluded. The six nominal hourly caps are 5,4,3,2,1,1, but actual selected data are much smaller. Presented target audio is 50.56+43.23+35.87+25.46+11.66+11.66 minutes, about 2.97 hours including repetitions. This must not be described as 50–100 hours of unique training audio. +4. Four GPUs, effective batch four, 652 total updates. Phase lengths are 183,160,132,91,43,43. Linear warmup reaches peak 3e-5 at the end of phase 1; one cosine decay spans the remaining phases to 3e-6. Optimizer is AdamW, betas .9/.95, epsilon 1e-8, no weight decay. Fresh-process optimizer/RNG/adapter replay smoke passed before production. Final update 652 is the release adapter. +5. Merge the rank-64 residual B@A at scale 1 into its actual full-fine-tuned podcast parent. Root weights are BF16 for inference. Every adapted Linear is checked against the trained residual definition, followed by exact export/reload equality for the merged model's semantic prefix. BF16 merging is not claimed bit-identical to evaluating separate FP32 residual adapters. The unmerged original adapter is retained under adapter/ for provenance and must not be applied again to root weights. + +Original training scripts and run statistics are included under training/ and stats/. They preserve exact JUPITER paths and environment entrypoints as historical evidence; relocate those paths and stage their dependencies before rerunning on another cluster. They are not falsely advertised as a standalone one-command portable training pipeline. The root inference script is portable and is tested against the merged export on allocated compute. External codec and scorer weights are separate dependencies, not silently bundled. + +The held-out evaluation and the new-text HQ-reference demonstration answer different questions. The latter intentionally uses in-domain training-source voices. Full outputs are EARS-scored in <=30-second windows, weighted by duration; ASR chunks are concatenated for mean per-clip WER. Chunk boundaries may introduce errors. Speaker-switch correctness is not computed from the requested script. Inspect actual speaker changes by listening. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2dcfd72a091e0ecacd0cc44d05d3bc8b8fdb69de --- /dev/null +++ b/__init__.py @@ -0,0 +1,9 @@ +from .configuration_moss_tts import MossTTSLocalConfig +from .modeling_moss_tts import MossTTSLocalModel +from .processing_moss_tts import MossTTSLocalProcessor + +__all__ = [ + "MossTTSLocalConfig", + "MossTTSLocalModel", + "MossTTSLocalProcessor", +] diff --git a/adapter/README.txt b/adapter/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..26909773dbfcb3cc80fde6e77578744c25b7ec24 --- /dev/null +++ b/adapter/README.txt @@ -0,0 +1 @@ +Rank 64, alpha 64, scale 1, final update 652. Already merged in root weights. Do not apply twice. Original parent is NotebookLM full FT step 5613, not unmodified Large 1.3. diff --git a/adapter/READY.json b/adapter/READY.json new file mode 100644 index 0000000000000000000000000000000000000000..d920030a312f3650349eefcce4961e246bac388a --- /dev/null +++ b/adapter/READY.json @@ -0,0 +1,8 @@ +{ + "step": 652, + "reason": "final", + "contract_sha256": "88d1284099df084d941f8274e953612a864c607e9ae4380af738280ef9dfc517", + "training_sha256": "86a8fa8f5ff1c3054278e0ea4f4fbb41b5060c7fe900bd5d46ab1f2dce65a99a", + "adapter_sha256": "844cf1707ab8e3980e9aab09131132de955be68fc0eab62dd933d24a810744e0", + "optimizer_rng_sampler_schedule_saved": true +} diff --git a/adapter/adapter.safetensors b/adapter/adapter.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7b3a72392427d90d692329eb2ac3810cb6f7695c --- /dev/null +++ b/adapter/adapter.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:844cf1707ab8e3980e9aab09131132de955be68fc0eab62dd933d24a810744e0 +size 192708704 diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5f284c0d74ecf9b9c006194bd5040b1fe3b640 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,31 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151671, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/base_runtime.py b/base_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..b9f7a0ad618ea669d9f25cbcc18643f831b8f298 --- /dev/null +++ b/base_runtime.py @@ -0,0 +1,144 @@ +"""Humaneness Voice Large 1.3 Alpha: portable full-model inference. + +Use a downloaded model directory, or a Hugging Face model ID. This file has no +JUPITER-specific runtime dependency. Twelve MOSS-v2 codebooks,48kHz mono. +""" +import argparse +import json +from pathlib import Path + +LANGUAGES = {'en': 'English', 'de': 'German', 'nl': 'Dutch', 'fr': 'French', 'es': 'Spanish'} + + +def load_model(directory, device='cuda:0'): + import torch + from transformers import AutoModel, AutoTokenizer + from transformers.dynamic_module_utils import get_class_from_dynamic_module + model = AutoModel.from_pretrained(str(directory), trust_remote_code=True, + dtype=torch.bfloat16, attn_implementation='sdpa').to(device).eval() + Processor = get_class_from_dynamic_module('processing_moss_tts.MossTTSLocalProcessor', str(directory)) + tokenizer = AutoTokenizer.from_pretrained(str(directory), trust_remote_code=True) + proc = Processor(tokenizer=tokenizer, audio_tokenizer=None, model_config=model.config) + return model, proc + + +def load_codec(proc, path, device='cuda:0'): + from transformers import AutoModel + codec = AutoModel.from_pretrained(str(path), trust_remote_code=True, codec_weight_dtype='fp32') + if hasattr(codec, 'set_compute_dtype'): + codec.set_compute_dtype('bf16') + proc.audio_tokenizer = codec.to(device).eval() + return proc + + +def build_user(proc, *, text, prompt, frames, language, reference=None, alias=None): + if not text.strip() or not prompt.strip() or language not in LANGUAGES: + raise ValueError('Nonempty literal text, English directions and a supported language required') + if reference is not None and (reference.ndim != 2 or reference.shape[1] != 12 + or not 63 <= len(reference) <= 187): + raise ValueError('Reference codes must contain12codebooks over5.04–14.96seconds') + if alias is not None: + if reference is not None: + raise ValueError('Choose either a reference recording or a registered voice name, not both') + catalog = json.loads(Path(__file__).with_name('voices.json').read_text()) + names = {v['speaker_alias'] for v in catalog['voices']} + if alias not in names: + raise ValueError('Unknown voice name; see VOICE_ALIASES.md and voices.json') + user = proc.build_user_message(text=text, instruction=prompt, tokens=int(frames), + language=LANGUAGES[language], reference=[reference] if reference is not None else None) + if alias is not None: + # Exact alias representation used by training/legacy/acting_data.py. + # It is not an invented prefix in the spoken text or a new special token. + marker = '- Reference(s):\nNone\n' + if user['content'].count(marker) != 1 or user.get('audio_codes_list'): + raise RuntimeError('Processor wrapper no longer matches the trained voice-name format') + user['content'] = user['content'].replace(marker, f'- Reference(s):\nSpeaker: {alias}\n', 1) + return user + + +def generate(model, proc, user, *, seed=777, max_frames=625, device='cuda:0'): + import torch + batch = proc([[user]], mode='generation') + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): + output = model.generate(input_ids=batch['input_ids'].to(device), + attention_mask=batch['attention_mask'].to(device), max_new_frames=max_frames, + do_sample=True, audio_temperature=1., audio_top_p=.95, audio_top_k=50, + audio_repetition_penalty=1., use_kv_cache=True) + prefix, sequence = output[0] + if prefix != 0 or sequence.ndim != 2 or sequence.shape[1] != 13: + raise RuntimeError('Unexpected MOSS frame layout') + codes = sequence[1:, 1:] + codes = codes[(codes[:, 0] >= 0) & (codes[:, 0] != model.config.audio_pad_token_id)] + if len(codes) < 2 or int(codes.min()) < 0 or int(codes.max()) > 1023: + raise RuntimeError('Too few frames or corrupt codec indices') + return codes.cpu() + + +def decode(proc, codes, device='cuda:0'): + import numpy as np + import torch + with torch.inference_mode(): + wave = proc.decode_audio_codes([codes.to(device)], return_stereo=False)[0] + wave = wave.float().cpu().numpy().reshape(-1) + rate = int(proc.model_config.sampling_rate) + if not np.isfinite(wave).all() or abs(len(wave) / len(codes) - rate / 12.5) > .01 * rate / 12.5: + raise RuntimeError('Codec frame-rate/sample-rate canary failed') + return wave, rate + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--model', default='laion/Humaneness-Voice-Large-1.3-Alpha') + p.add_argument('--codec', default='OpenMOSS-Team/MOSS-Audio-Tokenizer-v2') + p.add_argument('--text', required=True, help='Literal spoken words, without directing tags') + p.add_argument('--prompt', help='GENERAL/SCRIPT directions; literal \\n accepted') + p.add_argument('--prompt-file', type=Path) + p.add_argument('--language', choices=LANGUAGES, default='en') + voice = p.add_mutually_exclusive_group() + voice.add_argument('--reference-wav', type=Path) + voice.add_argument('--voice-name', help='Registered fictional voice alias; see VOICE_ALIASES.md') + p.add_argument('--seconds', type=float, default=8.) + p.add_argument('--seed', type=int, default=777) + p.add_argument('--output', type=Path, required=True) + args = p.parse_args() + if bool(args.prompt) == bool(args.prompt_file): + p.error('Supply exactly one of --prompt or --prompt-file') + if not 1 <= args.seconds <= 60: + p.error('Target seconds must be between1and60') + import numpy as np + import soundfile as sf + import torch + if not torch.cuda.is_available(): + raise RuntimeError('The tested path requires a CUDA GPU') + device = 'cuda:0' + model, proc = load_model(args.model, device) + load_codec(proc, args.codec, device) + reference = None + if args.reference_wav: + wave, sr = sf.read(args.reference_wav, dtype='float32', always_2d=True) + if not np.isfinite(wave).all() or len(wave) / sr < 5.04: + raise ValueError('Use at least5.04seconds of clean single-speaker reference speech') + # Never reinstate37frames/2.96s. Crop waveform BEFORE encoding. + wave = wave[:int(sr * 14.96)] + with torch.inference_mode(): + reference = proc.encode_audios_from_wav([torch.from_numpy(wave.T.copy())], int(sr), n_vq=12)[0][:187] + prompt = args.prompt_file.read_text() if args.prompt_file else args.prompt.replace('\\n', '\n') + frames = round(args.seconds * 12.5) + user = build_user(proc, text=args.text, prompt=prompt, frames=frames, + language=args.language, reference=reference, alias=args.voice_name) + codes = generate(model, proc, user, seed=args.seed, max_frames=min(1000, frames + 190)) + wave, sr = decode(proc, codes) + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.output.exists(): + raise FileExistsError('Refusing to overwrite ' + str(args.output)) + sf.write(args.output, wave, sr, subtype='PCM_16') + print(json.dumps({'output': str(args.output), 'duration_s': len(wave) / sr, + 'requested_s': args.seconds, 'frames': len(codes), 'language': args.language, + 'reference_frames': len(reference) if reference is not None else 0, + 'voice_name': args.voice_name, 'seed': args.seed})) + + +if __name__ == '__main__': + main() diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d7214cb4cb0bd0b2fec95192100fbfee69085d64 --- /dev/null +++ b/config.json @@ -0,0 +1,267 @@ +{ + "architectures": [ + "MossTTSLocalModel" + ], + "attn_implementation": "sdpa", + "audio_assistant_gen_slot_token_id": 151656, + "audio_assistant_slot_token_id": 151656, + "audio_codebook_sizes": [ + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024 + ], + "audio_end_token_id": 151670, + "audio_pad_code": 1024, + "audio_pad_token_id": 1024, + "audio_start_token_id": 151669, + "audio_tokenizer_name_or_path": "OpenMOSS-Team/MOSS-Audio-Tokenizer-v2", + "audio_user_slot_token_id": 151654, + "audio_vocab_size": 1024, + "auto_map": { + "AutoConfig": "configuration_moss_tts.MossTTSLocalConfig", + "AutoModel": "modeling_moss_tts.MossTTSLocalModel", + "AutoProcessor": "processing_moss_tts.MossTTSLocalProcessor" + }, + "dtype": "bfloat16", + "gpt2_config": { + "activation_function": "silu", + "add_cross_attention": false, + "attn_pdrop": 0.0, + "bos_token_id": null, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "embd_pdrop": 0.0, + "eos_token_id": 151645, + "finetuning_task": null, + "initializer_range": 0.02, + "is_decoder": false, + "layer_norm_epsilon": 1e-06, + "model_type": "gpt2", + "n_ctx": 10240, + "n_embd": 2560, + "n_head": 32, + "n_inner": 9728, + "n_layer": 1, + "n_positions": 10240, + "pad_token_id": null, + "position_embedding_type": "rope", + "prefix": null, + "pruned_heads": {}, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.0, + "rope_base": 1000000.0, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "sep_token_id": null, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": null, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "torchscript": false, + "use_bfloat16": false, + "use_cache": true, + "vocab_size": 151936 + }, + "hidden_size": 2560, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "initializer_range": 0.02, + "language_config": { + "add_cross_attention": false, + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "finetuning_task": null, + "gradient_checkpointing_use_reentrant": false, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "is_decoder": false, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "prefix": null, + "pruned_heads": {}, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sep_token_id": null, + "sliding_window": null, + "task_specific_params": null, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "torchscript": false, + "use_bfloat16": false, + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 + }, + "local_hidden_size": 2560, + "local_text_head_mode": "binary", + "local_transformer_attn_implementation": "sdpa", + "local_transformer_layers": 1, + "model_type": "moss_tts_local", + "n_vq": 12, + "pad_token_id": 151643, + "processor_class": "MossTTSLocalProcessor", + "qwen3_config": { + "add_cross_attention": false, + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "finetuning_task": null, + "gradient_checkpointing_use_reentrant": false, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "is_decoder": false, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "prefix": null, + "pruned_heads": {}, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sep_token_id": null, + "sliding_window": null, + "task_specific_params": null, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "torchscript": false, + "use_bfloat16": false, + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 + }, + "sampling_rate": 48000, + "tie_word_embeddings": true, + "transformers_version": "5.14.1", + "use_static_local_kv_cache": true, + "vocab_size": 151936 +} diff --git a/configuration_moss_tts.py b/configuration_moss_tts.py new file mode 100644 index 0000000000000000000000000000000000000000..04b453edd7077170ebbd6a3ffb8167d9af8ac458 --- /dev/null +++ b/configuration_moss_tts.py @@ -0,0 +1,158 @@ +# coding=utf-8 +"""Configuration for the MOSS-TTS-Local-Transformer-v1.5 release.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Union + +from transformers.configuration_utils import PretrainedConfig +from transformers.models.gpt2.configuration_gpt2 import GPT2Config +from transformers.models.qwen3.configuration_qwen3 import Qwen3Config + + +SUPPORTED_ATTENTION_IMPLEMENTATIONS = {"flash_attention_2", "sdpa", "eager"} + + +def _normalize_attention_implementation(value: Optional[str], default: str = "flash_attention_2") -> str: + normalized = str(value or default).strip().lower() + if normalized in {"flash", "flash_attn", "flash-attn", "flash_attention"}: + normalized = "flash_attention_2" + if normalized not in SUPPORTED_ATTENTION_IMPLEMENTATIONS: + raise ValueError( + "attn_implementation must be one of " + f"{sorted(SUPPORTED_ATTENTION_IMPLEMENTATIONS)}, got {value!r}." + ) + return normalized + + +class MossTTSLocalConfig(PretrainedConfig): + model_type = "moss_tts_local" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + qwen3_config: Optional[Union[Qwen3Config, Dict[str, Any]]] = None, + gpt2_config: Optional[Union[GPT2Config, Dict[str, Any]]] = None, + language_config: Optional[Union[Qwen3Config, Dict[str, Any]]] = None, + n_vq: int = 12, + audio_vocab_size: int = 1024, + audio_codebook_sizes: Optional[list[int]] = None, + audio_pad_token_id: int = 1024, + audio_pad_code: Optional[int] = None, + pad_token_id: int = 151643, + im_start_token_id: int = 151644, + im_end_token_id: int = 151645, + audio_start_token_id: int = 151669, + audio_end_token_id: int = 151670, + audio_user_slot_token_id: int = 151654, + audio_assistant_slot_token_id: int = 151656, + audio_assistant_gen_slot_token_id: Optional[int] = None, + sampling_rate: int = 48000, + audio_tokenizer_name_or_path: Optional[str] = None, + attn_implementation: str = "flash_attention_2", + local_transformer_attn_implementation: Optional[str] = None, + local_text_head_mode: str = "binary", + initializer_range: float = 0.02, + **kwargs: Any, + ) -> None: + if qwen3_config is None and language_config is not None: + qwen3_config = language_config + if isinstance(qwen3_config, dict): + self.qwen3_config = Qwen3Config(**qwen3_config) + elif qwen3_config is None: + self.qwen3_config = Qwen3Config() + else: + self.qwen3_config = qwen3_config + + if isinstance(gpt2_config, dict): + self.gpt2_config = GPT2Config(**gpt2_config) + elif gpt2_config is None: + self.gpt2_config = GPT2Config( + vocab_size=int(self.qwen3_config.vocab_size), + n_embd=int(self.qwen3_config.hidden_size), + n_layer=1, + n_head=max(1, int(self.qwen3_config.hidden_size) // 80), + n_positions=int(n_vq) + 1, + n_ctx=int(n_vq) + 1, + activation_function="silu", + layer_norm_epsilon=1e-6, + resid_pdrop=0.0, + embd_pdrop=0.0, + attn_pdrop=0.0, + ) + else: + self.gpt2_config = gpt2_config + + self.n_vq = int(n_vq) + if self.n_vq <= 0: + raise ValueError("n_vq must be positive.") + if audio_codebook_sizes is None: + self.audio_codebook_sizes = [int(audio_vocab_size)] * self.n_vq + else: + self.audio_codebook_sizes = [int(size) for size in audio_codebook_sizes] + if len(self.audio_codebook_sizes) != self.n_vq: + raise ValueError( + f"audio_codebook_sizes must have length n_vq={self.n_vq}, " + f"got {len(self.audio_codebook_sizes)}." + ) + if any(size <= 0 for size in self.audio_codebook_sizes): + raise ValueError("audio_codebook_sizes must contain positive integers.") + self.audio_vocab_size = int(max(int(audio_vocab_size), max(self.audio_codebook_sizes))) + self.audio_pad_token_id = int(audio_pad_code if audio_pad_code is not None else audio_pad_token_id) + self.audio_pad_code = self.audio_pad_token_id + if self.audio_pad_token_id < self.audio_vocab_size: + raise ValueError("audio_pad_token_id/audio_pad_code must be outside the audio vocab.") + + self.pad_token_id = int(pad_token_id) + self.im_start_token_id = int(im_start_token_id) + self.im_end_token_id = int(im_end_token_id) + self.audio_start_token_id = int(audio_start_token_id) + self.audio_end_token_id = int(audio_end_token_id) + self.audio_user_slot_token_id = int(audio_user_slot_token_id) + self.audio_assistant_slot_token_id = int( + audio_assistant_slot_token_id + if audio_assistant_gen_slot_token_id is None + else audio_assistant_gen_slot_token_id + ) + self.audio_assistant_gen_slot_token_id = self.audio_assistant_slot_token_id + + self.sampling_rate = int(sampling_rate) + self.audio_tokenizer_name_or_path = audio_tokenizer_name_or_path + self.attn_implementation = _normalize_attention_implementation(attn_implementation) + self.local_transformer_attn_implementation = _normalize_attention_implementation( + local_transformer_attn_implementation, + default=self.attn_implementation, + ) + self.initializer_range = float(initializer_range) + + self.hidden_size = int(self.qwen3_config.hidden_size) + self.vocab_size = int(self.qwen3_config.vocab_size) + self.local_hidden_size = int(self.gpt2_config.hidden_size) + if self.local_hidden_size != self.hidden_size: + raise ValueError( + "This MOSS-TTS-Local-Transformer-v1.5 release expects local hidden size to " + "match Qwen3 hidden size so audio embeddings and heads are tied." + ) + + normalized_text_head_mode = str(local_text_head_mode or "full_vocab").strip().lower() + if normalized_text_head_mode in {"full", "full-vocab", "vocab"}: + normalized_text_head_mode = "full_vocab" + if normalized_text_head_mode not in {"full_vocab", "binary"}: + raise ValueError("local_text_head_mode must be 'full_vocab' or 'binary'.") + self.local_text_head_mode = normalized_text_head_mode + + kwargs.setdefault("tie_word_embeddings", True) + super().__init__(pad_token_id=self.pad_token_id, **kwargs) + + @property + def language_config(self) -> Qwen3Config: + return self.qwen3_config + + def to_dict(self) -> Dict[str, Any]: + output = super().to_dict() + output["qwen3_config"] = self.qwen3_config.to_dict() + output["language_config"] = self.qwen3_config.to_dict() + output["gpt2_config"] = self.gpt2_config.to_dict() + output["audio_pad_code"] = self.audio_pad_token_id + output["audio_assistant_gen_slot_token_id"] = self.audio_assistant_slot_token_id + return output diff --git a/examples/all_dialogues.json b/examples/all_dialogues.json new file mode 100644 index 0000000000000000000000000000000000000000..ad7bce470159ae4886bbab8197e59e6e19ace965 --- /dev/null +++ b/examples/all_dialogues.json @@ -0,0 +1,1292 @@ +[ + { + "task_id": "short-en-coffee", + "group": "short", + "language": "en", + "topic": "coffee", + "seed": 20261103, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I tried making coffee without looking at my phone this morning. It felt oddly peaceful, actually.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That sounds lovely. Although I would probably spend the whole time wondering whether the kettle was taking longer.", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-en-walking", + "group": "short", + "language": "en", + "topic": "walking", + "seed": 20261104, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Do you ever take a different route home just because the usual one has started to feel invisible?", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Yes, and then I discover a tiny shop that has apparently been there for about twenty years.", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-en-learning", + "group": "short", + "language": "en", + "topic": "learning", + "seed": 20261105, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I finally played that difficult passage on the piano without stopping. Only once, but I am counting it.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "You absolutely should. One good attempt means your hands have discovered that the thing is possible.", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-de-cooking", + "group": "short", + "language": "de", + "topic": "cooking", + "seed": 20261106, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ich habe gestern endlich das Brot selbst gebacken. Die Küche sah danach allerdings ziemlich abenteuerlich aus.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das gehört dazu. Die entscheidende Frage ist doch: Hat es gut geschmeckt, oder war es eher ein Türstopper?", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-de-train", + "group": "short", + "language": "de", + "topic": "train", + "seed": 20261107, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Im Zug hat mich heute ein Kind gefragt, warum die Wolken eigentlich nicht einfach vom Himmel fallen.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das ist eine ausgezeichnete Frage. Hast du wissenschaftlich geantwortet oder erst einmal sehr lange aus dem Fenster geschaut?", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-de-garden", + "group": "short", + "language": "de", + "topic": "garden", + "seed": 20261108, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Auf meinem Balkon wächst jetzt tatsächlich etwas Grünes. Ich hätte nicht gedacht, dass mich so ein kleiner Trieb so freut.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Was hast du gepflanzt? Und bitte sag mir, dass du diesmal noch weißt, in welchen Topf die Samen gekommen sind.", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-fr-jardin", + "group": "short", + "language": "fr", + "topic": "jardin", + "seed": 20261109, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ce matin, j’ai découvert une petite fleur sur mon balcon. Je pensais vraiment que cette plante avait abandonné.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Comme quoi, elle préparait peut-être sa surprise en silence. Tu lui as donné un peu plus de lumière ?", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-fr-chance", + "group": "short", + "language": "fr", + "topic": "chance", + "seed": 20261110, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "J’ai retrouvé un vieux carnet dans une boîte. Il était rempli de projets que j’avais complètement oubliés.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Et alors, tu as eu envie de les reprendre, ou surtout de remercier ton ancien toi de ne pas avoir tout essayé ?", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-es-radio", + "group": "short", + "language": "es", + "topic": "radio", + "seed": 20261111, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Esta mañana escuché una canción que no oía desde el colegio. De repente recordé hasta el camino de vuelta a casa.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Es increíble cómo pasa eso. A veces una melodía guarda recuerdos que ni siquiera sabíamos que seguían allí.", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "short-es-friends", + "group": "short", + "language": "es", + "topic": "friends", + "seed": 20261112, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ayer cociné para unos amigos y olvidé poner sal. Me di cuenta cuando todos estaban intentando ser demasiado amables.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Ese silencio tan educado lo dice todo. ¿Alguien pidió el salero o decidiste presentar el plato como una experiencia delicada?", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-en-coffee", + "group": "medium", + "language": "en", + "topic": "coffee", + "seed": 20261103, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I tried making coffee without looking at my phone this morning. It felt oddly peaceful, actually.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That sounds lovely. Although I would probably spend the whole time wondering whether the kettle was taking longer.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Exactly! At first I kept reaching for a notification that was not even there. Then I noticed the rain.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "We fill every little gap, don’t we? Sometimes the nicest part of a morning is that nothing happens.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "And the coffee tasted better. Not because I made it differently, but because I finally paid attention.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Well, now you have turned a cup of coffee into a small philosophy lesson. I am surprisingly convinced.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-en-walking", + "group": "medium", + "language": "en", + "topic": "walking", + "seed": 20261104, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Do you ever take a different route home just because the usual one has started to feel invisible?", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Yes, and then I discover a tiny shop that has apparently been there for about twenty years.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "That happened yesterday. There was a bookshop with a sleeping dog right beside the door. I almost missed it.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "A sleeping dog is an excellent recommendation. It suggests the atmosphere is relaxed and the customers are trustworthy.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "The owner asked what I liked reading, and I realised I had not answered that question in ages.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "What did you say? Please tell me you did not just point to the nearest shelf and panic.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-en-learning", + "group": "medium", + "language": "en", + "topic": "learning", + "seed": 20261105, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I finally played that difficult passage on the piano without stopping. Only once, but I am counting it.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "You absolutely should. One good attempt means your hands have discovered that the thing is possible.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "The funny part is that I had been practising it faster and faster, which mostly made it sound worse.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Then somebody told you to slow down, and you were deeply offended by this very sensible advice?", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "A little! But after a few quiet minutes, I could actually hear where I was rushing ahead.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That is the awkward secret of learning. Sometimes trying harder just means repeating the mistake with more confidence.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-de-cooking", + "group": "medium", + "language": "de", + "topic": "cooking", + "seed": 20261106, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ich habe gestern endlich das Brot selbst gebacken. Die Küche sah danach allerdings ziemlich abenteuerlich aus.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das gehört dazu. Die entscheidende Frage ist doch: Hat es gut geschmeckt, oder war es eher ein Türstopper?", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Überraschend gut! Die Kruste war ein bisschen dunkel, aber innen war es weich. Ich war richtig stolz darauf.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Dann war das keine dunkle Kruste, sondern ein bewusst gewähltes Röstaroma. So würde ich es jedenfalls verkaufen.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Genau das habe ich auch gesagt. Beim zweiten Versuch möchte ich trotzdem vorher schauen, wie heiß der Ofen wirklich ist.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Vernünftig. Ich finde sowieso schön, wenn man etwas macht, bei dem man das Ergebnis hinterher gemeinsam essen kann.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-de-train", + "group": "medium", + "language": "de", + "topic": "train", + "seed": 20261107, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Im Zug hat mich heute ein Kind gefragt, warum die Wolken eigentlich nicht einfach vom Himmel fallen.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das ist eine ausgezeichnete Frage. Hast du wissenschaftlich geantwortet oder erst einmal sehr lange aus dem Fenster geschaut?", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Beides. Ich wollte mit winzigen Wassertröpfchen anfangen, aber dann fragte es sofort, warum die nicht alle zusammenkleben.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Kinder haben dieses Talent, jede vermeintlich einfache Erklärung genau an der spannendsten Stelle auseinanderzunehmen.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Ja, und plötzlich saßen drei Erwachsene da und überlegten gemeinsam. Das war viel netter als auf das Handy zu schauen.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Dann war die eigentliche Entdeckung vielleicht gar nicht das Wetter, sondern dass ein fremder Mensch ein Gespräch anfangen kann.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-de-garden", + "group": "medium", + "language": "de", + "topic": "garden", + "seed": 20261108, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Auf meinem Balkon wächst jetzt tatsächlich etwas Grünes. Ich hätte nicht gedacht, dass mich so ein kleiner Trieb so freut.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Was hast du gepflanzt? Und bitte sag mir, dass du diesmal noch weißt, in welchen Topf die Samen gekommen sind.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Basilikum, glaube ich. Die Beschriftung hat der Regen mitgenommen. Es könnte also auch eine sehr überzeugende andere Pflanze sein.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Dann betrachten wir das vorerst als botanisches Rätsel. Du musst nichts überstürzen und solltest vielleicht noch nicht daran knabbern.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Keine Sorge. Ich finde es einfach angenehm, morgens kurz nachzusehen, ob wieder ein neues Blatt dazugekommen ist.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das verstehe ich. Es ist eine kleine Veränderung, die nicht drängelt und trotzdem zeigt, dass etwas vorangeht.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-fr-jardin", + "group": "medium", + "language": "fr", + "topic": "jardin", + "seed": 20261109, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ce matin, j’ai découvert une petite fleur sur mon balcon. Je pensais vraiment que cette plante avait abandonné.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Comme quoi, elle préparait peut-être sa surprise en silence. Tu lui as donné un peu plus de lumière ?", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Oui, et surtout un peu moins d’eau. Apparemment, mon enthousiasme dépassait largement ses besoins. Je voulais trop bien faire.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "C’est assez humain, finalement. On croit aider en faisant davantage, alors que parfois il faut simplement laisser respirer.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Tu as raison. Maintenant, je regarde les nouvelles feuilles avant de prendre mon téléphone. Cela change le début de ma journée.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Et tu ne peux pas accélérer la prochaine fleur en actualisant la page. C’est peut-être exactement ce qui fait du bien.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-fr-chance", + "group": "medium", + "language": "fr", + "topic": "chance", + "seed": 20261110, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "J’ai retrouvé un vieux carnet dans une boîte. Il était rempli de projets que j’avais complètement oubliés.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Et alors, tu as eu envie de les reprendre, ou surtout de remercier ton ancien toi de ne pas avoir tout essayé ?", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Un peu des deux. Il y avait une idée de voyage magnifique, puis une recette de soupe franchement inquiétante.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "C’est rassurant. Un bon carnet devrait contenir de grandes ambitions et quelques décisions culinaires discutables.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Ce qui m’a touché, c’est de voir combien je pouvais être enthousiaste. Pas forcément réaliste, mais vraiment curieux.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Peut-être que tu peux garder cette curiosité sans reprendre chaque projet. On a le droit de changer de direction.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-es-radio", + "group": "medium", + "language": "es", + "topic": "radio", + "seed": 20261111, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Esta mañana escuché una canción que no oía desde el colegio. De repente recordé hasta el camino de vuelta a casa.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Es increíble cómo pasa eso. A veces una melodía guarda recuerdos que ni siquiera sabíamos que seguían allí.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Sí, y no era una canción especialmente importante. Creo que simplemente sonaba siempre en la pequeña tienda de la esquina.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Entonces también recuerdas la tienda, el ruido de la calle, quizá alguna conversación. Todo aparece junto, sin pedir permiso.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Exactamente. Me dieron ganas de escribirle a un amigo de aquella época, aunque llevamos bastante tiempo sin hablar.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Hazlo. No necesitas un mensaje perfecto. Puedes empezar diciendo que escuchaste algo y te acordaste de él.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "medium-es-friends", + "group": "medium", + "language": "es", + "topic": "friends", + "seed": 20261112, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ayer cociné para unos amigos y olvidé poner sal. Me di cuenta cuando todos estaban intentando ser demasiado amables.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Ese silencio tan educado lo dice todo. ¿Alguien pidió el salero o decidiste presentar el plato como una experiencia delicada?", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Al final me reí y lo dije yo. Fue un alivio, porque en cuanto lo reconocí todos empezaron a contar sus propios desastres.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Eso suele mejorar una cena más que una receta perfecta. De repente nadie tiene que fingir que siempre le sale todo bien.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Una amiga contó que una vez horneó un pastel sin harina. Todavía no entiendo muy bien qué fue lo que terminó sirviendo.", + "seconds": 6.2, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Quizá inventó un postre nuevo. Hay errores que suenan terribles hasta que alguien les pone un nombre elegante.", + "seconds": 6.2, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-en-coffee", + "group": "long", + "language": "en", + "topic": "coffee", + "seed": 20261103, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I tried making coffee without looking at my phone this morning. It felt oddly peaceful, actually.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That sounds lovely. Although I would probably spend the whole time wondering whether the kettle was taking longer.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Exactly! At first I kept reaching for a notification that was not even there. Then I noticed the rain.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "We fill every little gap, don’t we? Sometimes the nicest part of a morning is that nothing happens.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "And the coffee tasted better. Not because I made it differently, but because I finally paid attention.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Well, now you have turned a cup of coffee into a small philosophy lesson. I am surprisingly convinced.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Let’s not get carried away. Tomorrow I might forget the water and just stare thoughtfully at the empty cup.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Honestly, that would still be a very committed experiment. We should report back next week.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-en-walking", + "group": "long", + "language": "en", + "topic": "walking", + "seed": 20261104, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Do you ever take a different route home just because the usual one has started to feel invisible?", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Yes, and then I discover a tiny shop that has apparently been there for about twenty years.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "That happened yesterday. There was a bookshop with a sleeping dog right beside the door. I almost missed it.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "A sleeping dog is an excellent recommendation. It suggests the atmosphere is relaxed and the customers are trustworthy.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "The owner asked what I liked reading, and I realised I had not answered that question in ages.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "What did you say? Please tell me you did not just point to the nearest shelf and panic.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Almost. Then I said I wanted something that would surprise me without making the whole week feel heavier.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That is a wonderful request. Perhaps taking the longer route is sometimes the quickest way to find something new.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-en-learning", + "group": "long", + "language": "en", + "topic": "learning", + "seed": 20261105, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I finally played that difficult passage on the piano without stopping. Only once, but I am counting it.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "You absolutely should. One good attempt means your hands have discovered that the thing is possible.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "The funny part is that I had been practising it faster and faster, which mostly made it sound worse.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Then somebody told you to slow down, and you were deeply offended by this very sensible advice?", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "A little! But after a few quiet minutes, I could actually hear where I was rushing ahead.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That is the awkward secret of learning. Sometimes trying harder just means repeating the mistake with more confidence.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "So tomorrow I will be patient, start slowly, and resist the urge to perform a heroic concert for the neighbours.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "A thoughtful decision. Although I think the neighbours would appreciate a short victory fanfare at the end.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-de-cooking", + "group": "long", + "language": "de", + "topic": "cooking", + "seed": 20261106, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ich habe gestern endlich das Brot selbst gebacken. Die Küche sah danach allerdings ziemlich abenteuerlich aus.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das gehört dazu. Die entscheidende Frage ist doch: Hat es gut geschmeckt, oder war es eher ein Türstopper?", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Überraschend gut! Die Kruste war ein bisschen dunkel, aber innen war es weich. Ich war richtig stolz darauf.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Dann war das keine dunkle Kruste, sondern ein bewusst gewähltes Röstaroma. So würde ich es jedenfalls verkaufen.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Genau das habe ich auch gesagt. Beim zweiten Versuch möchte ich trotzdem vorher schauen, wie heiß der Ofen wirklich ist.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Vernünftig. Ich finde sowieso schön, wenn man etwas macht, bei dem man das Ergebnis hinterher gemeinsam essen kann.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Und wenn es schiefgeht, hat man wenigstens eine Geschichte. Vielleicht brauche ich beim nächsten Mal weniger Mehl auf dem Boden.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Oder mehr Gäste zum Aufräumen. Du lädst zum Brotessen ein und erwähnst den kleinen Arbeitseinsatz erst später.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-de-train", + "group": "long", + "language": "de", + "topic": "train", + "seed": 20261107, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Im Zug hat mich heute ein Kind gefragt, warum die Wolken eigentlich nicht einfach vom Himmel fallen.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das ist eine ausgezeichnete Frage. Hast du wissenschaftlich geantwortet oder erst einmal sehr lange aus dem Fenster geschaut?", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Beides. Ich wollte mit winzigen Wassertröpfchen anfangen, aber dann fragte es sofort, warum die nicht alle zusammenkleben.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Kinder haben dieses Talent, jede vermeintlich einfache Erklärung genau an der spannendsten Stelle auseinanderzunehmen.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Ja, und plötzlich saßen drei Erwachsene da und überlegten gemeinsam. Das war viel netter als auf das Handy zu schauen.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Dann war die eigentliche Entdeckung vielleicht gar nicht das Wetter, sondern dass ein fremder Mensch ein Gespräch anfangen kann.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Am Ende hat das Kind gesagt: Vielleicht bleiben die Wolken oben, weil sie noch irgendwo hinwollen. Das fand ich wunderschön.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das gefällt mir auch. Die Erklärung würde ich behalten, selbst wenn du später noch die genaue Physik nachliest.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-de-garden", + "group": "long", + "language": "de", + "topic": "garden", + "seed": 20261108, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Auf meinem Balkon wächst jetzt tatsächlich etwas Grünes. Ich hätte nicht gedacht, dass mich so ein kleiner Trieb so freut.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Was hast du gepflanzt? Und bitte sag mir, dass du diesmal noch weißt, in welchen Topf die Samen gekommen sind.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Basilikum, glaube ich. Die Beschriftung hat der Regen mitgenommen. Es könnte also auch eine sehr überzeugende andere Pflanze sein.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Dann betrachten wir das vorerst als botanisches Rätsel. Du musst nichts überstürzen und solltest vielleicht noch nicht daran knabbern.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Keine Sorge. Ich finde es einfach angenehm, morgens kurz nachzusehen, ob wieder ein neues Blatt dazugekommen ist.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Das verstehe ich. Es ist eine kleine Veränderung, die nicht drängelt und trotzdem zeigt, dass etwas vorangeht.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Genau. Und ich merke, dass ich dabei geduldiger werde. Obwohl ich immer noch dreimal täglich nach dem angeblichen Basilikum schaue.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Geduld mit regelmäßiger Kontrolle. Das klingt nach einem guten Anfang. Jetzt fehlt nur noch eine wetterfeste Beschriftung.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-fr-jardin", + "group": "long", + "language": "fr", + "topic": "jardin", + "seed": 20261109, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ce matin, j’ai découvert une petite fleur sur mon balcon. Je pensais vraiment que cette plante avait abandonné.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Comme quoi, elle préparait peut-être sa surprise en silence. Tu lui as donné un peu plus de lumière ?", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Oui, et surtout un peu moins d’eau. Apparemment, mon enthousiasme dépassait largement ses besoins. Je voulais trop bien faire.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "C’est assez humain, finalement. On croit aider en faisant davantage, alors que parfois il faut simplement laisser respirer.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Tu as raison. Maintenant, je regarde les nouvelles feuilles avant de prendre mon téléphone. Cela change le début de ma journée.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Et tu ne peux pas accélérer la prochaine fleur en actualisant la page. C’est peut-être exactement ce qui fait du bien.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Voilà. Même si je vérifie encore beaucoup trop souvent. J’appelle cela de la patience, mais une patience très attentive.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "C’est une jolie manière de le dire. Garde cette attention, et pense seulement à poser l’arrosoir de temps en temps.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-fr-chance", + "group": "long", + "language": "fr", + "topic": "chance", + "seed": 20261110, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "J’ai retrouvé un vieux carnet dans une boîte. Il était rempli de projets que j’avais complètement oubliés.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Et alors, tu as eu envie de les reprendre, ou surtout de remercier ton ancien toi de ne pas avoir tout essayé ?", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Un peu des deux. Il y avait une idée de voyage magnifique, puis une recette de soupe franchement inquiétante.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "C’est rassurant. Un bon carnet devrait contenir de grandes ambitions et quelques décisions culinaires discutables.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Ce qui m’a touché, c’est de voir combien je pouvais être enthousiaste. Pas forcément réaliste, mais vraiment curieux.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Peut-être que tu peux garder cette curiosité sans reprendre chaque projet. On a le droit de changer de direction.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Oui. Je vais choisir une seule petite chose, quelque chose que je pourrais faire ce week-end, sans organisation compliquée.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Très bonne idée. Et pour la soupe, nous attendrons prudemment une deuxième opinion avant de rallumer le feu.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-es-radio", + "group": "long", + "language": "es", + "topic": "radio", + "seed": 20261111, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Esta mañana escuché una canción que no oía desde el colegio. De repente recordé hasta el camino de vuelta a casa.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Es increíble cómo pasa eso. A veces una melodía guarda recuerdos que ni siquiera sabíamos que seguían allí.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Sí, y no era una canción especialmente importante. Creo que simplemente sonaba siempre en la pequeña tienda de la esquina.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Entonces también recuerdas la tienda, el ruido de la calle, quizá alguna conversación. Todo aparece junto, sin pedir permiso.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Exactamente. Me dieron ganas de escribirle a un amigo de aquella época, aunque llevamos bastante tiempo sin hablar.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Hazlo. No necesitas un mensaje perfecto. Puedes empezar diciendo que escuchaste algo y te acordaste de él.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Eso voy a hacer. A veces complico demasiado estas cosas, como si retomar una amistad necesitara una gran explicación.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Y muchas veces basta con una frase sincera. Bueno, y con no enviarle veinte canciones antes de que conteste.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + }, + { + "task_id": "long-es-friends", + "group": "long", + "language": "es", + "topic": "friends", + "seed": 20261112, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "Ayer cociné para unos amigos y olvidé poner sal. Me di cuenta cuando todos estaban intentando ser demasiado amables.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Ese silencio tan educado lo dice todo. ¿Alguien pidió el salero o decidiste presentar el plato como una experiencia delicada?", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Al final me reí y lo dije yo. Fue un alivio, porque en cuanto lo reconocí todos empezaron a contar sus propios desastres.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Eso suele mejorar una cena más que una receta perfecta. De repente nadie tiene que fingir que siempre le sale todo bien.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "Una amiga contó que una vez horneó un pastel sin harina. Todavía no entiendo muy bien qué fue lo que terminó sirviendo.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Quizá inventó un postre nuevo. Hay errores que suenan terribles hasta que alguien les pone un nombre elegante.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + }, + { + "speaker": "S1", + "text": "En cualquier caso, terminamos hablando durante horas. La próxima vez pondré sal, pero espero que la conversación sea igual de buena.", + "seconds": 7.19, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "Seguro que sí. Y si no, siempre puedes olvidar las cucharas. Aunque yo preferiría conservar la comida y buscar otra anécdota.", + "seconds": 7.19, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] + } +] diff --git a/examples/de-S1.wav b/examples/de-S1.wav new file mode 100644 index 0000000000000000000000000000000000000000..3f7ca4ea3e5c2ae3dabdecedb5af77bf5c31956a --- /dev/null +++ b/examples/de-S1.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9966c27d168196d1d9653cd9b23f9c547ef36aa66a52cefe0685f15e0d504abc +size 576044 diff --git a/examples/de-S2.wav b/examples/de-S2.wav new file mode 100644 index 0000000000000000000000000000000000000000..4f6ecc1041193d45d23aa01000c19dddd8e2f3c1 --- /dev/null +++ b/examples/de-S2.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4672ee492a4b4a27e524fcd8b748e804e0c3b3b94e86c8a1dba02be6204a4481 +size 537644 diff --git a/examples/dialogue.json b/examples/dialogue.json new file mode 100644 index 0000000000000000000000000000000000000000..f9889f7533ef84826d1362f7faab69fc73b898e0 --- /dev/null +++ b/examples/dialogue.json @@ -0,0 +1,23 @@ +{ + "task_id": "short-en-coffee", + "group": "short", + "language": "en", + "topic": "coffee", + "seed": 20261103, + "pause_s": 0.24, + "general": "A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.", + "turns": [ + { + "speaker": "S1", + "text": "I tried making coffee without looking at my phone this morning. It felt oddly peaceful, actually.", + "seconds": 8.88, + "direction": "warm, curious, relaxed conversational delivery" + }, + { + "speaker": "S2", + "text": "That sounds lovely. Although I would probably spend the whole time wondering whether the kettle was taking longer.", + "seconds": 8.88, + "direction": "engaged, friendly, spontaneous conversational reply" + } + ] +} diff --git a/examples/en-S1.wav b/examples/en-S1.wav new file mode 100644 index 0000000000000000000000000000000000000000..2d6772988e1acc435f5d83a3908e5aade31b89a3 --- /dev/null +++ b/examples/en-S1.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85fb8b03e2c12b502917e2d2e0da72048ec335d8c59bf51dd53263870b31625 +size 852524 diff --git a/examples/en-S2.wav b/examples/en-S2.wav new file mode 100644 index 0000000000000000000000000000000000000000..ee5640aa24fa15bf006d0adcd70198ce924c9574 --- /dev/null +++ b/examples/en-S2.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cfd4e50d4f08e92d95d6e6824cc1d9d309844506ba6f4b24335e5c69ee2953b +size 721964 diff --git a/examples/es-S1.wav b/examples/es-S1.wav new file mode 100644 index 0000000000000000000000000000000000000000..6454645f0ecedb520def4ec5ab944845ca0afe6e --- /dev/null +++ b/examples/es-S1.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e0d9ed986b8b62d0a6b9f3c944020951747695e8a8f5966fd2ed57b4d54c6e0 +size 568364 diff --git a/examples/es-S2.wav b/examples/es-S2.wav new file mode 100644 index 0000000000000000000000000000000000000000..d058b67eaf12e041d93f17eff191946337bd501d --- /dev/null +++ b/examples/es-S2.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d320996367763baa6e99f589d8a95a5c6a8375d52e1098bf9ac5da74ac9e459 +size 560684 diff --git a/examples/fr-S1.wav b/examples/fr-S1.wav new file mode 100644 index 0000000000000000000000000000000000000000..85a3e2d3013a108098bf95351368f68e2c937694 --- /dev/null +++ b/examples/fr-S1.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f5a721525d0a0405d153136e37f883bd9bc54699b733e70ccbc8a2e2cc55ad6 +size 499244 diff --git a/examples/fr-S2.wav b/examples/fr-S2.wav new file mode 100644 index 0000000000000000000000000000000000000000..18b4028677dea2661b0ca9eed097b65cca5ebd83 --- /dev/null +++ b/examples/fr-S2.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c78996dd65b14b3c5e2df8c9accbd1540e092a6f28fa4ee48558403c0cf6505b +size 545324 diff --git a/examples/reference-provenance.json b/examples/reference-provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..84bbe1f7e53711c0aac1c1c044148979b7280ff9 --- /dev/null +++ b/examples/reference-provenance.json @@ -0,0 +1,1754 @@ +{ + "en": { + "S1": { + "episode": "47f242531f7d85bda216c3debaec5c369935e9fd0a89ec36aba025e80057a30b", + "original_speaker": "S2", + "start_frame": 4535, + "end_frame": 4646, + "language": "en", + "window": { + "episode": "47f242531f7d85bda216c3debaec5c369935e9fd0a89ec36aba025e80057a30b", + "lang": "en", + "window": "middle", + "start": 347.540408, + "duration": 30.0, + "ce": 6.411804489791393 + }, + "text": "Recognizing that just installing the software achieves nothing unless the operating model, the human skills, adapt at the same time. It's as much a change management challenge as a tech", + "scores": { + "emo_Affection": 0.044916778802871704, + "emo_Amusement": 0.0896710455417633, + "emo_Anger": 0.020686715841293335, + "emo_Astonishment_Surprise": 0.021077096462249756, + "emo_Awe": 0.00831460952758789, + "emo_Bitterness": 0.01997998356819153, + "emo_Concentration": 2.3828930854797363, + "emo_Confusion": 0.019798070192337036, + "emo_Contemplation": 1.4535179138183594, + "emo_Contempt": 0.005230426788330078, + "emo_Contentment": 1.8263260126113892, + "emo_Disappointment": -0.02528536319732666, + "emo_Disgust": 0.011815264821052551, + "emo_Distress": 0.015124201774597168, + "emo_Doubt": 0.021136879920959473, + "emo_Elation": 0.00012952089309692383, + "emo_Embarrassment": -0.018539071083068848, + "emo_Emotional_Numbness": 0.29845017194747925, + "emo_Fatigue_Exhaustion": -0.015124976634979248, + "emo_Fear": 0.029022395610809326, + "emo_Helplessness": -0.05028754472732544, + "emo_Hope_Enthusiasm_Optimism": 1.1021933555603027, + "emo_Impatience_and_Irritability": 0.049224674701690674, + "emo_Infatuation": -0.0066391825675964355, + "emo_Interest": 2.3918237686157227, + "emo_Intoxication_Altered_States_of_Consciousness": -0.023080214858055115, + "emo_Jealousy_and_Envy": 0.003954067826271057, + "emo_Longing": 0.035218000411987305, + "emo_Malevolence_Malice": 0.016138404607772827, + "emo_Pain": -0.028374433517456055, + "emo_Pleasure_Ecstasy": -0.017789393663406372, + "emo_Pride": 0.8620995879173279, + "emo_Relief": -0.014433503150939941, + "emo_Sadness": -0.00990837812423706, + "emo_Sexual_Lust": 0.004031270742416382, + "emo_Shame": -0.009914115071296692, + "emo_Sourness": -0.004457250237464905, + "emo_Teasing": 0.028293877840042114, + "emo_Thankfulness_Gratitude": 0.02457219362258911, + "emo_Triumph": 0.0600743293762207, + "vn_AGEV_reg": 3.3016669750213623, + "vn_AROU_reg": 2.8899264335632324, + "vn_ARSH_reg": 3.131937026977539, + "vn_ATCK_reg": 3.188565731048584, + "vn_BKGN_reg": 3.993759870529175, + "vn_BRGT_reg": 3.312556743621826, + "vn_CHNK_reg": 3.391357421875, + "vn_CLRT_reg": 4.476703643798828, + "vn_COGL_reg": 0.7366646528244019, + "vn_DARC_reg": 3.081094741821289, + "vn_DFLU_reg": 0.2404625415802002, + "vn_EMPH_reg": 3.1535863876342773, + "vn_ESTH_reg": 4.101387977600098, + "vn_EXPL_reg": 0.04618470370769501, + "vn_FOCS_reg": 4.183499336242676, + "vn_FULL_reg": 3.8218843936920166, + "vn_GEND_reg": 4.951261043548584, + "vn_HARM_reg": 3.733027219772339, + "vn_METL_reg": 1.5914198160171509, + "vn_RANG_reg": 2.639007329940796, + "vn_RCQL_reg": 4.19329309463501, + "vn_REGS_reg": 1.3563289642333984, + "vn_RESP_reg": 1.4511476755142212, + "vn_ROUG_reg": 1.4923737049102783, + "vn_R_CHST_reg": 3.706143856048584, + "vn_R_HEAD_reg": 1.6914302110671997, + "vn_R_MASK_reg": 2.9622108936309814, + "vn_R_MIXD_reg": 3.782395601272583, + "vn_R_NASL_reg": 1.6039230823516846, + "vn_R_ORAL_reg": 3.5968661308288574, + "vn_R_THRT_reg": 2.590158224105835, + "vn_SMTH_reg": 4.129175186157227, + "vn_STNC_reg": 4.157435417175293, + "vn_STRU_reg": 4.648323059082031, + "vn_S_ASMR_reg": -0.11486732959747314, + "vn_S_AUTH_reg": 4.035935401916504, + "vn_S_CART_reg": -0.20317816734313965, + "vn_S_CASU_reg": 1.9761770963668823, + "vn_S_CONV_reg": 2.6582865715026855, + "vn_S_DRAM_reg": 1.777975082397461, + "vn_S_FORM_reg": 3.69545316696167, + "vn_S_MONO_reg": 3.1506378650665283, + "vn_S_NARR_reg": 3.8744468688964844, + "vn_S_NEWS_reg": 2.7603554725646973, + "vn_S_PLAY_reg": 1.0244368314743042, + "vn_S_RANT_reg": 0.15826869010925293, + "vn_S_STRY_reg": 2.047847270965576, + "vn_S_TECH_reg": 3.592550039291382, + "vn_S_WHIS_reg": -0.05654406547546387, + "vn_TEMP_reg": 3.615788221359253, + "vn_TENS_reg": 1.8553197383880615, + "vn_VALN_reg": 2.870231866836548, + "vn_VALS_reg": 3.128520965576172, + "vn_VFLX_reg": 2.970663547515869, + "vn_VOLT_reg": 1.2053163051605225, + "vn_VULN_reg": 0.9679520130157471, + "vn_WARM_reg": 3.524272918701172, + "genuineness_0_6": 4.036643981933594, + "blend_0_10": 6.994388103485107, + "R_quality": 0.49497437477111816, + "eiv_extra:Age": 3.3220932483673096, + "eiv_extra:Arousal": 1.2319591045379639, + "eiv_extra:Authenticity": 2.771702289581299, + "eiv_extra:Background_Noise": 0.11300861835479736, + "eiv_extra:Confident_vs._Hesitant": 0.9623448848724365, + "eiv_extra:Gender": -1.0594953298568726, + "eiv_extra:High-Pitched_vs._Low-Pitched": 2.068246603012085, + "eiv_extra:Monotone_vs._Expressive": 1.926389217376709, + "eiv_extra:Recording_Quality": 2.764446258544922, + "eiv_extra:Serious_vs._Humorous": 0.9570503830909729, + "eiv_extra:Soft_vs._Harsh": 0.08383968472480774, + "eiv_extra:Submissive_vs._Dominant": 0.5415263175964355, + "eiv_extra:Valence": -0.07493813335895538, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 2.6135826110839844, + "eiv_extra:Warm_vs._Cold": 0.03427967429161072, + "eiv_extra:score_background_quality": 3.7240524291992188, + "eiv_extra:score_content_enjoyment": 4.916215896606445, + "eiv_extra:score_overall_quality": 3.0479235649108887, + "eiv_extra:score_speech_quality": 1.9263687133789062, + "audiobox:CE": 6.475345611572266, + "audiobox:CU": 7.470410346984863, + "audiobox:PC": 1.479453682899475, + "audiobox:PQ": 8.056504249572754, + "dnsmos:SIG_raw": 4.183594226837158, + "dnsmos:BAK_raw": 4.429921627044678, + "dnsmos:OVRL_raw": 3.986091375350952, + "dnsmos:SIG": 3.635957717895508, + "dnsmos:BAK": 4.14784574508667, + "dnsmos:OVRL": 3.4156992435455322, + "dnsmos:P808_MOS": 4.070035934448242, + "burst_count_log1p": 0.1443398892879486, + "voiceclap_attribute:Affection": -0.20595383644104004, + "voiceclap_attribute:Age": 3.251166343688965, + "voiceclap_attribute:Amusement": 0.2761256992816925, + "voiceclap_attribute:Anger": -0.0215071439743042, + "voiceclap_attribute:Arousal": 1.4376707077026367, + "voiceclap_attribute:Astonishment_Surprise": 0.17212054133415222, + "voiceclap_attribute:Authenticity": 2.892148494720459, + "voiceclap_attribute:Awe": 0.13784077763557434, + "voiceclap_attribute:Background_Noise": 0.24468371272087097, + "voiceclap_attribute:Bitterness": 0.018249094486236572, + "voiceclap_attribute:Concentration": 2.494945526123047, + "voiceclap_attribute:Confident_vs._Hesitant": 0.8552287220954895, + "voiceclap_attribute:Confusion": -0.09103381633758545, + "voiceclap_attribute:Contemplation": 1.528646469116211, + "voiceclap_attribute:Contempt": 0.06687861680984497, + "voiceclap_attribute:Contentment": 1.4767652750015259, + "voiceclap_attribute:Disappointment": 0.011530518531799316, + "voiceclap_attribute:Disgust": 0.029055893421173096, + "voiceclap_attribute:Distress": -0.06888538599014282, + "voiceclap_attribute:Doubt": 0.03893560171127319, + "voiceclap_attribute:Elation": 0.09965866804122925, + "voiceclap_attribute:Embarrassment": -0.013739347457885742, + "voiceclap_attribute:Emotional_Numbness": 0.08770185708999634, + "voiceclap_attribute:Fatigue_Exhaustion": -0.19961676001548767, + "voiceclap_attribute:Fear": 0.07311379909515381, + "voiceclap_attribute:Gender": -1.0299090147018433, + "voiceclap_attribute:Helplessness": -0.0892876386642456, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.9851802587509155, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 0.9585894346237183, + "voiceclap_attribute:Impatience_and_Irritability": 0.027670860290527344, + "voiceclap_attribute:Infatuation": -0.15207678079605103, + "voiceclap_attribute:Interest": 2.5115389823913574, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": -0.08464688062667847, + "voiceclap_attribute:Jealousy_&_Envy": -0.06632769107818604, + "voiceclap_attribute:Longing": -0.20924341678619385, + "voiceclap_attribute:Malevolence_Malice": -0.0024952292442321777, + "voiceclap_attribute:Monotone_vs._Expressive": 1.8484630584716797, + "voiceclap_attribute:Pain": -0.09224006533622742, + "voiceclap_attribute:Pleasure_Ecstasy": -0.03208339214324951, + "voiceclap_attribute:Pride": 0.7150555849075317, + "voiceclap_attribute:Recording_Quality": 2.73024845123291, + "voiceclap_attribute:Relief": -0.08085215091705322, + "voiceclap_attribute:Sadness": -0.03938835859298706, + "voiceclap_attribute:Serious_vs._Humorous": 0.8458406925201416, + "voiceclap_attribute:Sexual_Lust": -0.09768033027648926, + "voiceclap_attribute:Shame": -0.02619558572769165, + "voiceclap_attribute:Soft_vs._Harsh": 0.062218621373176575, + "voiceclap_attribute:Sourness": 0.07866406440734863, + "voiceclap_attribute:Submissive_vs._Dominant": 0.5844260454177856, + "voiceclap_attribute:Teasing": 0.027302980422973633, + "voiceclap_attribute:Thankfulness_Gratitude": -0.32050418853759766, + "voiceclap_attribute:Triumph": 0.04176449775695801, + "voiceclap_attribute:Valence": -0.10700663924217224, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 2.4343531131744385, + "voiceclap_attribute:Warm_vs._Cold": 0.15293753147125244, + "voiceclap_attribute:duration": 8.938613891601562, + "voiceclap_attribute:score_background_quality": 3.575244903564453, + "voiceclap_attribute:score_content_enjoyment": 4.874354362487793, + "voiceclap_attribute:score_overall_quality": 2.993528366088867, + "voiceclap_attribute:score_speech_quality": 1.9786185026168823, + "voiceclap_attribute:talking_speed": 20.73843002319336 + }, + "asr": "Recognizing that just installing the software achieves nothing unless the operating model, the human skills, adapt at the same time. It's as much a change management challenge as a technological.", + "audio_sha256": "46f1c0d07283c73d4b950929a37fc1889902311e452be94d5a7675f4f209aa89", + "speaker": "S1", + "seconds": 8.88, + "media_member": "en-S1.mp3", + "wav_member": "en-S1.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + }, + "S2": { + "episode": "34b24e223b7c76afeb48760fbf721bed891e92ec9f3ff3cefdeb086d87d3f359", + "original_speaker": "S2", + "start_frame": 1854, + "end_frame": 1948, + "language": "en", + "window": { + "episode": "34b24e223b7c76afeb48760fbf721bed891e92ec9f3ff3cefdeb086d87d3f359", + "lang": "en", + "window": "middle", + "start": 128.430628, + "duration": 30.0, + "ce": 6.403861818835139 + }, + "text": "The pressure is immense down there, and that oil-filled tube helps them manage booyancy without using a lot of energy.", + "scores": { + "emo_Affection": 0.06828047335147858, + "emo_Amusement": 0.0744192898273468, + "emo_Anger": 0.0909375548362732, + "emo_Astonishment_Surprise": 0.03833329677581787, + "emo_Awe": 0.02389206737279892, + "emo_Bitterness": 0.070845827460289, + "emo_Concentration": 2.4285545349121094, + "emo_Confusion": 0.013547241687774658, + "emo_Contemplation": 1.4995293617248535, + "emo_Contempt": 0.07395827770233154, + "emo_Contentment": 1.8735581636428833, + "emo_Disappointment": 0.12265381217002869, + "emo_Disgust": 0.021390840411186218, + "emo_Distress": 0.08856144547462463, + "emo_Doubt": 0.04190543293952942, + "emo_Elation": 0.06467863917350769, + "emo_Embarrassment": -0.011710166931152344, + "emo_Emotional_Numbness": 0.20080125331878662, + "emo_Fatigue_Exhaustion": 0.016562193632125854, + "emo_Fear": 0.018269121646881104, + "emo_Helplessness": 0.01609298586845398, + "emo_Hope_Enthusiasm_Optimism": 0.6414806842803955, + "emo_Impatience_and_Irritability": 0.07327625155448914, + "emo_Infatuation": -0.00029659271240234375, + "emo_Interest": 2.1490163803100586, + "emo_Intoxication_Altered_States_of_Consciousness": -0.007871195673942566, + "emo_Jealousy_and_Envy": 0.0004715770483016968, + "emo_Longing": 0.047636568546295166, + "emo_Malevolence_Malice": 0.010001450777053833, + "emo_Pain": 0.0075737833976745605, + "emo_Pleasure_Ecstasy": 0.03467804193496704, + "emo_Pride": 0.5371546149253845, + "emo_Relief": 0.05917331576347351, + "emo_Sadness": 0.058759450912475586, + "emo_Sexual_Lust": -0.0018015503883361816, + "emo_Shame": 0.006461545825004578, + "emo_Sourness": 0.05296625196933746, + "emo_Teasing": 0.05141165852546692, + "emo_Thankfulness_Gratitude": 0.016471177339553833, + "emo_Triumph": 0.03841741383075714, + "vn_AGEV_reg": 3.3508851528167725, + "vn_AROU_reg": 2.5334854125976562, + "vn_ARSH_reg": 3.196974277496338, + "vn_ATCK_reg": 3.075408697128296, + "vn_BKGN_reg": 4.070611953735352, + "vn_BRGT_reg": 3.185899257659912, + "vn_CHNK_reg": 2.919048309326172, + "vn_CLRT_reg": 4.656875133514404, + "vn_COGL_reg": 0.5225298404693604, + "vn_DARC_reg": 3.123661756515503, + "vn_DFLU_reg": -0.03637242317199707, + "vn_EMPH_reg": 3.319296360015869, + "vn_ESTH_reg": 4.308232307434082, + "vn_EXPL_reg": -0.0590214878320694, + "vn_FOCS_reg": 4.1661577224731445, + "vn_FULL_reg": 4.291335105895996, + "vn_GEND_reg": 5.013467788696289, + "vn_HARM_reg": 3.7588999271392822, + "vn_METL_reg": 1.6802643537521362, + "vn_RANG_reg": 2.4561781883239746, + "vn_RCQL_reg": 4.467576026916504, + "vn_REGS_reg": 0.9662236571311951, + "vn_RESP_reg": 1.200474739074707, + "vn_ROUG_reg": 1.7610195875167847, + "vn_R_CHST_reg": 3.997999668121338, + "vn_R_HEAD_reg": 1.3681161403656006, + "vn_R_MASK_reg": 2.7863144874572754, + "vn_R_MIXD_reg": 3.8346996307373047, + "vn_R_NASL_reg": 1.4306567907333374, + "vn_R_ORAL_reg": 3.451535224914551, + "vn_R_THRT_reg": 2.992795944213867, + "vn_SMTH_reg": 4.198636054992676, + "vn_STNC_reg": 4.255288600921631, + "vn_STRU_reg": 5.033507823944092, + "vn_S_ASMR_reg": 0.014387369155883789, + "vn_S_AUTH_reg": 4.13332986831665, + "vn_S_CART_reg": -0.09621000289916992, + "vn_S_CASU_reg": 1.5973947048187256, + "vn_S_CONV_reg": 2.243990898132324, + "vn_S_DRAM_reg": 2.1295599937438965, + "vn_S_FORM_reg": 3.6796975135803223, + "vn_S_MONO_reg": 3.6004204750061035, + "vn_S_NARR_reg": 4.25568151473999, + "vn_S_NEWS_reg": 2.4159774780273438, + "vn_S_PLAY_reg": 0.7451401948928833, + "vn_S_RANT_reg": -0.08096981048583984, + "vn_S_STRY_reg": 2.418037176132202, + "vn_S_TECH_reg": 3.269036293029785, + "vn_S_WHIS_reg": -0.15646040439605713, + "vn_TEMP_reg": 2.780953884124756, + "vn_TENS_reg": 1.828498125076294, + "vn_VALN_reg": 2.759892225265503, + "vn_VALS_reg": 2.939750909805298, + "vn_VFLX_reg": 2.9329519271850586, + "vn_VOLT_reg": 0.9814172983169556, + "vn_VULN_reg": 1.17689049243927, + "vn_WARM_reg": 3.7787632942199707, + "genuineness_0_6": 4.124508857727051, + "blend_0_10": 6.564396381378174, + "R_quality": 0.2776834964752197, + "eiv_extra:Age": 3.3088409900665283, + "eiv_extra:Arousal": 1.030998945236206, + "eiv_extra:Authenticity": 2.780491352081299, + "eiv_extra:Background_Noise": -0.031336963176727295, + "eiv_extra:Confident_vs._Hesitant": 0.8572136163711548, + "eiv_extra:Gender": -1.0594953298568726, + "eiv_extra:High-Pitched_vs._Low-Pitched": 2.024301290512085, + "eiv_extra:Monotone_vs._Expressive": 1.506665587425232, + "eiv_extra:Recording_Quality": 2.7540531158447266, + "eiv_extra:Serious_vs._Humorous": 0.9053643941879272, + "eiv_extra:Soft_vs._Harsh": 0.24291978776454926, + "eiv_extra:Submissive_vs._Dominant": 0.4815673232078552, + "eiv_extra:Valence": -0.04363267868757248, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 2.402489185333252, + "eiv_extra:Warm_vs._Cold": 0.056246623396873474, + "eiv_extra:score_background_quality": 3.7944955825805664, + "eiv_extra:score_content_enjoyment": 4.8711256980896, + "eiv_extra:score_overall_quality": 3.034045457839966, + "eiv_extra:score_speech_quality": 1.9654312133789062, + "audiobox:CE": 6.570657730102539, + "audiobox:CU": 7.415783882141113, + "audiobox:PC": 1.5173166990280151, + "audiobox:PQ": 8.0977201461792, + "dnsmos:SIG_raw": 4.26984167098999, + "dnsmos:BAK_raw": 4.570577621459961, + "dnsmos:OVRL_raw": 4.081363677978516, + "dnsmos:SIG": 3.6966428756713867, + "dnsmos:BAK": 4.191372394561768, + "dnsmos:OVRL": 3.4678335189819336, + "dnsmos:P808_MOS": 4.209654808044434, + "burst_count_log1p": 0.09367421269416809, + "voiceclap_attribute:Affection": -0.006678104400634766, + "voiceclap_attribute:Age": 3.3191351890563965, + "voiceclap_attribute:Amusement": 0.18645429611206055, + "voiceclap_attribute:Anger": -0.05321788787841797, + "voiceclap_attribute:Arousal": 1.1774439811706543, + "voiceclap_attribute:Astonishment_Surprise": 0.05988532304763794, + "voiceclap_attribute:Authenticity": 2.904355525970459, + "voiceclap_attribute:Awe": 0.2330930233001709, + "voiceclap_attribute:Background_Noise": 0.10025382041931152, + "voiceclap_attribute:Bitterness": -0.005188405513763428, + "voiceclap_attribute:Concentration": 2.4700779914855957, + "voiceclap_attribute:Confident_vs._Hesitant": 0.7343697547912598, + "voiceclap_attribute:Confusion": -0.13817310333251953, + "voiceclap_attribute:Contemplation": 1.8421540260314941, + "voiceclap_attribute:Contempt": 0.06687861680984497, + "voiceclap_attribute:Contentment": 1.620650291442871, + "voiceclap_attribute:Disappointment": 0.025149106979370117, + "voiceclap_attribute:Disgust": -0.021725356578826904, + "voiceclap_attribute:Distress": -0.11263149976730347, + "voiceclap_attribute:Doubt": 0.0779981017112732, + "voiceclap_attribute:Elation": 0.0685073733329773, + "voiceclap_attribute:Embarrassment": -0.028491556644439697, + "voiceclap_attribute:Emotional_Numbness": 0.04195237159729004, + "voiceclap_attribute:Fatigue_Exhaustion": -0.08572903275489807, + "voiceclap_attribute:Fear": 0.006707549095153809, + "voiceclap_attribute:Gender": -1.1005455255508423, + "voiceclap_attribute:Helplessness": -0.034795284271240234, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.8932565450668335, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 0.9213427901268005, + "voiceclap_attribute:Impatience_and_Irritability": -0.1671210527420044, + "voiceclap_attribute:Infatuation": -0.13648593425750732, + "voiceclap_attribute:Interest": 2.2541165351867676, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": -0.019401192665100098, + "voiceclap_attribute:Jealousy_&_Envy": -0.042890191078186035, + "voiceclap_attribute:Longing": -0.03346216678619385, + "voiceclap_attribute:Malevolence_Malice": -0.0024952292442321777, + "voiceclap_attribute:Monotone_vs._Expressive": 1.553143858909607, + "voiceclap_attribute:Pain": -0.09028694033622742, + "voiceclap_attribute:Pleasure_Ecstasy": 0.04086393117904663, + "voiceclap_attribute:Pride": 0.7654954195022583, + "voiceclap_attribute:Recording_Quality": 2.771263599395752, + "voiceclap_attribute:Relief": -0.08735132217407227, + "voiceclap_attribute:Sadness": 0.0046160221099853516, + "voiceclap_attribute:Serious_vs._Humorous": 0.6564277410507202, + "voiceclap_attribute:Sexual_Lust": -0.05524212121963501, + "voiceclap_attribute:Shame": -0.035001397132873535, + "voiceclap_attribute:Soft_vs._Harsh": 0.21556827425956726, + "voiceclap_attribute:Sourness": 0.06958633661270142, + "voiceclap_attribute:Submissive_vs._Dominant": 0.48485857248306274, + "voiceclap_attribute:Teasing": -0.04382467269897461, + "voiceclap_attribute:Thankfulness_Gratitude": -0.13853907585144043, + "voiceclap_attribute:Triumph": 0.06699061393737793, + "voiceclap_attribute:Valence": 0.13324740529060364, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 2.2106969356536865, + "voiceclap_attribute:Warm_vs._Cold": 0.22859765589237213, + "voiceclap_attribute:duration": 8.196389198303223, + "voiceclap_attribute:score_background_quality": 3.795308828353882, + "voiceclap_attribute:score_content_enjoyment": 4.852869987487793, + "voiceclap_attribute:score_overall_quality": 3.0869405269622803, + "voiceclap_attribute:score_speech_quality": 1.9883841276168823, + "voiceclap_attribute:talking_speed": 15.080714225769043 + }, + "asr": "The pressure is immense down there, and that oil filled tube helps them manage buoyancy without using a lot of energy.", + "audio_sha256": "d97bf585de5e13c1fe7196bd172e31a5d50cc27356afadec04e09d783d9e6ad8", + "speaker": "S2", + "seconds": 7.52, + "media_member": "en-S2.mp3", + "wav_member": "en-S2.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + } + }, + "de": { + "S1": { + "episode": "506e1ff553b396e74b5e11ac851ebb2d8abbac980f1c9e9c1f5d2147ccdd361a", + "original_speaker": "S1", + "start_frame": 7511, + "end_frame": 7586, + "language": "de", + "window": { + "episode": "506e1ff553b396e74b5e11ac851ebb2d8abbac980f1c9e9c1f5d2147ccdd361a", + "lang": "de", + "window": "middle", + "start": 600.828027, + "duration": 30.0, + "ce": 6.332377780228853 + }, + "text": "ganz oben mit einer extrem hohen Gewichtung, wie du schon meintest, das Google Business Profil inklusive echter", + "scores": { + "emo_Affection": 0.003499329090118408, + "emo_Amusement": 0.15237268805503845, + "emo_Anger": 0.08867141604423523, + "emo_Astonishment_Surprise": 0.02754819393157959, + "emo_Awe": 0.05463704466819763, + "emo_Bitterness": 0.06925627589225769, + "emo_Concentration": 2.2230780124664307, + "emo_Confusion": 0.015630871057510376, + "emo_Contemplation": 1.1912533044815063, + "emo_Contempt": 0.05897912383079529, + "emo_Contentment": 1.519317388534546, + "emo_Disappointment": 0.09251806139945984, + "emo_Disgust": 0.046287357807159424, + "emo_Distress": 0.10292959213256836, + "emo_Doubt": 0.0494576096534729, + "emo_Elation": 0.07590454816818237, + "emo_Embarrassment": -0.004881232976913452, + "emo_Emotional_Numbness": 0.15009891986846924, + "emo_Fatigue_Exhaustion": 0.025012105703353882, + "emo_Fear": 0.02065873146057129, + "emo_Helplessness": 0.01056128740310669, + "emo_Hope_Enthusiasm_Optimism": 0.6577759981155396, + "emo_Impatience_and_Irritability": 0.11068978905677795, + "emo_Infatuation": -0.027781158685684204, + "emo_Interest": 2.1954352855682373, + "emo_Intoxication_Altered_States_of_Consciousness": -0.028512001037597656, + "emo_Jealousy_and_Envy": -0.0012696832418441772, + "emo_Longing": -0.021907448768615723, + "emo_Malevolence_Malice": 0.025343775749206543, + "emo_Pain": 0.021054372191429138, + "emo_Pleasure_Ecstasy": 0.00844430923461914, + "emo_Pride": 0.7768015265464783, + "emo_Relief": -0.01715967059135437, + "emo_Sadness": 0.04681721329689026, + "emo_Sexual_Lust": -0.0018015503883361816, + "emo_Shame": 0.016696348786354065, + "emo_Sourness": 0.06309981644153595, + "emo_Teasing": 0.06191974878311157, + "emo_Thankfulness_Gratitude": 0.021871864795684814, + "emo_Triumph": 0.07605919241905212, + "vn_AGEV_reg": 3.3902597427368164, + "vn_AROU_reg": 2.634697198867798, + "vn_ARSH_reg": 3.2585883140563965, + "vn_ATCK_reg": 3.3325839042663574, + "vn_BKGN_reg": 3.899172782897949, + "vn_BRGT_reg": 3.3822181224823, + "vn_CHNK_reg": 2.9287867546081543, + "vn_CLRT_reg": 4.611832141876221, + "vn_COGL_reg": 0.5637096166610718, + "vn_DARC_reg": 3.1023783683776855, + "vn_DFLU_reg": -0.12865090370178223, + "vn_EMPH_reg": 3.4788684844970703, + "vn_ESTH_reg": 4.33256721496582, + "vn_EXPL_reg": -0.02448570728302002, + "vn_FOCS_reg": 4.200840950012207, + "vn_FULL_reg": 4.125646591186523, + "vn_GEND_reg": 5.08811616897583, + "vn_HARM_reg": 3.96588134765625, + "vn_METL_reg": 1.7382471561431885, + "vn_RANG_reg": 2.7016913890838623, + "vn_RCQL_reg": 4.312852382659912, + "vn_REGS_reg": 0.8975258469581604, + "vn_RESP_reg": 1.0082920789718628, + "vn_ROUG_reg": 1.464820384979248, + "vn_R_CHST_reg": 3.7990071773529053, + "vn_R_HEAD_reg": 1.5772247314453125, + "vn_R_MASK_reg": 2.9542155265808105, + "vn_R_MIXD_reg": 4.08314323425293, + "vn_R_NASL_reg": 1.591546893119812, + "vn_R_ORAL_reg": 3.6143059730529785, + "vn_R_THRT_reg": 2.873262882232666, + "vn_SMTH_reg": 3.9902539253234863, + "vn_STNC_reg": 3.837189197540283, + "vn_STRU_reg": 4.632915496826172, + "vn_S_ASMR_reg": 0.2944391965866089, + "vn_S_AUTH_reg": 3.9060754776000977, + "vn_S_CART_reg": 0.09633278846740723, + "vn_S_CASU_reg": 1.5774586200714111, + "vn_S_CONV_reg": 2.479820728302002, + "vn_S_DRAM_reg": 2.218568801879883, + "vn_S_FORM_reg": 3.8530054092407227, + "vn_S_MONO_reg": 3.2624623775482178, + "vn_S_NARR_reg": 4.053851127624512, + "vn_S_NEWS_reg": 2.5881664752960205, + "vn_S_PLAY_reg": 0.623265266418457, + "vn_S_RANT_reg": 0.45731687545776367, + "vn_S_STRY_reg": 2.52691650390625, + "vn_S_TECH_reg": 2.9835829734802246, + "vn_S_WHIS_reg": -0.25637686252593994, + "vn_TEMP_reg": 3.109259605407715, + "vn_TENS_reg": 2.1080920696258545, + "vn_VALN_reg": 3.0070528984069824, + "vn_VALS_reg": 3.415966033935547, + "vn_VFLX_reg": 2.7750344276428223, + "vn_VOLT_reg": 1.0560503005981445, + "vn_VULN_reg": 1.068552017211914, + "vn_WARM_reg": 3.679795026779175, + "genuineness_0_6": 4.174716949462891, + "blend_0_10": 6.4024858474731445, + "R_quality": 0.2621626853942871, + "eiv_extra:Age": 3.2742700576782227, + "eiv_extra:Arousal": 1.241325855255127, + "eiv_extra:Authenticity": 2.724339008331299, + "eiv_extra:Background_Noise": 0.1460018754005432, + "eiv_extra:Confident_vs._Hesitant": 0.9474958181381226, + "eiv_extra:Gender": -1.082871437072754, + "eiv_extra:High-Pitched_vs._Low-Pitched": 1.923715353012085, + "eiv_extra:Monotone_vs._Expressive": 1.6315723657608032, + "eiv_extra:Recording_Quality": 2.429269313812256, + "eiv_extra:Serious_vs._Humorous": 0.9710330963134766, + "eiv_extra:Soft_vs._Harsh": 0.11378416419029236, + "eiv_extra:Submissive_vs._Dominant": 0.49103453755378723, + "eiv_extra:Valence": -0.04065120965242386, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 2.3718464374542236, + "eiv_extra:Warm_vs._Cold": -0.017269432544708252, + "eiv_extra:score_background_quality": 3.5045952796936035, + "eiv_extra:score_content_enjoyment": 4.849766731262207, + "eiv_extra:score_overall_quality": 2.8258748054504395, + "eiv_extra:score_speech_quality": 1.9371109008789062, + "audiobox:CE": 6.427689552307129, + "audiobox:CU": 7.173866271972656, + "audiobox:PC": 1.4857641458511353, + "audiobox:PQ": 8.152673721313477, + "dnsmos:SIG_raw": 4.089506149291992, + "dnsmos:BAK_raw": 4.421130657196045, + "dnsmos:OVRL_raw": 3.9098737239837646, + "dnsmos:SIG": 3.5850608348846436, + "dnsmos:BAK": 4.133337020874023, + "dnsmos:OVRL": 3.363564968109131, + "dnsmos:P808_MOS": 4.023495674133301, + "burst_count_log1p": 0.07847452163696289, + "voiceclap_attribute:Affection": 0.11895221471786499, + "voiceclap_attribute:Age": 3.3403754234313965, + "voiceclap_attribute:Amusement": 0.22232285141944885, + "voiceclap_attribute:Anger": 0.07362490892410278, + "voiceclap_attribute:Arousal": 1.3780860900878906, + "voiceclap_attribute:Astonishment_Surprise": 0.11974412202835083, + "voiceclap_attribute:Authenticity": 2.805722713470459, + "voiceclap_attribute:Awe": 0.12770754098892212, + "voiceclap_attribute:Background_Noise": 0.09406396746635437, + "voiceclap_attribute:Bitterness": 0.08856159448623657, + "voiceclap_attribute:Concentration": 2.2960050106048584, + "voiceclap_attribute:Confident_vs._Hesitant": 0.8680471181869507, + "voiceclap_attribute:Confusion": -0.09627151489257812, + "voiceclap_attribute:Contemplation": 1.5919814109802246, + "voiceclap_attribute:Contempt": 0.15180200338363647, + "voiceclap_attribute:Contentment": 1.4623768329620361, + "voiceclap_attribute:Disappointment": 0.03422820568084717, + "voiceclap_attribute:Disgust": 0.0759308934211731, + "voiceclap_attribute:Distress": -0.13450461626052856, + "voiceclap_attribute:Doubt": 0.1658887267112732, + "voiceclap_attribute:Elation": 0.0498165488243103, + "voiceclap_attribute:Embarrassment": -0.04324379563331604, + "voiceclap_attribute:Emotional_Numbness": 0.058588504791259766, + "voiceclap_attribute:Fatigue_Exhaustion": -0.2077515423297882, + "voiceclap_attribute:Fear": -0.005011200904846191, + "voiceclap_attribute:Gender": -0.7826812863349915, + "voiceclap_attribute:Helplessness": -0.11862963438034058, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.7247297763824463, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 0.8216304779052734, + "voiceclap_attribute:Impatience_and_Irritability": 0.10812842845916748, + "voiceclap_attribute:Infatuation": -0.08451634645462036, + "voiceclap_attribute:Interest": 2.20949649810791, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": 0.002347409725189209, + "voiceclap_attribute:Jealousy_&_Envy": -0.046796441078186035, + "voiceclap_attribute:Longing": -0.04908716678619385, + "voiceclap_attribute:Malevolence_Malice": 0.035348474979400635, + "voiceclap_attribute:Monotone_vs._Expressive": 1.4692661762237549, + "voiceclap_attribute:Pain": -0.09419319033622742, + "voiceclap_attribute:Pleasure_Ecstasy": 0.04321706295013428, + "voiceclap_attribute:Pride": 0.6700686812400818, + "voiceclap_attribute:Recording_Quality": 2.5123560428619385, + "voiceclap_attribute:Relief": -0.16534173488616943, + "voiceclap_attribute:Sadness": -0.12739717960357666, + "voiceclap_attribute:Serious_vs._Humorous": 0.845283567905426, + "voiceclap_attribute:Sexual_Lust": -0.029126286506652832, + "voiceclap_attribute:Shame": -0.10104489326477051, + "voiceclap_attribute:Soft_vs._Harsh": 0.036191388964653015, + "voiceclap_attribute:Sourness": 0.14901649951934814, + "voiceclap_attribute:Submissive_vs._Dominant": 0.5605859756469727, + "voiceclap_attribute:Teasing": 0.053975820541381836, + "voiceclap_attribute:Thankfulness_Gratitude": -0.12199687957763672, + "voiceclap_attribute:Triumph": 0.050173163414001465, + "voiceclap_attribute:Valence": -0.05870813876390457, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 2.0304691791534424, + "voiceclap_attribute:Warm_vs._Cold": 0.054657384753227234, + "voiceclap_attribute:duration": 6.588235378265381, + "voiceclap_attribute:score_background_quality": 3.623931407928467, + "voiceclap_attribute:score_content_enjoyment": 4.760096549987793, + "voiceclap_attribute:score_overall_quality": 2.8598520755767822, + "voiceclap_attribute:score_speech_quality": 2.005962371826172, + "voiceclap_attribute:talking_speed": 17.220680236816406 + }, + "asr": "Ganz oben mit einer extrem hohen Gewichtung, wie du schon meintest, das Google Business Profil inklusive Echtspiel.", + "audio_sha256": "8e532d43e660a9f87fb0727755ddafae0af691f86f1582cf3f37e052a528c641", + "speaker": "S1", + "seconds": 6.0, + "media_member": "de-S1.mp3", + "wav_member": "de-S1.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + }, + "S2": { + "episode": "506e1ff553b396e74b5e11ac851ebb2d8abbac980f1c9e9c1f5d2147ccdd361a", + "original_speaker": "S2", + "start_frame": 7603, + "end_frame": 7673, + "language": "de", + "window": { + "episode": "506e1ff553b396e74b5e11ac851ebb2d8abbac980f1c9e9c1f5d2147ccdd361a", + "lang": "de", + "window": "middle", + "start": 600.828027, + "duration": 30.0, + "ce": 6.332377780228853 + }, + "text": "es eben der stärkste Indikator für lokale Relevanz und echte physische Existenz", + "scores": { + "emo_Affection": 0.08421026170253754, + "emo_Amusement": 0.488334983587265, + "emo_Anger": 0.1056675910949707, + "emo_Astonishment_Surprise": 0.07823824882507324, + "emo_Awe": 0.13047467172145844, + "emo_Bitterness": 0.035080790519714355, + "emo_Concentration": 2.2230780124664307, + "emo_Confusion": 0.017714470624923706, + "emo_Contemplation": 1.4581191539764404, + "emo_Contempt": 0.05193011462688446, + "emo_Contentment": 1.7397338151931763, + "emo_Disappointment": 0.059642672538757324, + "emo_Disgust": 0.03096643090248108, + "emo_Distress": 0.07259681820869446, + "emo_Doubt": 0.1759568750858307, + "emo_Elation": 0.17693793773651123, + "emo_Embarrassment": 0.005362153053283691, + "emo_Emotional_Numbness": 0.09188514947891235, + "emo_Fatigue_Exhaustion": -0.015124976634979248, + "emo_Fear": 0.050528913736343384, + "emo_Helplessness": 0.005029559135437012, + "emo_Hope_Enthusiasm_Optimism": 1.0370121002197266, + "emo_Impatience_and_Irritability": 0.08129343390464783, + "emo_Infatuation": -0.03306666016578674, + "emo_Interest": 2.484661817550659, + "emo_Intoxication_Altered_States_of_Consciousness": -0.010043919086456299, + "emo_Jealousy_and_Envy": 0.008307188749313354, + "emo_Longing": -0.007005155086517334, + "emo_Malevolence_Malice": 0.039663299918174744, + "emo_Pain": -0.0025366544723510742, + "emo_Pleasure_Ecstasy": -0.001095205545425415, + "emo_Pride": 0.8417905569076538, + "emo_Relief": 0.03600078821182251, + "emo_Sadness": 0.013976067304611206, + "emo_Sexual_Lust": -0.011522918939590454, + "emo_Shame": 0.00032067298889160156, + "emo_Sourness": 0.028476819396018982, + "emo_Teasing": 0.10710452497005463, + "emo_Thankfulness_Gratitude": 0.012420654296875, + "emo_Triumph": 0.16010864078998566, + "vn_AGEV_reg": 3.7052559852600098, + "vn_AROU_reg": 2.6930038928985596, + "vn_ARSH_reg": 3.1661670207977295, + "vn_ATCK_reg": 3.2297139167785645, + "vn_BKGN_reg": 4.041053295135498, + "vn_BRGT_reg": 3.9268441200256348, + "vn_CHNK_reg": 2.5246458053588867, + "vn_CLRT_reg": 4.521746635437012, + "vn_COGL_reg": 0.6378332376480103, + "vn_DARC_reg": 2.9746766090393066, + "vn_DFLU_reg": -0.16556215286254883, + "vn_EMPH_reg": 3.2701969146728516, + "vn_ESTH_reg": 4.490742206573486, + "vn_EXPL_reg": -0.02832302451133728, + "vn_FOCS_reg": 3.854011058807373, + "vn_FULL_reg": 3.352433681488037, + "vn_GEND_reg": 0.9078109264373779, + "vn_HARM_reg": 4.289289951324463, + "vn_METL_reg": 1.6952276229858398, + "vn_RANG_reg": 3.210130453109741, + "vn_RCQL_reg": 4.678562164306641, + "vn_REGS_reg": 4.072345733642578, + "vn_RESP_reg": 1.459503412246704, + "vn_ROUG_reg": 0.844868540763855, + "vn_R_CHST_reg": 2.1125450134277344, + "vn_R_HEAD_reg": 3.25652813911438, + "vn_R_MASK_reg": 3.929640293121338, + "vn_R_MIXD_reg": 4.004687309265137, + "vn_R_NASL_reg": 1.9958350658416748, + "vn_R_ORAL_reg": 4.160750389099121, + "vn_R_THRT_reg": 1.8289213180541992, + "vn_SMTH_reg": 3.972888469696045, + "vn_STNC_reg": 3.71264910697937, + "vn_STRU_reg": 4.540471076965332, + "vn_S_ASMR_reg": 0.3644522428512573, + "vn_S_AUTH_reg": 3.532729148864746, + "vn_S_CART_reg": 0.5135087966918945, + "vn_S_CASU_reg": 1.9263372421264648, + "vn_S_CONV_reg": 2.7615418434143066, + "vn_S_DRAM_reg": 2.0850555896759033, + "vn_S_FORM_reg": 3.474879741668701, + "vn_S_MONO_reg": 2.640594482421875, + "vn_S_NARR_reg": 4.031425952911377, + "vn_S_NEWS_reg": 2.1663033962249756, + "vn_S_PLAY_reg": 1.517014503479004, + "vn_S_RANT_reg": 0.2778879404067993, + "vn_S_STRY_reg": 2.4507009983062744, + "vn_S_TECH_reg": 3.002613067626953, + "vn_S_WHIS_reg": -0.10650229454040527, + "vn_TEMP_reg": 2.8982059955596924, + "vn_TENS_reg": 1.997554898262024, + "vn_VALN_reg": 3.333658218383789, + "vn_VALS_reg": 3.16713285446167, + "vn_VFLX_reg": 2.767963409423828, + "vn_VOLT_reg": 1.32744300365448, + "vn_VULN_reg": 1.4090443849563599, + "vn_WARM_reg": 3.227367639541626, + "genuineness_0_6": 3.9801597595214844, + "blend_0_10": 6.28304386138916, + "R_quality": 0.29320424795150757, + "eiv_extra:Age": 3.321516990661621, + "eiv_extra:Arousal": 1.381827712059021, + "eiv_extra:Authenticity": 2.858616352081299, + "eiv_extra:Background_Noise": 0.0016562938690185547, + "eiv_extra:Confident_vs._Hesitant": 0.7503005266189575, + "eiv_extra:Gender": 1.2936965227127075, + "eiv_extra:High-Pitched_vs._Low-Pitched": 1.409066915512085, + "eiv_extra:Monotone_vs._Expressive": 1.4460489749908447, + "eiv_extra:Recording_Quality": 2.7514548301696777, + "eiv_extra:Serious_vs._Humorous": 0.8154757022857666, + "eiv_extra:Soft_vs._Harsh": 0.2709927558898926, + "eiv_extra:Submissive_vs._Dominant": 0.28906723856925964, + "eiv_extra:Valence": 0.11885757744312286, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 1.9334872961044312, + "eiv_extra:Warm_vs._Cold": 0.09527456760406494, + "eiv_extra:score_background_quality": 3.64819073677063, + "eiv_extra:score_content_enjoyment": 4.763145923614502, + "eiv_extra:score_overall_quality": 2.9170734882354736, + "eiv_extra:score_speech_quality": 1.9751968383789062, + "audiobox:CE": 6.205295085906982, + "audiobox:CU": 7.017790794372559, + "audiobox:PC": 1.479453682899475, + "audiobox:PQ": 7.946595668792725, + "dnsmos:SIG_raw": 4.113028049468994, + "dnsmos:BAK_raw": 4.324429988861084, + "dnsmos:OVRL_raw": 3.8781161308288574, + "dnsmos:SIG": 3.6007213592529297, + "dnsmos:BAK": 4.071674346923828, + "dnsmos:OVRL": 3.349346399307251, + "dnsmos:P808_MOS": 3.9653210639953613, + "burst_count_log1p": 0.06580808758735657, + "voiceclap_attribute:Affection": 0.14494472742080688, + "voiceclap_attribute:Age": 3.4465768337249756, + "voiceclap_attribute:Amusement": 0.4173581600189209, + "voiceclap_attribute:Anger": -0.00565183162689209, + "voiceclap_attribute:Arousal": 1.3270134925842285, + "voiceclap_attribute:Astonishment_Surprise": 0.39284974336624146, + "voiceclap_attribute:Authenticity": 3.018613338470459, + "voiceclap_attribute:Awe": 0.19864007830619812, + "voiceclap_attribute:Background_Noise": 0.03835529088973999, + "voiceclap_attribute:Bitterness": 0.03387409448623657, + "voiceclap_attribute:Concentration": 2.1219322681427, + "voiceclap_attribute:Confident_vs._Hesitant": 0.9605225324630737, + "voiceclap_attribute:Confusion": -0.007230699062347412, + "voiceclap_attribute:Contemplation": 1.357642412185669, + "voiceclap_attribute:Contempt": 0.10403263568878174, + "voiceclap_attribute:Contentment": 1.5391154289245605, + "voiceclap_attribute:Disappointment": -0.08833938837051392, + "voiceclap_attribute:Disgust": 0.048587143421173096, + "voiceclap_attribute:Distress": -0.16731411218643188, + "voiceclap_attribute:Doubt": 0.1619824767112732, + "voiceclap_attribute:Elation": 0.24295487999916077, + "voiceclap_attribute:Embarrassment": -0.013739347457885742, + "voiceclap_attribute:Emotional_Numbness": -0.06618279218673706, + "voiceclap_attribute:Fatigue_Exhaustion": -0.2524931728839874, + "voiceclap_attribute:Fear": -0.05969870090484619, + "voiceclap_attribute:Gender": 1.356034517288208, + "voiceclap_attribute:Helplessness": -0.10605448484420776, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.262193202972412, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 1.1029201745986938, + "voiceclap_attribute:Impatience_and_Irritability": 0.09542453289031982, + "voiceclap_attribute:Infatuation": -0.10530418157577515, + "voiceclap_attribute:Interest": 2.3811116218566895, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": 0.07484263181686401, + "voiceclap_attribute:Jealousy_&_Envy": -0.06632769107818604, + "voiceclap_attribute:Longing": -0.08033716678619385, + "voiceclap_attribute:Malevolence_Malice": 0.013723492622375488, + "voiceclap_attribute:Monotone_vs._Expressive": 1.473926067352295, + "voiceclap_attribute:Pain": -0.11372444033622742, + "voiceclap_attribute:Pleasure_Ecstasy": 0.09263297915458679, + "voiceclap_attribute:Pride": 0.6468936204910278, + "voiceclap_attribute:Recording_Quality": 2.771263599395752, + "voiceclap_attribute:Relief": 0.09462636709213257, + "voiceclap_attribute:Sadness": -0.09317153692245483, + "voiceclap_attribute:Serious_vs._Humorous": 0.819100022315979, + "voiceclap_attribute:Sexual_Lust": 0.055750131607055664, + "voiceclap_attribute:Shame": -0.11425364017486572, + "voiceclap_attribute:Soft_vs._Harsh": 0.3696213662624359, + "voiceclap_attribute:Sourness": 0.046891987323760986, + "voiceclap_attribute:Submissive_vs._Dominant": 0.3453238904476166, + "voiceclap_attribute:Teasing": 0.2718041241168976, + "voiceclap_attribute:Thankfulness_Gratitude": -0.0889122486114502, + "voiceclap_attribute:Triumph": 0.11463993787765503, + "voiceclap_attribute:Valence": 0.12953214347362518, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 1.7557846307754517, + "voiceclap_attribute:Warm_vs._Cold": 0.19583760201931, + "voiceclap_attribute:duration": 6.340827465057373, + "voiceclap_attribute:score_background_quality": 3.647301197052002, + "voiceclap_attribute:score_content_enjoyment": 4.833338737487793, + "voiceclap_attribute:score_overall_quality": 2.9065582752227783, + "voiceclap_attribute:score_speech_quality": 2.027935028076172, + "voiceclap_attribute:talking_speed": 15.66700553894043 + }, + "asr": "Der stärkste Indikator für lokale Relevanz und echte physische Existenz ist?", + "audio_sha256": "ba164cef51bc66522c22d03244943bc3e7c6cdebb3f88b753f0cba9d9963ea3d", + "speaker": "S2", + "seconds": 5.6, + "media_member": "de-S2.mp3", + "wav_member": "de-S2.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + } + }, + "fr": { + "S1": { + "episode": "5712265f42d9ba1a2627b39ffd2285358f5c693838a4c77eea13a50032832edf", + "original_speaker": "S2", + "start_frame": 7842, + "end_frame": 7907, + "language": "fr", + "window": { + "episode": "5712265f42d9ba1a2627b39ffd2285358f5c693838a4c77eea13a50032832edf", + "lang": "fr", + "window": "middle", + "start": 619.7232655, + "duration": 30.0, + "ce": 6.062326967716217 + }, + "text": "le phénomène hashtag Booktock, des jeunes qui partagent leur amour des livres et font décoller les", + "scores": { + "emo_Affection": 0.10120202600955963, + "emo_Amusement": 0.36081334948539734, + "emo_Anger": 0.07734063267707825, + "emo_Astonishment_Surprise": 0.07715973258018494, + "emo_Awe": 0.014053672552108765, + "emo_Bitterness": 0.0517711341381073, + "emo_Concentration": 2.2230780124664307, + "emo_Confusion": 0.03855052590370178, + "emo_Contemplation": 1.3430907726287842, + "emo_Contempt": 0.050167858600616455, + "emo_Contentment": 1.5586774349212646, + "emo_Disappointment": 0.15278956294059753, + "emo_Disgust": 0.04532980918884277, + "emo_Distress": 0.13006946444511414, + "emo_Doubt": 0.0796663910150528, + "emo_Elation": 0.13343745470046997, + "emo_Embarrassment": 0.020727217197418213, + "emo_Emotional_Numbness": 0.10127449035644531, + "emo_Fatigue_Exhaustion": 0.04402439296245575, + "emo_Fear": 0.0338016152381897, + "emo_Helplessness": 0.0631125271320343, + "emo_Hope_Enthusiasm_Optimism": 0.9481286406517029, + "emo_Impatience_and_Irritability": 0.09465542435646057, + "emo_Infatuation": 0.010274410247802734, + "emo_Interest": 2.6132068634033203, + "emo_Intoxication_Altered_States_of_Consciousness": -0.013302981853485107, + "emo_Jealousy_and_Envy": 0.017013445496559143, + "emo_Longing": 0.03770172595977783, + "emo_Malevolence_Malice": 0.029435068368911743, + "emo_Pain": 0.04239863157272339, + "emo_Pleasure_Ecstasy": -0.001095205545425415, + "emo_Pride": 0.4234238266944885, + "emo_Relief": 0.007375925779342651, + "emo_Sadness": 0.04383164644241333, + "emo_Sexual_Lust": 0.022501900792121887, + "emo_Shame": -0.009914115071296692, + "emo_Sourness": 0.05887749791145325, + "emo_Teasing": 0.11866341531276703, + "emo_Thankfulness_Gratitude": 0.05292579531669617, + "emo_Triumph": 0.032229721546173096, + "vn_AGEV_reg": 3.033428192138672, + "vn_AROU_reg": 2.7447097301483154, + "vn_ARSH_reg": 3.0292468070983887, + "vn_ATCK_reg": 3.0239737033843994, + "vn_BKGN_reg": 3.769115924835205, + "vn_BRGT_reg": 3.5595383644104004, + "vn_CHNK_reg": 2.6999359130859375, + "vn_CLRT_reg": 3.831089496612549, + "vn_COGL_reg": 2.0595650672912598, + "vn_DARC_reg": 2.8789005279541016, + "vn_DFLU_reg": 1.495448350906372, + "vn_EMPH_reg": 3.190410852432251, + "vn_ESTH_reg": 3.7850375175476074, + "vn_EXPL_reg": -0.04559090733528137, + "vn_FOCS_reg": 3.836669445037842, + "vn_FULL_reg": 3.062478542327881, + "vn_GEND_reg": 2.2514805793762207, + "vn_HARM_reg": 3.487236499786377, + "vn_METL_reg": 1.7569512128829956, + "vn_RANG_reg": 2.8653669357299805, + "vn_RCQL_reg": 3.8838460445404053, + "vn_REGS_reg": 3.0713205337524414, + "vn_RESP_reg": 1.8271570205688477, + "vn_ROUG_reg": 1.444155216217041, + "vn_R_CHST_reg": 2.3961093425750732, + "vn_R_HEAD_reg": 2.7257137298583984, + "vn_R_MASK_reg": 3.2660317420959473, + "vn_R_MIXD_reg": 3.442420482635498, + "vn_R_NASL_reg": 2.103095293045044, + "vn_R_ORAL_reg": 3.9165945053100586, + "vn_R_THRT_reg": 2.2284133434295654, + "vn_SMTH_reg": 3.3303771018981934, + "vn_STNC_reg": 3.3390283584594727, + "vn_STRU_reg": 3.616027355194092, + "vn_S_ASMR_reg": 0.27289676666259766, + "vn_S_AUTH_reg": 3.2161965370178223, + "vn_S_CART_reg": 0.22469472885131836, + "vn_S_CASU_reg": 2.863325357437134, + "vn_S_CONV_reg": 3.5582644939422607, + "vn_S_DRAM_reg": 1.7913264036178589, + "vn_S_FORM_reg": 2.5138094425201416, + "vn_S_MONO_reg": 2.1864633560180664, + "vn_S_NARR_reg": 3.2689566612243652, + "vn_S_NEWS_reg": 1.266615867614746, + "vn_S_PLAY_reg": 1.4408427476882935, + "vn_S_RANT_reg": 0.3327134847640991, + "vn_S_STRY_reg": 2.2792160511016846, + "vn_S_TECH_reg": 2.7742505073547363, + "vn_S_WHIS_reg": 0.16826772689819336, + "vn_TEMP_reg": 3.109259605407715, + "vn_TENS_reg": 2.1861181259155273, + "vn_VALN_reg": 3.0070528984069824, + "vn_VALS_reg": 3.0813283920288086, + "vn_VFLX_reg": 2.9235239028930664, + "vn_VOLT_reg": 1.4224305152893066, + "vn_VULN_reg": 1.48642897605896, + "vn_WARM_reg": 3.2874555587768555, + "genuineness_0_6": 4.419482707977295, + "blend_0_10": 7.849062919616699, + "R_quality": 0.7743484377861023, + "eiv_extra:Age": 3.285217523574829, + "eiv_extra:Arousal": 1.4048188924789429, + "eiv_extra:Authenticity": 2.803928852081299, + "eiv_extra:Background_Noise": 0.4016996920108795, + "eiv_extra:Confident_vs._Hesitant": 0.9445260167121887, + "eiv_extra:Gender": 0.9742235541343689, + "eiv_extra:High-Pitched_vs._Low-Pitched": 1.577035665512085, + "eiv_extra:Monotone_vs._Expressive": 1.8363828659057617, + "eiv_extra:Recording_Quality": 2.4942259788513184, + "eiv_extra:Serious_vs._Humorous": 1.032540202140808, + "eiv_extra:Soft_vs._Harsh": 0.32526713609695435, + "eiv_extra:Submissive_vs._Dominant": 0.1489524394273758, + "eiv_extra:Valence": 0.07376280426979065, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 1.954766869544983, + "eiv_extra:Warm_vs._Cold": 0.1129213497042656, + "eiv_extra:score_background_quality": 3.6752843856811523, + "eiv_extra:score_content_enjoyment": 4.70025634765625, + "eiv_extra:score_overall_quality": 2.9131083488464355, + "eiv_extra:score_speech_quality": 1.9312515258789062, + "audiobox:CE": 5.919358730316162, + "audiobox:CU": 6.506642818450928, + "audiobox:PC": 1.6182845830917358, + "audiobox:PQ": 7.788601875305176, + "dnsmos:SIG_raw": 3.8372974395751953, + "dnsmos:BAK_raw": 4.210146903991699, + "dnsmos:OVRL_raw": 3.595475196838379, + "dnsmos:SIG": 3.440199851989746, + "dnsmos:BAK": 4.031774997711182, + "dnsmos:OVRL": 3.1716156005859375, + "dnsmos:P808_MOS": 3.792736291885376, + "burst_count_log1p": 0.13674002885818481, + "voiceclap_attribute:Affection": 0.0886276364326477, + "voiceclap_attribute:Age": 3.245856285095215, + "voiceclap_attribute:Amusement": 0.3837313652038574, + "voiceclap_attribute:Anger": -0.05321788787841797, + "voiceclap_attribute:Arousal": 1.4887431859970093, + "voiceclap_attribute:Astonishment_Surprise": 0.23759105801582336, + "voiceclap_attribute:Authenticity": 2.817929744720459, + "voiceclap_attribute:Awe": 0.07096153497695923, + "voiceclap_attribute:Background_Noise": 0.3189619481563568, + "voiceclap_attribute:Bitterness": -0.03643840551376343, + "voiceclap_attribute:Concentration": 2.159233570098877, + "voiceclap_attribute:Confident_vs._Hesitant": 1.426561951637268, + "voiceclap_attribute:Confusion": 0.17346984148025513, + "voiceclap_attribute:Contemplation": 1.5476469993591309, + "voiceclap_attribute:Contempt": 0.04034006595611572, + "voiceclap_attribute:Contentment": 1.4288036823272705, + "voiceclap_attribute:Disappointment": -0.006627678871154785, + "voiceclap_attribute:Disgust": 0.032962143421173096, + "voiceclap_attribute:Distress": 0.002201974391937256, + "voiceclap_attribute:Doubt": 0.4246777892112732, + "voiceclap_attribute:Elation": 0.1432705819606781, + "voiceclap_attribute:Embarrassment": -0.023574143648147583, + "voiceclap_attribute:Emotional_Numbness": 0.025316178798675537, + "voiceclap_attribute:Fatigue_Exhaustion": -0.11420097947120667, + "voiceclap_attribute:Fear": 0.006707549095153809, + "voiceclap_attribute:Gender": 0.8380336165428162, + "voiceclap_attribute:Helplessness": -0.001261591911315918, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.3789217472076416, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 0.9834205508232117, + "voiceclap_attribute:Impatience_and_Irritability": 0.027670860290527344, + "voiceclap_attribute:Infatuation": -0.06892544031143188, + "voiceclap_attribute:Interest": 2.401705265045166, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": -0.08102214336395264, + "voiceclap_attribute:Jealousy_&_Envy": -0.015546441078186035, + "voiceclap_attribute:Longing": -0.08033716678619385, + "voiceclap_attribute:Malevolence_Malice": 0.06237971782684326, + "voiceclap_attribute:Monotone_vs._Expressive": 1.6579909324645996, + "voiceclap_attribute:Pain": -0.05513069033622742, + "voiceclap_attribute:Pleasure_Ecstasy": 0.0032137036323547363, + "voiceclap_attribute:Pride": 0.35379719734191895, + "voiceclap_attribute:Recording_Quality": 2.3655989170074463, + "voiceclap_attribute:Relief": -0.0548553466796875, + "voiceclap_attribute:Sadness": -0.005162715911865234, + "voiceclap_attribute:Serious_vs._Humorous": 0.7778748869895935, + "voiceclap_attribute:Sexual_Lust": -0.022597312927246094, + "voiceclap_attribute:Shame": -0.061418771743774414, + "voiceclap_attribute:Soft_vs._Harsh": 0.31193938851356506, + "voiceclap_attribute:Sourness": -0.03480762243270874, + "voiceclap_attribute:Submissive_vs._Dominant": 0.37056636810302734, + "voiceclap_attribute:Teasing": 0.1517762839794159, + "voiceclap_attribute:Thankfulness_Gratitude": -0.05582773685455322, + "voiceclap_attribute:Triumph": 0.050173163414001465, + "voiceclap_attribute:Valence": -0.12186771631240845, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 1.7291847467422485, + "voiceclap_attribute:Warm_vs._Cold": 0.01624232903122902, + "voiceclap_attribute:duration": 6.835643768310547, + "voiceclap_attribute:score_background_quality": 3.5401902198791504, + "voiceclap_attribute:score_content_enjoyment": 4.682459831237793, + "voiceclap_attribute:score_overall_quality": 2.8195881843566895, + "voiceclap_attribute:score_speech_quality": 1.9922903776168823, + "voiceclap_attribute:talking_speed": 15.168658256530762 + }, + "asr": "phénomène hashtag book talk des jeunes qui partagent leur amour des livres et font décoller les ventes", + "audio_sha256": "f20e89d2454ceec9389056dfdfa255fa66c449d4817aec5d5ee287eb4bddd52e", + "speaker": "S1", + "seconds": 5.2, + "media_member": "fr-S1.mp3", + "wav_member": "fr-S1.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + }, + "S2": { + "episode": "2d26cd6d5595ee967e580f0838c8e6407f10dde0be2a589db4b0c8c65f6e27fb", + "original_speaker": "S2", + "start_frame": 6544, + "end_frame": 6615, + "language": "fr", + "window": { + "episode": "2d26cd6d5595ee967e580f0838c8e6407f10dde0be2a589db4b0c8c65f6e27fb", + "lang": "fr", + "window": "middle", + "start": 522.939592, + "duration": 30.0, + "ce": 5.824046839028597 + }, + "text": "analysant le contenu exact de ces charterabs sur l'IA, on découvre une cartographie fascinante de ce qui est", + "scores": { + "emo_Affection": 0.08421026170253754, + "emo_Amusement": 0.3438669443130493, + "emo_Anger": 0.0002913177013397217, + "emo_Astonishment_Surprise": 0.15427334606647491, + "emo_Awe": 0.4297258257865906, + "emo_Bitterness": -0.030091077089309692, + "emo_Concentration": 2.348647117614746, + "emo_Confusion": 0.003129243850708008, + "emo_Contemplation": 1.241865873336792, + "emo_Contempt": 0.0008248090744018555, + "emo_Contentment": 2.0231266021728516, + "emo_Disappointment": -0.04446268081665039, + "emo_Disgust": -0.02457195520401001, + "emo_Distress": -0.026383817195892334, + "emo_Doubt": 0.023024946451187134, + "emo_Elation": 0.24710001051425934, + "emo_Embarrassment": -0.04414752125740051, + "emo_Emotional_Numbness": 0.10503020882606506, + "emo_Fatigue_Exhaustion": -0.04047468304634094, + "emo_Fear": -0.010406255722045898, + "emo_Helplessness": -0.10283881425857544, + "emo_Hope_Enthusiasm_Optimism": 1.2207045555114746, + "emo_Impatience_and_Irritability": 0.011811137199401855, + "emo_Infatuation": -0.05737994611263275, + "emo_Interest": 2.963135242462158, + "emo_Intoxication_Altered_States_of_Consciousness": -0.045893728733062744, + "emo_Jealousy_and_Envy": 0.00308343768119812, + "emo_Longing": 0.03273427486419678, + "emo_Malevolence_Malice": -0.01454627513885498, + "emo_Pain": -0.06207588315010071, + "emo_Pleasure_Ecstasy": 0.06568154692649841, + "emo_Pride": 0.8052342534065247, + "emo_Relief": 0.06462568044662476, + "emo_Sadness": -0.09350407123565674, + "emo_Sexual_Lust": -0.030965656042099, + "emo_Shame": -0.044712409377098083, + "emo_Sourness": -0.014590814709663391, + "emo_Teasing": 0.030395478010177612, + "emo_Thankfulness_Gratitude": 0.04347461462020874, + "emo_Triumph": 0.0997786670923233, + "vn_AGEV_reg": 3.0162017345428467, + "vn_AROU_reg": 2.6875033378601074, + "vn_ARSH_reg": 3.323625326156616, + "vn_ATCK_reg": 3.0136866569519043, + "vn_BKGN_reg": 3.875526189804077, + "vn_BRGT_reg": 3.2365622520446777, + "vn_CHNK_reg": 2.899571657180786, + "vn_CLRT_reg": 4.4466753005981445, + "vn_COGL_reg": 0.7201927900314331, + "vn_DARC_reg": 3.1555871963500977, + "vn_DFLU_reg": 0.2681460380554199, + "vn_EMPH_reg": 3.2701969146728516, + "vn_ESTH_reg": 4.235228538513184, + "vn_EXPL_reg": 0.02923659235239029, + "vn_FOCS_reg": 4.1488165855407715, + "vn_FULL_reg": 4.111839294433594, + "vn_GEND_reg": 5.573329925537109, + "vn_HARM_reg": 3.771836280822754, + "vn_METL_reg": 1.526890754699707, + "vn_RANG_reg": 2.776564359664917, + "vn_RCQL_reg": 4.284720420837402, + "vn_REGS_reg": 0.804293155670166, + "vn_RESP_reg": 1.5179938077926636, + "vn_ROUG_reg": 1.347718358039856, + "vn_R_CHST_reg": 3.9051365852355957, + "vn_R_HEAD_reg": 1.783116340637207, + "vn_R_MASK_reg": 3.0101826190948486, + "vn_R_MIXD_reg": 4.226978778839111, + "vn_R_NASL_reg": 1.6658039093017578, + "vn_R_ORAL_reg": 3.6143059730529785, + "vn_R_THRT_reg": 2.6813807487487793, + "vn_SMTH_reg": 4.094444751739502, + "vn_STNC_reg": 3.5792131423950195, + "vn_STRU_reg": 4.478841304779053, + "vn_S_ASMR_reg": 0.48832130432128906, + "vn_S_AUTH_reg": 3.5164966583251953, + "vn_S_CART_reg": -0.11760354042053223, + "vn_S_CASU_reg": 2.035984754562378, + "vn_S_CONV_reg": 3.2013328075408936, + "vn_S_DRAM_reg": 1.631110429763794, + "vn_S_FORM_reg": 3.411858558654785, + "vn_S_MONO_reg": 2.9655065536499023, + "vn_S_NARR_reg": 4.132340908050537, + "vn_S_NEWS_reg": 2.4504151344299316, + "vn_S_PLAY_reg": 1.3900614976882935, + "vn_S_RANT_reg": 0.0585860013961792, + "vn_S_STRY_reg": 2.4500203132629395, + "vn_S_TECH_reg": 2.7171597480773926, + "vn_S_WHIS_reg": 0.11830949783325195, + "vn_TEMP_reg": 3.006077766418457, + "vn_TENS_reg": 1.7976127862930298, + "vn_VALN_reg": 3.96921443939209, + "vn_VALS_reg": 3.613316535949707, + "vn_VFLX_reg": 2.82924485206604, + "vn_VOLT_reg": 1.1646074056625366, + "vn_VULN_reg": 1.269752025604248, + "vn_WARM_reg": 3.799971103668213, + "genuineness_0_6": 4.111956596374512, + "blend_0_10": 6.766120910644531, + "R_quality": 0.38632893562316895, + "eiv_extra:Age": 3.274846315383911, + "eiv_extra:Arousal": 1.180016040802002, + "eiv_extra:Authenticity": 2.870335102081299, + "eiv_extra:Background_Noise": 0.1460018754005432, + "eiv_extra:Confident_vs._Hesitant": 0.7645555734634399, + "eiv_extra:Gender": -1.0517033338546753, + "eiv_extra:High-Pitched_vs._Low-Pitched": 1.845224142074585, + "eiv_extra:Monotone_vs._Expressive": 1.3946168422698975, + "eiv_extra:Recording_Quality": 2.868377208709717, + "eiv_extra:Serious_vs._Humorous": 1.0205551385879517, + "eiv_extra:Soft_vs._Harsh": 0.2398005723953247, + "eiv_extra:Submissive_vs._Dominant": 0.1262311339378357, + "eiv_extra:Valence": 0.33501434326171875, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 2.589749336242676, + "eiv_extra:Warm_vs._Cold": 0.21645890176296234, + "eiv_extra:score_background_quality": 3.531688928604126, + "eiv_extra:score_content_enjoyment": 4.7951836585998535, + "eiv_extra:score_overall_quality": 2.7941536903381348, + "eiv_extra:score_speech_quality": 1.9283218383789062, + "audiobox:CE": 6.149696350097656, + "audiobox:CU": 6.619797706604004, + "audiobox:PC": 1.6245951652526855, + "audiobox:PQ": 7.822948455810547, + "dnsmos:SIG_raw": 4.052916049957275, + "dnsmos:BAK_raw": 4.315639019012451, + "dnsmos:OVRL_raw": 3.814601421356201, + "dnsmos:SIG": 3.5654850006103516, + "dnsmos:BAK": 4.097064971923828, + "dnsmos:OVRL": 3.3114304542541504, + "dnsmos:P808_MOS": 3.9187815189361572, + "burst_count_log1p": 0.05060839653015137, + "voiceclap_attribute:Affection": 0.17960137128829956, + "voiceclap_attribute:Age": 3.3828558921813965, + "voiceclap_attribute:Amusement": 0.37364333868026733, + "voiceclap_attribute:Anger": -0.16420531272888184, + "voiceclap_attribute:Arousal": 1.3707900047302246, + "voiceclap_attribute:Astonishment_Surprise": 0.2581675052642822, + "voiceclap_attribute:Authenticity": 2.777158260345459, + "voiceclap_attribute:Awe": 0.31922537088394165, + "voiceclap_attribute:Background_Noise": 0.25293686985969543, + "voiceclap_attribute:Bitterness": -0.18878215551376343, + "voiceclap_attribute:Concentration": 2.1219322681427, + "voiceclap_attribute:Confident_vs._Hesitant": 0.9028398394584656, + "voiceclap_attribute:Confusion": -0.017706096172332764, + "voiceclap_attribute:Contemplation": 1.3703093528747559, + "voiceclap_attribute:Contempt": -0.03927558660507202, + "voiceclap_attribute:Contentment": 1.778923749923706, + "voiceclap_attribute:Disappointment": -0.13827431201934814, + "voiceclap_attribute:Disgust": -0.037350356578826904, + "voiceclap_attribute:Distress": -0.161845862865448, + "voiceclap_attribute:Doubt": 0.1053418517112732, + "voiceclap_attribute:Elation": 0.2927970588207245, + "voiceclap_attribute:Embarrassment": -0.07274824380874634, + "voiceclap_attribute:Emotional_Numbness": 0.03779333829879761, + "voiceclap_attribute:Fatigue_Exhaustion": -0.28909990191459656, + "voiceclap_attribute:Fear": 0.0028012990951538086, + "voiceclap_attribute:Gender": -0.7434388399124146, + "voiceclap_attribute:Helplessness": -0.1312047839164734, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.7819997072219849, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 1.1261993646621704, + "voiceclap_attribute:Impatience_and_Irritability": -0.0824289321899414, + "voiceclap_attribute:Infatuation": -0.04294067621231079, + "voiceclap_attribute:Interest": 2.3879761695861816, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": -0.12814408540725708, + "voiceclap_attribute:Jealousy_&_Envy": -0.015546441078186035, + "voiceclap_attribute:Longing": -0.05689966678619385, + "voiceclap_attribute:Malevolence_Malice": -0.13224518299102783, + "voiceclap_attribute:Monotone_vs._Expressive": 1.4436368942260742, + "voiceclap_attribute:Pain": -0.018021315336227417, + "voiceclap_attribute:Pleasure_Ecstasy": 0.17499282956123352, + "voiceclap_attribute:Pride": 0.779127836227417, + "voiceclap_attribute:Recording_Quality": 2.443143129348755, + "voiceclap_attribute:Relief": 0.11412400007247925, + "voiceclap_attribute:Sadness": -0.1958484649658203, + "voiceclap_attribute:Serious_vs._Humorous": 0.9115780591964722, + "voiceclap_attribute:Sexual_Lust": -0.035655200481414795, + "voiceclap_attribute:Shame": -0.07903039455413818, + "voiceclap_attribute:Soft_vs._Harsh": 0.129045307636261, + "voiceclap_attribute:Sourness": -0.09835177659988403, + "voiceclap_attribute:Submissive_vs._Dominant": 0.559884786605835, + "voiceclap_attribute:Teasing": 0.11028516292572021, + "voiceclap_attribute:Thankfulness_Gratitude": 0.010341405868530273, + "voiceclap_attribute:Triumph": 0.19312119483947754, + "voiceclap_attribute:Valence": -0.05127760022878647, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 2.1629257202148438, + "voiceclap_attribute:Warm_vs._Cold": 0.06791740655899048, + "voiceclap_attribute:duration": 7.726313591003418, + "voiceclap_attribute:score_background_quality": 3.5207157135009766, + "voiceclap_attribute:score_content_enjoyment": 4.806971549987793, + "voiceclap_attribute:score_overall_quality": 2.8228092193603516, + "voiceclap_attribute:score_speech_quality": 2.007915496826172, + "voiceclap_attribute:talking_speed": 16.986162185668945 + }, + "asr": "En analysant le contenu exact de ces chartes arabes sur l'IA, on découvre une cartographie fascinante de ce qui est dit.", + "audio_sha256": "95179851c938d48465784c2ae6f38fee680d53399256345ead837fa0f2378ff8", + "speaker": "S2", + "seconds": 5.68, + "media_member": "fr-S2.mp3", + "wav_member": "fr-S2.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + } + }, + "es": { + "S1": { + "episode": "cfe4a35611ec05423d3beeea57019184380e593676edee760567e1ce49c6e8f4", + "original_speaker": "S2", + "start_frame": 8604, + "end_frame": 8678, + "language": "es", + "window": { + "episode": "cfe4a35611ec05423d3beeea57019184380e593676edee760567e1ce49c6e8f4", + "lang": "es", + "window": "middle", + "start": 686.76, + "duration": 30.0, + "ce": 6.316492438316345 + }, + "text": "be absolutely certain, maybe it needs to convert the solar system into computronium perfect computing matter", + "scores": { + "emo_Affection": 0.09695407748222351, + "emo_Amusement": 0.22863148152828217, + "emo_Anger": 0.02521902322769165, + "emo_Astonishment_Surprise": 0.04588288068771362, + "emo_Awe": 0.009954340755939484, + "emo_Bitterness": 0.003289639949798584, + "emo_Concentration": 2.6226155757904053, + "emo_Confusion": 0.032299697399139404, + "emo_Contemplation": 1.4167089462280273, + "emo_Contempt": -0.023846730589866638, + "emo_Contentment": 1.991638422012329, + "emo_Disappointment": 0.056903064250946045, + "emo_Disgust": 0.005112349987030029, + "emo_Distress": 0.04386052489280701, + "emo_Doubt": 0.04379346966743469, + "emo_Elation": 0.07871103286743164, + "emo_Embarrassment": -0.0014667809009552002, + "emo_Emotional_Numbness": 0.09751874208450317, + "emo_Fatigue_Exhaustion": -0.059486985206604004, + "emo_Fear": 0.025437980890274048, + "emo_Helplessness": -0.022628992795944214, + "emo_Hope_Enthusiasm_Optimism": 1.5999407768249512, + "emo_Impatience_and_Irritability": 0.027845531702041626, + "emo_Infatuation": -0.04152345657348633, + "emo_Interest": 2.713186502456665, + "emo_Intoxication_Altered_States_of_Consciousness": -0.020907491445541382, + "emo_Jealousy_and_Envy": 0.007436573505401611, + "emo_Longing": 0.022799432277679443, + "emo_Malevolence_Malice": -0.035002708435058594, + "emo_Pain": -0.017140626907348633, + "emo_Pleasure_Ecstasy": -0.03209871053695679, + "emo_Pride": 0.4985673427581787, + "emo_Relief": 0.004649758338928223, + "emo_Sadness": -0.00990837812423706, + "emo_Sexual_Lust": -0.021244287490844727, + "emo_Shame": -0.003773242235183716, + "emo_Sourness": -0.014590814709663391, + "emo_Teasing": 0.03670033812522888, + "emo_Thankfulness_Gratitude": 0.05562615394592285, + "emo_Triumph": 0.02604202926158905, + "vn_AGEV_reg": 3.217996120452881, + "vn_AROU_reg": 2.6115944385528564, + "vn_ARSH_reg": 3.2106661796569824, + "vn_ATCK_reg": 2.972538709640503, + "vn_BKGN_reg": 4.182933807373047, + "vn_BRGT_reg": 3.863515615463257, + "vn_CHNK_reg": 3.2160675525665283, + "vn_CLRT_reg": 4.116360664367676, + "vn_COGL_reg": 0.6460691690444946, + "vn_DARC_reg": 3.027885675430298, + "vn_DFLU_reg": 0.25891828536987305, + "vn_EMPH_reg": 3.245647430419922, + "vn_ESTH_reg": 4.308232307434082, + "vn_EXPL_reg": -0.016171544790267944, + "vn_FOCS_reg": 3.680595874786377, + "vn_FULL_reg": 3.3248186111450195, + "vn_GEND_reg": 1.529880166053772, + "vn_HARM_reg": 4.27635383605957, + "vn_METL_reg": 1.7242189645767212, + "vn_RANG_reg": 2.764375686645508, + "vn_RCQL_reg": 4.48164176940918, + "vn_REGS_reg": 3.562018871307373, + "vn_RESP_reg": 1.3174554109573364, + "vn_ROUG_reg": 1.085960865020752, + "vn_R_CHST_reg": 2.4889726638793945, + "vn_R_HEAD_reg": 2.9283883571624756, + "vn_R_MASK_reg": 3.3219985961914062, + "vn_R_MIXD_reg": 4.070067405700684, + "vn_R_NASL_reg": 1.78544020652771, + "vn_R_ORAL_reg": 4.00960636138916, + "vn_R_THRT_reg": 2.159209966659546, + "vn_SMTH_reg": 4.146540641784668, + "vn_STNC_reg": 3.8994593620300293, + "vn_STRU_reg": 4.602100849151611, + "vn_S_ASMR_reg": 0.4183082580566406, + "vn_S_AUTH_reg": 3.2405447959899902, + "vn_S_CART_reg": 0.053545594215393066, + "vn_S_CASU_reg": 2.035984754562378, + "vn_S_CONV_reg": 2.775564193725586, + "vn_S_DRAM_reg": 1.3017776012420654, + "vn_S_FORM_reg": 3.0337324142456055, + "vn_S_MONO_reg": 3.033843517303467, + "vn_S_NARR_reg": 3.8295958042144775, + "vn_S_NEWS_reg": 2.5881664752960205, + "vn_S_PLAY_reg": 1.059983730316162, + "vn_S_RANT_reg": -0.18065261840820312, + "vn_S_STRY_reg": 1.832810640335083, + "vn_S_TECH_reg": 3.1167943477630615, + "vn_S_WHIS_reg": -0.13148140907287598, + "vn_TEMP_reg": 2.968557119369507, + "vn_TENS_reg": 1.9650440216064453, + "vn_VALN_reg": 3.3160037994384766, + "vn_VALS_reg": 3.2529375553131104, + "vn_VFLX_reg": 2.8481006622314453, + "vn_VOLT_reg": 0.9882020950317383, + "vn_VULN_reg": 1.385828971862793, + "vn_WARM_reg": 3.305128574371338, + "genuineness_0_6": 4.074300289154053, + "blend_0_10": 6.4104485511779785, + "R_quality": 0.344940185546875, + "eiv_extra:Age": 3.2195329666137695, + "eiv_extra:Arousal": 1.4652773141860962, + "eiv_extra:Authenticity": 2.959202289581299, + "eiv_extra:Background_Noise": -0.179806649684906, + "eiv_extra:Confident_vs._Hesitant": 0.9332407712936401, + "eiv_extra:Gender": 0.8807191848754883, + "eiv_extra:High-Pitched_vs._Low-Pitched": 1.623910665512085, + "eiv_extra:Monotone_vs._Expressive": 1.7812769412994385, + "eiv_extra:Recording_Quality": 2.9775044918060303, + "eiv_extra:Serious_vs._Humorous": 0.9033668637275696, + "eiv_extra:Soft_vs._Harsh": 0.46313655376434326, + "eiv_extra:Submissive_vs._Dominant": 0.3300918638706207, + "eiv_extra:Valence": 0.215755432844162, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 2.259490489959717, + "eiv_extra:Warm_vs._Cold": 0.2791379392147064, + "eiv_extra:score_background_quality": 3.645481586456299, + "eiv_extra:score_content_enjoyment": 4.828408241271973, + "eiv_extra:score_overall_quality": 2.8694915771484375, + "eiv_extra:score_speech_quality": 1.9673843383789062, + "audiobox:CE": 6.41180419921875, + "audiobox:CU": 7.486018180847168, + "audiobox:PC": 1.460522174835205, + "audiobox:PQ": 8.00154972076416, + "dnsmos:SIG_raw": 4.141777038574219, + "dnsmos:BAK_raw": 4.254101753234863, + "dnsmos:OVRL_raw": 3.8781161308288574, + "dnsmos:SIG": 3.612466812133789, + "dnsmos:BAK": 4.024520397186279, + "dnsmos:OVRL": 3.335127830505371, + "dnsmos:P808_MOS": 4.101062297821045, + "burst_count_log1p": 0.055674970149993896, + "voiceclap_attribute:Affection": 0.06263518333435059, + "voiceclap_attribute:Age": 3.293646812438965, + "voiceclap_attribute:Amusement": 0.2895764112472534, + "voiceclap_attribute:Anger": -0.045290231704711914, + "voiceclap_attribute:Arousal": 1.2297325134277344, + "voiceclap_attribute:Astonishment_Surprise": 0.2750028073787689, + "voiceclap_attribute:Authenticity": 3.106503963470459, + "voiceclap_attribute:Awe": 0.1925601363182068, + "voiceclap_attribute:Background_Noise": -0.027669787406921387, + "voiceclap_attribute:Bitterness": 0.018249094486236572, + "voiceclap_attribute:Concentration": 2.2338361740112305, + "voiceclap_attribute:Confident_vs._Hesitant": 0.884527862071991, + "voiceclap_attribute:Confusion": -0.0019930005073547363, + "voiceclap_attribute:Contemplation": 1.3893098831176758, + "voiceclap_attribute:Contempt": 0.04564779996871948, + "voiceclap_attribute:Contentment": 1.5583001375198364, + "voiceclap_attribute:Disappointment": -0.03386491537094116, + "voiceclap_attribute:Disgust": 0.048587143421173096, + "voiceclap_attribute:Distress": -0.08529025316238403, + "voiceclap_attribute:Doubt": 0.1033887267112732, + "voiceclap_attribute:Elation": 0.2367246150970459, + "voiceclap_attribute:Embarrassment": 0.05756309628486633, + "voiceclap_attribute:Emotional_Numbness": 0.0669066309928894, + "voiceclap_attribute:Fatigue_Exhaustion": -0.08572903275489807, + "voiceclap_attribute:Fear": 0.006707549095153809, + "voiceclap_attribute:Gender": 0.9793065786361694, + "voiceclap_attribute:Helplessness": -0.05156219005584717, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.43144953250885, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 1.1618940830230713, + "voiceclap_attribute:Impatience_and_Irritability": -0.006205916404724121, + "voiceclap_attribute:Infatuation": -0.1780616044998169, + "voiceclap_attribute:Interest": 2.2884392738342285, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": 0.08571690320968628, + "voiceclap_attribute:Jealousy_&_Envy": -0.054608941078186035, + "voiceclap_attribute:Longing": -0.12330591678619385, + "voiceclap_attribute:Malevolence_Malice": -0.013307750225067139, + "voiceclap_attribute:Monotone_vs._Expressive": 1.4972254037857056, + "voiceclap_attribute:Pain": -0.05317756533622742, + "voiceclap_attribute:Pleasure_Ecstasy": 0.05968901515007019, + "voiceclap_attribute:Pride": 0.6455303430557251, + "voiceclap_attribute:Recording_Quality": 2.914816379547119, + "voiceclap_attribute:Relief": 0.11412400007247925, + "voiceclap_attribute:Sadness": -0.05894589424133301, + "voiceclap_attribute:Serious_vs._Humorous": 0.585119366645813, + "voiceclap_attribute:Sexual_Lust": -0.035655200481414795, + "voiceclap_attribute:Shame": -0.05261296033859253, + "voiceclap_attribute:Soft_vs._Harsh": 0.5117160081863403, + "voiceclap_attribute:Sourness": 0.006042182445526123, + "voiceclap_attribute:Submissive_vs._Dominant": 0.36074984073638916, + "voiceclap_attribute:Teasing": -0.02604275941848755, + "voiceclap_attribute:Thankfulness_Gratitude": -0.006200909614562988, + "voiceclap_attribute:Triumph": 0.05297607183456421, + "voiceclap_attribute:Valence": 0.244705468416214, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 1.7818416357040405, + "voiceclap_attribute:Warm_vs._Cold": 0.42359793186187744, + "voiceclap_attribute:duration": 6.835643768310547, + "voiceclap_attribute:score_background_quality": 3.6317214965820312, + "voiceclap_attribute:score_content_enjoyment": 4.847010612487793, + "voiceclap_attribute:score_overall_quality": 2.9790332317352295, + "voiceclap_attribute:score_speech_quality": 2.006450653076172, + "voiceclap_attribute:talking_speed": 17.04479217529297 + }, + "asr": "Absolutely certain maybe it needs to convert the solar system into computronium perfect computing matter.", + "audio_sha256": "0f122cd76b54b921af91818f0e99e99455e78863d137b9e64fc7fb048fac60bf", + "speaker": "S1", + "seconds": 5.92, + "media_member": "es-S1.mp3", + "wav_member": "es-S1.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + }, + "S2": { + "episode": "41aba91cc5ca6308be7ec4f3e27dcba37cfb310eaf14c938aa66359c048499a6", + "original_speaker": "S2", + "start_frame": 5486, + "end_frame": 5559, + "language": "es", + "window": { + "episode": "41aba91cc5ca6308be7ec4f3e27dcba37cfb310eaf14c938aa66359c048499a6", + "lang": "es", + "window": "middle", + "start": 414.97551, + "duration": 30.0, + "ce": 6.284721754491329 + }, + "text": "lo largo de 150 mil kilómetros, la reducción total de emisiones es de hasta un 73%.", + "scores": { + "emo_Affection": -0.07296362519264221, + "emo_Amusement": -0.04420545697212219, + "emo_Anger": -0.00197485089302063, + "emo_Astonishment_Surprise": 0.013527512550354004, + "emo_Awe": -0.030219100415706635, + "emo_Bitterness": 0.016800880432128906, + "emo_Concentration": 2.4742159843444824, + "emo_Confusion": -0.034375667572021484, + "emo_Contemplation": 0.69548100233078, + "emo_Contempt": 0.004349306225776672, + "emo_Contentment": 0.8344520330429077, + "emo_Disappointment": 0.059642672538757324, + "emo_Disgust": -0.00733591616153717, + "emo_Distress": 0.031088829040527344, + "emo_Doubt": 0.0192488431930542, + "emo_Elation": 0.047839730978012085, + "emo_Embarrassment": -0.04414752125740051, + "emo_Emotional_Numbness": 0.5744961500167847, + "emo_Fatigue_Exhaustion": -0.05103707313537598, + "emo_Fear": -0.0247439444065094, + "emo_Helplessness": -0.03369241952896118, + "emo_Hope_Enthusiasm_Optimism": 0.3185374140739441, + "emo_Impatience_and_Irritability": 0.011811137199401855, + "emo_Infatuation": -0.03623795509338379, + "emo_Interest": 1.813370704650879, + "emo_Intoxication_Altered_States_of_Consciousness": -0.03937558829784393, + "emo_Jealousy_and_Envy": -0.004752174019813538, + "emo_Longing": -0.08151662349700928, + "emo_Malevolence_Malice": -0.04727655649185181, + "emo_Pain": -0.0373615026473999, + "emo_Pleasure_Ecstasy": -0.0654870867729187, + "emo_Pride": 0.31781667470932007, + "emo_Relief": -0.12075445055961609, + "emo_Sadness": 0.04084610939025879, + "emo_Sexual_Lust": -0.011522918939590454, + "emo_Shame": -0.030383706092834473, + "emo_Sourness": 0.007365241646766663, + "emo_Teasing": -0.04316112399101257, + "emo_Thankfulness_Gratitude": -0.0915425717830658, + "emo_Triumph": -0.0059276968240737915, + "vn_AGEV_reg": 3.6166632175445557, + "vn_AROU_reg": 2.3294119834899902, + "vn_ARSH_reg": 3.111398935317993, + "vn_ATCK_reg": 2.972538709640503, + "vn_BKGN_reg": 3.999671459197998, + "vn_BRGT_reg": 2.9515833854675293, + "vn_CHNK_reg": 3.1138148307800293, + "vn_CLRT_reg": 4.5367608070373535, + "vn_COGL_reg": 0.34133875370025635, + "vn_DARC_reg": 2.8256916999816895, + "vn_DFLU_reg": -0.20247364044189453, + "vn_EMPH_reg": 3.0124263763427734, + "vn_ESTH_reg": 4.259562969207764, + "vn_EXPL_reg": -0.028962567448616028, + "vn_FOCS_reg": 4.070779800415039, + "vn_FULL_reg": 4.028995037078857, + "vn_GEND_reg": 4.920157432556152, + "vn_HARM_reg": 3.9141359329223633, + "vn_METL_reg": 1.5418540239334106, + "vn_RANG_reg": 2.0278356075286865, + "vn_RCQL_reg": 4.340983867645264, + "vn_REGS_reg": 0.9269677996635437, + "vn_RESP_reg": 0.9665132761001587, + "vn_ROUG_reg": 1.5543689727783203, + "vn_R_CHST_reg": 3.7326765060424805, + "vn_R_HEAD_reg": 1.5257518291473389, + "vn_R_MASK_reg": 2.8902533054351807, + "vn_R_MIXD_reg": 3.9654593467712402, + "vn_R_NASL_reg": 1.7606878280639648, + "vn_R_ORAL_reg": 3.4108426570892334, + "vn_R_THRT_reg": 2.9298839569091797, + "vn_SMTH_reg": 4.407018184661865, + "vn_STNC_reg": 3.7393362522125244, + "vn_STRU_reg": 4.8640265464782715, + "vn_S_ASMR_reg": 0.014387369155883789, + "vn_S_AUTH_reg": 4.1170973777771, + "vn_S_CART_reg": -0.4385082721710205, + "vn_S_CASU_reg": 0.9195735454559326, + "vn_S_CONV_reg": 1.895982265472412, + "vn_S_DRAM_reg": 1.702317476272583, + "vn_S_FORM_reg": 4.514725685119629, + "vn_S_MONO_reg": 3.77933931350708, + "vn_S_NARR_reg": 4.502363204956055, + "vn_S_NEWS_reg": 3.4835493564605713, + "vn_S_PLAY_reg": -0.1282966136932373, + "vn_S_RANT_reg": 0.08849072456359863, + "vn_S_STRY_reg": 2.1363117694854736, + "vn_S_TECH_reg": 3.6496405601501465, + "vn_S_WHIS_reg": -0.30633509159088135, + "vn_TEMP_reg": 3.2218213081359863, + "vn_TENS_reg": 1.9000221490859985, + "vn_VALN_reg": 2.6230711936950684, + "vn_VALS_reg": 2.948331356048584, + "vn_VFLX_reg": 2.8669564723968506, + "vn_VOLT_reg": 0.9067842960357666, + "vn_VULN_reg": 0.9292596578598022, + "vn_WARM_reg": 3.5525498390197754, + "genuineness_0_6": 3.9927117824554443, + "blend_0_10": 6.5511250495910645, + "R_quality": 0.3087250590324402, + "eiv_extra:Age": 3.3105695247650146, + "eiv_extra:Arousal": 0.9049730896949768, + "eiv_extra:Authenticity": 2.779514789581299, + "eiv_extra:Background_Noise": -0.0024678707122802734, + "eiv_extra:Confident_vs._Hesitant": 0.7455488443374634, + "eiv_extra:Gender": -0.8569026589393616, + "eiv_extra:High-Pitched_vs._Low-Pitched": 1.942270040512085, + "eiv_extra:Monotone_vs._Expressive": 0.7241610288619995, + "eiv_extra:Recording_Quality": 2.644925832748413, + "eiv_extra:Serious_vs._Humorous": 0.5850942730903625, + "eiv_extra:Soft_vs._Harsh": 0.09257349371910095, + "eiv_extra:Submissive_vs._Dominant": 0.0757392942905426, + "eiv_extra:Valence": 0.07790891081094742, + "eiv_extra:Vulnerable_vs._Emotionally_Detached": 2.74296236038208, + "eiv_extra:Warm_vs._Cold": 0.031936533749103546, + "eiv_extra:score_background_quality": 3.7538554668426514, + "eiv_extra:score_content_enjoyment": 4.738227367401123, + "eiv_extra:score_overall_quality": 2.980515956878662, + "eiv_extra:score_speech_quality": 2.0064468383789062, + "audiobox:CE": 6.332377910614014, + "audiobox:CU": 7.166062831878662, + "audiobox:PC": 1.473143219947815, + "audiobox:PQ": 8.235105514526367, + "dnsmos:SIG_raw": 4.102573871612549, + "dnsmos:BAK_raw": 4.359593868255615, + "dnsmos:OVRL_raw": 3.884467601776123, + "dnsmos:SIG": 3.5948486328125, + "dnsmos:BAK": 4.1043195724487305, + "dnsmos:OVRL": 3.344606876373291, + "dnsmos:P808_MOS": 4.108819007873535, + "burst_count_log1p": 0.04047524929046631, + "voiceclap_attribute:Affection": -0.18862557411193848, + "voiceclap_attribute:Age": 3.4040963649749756, + "voiceclap_attribute:Amusement": -0.19016557931900024, + "voiceclap_attribute:Anger": -0.16420531272888184, + "voiceclap_attribute:Arousal": 0.9585616588592529, + "voiceclap_attribute:Astonishment_Surprise": -0.10098510980606079, + "voiceclap_attribute:Authenticity": 2.788632869720459, + "voiceclap_attribute:Awe": -0.03645056486129761, + "voiceclap_attribute:Background_Noise": 0.07136785984039307, + "voiceclap_attribute:Bitterness": 0.018249094486236572, + "voiceclap_attribute:Concentration": 2.511523723602295, + "voiceclap_attribute:Confident_vs._Hesitant": 0.846072793006897, + "voiceclap_attribute:Confusion": -0.07008308172225952, + "voiceclap_attribute:Contemplation": 1.5571472644805908, + "voiceclap_attribute:Contempt": -0.06581413745880127, + "voiceclap_attribute:Contentment": 1.2753262519836426, + "voiceclap_attribute:Disappointment": 0.15679579973220825, + "voiceclap_attribute:Disgust": -0.049069106578826904, + "voiceclap_attribute:Distress": 0.024075031280517578, + "voiceclap_attribute:Doubt": 0.1326856017112732, + "voiceclap_attribute:Elation": -0.16824287176132202, + "voiceclap_attribute:Embarrassment": 0.018223822116851807, + "voiceclap_attribute:Emotional_Numbness": 0.13345128297805786, + "voiceclap_attribute:Fatigue_Exhaustion": -0.0531897246837616, + "voiceclap_attribute:Fear": 0.08092629909515381, + "voiceclap_attribute:Gender": -0.9553482532501221, + "voiceclap_attribute:Helplessness": 0.08257275819778442, + "voiceclap_attribute:High-Pitched_vs._Low-Pitched": 1.8002386093139648, + "voiceclap_attribute:Hope_Enthusiasm_Optimism": 0.5178375244140625, + "voiceclap_attribute:Impatience_and_Irritability": -0.14171338081359863, + "voiceclap_attribute:Infatuation": -0.14687985181808472, + "voiceclap_attribute:Interest": 2.068772315979004, + "voiceclap_attribute:Intoxication_Altered_States_of_Consciousness": -0.0049021244049072266, + "voiceclap_attribute:Jealousy_&_Envy": -0.031171441078186035, + "voiceclap_attribute:Longing": -0.09596216678619385, + "voiceclap_attribute:Malevolence_Malice": -0.09980767965316772, + "voiceclap_attribute:Monotone_vs._Expressive": 1.0708472728729248, + "voiceclap_attribute:Pain": -0.04731819033622742, + "voiceclap_attribute:Pleasure_Ecstasy": -0.029730260372161865, + "voiceclap_attribute:Pride": 0.6441671252250671, + "voiceclap_attribute:Recording_Quality": 2.5456807613372803, + "voiceclap_attribute:Relief": -0.1848393678665161, + "voiceclap_attribute:Sadness": 0.09262484312057495, + "voiceclap_attribute:Serious_vs._Humorous": 0.5472368001937866, + "voiceclap_attribute:Sexual_Lust": -0.12053167819976807, + "voiceclap_attribute:Shame": -0.04380720853805542, + "voiceclap_attribute:Soft_vs._Harsh": 0.1730102300643921, + "voiceclap_attribute:Sourness": 0.046891987323760986, + "voiceclap_attribute:Submissive_vs._Dominant": 0.37337106466293335, + "voiceclap_attribute:Teasing": -0.26313477754592896, + "voiceclap_attribute:Thankfulness_Gratitude": -0.32050418853759766, + "voiceclap_attribute:Triumph": 0.0025238394737243652, + "voiceclap_attribute:Valence": -0.1255829781293869, + "voiceclap_attribute:Vulnerable_vs._Emotionally_Detached": 2.356182098388672, + "voiceclap_attribute:Warm_vs._Cold": 0.03749736398458481, + "voiceclap_attribute:duration": 5.944973945617676, + "voiceclap_attribute:score_background_quality": 3.6200366020202637, + "voiceclap_attribute:score_content_enjoyment": 4.772791862487793, + "voiceclap_attribute:score_overall_quality": 2.8904526233673096, + "voiceclap_attribute:score_speech_quality": 2.051372528076172, + "voiceclap_attribute:talking_speed": 18.39326286315918 + }, + "asr": "A lo largo de ciento cincuenta mil kilómetros, la reducción total de emisiones es de hasta un setenta y tres por ciento.", + "audio_sha256": "f2485b9712ff457f59f374b95ce1e98c9bcb5b21acbc8c8c38a637d3781ef4e6", + "speaker": "S2", + "seconds": 5.84, + "media_member": "es-S2.mp3", + "wav_member": "es-S2.wav", + "source_split": "train", + "source_audio": "Original media crop, not a codec reconstruction" + } + } +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5223f3a50fc9ddcde83df3e1293a2a08069692b5 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "output_attentions": false, + "output_hidden_states": false, + "pad_token_id": 151643, + "transformers_version": "5.14.1" +} diff --git a/gpt2_decoder.py b/gpt2_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..84c597cf1bcca240562dec1417a53fb19f47bfca --- /dev/null +++ b/gpt2_decoder.py @@ -0,0 +1,721 @@ +# coding=utf-8 +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from transformers.activations import ACT2FN +from transformers.modeling_outputs import BaseModelOutputWithPast +from transformers.models.gpt2.configuration_gpt2 import GPT2Config + +try: + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import pad_input, unpad_input + + _FLASH_ATTN_AVAILABLE = True +except Exception: + flash_attn_func = None + flash_attn_varlen_func = None + pad_input = None + unpad_input = None + _FLASH_ATTN_AVAILABLE = False + + +@dataclass +class PackedSequenceMetadata: + cu_seqlens: torch.Tensor + max_seqlen: int + indices: Optional[torch.Tensor] = None + batch_size: Optional[int] = None + seq_len: Optional[int] = None + + +def _is_static_kv_cache_layer(layer_past: object) -> bool: + return isinstance(layer_past, dict) and bool(layer_past.get("static_kv_cache", False)) + + +class MossTTSNanoGPT2RotaryEmbedding(nn.Module): + def __init__(self, dim: int, base: float = 10000.0) -> None: + super().__init__() + if dim % 2 != 0: + raise ValueError(f"RoPE head_dim must be even, got {dim}") + self.dim = int(dim) + self.base = float(base) + self.register_buffer("inv_freq", self._compute_inv_freq(), persistent=False) + + def _compute_inv_freq(self, device: Optional[torch.device] = None) -> torch.Tensor: + return 1.0 / ( + self.base ** (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim) + ) + + def forward( + self, + position_ids: torch.LongTensor, + *, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + if position_ids.ndim == 1: + position_ids = position_ids.unsqueeze(0) + inv_freq = self._compute_inv_freq(device=device) + freqs = torch.einsum("bs,d->bsd", position_ids.to(device=device, dtype=inv_freq.dtype), inv_freq) + cos = freqs.cos().repeat_interleave(2, dim=-1).unsqueeze(2).to(dtype=dtype) + sin = freqs.sin().repeat_interleave(2, dim=-1).unsqueeze(2).to(dtype=dtype) + return cos, sin + + +def rotate_half(hidden_states: torch.Tensor) -> torch.Tensor: + even = hidden_states[..., ::2] + odd = hidden_states[..., 1::2] + return torch.stack((-odd, even), dim=-1).reshape_as(hidden_states) + + +def apply_rotary_pos_emb( + hidden_states: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, +) -> torch.Tensor: + return (hidden_states * cos) + (rotate_half(hidden_states) * sin) + + +class MossTTSNanoGPT2MLP(nn.Module): + def __init__(self, config: GPT2Config) -> None: + super().__init__() + hidden_size = int(config.hidden_size) + inner_size = int(config.n_inner or 4 * hidden_size) + self.fc_in = nn.Linear(hidden_size, inner_size) + self.fc_out = nn.Linear(inner_size, hidden_size) + self.act = ACT2FN[config.activation_function] + self.dropout = nn.Dropout(config.resid_pdrop) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc_in(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.fc_out(hidden_states) + return self.dropout(hidden_states) + + +class MossTTSNanoGPT2Attention(nn.Module): + def __init__(self, config: GPT2Config, layer_idx: int, attn_implementation: str) -> None: + super().__init__() + hidden_size = int(config.hidden_size) + num_heads = int(config.num_attention_heads) + if hidden_size % num_heads != 0: + raise ValueError(f"hidden_size={hidden_size} must be divisible by num_attention_heads={num_heads}") + + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.embed_dim = hidden_size + self.layer_idx = layer_idx + self.attn_implementation = attn_implementation + self.attn_dropout = float(config.attn_pdrop) + self.resid_dropout = nn.Dropout(config.resid_pdrop) + self.scale_attn_weights = bool(getattr(config, "scale_attn_weights", True)) + self.scale_attn_by_inverse_layer_idx = bool(getattr(config, "scale_attn_by_inverse_layer_idx", False)) + self.position_embedding_type = str(getattr(config, "position_embedding_type", "absolute")).lower() + if self.position_embedding_type not in {"absolute", "rope"}: + raise ValueError(f"Unsupported position_embedding_type={self.position_embedding_type!r}") + + self.c_attn = nn.Linear(hidden_size, 3 * hidden_size) + self.c_proj = nn.Linear(hidden_size, hidden_size) + self.rotary_emb = None + if self.position_embedding_type == "rope": + self.rotary_emb = MossTTSNanoGPT2RotaryEmbedding( + self.head_dim, + base=float(getattr(config, "rope_base", 10000.0)), + ) + + def _split_heads(self, tensor: torch.Tensor) -> torch.Tensor: + if tensor.ndim == 3: + batch_size, seq_len, _ = tensor.shape + return tensor.view(batch_size, seq_len, self.num_heads, self.head_dim) + if tensor.ndim == 2: + total_tokens, _ = tensor.shape + return tensor.view(total_tokens, self.num_heads, self.head_dim) + raise ValueError(f"Unsupported tensor rank for attention split: {tensor.ndim}") + + def _merge_heads(self, tensor: torch.Tensor) -> torch.Tensor: + if tensor.ndim == 4: + batch_size, seq_len, _, _ = tensor.shape + return tensor.reshape(batch_size, seq_len, self.embed_dim) + if tensor.ndim == 3: + total_tokens, _, _ = tensor.shape + return tensor.reshape(total_tokens, self.embed_dim) + raise ValueError(f"Unsupported tensor rank for attention merge: {tensor.ndim}") + + def _causal_attention_mask( + self, + attention_mask: Optional[torch.Tensor], + query_length: int, + key_length: int, + device: torch.device, + ) -> torch.Tensor: + query_positions = torch.arange(query_length, device=device, dtype=torch.long) + query_positions = query_positions + max(key_length - query_length, 0) + key_positions = torch.arange(key_length, device=device, dtype=torch.long) + causal = key_positions.unsqueeze(0) <= query_positions.unsqueeze(1) + causal = causal.unsqueeze(0).unsqueeze(0) + if attention_mask is None: + return causal + key_mask = attention_mask[:, None, None, :].to(dtype=torch.bool) + return causal & key_mask + + def _eager_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + scale = 1.0 + if self.scale_attn_weights: + scale /= self.head_dim ** 0.5 + if self.scale_attn_by_inverse_layer_idx: + scale /= float(self.layer_idx + 1) + + scores = torch.matmul(query, key.transpose(-1, -2)) * scale + causal_mask = self._causal_attention_mask( + attention_mask=attention_mask, + query_length=query.shape[-2], + key_length=key.shape[-2], + device=query.device, + ) + scores = scores.masked_fill(~causal_mask, torch.finfo(scores.dtype).min) + probs = torch.softmax(scores, dim=-1) + if self.training and self.attn_dropout > 0: + probs = torch.dropout(probs, self.attn_dropout, train=True) + output = torch.matmul(probs, value) + return output.transpose(1, 2).contiguous() + + def _sdpa_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + mask = None + if attention_mask is not None or query.shape[-2] != key.shape[-2]: + mask = self._causal_attention_mask( + attention_mask=attention_mask, + query_length=query.shape[-2], + key_length=key.shape[-2], + device=query.device, + ) + output = torch.nn.functional.scaled_dot_product_attention( + query, + key, + value, + attn_mask=mask, + dropout_p=self.attn_dropout if self.training else 0.0, + is_causal=mask is None, + ) + return output.transpose(1, 2).contiguous() + + def _flash_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + packed_metadata: Optional[PackedSequenceMetadata], + ) -> torch.Tensor: + if not _FLASH_ATTN_AVAILABLE: + raise ImportError("flash_attn is not installed, but attn_implementation='flash_attention_2' was requested.") + if query.device.type != "cuda": + raise ValueError("flash_attention_2 requires CUDA tensors.") + if query.dtype not in (torch.float16, torch.bfloat16): + raise ValueError( + f"flash_attention_2 requires fp16/bf16 tensors, but received dtype={query.dtype}." + ) + + dropout_p = self.attn_dropout if self.training else 0.0 + if packed_metadata is not None: + if packed_metadata.indices is not None: + query = query.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices) + key = key.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices) + value = value.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices) + output = flash_attn_varlen_func( + query, + key, + value, + packed_metadata.cu_seqlens, + packed_metadata.cu_seqlens, + packed_metadata.max_seqlen, + packed_metadata.max_seqlen, + dropout_p=dropout_p, + causal=True, + ) + if packed_metadata.indices is None: + return output + return pad_input( + output, + packed_metadata.indices, + packed_metadata.batch_size, + packed_metadata.seq_len, + ) + + if attention_mask is None or bool(attention_mask.all()): + return flash_attn_func( + query, + key, + value, + dropout_p=dropout_p, + causal=True, + ) + + if query.shape[1] != key.shape[1]: + query_attention_mask = attention_mask[:, -query.shape[1] :] + unpadded_query, query_indices, cu_seqlens_q, max_seqlen_q, _ = unpad_input( + query, + query_attention_mask, + ) + unpadded_key, _, cu_seqlens_k, max_seqlen_k, _ = unpad_input(key, attention_mask) + unpadded_value, _, _, _, _ = unpad_input(value, attention_mask) + output = flash_attn_varlen_func( + unpadded_query, + unpadded_key, + unpadded_value, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + dropout_p=dropout_p, + causal=True, + ) + return pad_input(output, query_indices, query.shape[0], query.shape[1]) + + unpadded_query, indices, cu_seqlens, max_seqlen, _ = unpad_input(query, attention_mask) + unpadded_key, _, _, _, _ = unpad_input(key, attention_mask) + unpadded_value, _, _, _, _ = unpad_input(value, attention_mask) + output = flash_attn_varlen_func( + unpadded_query, + unpadded_key, + unpadded_value, + cu_seqlens, + cu_seqlens, + max_seqlen, + max_seqlen, + dropout_p=dropout_p, + causal=True, + ) + return pad_input(output, indices, query.shape[0], query.shape[1]) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + packed_metadata: Optional[PackedSequenceMetadata] = None, + layer_past: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + ) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, torch.Tensor]]]: + qkv = self.c_attn(hidden_states) + query, key, value = qkv.split(self.embed_dim, dim=-1) + query = self._split_heads(query) + key = self._split_heads(key) + value = self._split_heads(value) + + if self.rotary_emb is not None: + if position_ids is None: + raise ValueError("position_ids must be provided when position_embedding_type='rope'.") + cos, sin = self.rotary_emb( + position_ids.to(device=query.device), + device=query.device, + dtype=query.dtype, + ) + query = apply_rotary_pos_emb(query, cos, sin) + key = apply_rotary_pos_emb(key, cos, sin) + + static_layer_past = layer_past is not None and _is_static_kv_cache_layer(layer_past) + if static_layer_past: + past_length = int(layer_past.get("length", 0)) + new_length = past_length + int(key.shape[1]) + key_cache = layer_past["key"] + value_cache = layer_past["value"] + if new_length > int(key_cache.shape[1]): + raise ValueError( + f"Static KV cache is too short: need {new_length}, capacity={int(key_cache.shape[1])}." + ) + key_cache[:, past_length:new_length].copy_(key) + value_cache[:, past_length:new_length].copy_(value) + key = key_cache[:, :new_length] + value = value_cache[:, :new_length] + layer_past["length"] = new_length + elif layer_past is not None: + past_key, past_value = layer_past + key = torch.cat([past_key.to(device=key.device, dtype=key.dtype), key], dim=1) + value = torch.cat([past_value.to(device=value.device, dtype=value.dtype), value], dim=1) + + present = layer_past if (use_cache and static_layer_past) else ((key, value) if use_cache else None) + + if self.attn_implementation == "flash_attention_2": + attn_output = self._flash_attention( + query=query, + key=key, + value=value, + attention_mask=attention_mask, + packed_metadata=packed_metadata, + ) + elif self.attn_implementation == "sdpa": + attn_output = self._sdpa_attention( + query=query, + key=key, + value=value, + attention_mask=attention_mask, + ) + else: + attn_output = self._eager_attention( + query=query, + key=key, + value=value, + attention_mask=attention_mask, + ) + + attn_output = self._merge_heads(attn_output) + attn_output = self.c_proj(attn_output) + return self.resid_dropout(attn_output), present + + +class MossTTSNanoGPT2Block(nn.Module): + def __init__(self, config: GPT2Config, layer_idx: int, attn_implementation: str) -> None: + super().__init__() + hidden_size = int(config.hidden_size) + self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.attn = MossTTSNanoGPT2Attention(config, layer_idx=layer_idx, attn_implementation=attn_implementation) + self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.mlp = MossTTSNanoGPT2MLP(config) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + packed_metadata: Optional[PackedSequenceMetadata] = None, + layer_past: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + ) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, torch.Tensor]]]: + attn_output, present = self.attn( + self.ln_1(hidden_states), + attention_mask=attention_mask, + position_ids=position_ids, + packed_metadata=packed_metadata, + layer_past=layer_past, + use_cache=use_cache, + ) + hidden_states = hidden_states + attn_output + hidden_states = hidden_states + self.mlp(self.ln_2(hidden_states)) + return hidden_states, present + + +class MossTTSNanoGPT2Model(nn.Module): + def __init__(self, config: GPT2Config, attn_implementation: str = "eager") -> None: + super().__init__() + self.config = config + self.attn_implementation = attn_implementation + self.position_embedding_type = str(getattr(config, "position_embedding_type", "absolute")).lower() + if self.position_embedding_type not in {"absolute", "rope"}: + raise ValueError(f"Unsupported position_embedding_type={self.position_embedding_type!r}") + hidden_size = int(config.hidden_size) + self.wte = nn.Embedding(config.vocab_size, hidden_size) + self.wpe = nn.Embedding(config.n_positions, hidden_size) if self.position_embedding_type == "absolute" else nn.Identity() + self.drop = nn.Dropout(config.embd_pdrop) + self.h = nn.ModuleList( + [MossTTSNanoGPT2Block(config, layer_idx=index, attn_implementation=attn_implementation) for index in range(config.n_layer)] + ) + self.ln_f = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.gradient_checkpointing = False + self._reset_parameters() + + def _reset_parameters(self) -> None: + init_std = float(self.config.initializer_range) + for module in self.modules(): + if isinstance(module, nn.Linear): + nn.init.normal_(module.weight, mean=0.0, std=init_std) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, nn.Embedding): + nn.init.normal_(module.weight, mean=0.0, std=init_std) + elif isinstance(module, nn.LayerNorm): + nn.init.ones_(module.weight) + nn.init.zeros_(module.bias) + + @staticmethod + def _normalize_num_sequences( + cu_seqlens: torch.Tensor, + num_sequences: Optional[torch.Tensor], + device: torch.device, + ) -> torch.Tensor: + if cu_seqlens.ndim == 1: + cu_seqlens = cu_seqlens.unsqueeze(0) + if num_sequences is None: + diffs = cu_seqlens[:, 1:] - cu_seqlens[:, :-1] + return diffs.gt(0).sum(dim=-1).to(device=device, dtype=torch.long) + if num_sequences.ndim == 0: + num_sequences = num_sequences.unsqueeze(0) + return num_sequences.to(device=device, dtype=torch.long) + + @staticmethod + def _packed_segments_from_cu_seqlens( + cu_seqlens: torch.Tensor, + num_sequences: Optional[torch.Tensor], + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if cu_seqlens.ndim == 1: + cu_seqlens = cu_seqlens.unsqueeze(0) + cu_seqlens = cu_seqlens.to(device=device) + batch_size, boundary_count = cu_seqlens.shape + segment_slots = boundary_count - 1 + if segment_slots <= 0: + empty = torch.empty(0, dtype=torch.long, device=device) + return empty, empty, empty + + counts = MossTTSNanoGPT2Model._normalize_num_sequences(cu_seqlens, num_sequences, device=device) + counts = counts.clamp(min=0, max=segment_slots) + segment_slots = int(counts.max().item()) if counts.numel() > 0 else 0 + if segment_slots <= 0: + empty = torch.empty(0, dtype=torch.long, device=device) + return empty, empty, empty + cu_seqlens = cu_seqlens[:, : segment_slots + 1] + slot_ids = torch.arange(segment_slots, device=device).unsqueeze(0) + valid_slots = slot_ids < counts.unsqueeze(1) + + starts = cu_seqlens[:, :-1].to(dtype=torch.long) + ends = cu_seqlens[:, 1:].to(dtype=torch.long) + lengths = (ends - starts).clamp_min(0) + lengths = torch.where(valid_slots, lengths, torch.zeros((), dtype=torch.long, device=device)) + + batch_ids = torch.arange(batch_size, device=device, dtype=torch.long).unsqueeze(1).expand(batch_size, segment_slots) + batch_ids = batch_ids.reshape(-1) + starts = starts.reshape(-1) + lengths = lengths.reshape(-1) + + valid_segments = lengths.gt(0) + valid_count = int(valid_segments.to(dtype=torch.long).sum().item()) + if valid_count <= 0: + empty = torch.empty(0, dtype=torch.long, device=device) + return empty, empty, empty + if valid_count == lengths.numel(): + return batch_ids, starts, lengths + + valid_order = torch.argsort(valid_segments.to(dtype=torch.long), descending=True, stable=True)[:valid_count] + return ( + batch_ids.index_select(0, valid_order), + starts.index_select(0, valid_order), + lengths.index_select(0, valid_order), + ) + + @staticmethod + def _packed_token_indices( + batch_ids: torch.Tensor, + starts: torch.Tensor, + lengths: torch.Tensor, + seq_len: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + total_tokens = int(lengths.sum().item()) + if total_tokens <= 0: + empty = torch.empty(0, dtype=torch.long, device=lengths.device) + return empty, empty + + segment_ids = torch.repeat_interleave( + torch.arange(lengths.numel(), device=lengths.device, dtype=torch.long), + lengths, + output_size=total_tokens, + ) + segment_starts = torch.cumsum(lengths, dim=0) - lengths + positions = torch.arange(total_tokens, device=lengths.device, dtype=torch.long) - segment_starts[segment_ids] + indices = batch_ids[segment_ids] * seq_len + starts[segment_ids] + positions + return indices, positions + + @staticmethod + def build_packed_position_ids( + attention_mask: Optional[torch.Tensor], + cu_seqlens: torch.Tensor, + num_sequences: Optional[torch.Tensor], + sequence_length: Optional[int] = None, + ) -> torch.Tensor: + if cu_seqlens.ndim == 1: + cu_seqlens = cu_seqlens.unsqueeze(0) + batch_size = cu_seqlens.shape[0] + seq_len = int(sequence_length or (cu_seqlens.shape[1] - 1)) + device = cu_seqlens.device + position_ids = torch.zeros((batch_size, seq_len), dtype=torch.long, device=device) + batch_ids, starts, lengths = MossTTSNanoGPT2Model._packed_segments_from_cu_seqlens( + cu_seqlens, + num_sequences, + device, + ) + if lengths.numel() > 0: + indices, positions = MossTTSNanoGPT2Model._packed_token_indices(batch_ids, starts, lengths, seq_len) + position_ids.view(-1).scatter_(0, indices, positions) + if attention_mask is not None: + position_ids = position_ids * attention_mask.to(dtype=position_ids.dtype) + return position_ids + + @staticmethod + def build_packed_metadata( + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + num_sequences: Optional[torch.Tensor], + ) -> PackedSequenceMetadata: + if cu_seqlens.ndim == 1: + cu_seqlens = cu_seqlens.unsqueeze(0) + device = hidden_states.device + seq_len = hidden_states.shape[1] + batch_ids, starts, lengths = MossTTSNanoGPT2Model._packed_segments_from_cu_seqlens( + cu_seqlens, + num_sequences, + device, + ) + if lengths.numel() == 0: + raise ValueError("cu_seqlens did not describe any non-empty packed sequences.") + + indices, _ = MossTTSNanoGPT2Model._packed_token_indices(batch_ids, starts, lengths, seq_len) + cumulative = torch.empty(lengths.numel() + 1, dtype=torch.int32, device=device) + cumulative[0] = 0 + cumulative[1:] = lengths.to(dtype=torch.int32).cumsum(dim=0) + return PackedSequenceMetadata( + cu_seqlens=cumulative, + max_seqlen=int(lengths.max().item()), + indices=indices, + batch_size=hidden_states.shape[0], + seq_len=hidden_states.shape[1], + ) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[tuple[tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: bool = True, + cu_seqlens: Optional[torch.Tensor] = None, + num_sequences: Optional[torch.Tensor] = None, + ) -> BaseModelOutputWithPast: + del input_ids, output_attentions + + if inputs_embeds is None: + raise ValueError("inputs_embeds must be provided.") + + use_cache = bool(use_cache) + if use_cache and cu_seqlens is not None: + raise ValueError("use_cache=True is not supported together with cu_seqlens packing.") + + hidden_states = inputs_embeds + query_attention_mask = None + if attention_mask is not None: + attention_mask = attention_mask.to(dtype=torch.bool, device=hidden_states.device) + query_attention_mask = attention_mask[:, -hidden_states.shape[1] :] + + packed_metadata = None + if position_ids is None: + if cu_seqlens is not None: + if attention_mask is None: + raise ValueError("attention_mask must be provided with cu_seqlens packing.") + position_ids = self.build_packed_position_ids( + attention_mask=attention_mask, + cu_seqlens=cu_seqlens.to(device=hidden_states.device), + num_sequences=num_sequences.to(device=hidden_states.device) if num_sequences is not None else None, + sequence_length=hidden_states.shape[1], + ) + elif attention_mask is not None: + position_ids = attention_mask.long().cumsum(dim=-1) - 1 + position_ids = position_ids.masked_fill(~attention_mask, 0) + position_ids = position_ids[:, -hidden_states.shape[1] :] + else: + past_length = 0 + if past_key_values is not None and len(past_key_values) > 0: + first_layer_past = past_key_values[0] + if _is_static_kv_cache_layer(first_layer_past): + past_length = int(first_layer_past.get("length", 0)) + else: + past_length = first_layer_past[0].shape[1] + position_ids = torch.arange(hidden_states.shape[1], device=hidden_states.device, dtype=torch.long) + position_ids = position_ids + past_length + position_ids = position_ids.unsqueeze(0).expand(hidden_states.shape[0], -1) + + if cu_seqlens is not None and self.attn_implementation == "flash_attention_2": + packed_metadata = self.build_packed_metadata( + hidden_states=hidden_states, + cu_seqlens=cu_seqlens.to(device=hidden_states.device), + num_sequences=num_sequences.to(device=hidden_states.device) if num_sequences is not None else None, + ) + + if self.position_embedding_type == "absolute": + hidden_states = hidden_states + self.wpe(position_ids) + hidden_states = self.drop(hidden_states) + if query_attention_mask is not None: + hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype) + + all_hidden_states = () if output_hidden_states else None + presents = [] if use_cache else None + for layer_index, block in enumerate(self.h): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if self.gradient_checkpointing and self.training: + if use_cache: + raise ValueError("use_cache=True is not supported when gradient checkpointing is enabled during training.") + + def custom_forward(*inputs): + output, _ = block( + inputs[0], + attention_mask=inputs[1], + position_ids=inputs[2], + packed_metadata=packed_metadata, + layer_past=None, + use_cache=False, + ) + return output + + hidden_states = torch.utils.checkpoint.checkpoint( + custom_forward, + hidden_states, + attention_mask, + position_ids, + use_reentrant=False, + ) + present = None + else: + hidden_states, present = block( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + packed_metadata=packed_metadata, + layer_past=None if past_key_values is None else past_key_values[layer_index], + use_cache=use_cache, + ) + if query_attention_mask is not None: + hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype) + if presents is not None: + presents.append(present) + + hidden_states = self.ln_f(hidden_states) + if query_attention_mask is not None: + hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype) + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return (hidden_states, tuple(presents) if presents is not None else None, all_hidden_states, None) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=tuple(presents) if presents is not None else None, + hidden_states=all_hidden_states, + attentions=None, + ) diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9abfef88d5699d3128768108671064350070bdd9 --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c8b9d4e555370073a298b34d65927b7956752de969aade484f05544c9d4acaa +size 3994689160 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd9ab2e25e7b7c96a18bbfe48732105c66db7be3 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9d4328d7b8a2f7e677f3e9aa35568f14de8f4e0fb7df615fc36adc5cfb8c2a0 +size 3961231200 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c1c06d7b4ffc252fdef55a706c3cd1f7d81beca7 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b87a011a7f1e37373e5a4489e9dbbffbb298c4760505e94474bfc63f37d1660c +size 304110936 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..93e8f4124c5cdf8f8564cb5d99938624f627ec19 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,433 @@ +{ + "metadata": { + "total_parameters": 4129990144, + "total_size": 8259980288 + }, + "weight_map": { + "audio_embeddings.0.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.1.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.10.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.11.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.2.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.3.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.4.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.5.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.6.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.7.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.8.weight": "model-00001-of-00003.safetensors", + "audio_embeddings.9.weight": "model-00001-of-00003.safetensors", + "local_text_lm_head.weight": "model-00001-of-00003.safetensors", + "local_transformer.h.0.attn.c_attn.bias": "model-00001-of-00003.safetensors", + "local_transformer.h.0.attn.c_attn.weight": "model-00001-of-00003.safetensors", + "local_transformer.h.0.attn.c_proj.bias": "model-00001-of-00003.safetensors", + "local_transformer.h.0.attn.c_proj.weight": "model-00001-of-00003.safetensors", + "local_transformer.h.0.ln_1.bias": "model-00001-of-00003.safetensors", + "local_transformer.h.0.ln_1.weight": "model-00001-of-00003.safetensors", + "local_transformer.h.0.ln_2.bias": "model-00001-of-00003.safetensors", + "local_transformer.h.0.ln_2.weight": "model-00001-of-00003.safetensors", + "local_transformer.h.0.mlp.fc_in.bias": "model-00001-of-00003.safetensors", + "local_transformer.h.0.mlp.fc_in.weight": "model-00001-of-00003.safetensors", + "local_transformer.h.0.mlp.fc_out.bias": "model-00001-of-00003.safetensors", + "local_transformer.h.0.mlp.fc_out.weight": "model-00001-of-00003.safetensors", + "local_transformer.ln_f.bias": "model-00001-of-00003.safetensors", + "local_transformer.ln_f.weight": "model-00001-of-00003.safetensors", + "transformer.embed_tokens.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors", + "transformer.layers.34.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.34.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "transformer.layers.34.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.self_attn.k_norm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.self_attn.q_norm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.34.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.self_attn.k_norm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.self_attn.q_norm.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.35.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "transformer.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "transformer.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "transformer.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/modeling_moss_tts.py b/modeling_moss_tts.py new file mode 100644 index 0000000000000000000000000000000000000000..4891b570440663b44f0304fe44e6df3d9baf55f8 --- /dev/null +++ b/modeling_moss_tts.py @@ -0,0 +1,623 @@ +# coding=utf-8 +"""Modeling code for the MOSS-TTS-Local-Transformer-v1.5 HuggingFace release.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional, Union + +import torch +import torch.nn as nn +from transformers.modeling_outputs import BaseModelOutputWithPast +from transformers.modeling_utils import PreTrainedModel +from transformers.models.gpt2.configuration_gpt2 import GPT2Config +from transformers.utils import ModelOutput + +from .configuration_moss_tts import MossTTSLocalConfig +from .gpt2_decoder import MossTTSNanoGPT2Model +from .qwen3_decoder import MossQwen3Model + + +@dataclass +class MossTTSLocalOutput(ModelOutput): + last_hidden_state: Optional[torch.FloatTensor] = None + past_key_values: Optional[tuple[tuple[torch.Tensor, torch.Tensor], ...]] = None + hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + + +def _find_last_equal(input_ids: torch.LongTensor, value: int) -> torch.LongTensor: + matches = input_ids.eq(int(value)) + if not bool(matches.any(dim=1).all().item()): + raise ValueError(f"Every sample must contain token id {int(value)}.") + positions = torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long) + masked_positions = positions.unsqueeze(0).masked_fill(~matches, -1) + return masked_positions.max(dim=1).values + + +class MossTTSLocalPreTrainedModel(PreTrainedModel): + config_class = MossTTSLocalConfig + base_model_prefix = "transformer" + supports_gradient_checkpointing = True + _no_split_modules = ["MossTTSNanoGPT2Block", "MossQwen3DecoderLayer"] + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_cache_class = True + + def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False) -> None: + if isinstance(module, MossTTSNanoGPT2Model) or isinstance(module, MossQwen3Model): + module.gradient_checkpointing = value + + +class MossTTSLocalModel(MossTTSLocalPreTrainedModel): + _tied_weights_keys = None + + def __init__(self, config: MossTTSLocalConfig) -> None: + super().__init__(config) + self._tied_weights_keys = self._build_tied_weights_keys(config) + + config.qwen3_config.pad_token_id = config.pad_token_id + config.qwen3_config._attn_implementation = config.attn_implementation + local_gpt2_config = config.gpt2_config.to_dict() + local_gpt2_config["n_layer"] = int(getattr(config, "local_transformer_layers", config.gpt2_config.n_layer)) + local_gpt2_config["n_positions"] = int(config.n_vq) + 1 + local_gpt2_config["n_ctx"] = int(config.n_vq) + 1 + local_gpt2_config = GPT2Config(**local_gpt2_config) + local_gpt2_config.pad_token_id = config.pad_token_id + local_gpt2_config._attn_implementation = config.local_transformer_attn_implementation + + self.transformer = MossQwen3Model(config.qwen3_config) + self.local_transformer = MossTTSNanoGPT2Model( + local_gpt2_config, + attn_implementation=config.local_transformer_attn_implementation, + ) + self.local_transformer.wte = nn.Identity() + + hidden_size = int(config.hidden_size) + self.audio_embeddings = nn.ModuleList( + [ + nn.Embedding(int(config.audio_codebook_sizes[index]), hidden_size) + for index in range(config.n_vq) + ] + ) + self.text_lm_head = nn.Linear(hidden_size, int(config.vocab_size), bias=False) + self.audio_lm_heads = nn.ModuleList( + [ + nn.Linear(hidden_size, int(config.audio_codebook_sizes[index]), bias=False) + for index in range(config.n_vq) + ] + ) + self.local_text_lm_head = ( + nn.Linear(hidden_size, 2, bias=False) + if self._use_binary_local_text_head() + else None + ) + + self.post_init() + self.tie_weights() + self.initialize_local_text_lm_head_from_text_lm_head() + + def can_generate(self) -> bool: + return True + + @staticmethod + def _build_tied_weights_keys(config: MossTTSLocalConfig) -> dict[str, str]: + tied_weights = {"text_lm_head.weight": "transformer.embed_tokens.weight"} + tied_weights.update( + { + f"audio_lm_heads.{index}.weight": f"audio_embeddings.{index}.weight" + for index in range(config.n_vq) + } + ) + return tied_weights + + def tie_weights(self, *args, **kwargs) -> None: + del args, kwargs + self.text_lm_head.weight = self.transformer.embed_tokens.weight + for embedding, head in zip(self.audio_embeddings, self.audio_lm_heads): + head.weight = embedding.weight + + def get_input_embeddings(self) -> nn.Embedding: + return self.transformer.embed_tokens + + def set_input_embeddings(self, value: nn.Embedding) -> None: + self.transformer.embed_tokens = value + self.tie_weights() + self.initialize_local_text_lm_head_from_text_lm_head() + + def get_output_embeddings(self) -> nn.Linear: + return self.text_lm_head + + def set_output_embeddings(self, new_embeddings: nn.Linear) -> None: + self.text_lm_head = new_embeddings + self.tie_weights() + self.initialize_local_text_lm_head_from_text_lm_head() + + def _use_binary_local_text_head(self) -> bool: + return str(getattr(self.config, "local_text_head_mode", "full_vocab")).strip().lower() == "binary" + + def _local_text_candidate_ids(self, device: torch.device) -> torch.LongTensor: + return torch.tensor( + [ + int(self.config.audio_assistant_slot_token_id), + int(self.config.audio_end_token_id), + ], + dtype=torch.long, + device=device, + ) + + def initialize_local_text_lm_head_from_text_lm_head(self) -> None: + if not self._use_binary_local_text_head() or self.local_text_lm_head is None: + return + candidate_ids = self._local_text_candidate_ids(self.text_lm_head.weight.device) + with torch.no_grad(): + source_weight = self.text_lm_head.weight.index_select(0, candidate_ids) + if tuple(source_weight.shape) == tuple(self.local_text_lm_head.weight.shape): + self.local_text_lm_head.weight.copy_( + source_weight.to( + device=self.local_text_lm_head.weight.device, + dtype=self.local_text_lm_head.weight.dtype, + ) + ) + + def _resolve_fixed_nq( + self, + n_vq_for_inference: Optional[int] = None, + nq: Optional[int] = None, + ) -> int: + requested = n_vq_for_inference if n_vq_for_inference is not None else nq + config_nq = int(self.config.n_vq) + if requested is not None and int(requested) != config_nq: + raise ValueError( + "This MOSS-TTS-Local-Transformer-v1.5 release is trained with a fixed RVQ depth. " + f"Expected n_vq={config_nq}, got {int(requested)}." + ) + return config_nq + + def _build_inputs_embeds(self, input_ids: torch.LongTensor) -> torch.FloatTensor: + if input_ids.ndim != 3 or input_ids.shape[-1] != self.config.n_vq + 1: + raise ValueError( + f"Expected input_ids shape [batch, seq, {self.config.n_vq + 1}], " + f"got {tuple(input_ids.shape)}." + ) + text_ids = input_ids[..., 0] + inputs_embeds = self.transformer.embed_tokens(text_ids) + for channel_index, embedding in enumerate(self.audio_embeddings): + channel_ids = input_ids[..., channel_index + 1] + valid_mask = channel_ids.ne(self.config.audio_pad_token_id) + safe_ids = channel_ids.masked_fill(~valid_mask, 0) + audio_embeds = embedding(safe_ids) * valid_mask.unsqueeze(-1) + inputs_embeds = inputs_embeds + audio_embeds + return inputs_embeds + + def _global_hidden_to_local(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + return hidden_states + + @staticmethod + def _local_past_length(past_key_values: Optional[tuple[Any, ...]]) -> int: + if past_key_values is None or len(past_key_values) == 0: + return 0 + first_layer_past = past_key_values[0] + if isinstance(first_layer_past, dict) and bool(first_layer_past.get("static_kv_cache", False)): + return int(first_layer_past.get("length", 0)) + return int(first_layer_past[0].shape[1]) + + def _new_static_local_past_key_values( + self, + batch_size: int, + max_length: int, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[dict[str, Any], ...]: + layers = [] + for block in self.local_transformer.h: + attn = block.attn + cache_shape = ( + int(batch_size), + int(max_length), + int(attn.num_heads), + int(attn.head_dim), + ) + layers.append( + { + "static_kv_cache": True, + "key": torch.empty(cache_shape, device=device, dtype=dtype), + "value": torch.empty(cache_shape, device=device, dtype=dtype), + "length": 0, + } + ) + return tuple(layers) + + def _decode_local_hidden_states_with_cache( + self, + local_inputs_embeds: torch.FloatTensor, + past_key_values: Optional[tuple[Any, ...]] = None, + ) -> tuple[torch.FloatTensor, Optional[tuple[Any, ...]]]: + if ( + past_key_values is None + and not self.training + and bool(getattr(self.config, "use_static_local_kv_cache", True)) + ): + max_length = max(int(getattr(self.config, "n_vq", 0)) + 1, int(local_inputs_embeds.shape[1])) + past_key_values = self._new_static_local_past_key_values( + batch_size=int(local_inputs_embeds.shape[0]), + max_length=max_length, + device=local_inputs_embeds.device, + dtype=local_inputs_embeds.dtype, + ) + past_length = self._local_past_length(past_key_values) + local_seq_len = int(local_inputs_embeds.shape[1]) + local_position_ids = torch.arange( + past_length, + past_length + local_seq_len, + device=local_inputs_embeds.device, + dtype=torch.long, + ).unsqueeze(0) + if int(local_inputs_embeds.shape[0]) != 1: + local_position_ids = local_position_ids.expand(int(local_inputs_embeds.shape[0]), -1) + local_outputs = self.local_transformer( + input_ids=None, + past_key_values=past_key_values, + attention_mask=None, + position_ids=local_position_ids, + inputs_embeds=local_inputs_embeds, + use_cache=True, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + cu_seqlens=None, + num_sequences=None, + ) + return local_outputs.last_hidden_state, local_outputs.past_key_values + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[tuple[tuple[torch.Tensor, torch.Tensor], ...]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = True, + **kwargs, + ) -> Union[tuple, MossTTSLocalOutput]: + del kwargs + if inputs_embeds is None: + if input_ids is None: + raise ValueError("Either input_ids or inputs_embeds must be provided.") + inputs_embeds = self._build_inputs_embeds(input_ids) + outputs = self.transformer( + input_ids=None, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=True, + cu_seqlens=None, + num_sequences=None, + ) + if not return_dict: + return ( + outputs.last_hidden_state, + outputs.past_key_values, + outputs.hidden_states, + outputs.attentions, + ) + return MossTTSLocalOutput( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def _decode_local_last_hidden_state( + self, + local_inputs_embeds: torch.FloatTensor, + ) -> torch.FloatTensor: + local_seq_len = int(local_inputs_embeds.shape[1]) + local_position_ids = torch.arange( + 0, + local_seq_len, + device=local_inputs_embeds.device, + dtype=torch.long, + ).unsqueeze(0) + if int(local_inputs_embeds.shape[0]) != 1: + local_position_ids = local_position_ids.expand(int(local_inputs_embeds.shape[0]), -1) + local_outputs = self.local_transformer( + input_ids=None, + attention_mask=None, + position_ids=local_position_ids, + inputs_embeds=local_inputs_embeds, + use_cache=False, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + cu_seqlens=None, + num_sequences=None, + ) + return local_outputs.last_hidden_state[:, -1, :] + + def _filter_logits( + self, + logits: torch.FloatTensor, + top_k: Optional[int], + top_p: Optional[float], + ) -> torch.FloatTensor: + scores = logits + if top_k is not None and int(top_k) > 0 and int(top_k) < scores.shape[-1]: + kth = torch.topk(scores, int(top_k), dim=-1).values[..., -1, None] + scores = scores.masked_fill(scores < kth, -torch.inf) + if top_p is not None and 0.0 < float(top_p) < 1.0: + sorted_scores, sorted_indices = torch.sort(scores, descending=True, dim=-1) + sorted_probs = torch.softmax(sorted_scores, dim=-1) + cumulative_probs = sorted_probs.cumsum(dim=-1) + sorted_mask = cumulative_probs > float(top_p) + sorted_mask[..., 1:] = sorted_mask[..., :-1].clone() + sorted_mask[..., 0] = False + remove_mask = torch.zeros_like(scores, dtype=torch.bool) + remove_mask.scatter_(dim=-1, index=sorted_indices, src=sorted_mask) + scores = scores.masked_fill(remove_mask, -torch.inf) + return scores + + def _apply_repetition_penalty( + self, + scores: torch.FloatTensor, + previous_token_ids: Optional[torch.LongTensor], + penalty: float, + ) -> torch.FloatTensor: + if previous_token_ids is None or float(penalty) == 1.0: + return scores + if previous_token_ids.ndim == 1: + previous_token_ids = previous_token_ids.unsqueeze(0) + updated = scores.clone() + for batch_index in range(updated.shape[0]): + unique_token_ids = torch.unique(previous_token_ids[batch_index]) + unique_token_ids = unique_token_ids[ + (unique_token_ids >= 0) & (unique_token_ids < updated.shape[-1]) + ] + if unique_token_ids.numel() == 0: + continue + token_scores = updated[batch_index].index_select(0, unique_token_ids) + token_scores = torch.where( + token_scores < 0, + token_scores * float(penalty), + token_scores / float(penalty), + ) + updated[batch_index].scatter_(0, unique_token_ids, token_scores) + return updated + + def _sample_next_token( + self, + logits: torch.FloatTensor, + do_sample: bool, + temperature: float, + top_k: Optional[int], + top_p: Optional[float], + previous_token_ids: Optional[torch.LongTensor] = None, + repetition_penalty: float = 1.0, + ) -> torch.LongTensor: + scores = logits.float() + scores = self._apply_repetition_penalty(scores, previous_token_ids, repetition_penalty) + if not do_sample: + return torch.argmax(scores, dim=-1) + if float(temperature) <= 0: + raise ValueError("temperature must be positive when do_sample=True.") + scores = scores / float(temperature) + scores = self._filter_logits(scores, top_k=top_k, top_p=top_p) + probs = torch.softmax(scores, dim=-1) + return torch.multinomial(probs, num_samples=1).squeeze(-1) + + def _sample_next_assistant_text_token( + self, + local_hidden_states: torch.FloatTensor, + do_sample: bool, + temperature: float, + top_k: Optional[int], + top_p: Optional[float], + ) -> torch.LongTensor: + if self._use_binary_local_text_head() and self.local_text_lm_head is not None: + logits = self.local_text_lm_head(local_hidden_states) + sampled_indices = self._sample_next_token( + logits=logits, + do_sample=do_sample, + temperature=temperature, + top_k=top_k, + top_p=top_p, + ) + candidate_ids = self._local_text_candidate_ids(logits.device) + return candidate_ids[sampled_indices] + + candidate_ids = self._local_text_candidate_ids(local_hidden_states.device) + logits = self.text_lm_head(local_hidden_states).index_select(dim=-1, index=candidate_ids) + sampled_indices = self._sample_next_token( + logits=logits, + do_sample=do_sample, + temperature=temperature, + top_k=top_k, + top_p=top_p, + ) + return candidate_ids[sampled_indices] + + def _build_generation_row( + self, + batch_size: int, + device: torch.device, + audio_token_ids: torch.LongTensor, + ) -> torch.LongTensor: + row = torch.full( + (batch_size, 1, self.config.n_vq + 1), + int(self.config.audio_pad_token_id), + dtype=torch.long, + device=device, + ) + row[:, :, 0] = int(self.config.audio_assistant_slot_token_id) + row[:, :, 1:] = audio_token_ids.unsqueeze(1) + return row + + @torch.inference_mode() + def generate( + self, + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + max_new_tokens: Optional[int] = None, + max_new_frames: Optional[int] = None, + do_sample: bool = True, + text_temperature: float = 1.0, + text_top_p: float = 1.0, + text_top_k: int = 50, + audio_temperature: Optional[float] = None, + audio_top_p: Optional[float] = None, + audio_top_k: Optional[int] = None, + audio_repetition_penalty: Optional[float] = None, + temperature: float = 1.0, + top_p: float = 0.95, + top_k: int = 50, + repetition_penalty: float = 1.0, + use_kv_cache: bool = True, + n_vq_for_inference: Optional[int] = None, + nq: Optional[int] = None, + **kwargs, + ) -> list[tuple[int, torch.LongTensor]]: + del kwargs + self._resolve_fixed_nq(n_vq_for_inference=n_vq_for_inference, nq=nq) + + if input_ids.ndim == 2: + input_ids = input_ids.unsqueeze(0) + if input_ids.ndim != 3: + raise ValueError(f"Expected input_ids with 3 dims, got {tuple(input_ids.shape)}.") + if input_ids.shape[-1] != self.config.n_vq + 1: + raise ValueError( + f"Expected {self.config.n_vq + 1} channels from config.n_vq, got {input_ids.shape[-1]}." + ) + if attention_mask is None: + attention_mask = torch.ones(input_ids.shape[:2], dtype=torch.bool, device=input_ids.device) + elif attention_mask.ndim == 1: + attention_mask = attention_mask.unsqueeze(0) + attention_mask = attention_mask.to(device=input_ids.device, dtype=torch.bool) + + frame_budget = max_new_frames if max_new_frames is not None else max_new_tokens + if frame_budget is None: + frame_budget = 4096 + frame_budget = int(frame_budget) + + audio_temperature = float(temperature if audio_temperature is None else audio_temperature) + audio_top_p = float(top_p if audio_top_p is None else audio_top_p) + audio_top_k = int(top_k if audio_top_k is None else audio_top_k) + audio_repetition_penalty = float( + repetition_penalty if audio_repetition_penalty is None else audio_repetition_penalty + ) + + batch_size = input_ids.shape[0] + input_ids_length = input_ids.shape[1] + current_input_ids = input_ids + current_attention_mask = attention_mask + current_model_input_ids = current_input_ids + generated_frames: list[torch.LongTensor] = [] + finished = torch.zeros(batch_size, dtype=torch.bool, device=input_ids.device) + past_key_values = None + local_dtype = self.local_transformer.ln_f.weight.dtype + + for _ in range(frame_budget): + generated_audio_history = torch.stack(generated_frames, dim=1) if generated_frames else None + global_inputs_embeds = self._build_inputs_embeds(current_model_input_ids) + global_outputs = self.transformer( + input_ids=None, + past_key_values=past_key_values, + attention_mask=current_attention_mask, + position_ids=None, + inputs_embeds=global_inputs_embeds, + use_cache=use_kv_cache, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + cu_seqlens=None, + num_sequences=None, + ) + global_hidden_states = global_outputs.last_hidden_state[:, -1, :] + local_global_hidden_states = self._global_hidden_to_local(global_hidden_states).to(dtype=local_dtype) + + local_prefix_hidden_states, local_prefix_past_key_values = self._decode_local_hidden_states_with_cache( + local_global_hidden_states.unsqueeze(1) + ) + local_hidden_states = local_prefix_hidden_states[:, -1, :] + next_text_tokens = self._sample_next_assistant_text_token( + local_hidden_states=local_hidden_states, + do_sample=do_sample, + temperature=text_temperature, + top_k=text_top_k, + top_p=text_top_p, + ) + should_continue = next_text_tokens.eq(int(self.config.audio_assistant_slot_token_id)) & ~finished + finished = finished | next_text_tokens.eq(int(self.config.audio_end_token_id)) + if not bool(should_continue.any().item()): + break + + next_frame_tokens = [] + for channel_index in range(int(self.config.n_vq)): + channel_logits = self.audio_lm_heads[channel_index](local_hidden_states) + channel_token = self._sample_next_token( + logits=channel_logits, + do_sample=do_sample, + temperature=audio_temperature, + top_k=audio_top_k, + top_p=audio_top_p, + previous_token_ids=( + None + if generated_audio_history is None + else generated_audio_history[:, :, channel_index] + ), + repetition_penalty=audio_repetition_penalty, + ) + next_frame_tokens.append(channel_token) + if channel_index + 1 < int(self.config.n_vq): + current_local_input = self.audio_embeddings[channel_index](channel_token).to(dtype=local_dtype) + local_token_hidden_states, local_prefix_past_key_values = ( + self._decode_local_hidden_states_with_cache( + current_local_input.unsqueeze(1), + past_key_values=local_prefix_past_key_values, + ) + ) + local_hidden_states = local_token_hidden_states[:, -1, :] + + next_frame = torch.stack(next_frame_tokens, dim=-1) + next_frame = next_frame.masked_fill( + ~should_continue.unsqueeze(-1), + int(self.config.audio_pad_token_id), + ) + generated_frames.append(next_frame) + + next_row = self._build_generation_row( + batch_size=batch_size, + device=input_ids.device, + audio_token_ids=next_frame, + ) + if bool((~should_continue).any().item()): + next_row[~should_continue, 0, 0] = int(self.config.pad_token_id) + next_row[~should_continue, 0, 1:] = int(self.config.audio_pad_token_id) + + current_input_ids = torch.cat([current_input_ids, next_row], dim=1) + current_attention_mask = torch.cat( + [current_attention_mask, should_continue.unsqueeze(1)], + dim=1, + ) + if use_kv_cache: + current_model_input_ids = next_row + past_key_values = global_outputs.past_key_values + else: + current_model_input_ids = current_input_ids + + start_indices = _find_last_equal(input_ids[..., 0], int(self.config.audio_start_token_id)) + start_lengths = input_ids_length - start_indices - 1 + outputs: list[tuple[int, torch.LongTensor]] = [] + for start_index, start_length, generation_ids in zip( + start_indices.tolist(), + start_lengths.tolist(), + current_input_ids, + ): + outputs.append((int(start_length), generation_ids[int(start_index):].detach().cpu())) + return outputs diff --git a/podcast_infer.py b/podcast_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..8cf24c0d67cdaa8827ac957c5afbd756c2866bd5 --- /dev/null +++ b/podcast_infer.py @@ -0,0 +1,84 @@ +"""Portable two-speaker podcast inference for the merged NotebookLM Edition.""" +import argparse,json,math +from pathlib import Path +import numpy as np +import torch +import base_runtime + +LANGUAGES={'en':'English','de':'German','fr':'French','es':'Spanish','nl':'Dutch'} + +def row_from_dialogue(dialogue,references,context_codes=None,context_description=''): + language=dialogue['language']; turns=dialogue['turns'] + if language not in LANGUAGES or not turns:raise ValueError('Supported language and nonempty turns required') + if set(references)!={'S1','S2'}:raise ValueError('Supply separate S1 and S2 identity references') + for codes in references.values(): + if codes.ndim!=2 or codes.shape[1]!=12 or not 63<=len(codes)<=187:raise ValueError('Each reference must have 12 codebooks and 5.04–14.96 seconds') + script=[];words=[];seconds=0.;pause=float(dialogue.get('pause_s',.24)) + if not 0<=pause<=3:raise ValueError('Pause out of range') + for i,t in enumerate(turns): + if t['speaker'] not in references or not t['text'].strip() or not .4<=t['seconds']<=60:raise ValueError('Invalid turn') + if '\n' in t['direction'] or '\n' in t['text']:raise ValueError('Use separate turns for separate lines') + script.append('(speaker='+t['speaker']+', '+t['direction']+') ['+format(t['seconds'],'.2f')+' seconds duration] '+t['text']) + words.append(t['text']);seconds+=t['seconds'] + if i750):raise ValueError('Conversation context must be <=60 seconds') + return dict(language=language,plain=' '.join(words),instruction='GENERAL: '+dialogue.get('general','')+'\nSCRIPT:\n'+'\n'.join(script), + frames=round(seconds*12.5),speaker_references=references,context_codes=context_codes,context_description=context_description) + +def generation_inputs(proc,row): + user=proc.build_user_message(text=row['plain'],instruction=row['instruction'],tokens=row['frames'],language=LANGUAGES[row['language']],reference=None) + blocks=[];audio=[] + for speaker in ['S1','S2']: + blocks.append('Speaker '+speaker+' identity reference:\n<|audio|>');audio.append(torch.as_tensor(row['speaker_references'][speaker],dtype=torch.long)) + context=row.get('context_codes') + if context is not None and len(context): + blocks.append('Previous conversation, chronological and strictly before target:\n'+row['context_description']+'\n<|audio|>');audio.append(torch.as_tensor(context,dtype=torch.long)) + marker='- Reference(s):\nNone\n' + if user['content'].count(marker)!=1:raise RuntimeError('Native processor reference wrapper changed') + user['content']=user['content'].replace(marker,'- Reference(s):\n'+'\n'.join(blocks)+'\n',1) + user['audio_codes_list']=audio;user.pop('text',None) + user['content']='<|im_start|>user\n'+user['content']+'<|im_end|>\n<|im_start|>assistant\n' + if user['content'].count('<|audio|>')!=len(audio):raise RuntimeError('Audio placeholders lost') + return proc([[user]],mode='generation',apply_chat_template=False) + +def encode_reference(proc,path,device='cuda:0',context=False): + import soundfile as sf + wave,sr=sf.read(path,dtype='float32',always_2d=True) + minimum=.08 if context else 5.04;maximum=60 if context else 14.96 + if len(wave)/sr30000:raise ValueError('Packed prefix too long; never silently truncate references') + torch.manual_seed(seed);torch.cuda.manual_seed_all(seed) + cap=min(1000,row['frames']+125) + with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16): + result=model.generate(input_ids=batch['input_ids'].to(device),attention_mask=batch['attention_mask'].to(device), + max_new_frames=cap,do_sample=True,audio_temperature=1.,audio_top_p=.95,audio_top_k=50,audio_repetition_penalty=1.,use_kv_cache=True) + prefix,sequence=result[0] + if prefix!=0 or sequence.ndim!=2 or sequence.shape[1]!=13:raise ValueError('Invalid MOSS output') + codes=sequence[1:,1:];codes=codes[(codes[:,0]>=0)&(codes[:,0]!=model.config.audio_pad_token_id)].cpu() + if len(codes)<2 or codes.min()<0 or codes.max()>1023:raise ValueError('Invalid generated codec indices') + return codes,{'frames':len(codes),'frame_cap':len(codes)>=cap,'requested_seconds':row['frames']/12.5,'sampling':{'temperature':1.,'top_p':.95,'top_k':50,'cfg':1.}} + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--model',default='laion/Humaneness-Voice-Large-NotebookLM-Edition') + p.add_argument('--codec',default='OpenMOSS-Team/MOSS-Audio-Tokenizer-v2') + p.add_argument('--dialogue',type=Path,required=True);p.add_argument('--reference-s1',required=True);p.add_argument('--reference-s2',required=True) + p.add_argument('--context');p.add_argument('--context-description',default='');p.add_argument('--seed',type=int,default=20261103);p.add_argument('--output',type=Path,required=True) + a=p.parse_args() + if a.output.exists():raise FileExistsError(a.output) + model,proc=base_runtime.load_model(a.model);base_runtime.load_codec(proc,a.codec) + refs={s:encode_reference(proc,f) for s,f in [('S1',a.reference_s1),('S2',a.reference_s2)]} + context=encode_reference(proc,a.context,context=True) if a.context else None + row=row_from_dialogue(json.loads(a.dialogue.read_text()),refs,context,a.context_description) + codes,meta=generate(model,proc,row,a.seed);wave,sr=base_runtime.decode(proc,codes) + import soundfile as sf + a.output.parent.mkdir(parents=True,exist_ok=True);sf.write(a.output,wave,sr,subtype='PCM_16');print(json.dumps(meta)) + +if __name__=='__main__':main() diff --git a/processing_moss_tts.py b/processing_moss_tts.py new file mode 100644 index 0000000000000000000000000000000000000000..94c33dea59269059772efe0df34acc3fe7d7d067 --- /dev/null +++ b/processing_moss_tts.py @@ -0,0 +1,914 @@ +# coding=utf-8 +"""Processor for the MOSS-TTS-Local-Transformer-v1.5 HuggingFace release.""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast + +import torch +import torchaudio +from transformers import ( + AutoConfig, + AutoModel, + AutoTokenizer, + BatchFeature, + PreTrainedTokenizerBase, + ProcessorMixin, + logging, + processing_utils, +) + +from .configuration_moss_tts import MossTTSLocalConfig + + +if hasattr(processing_utils, "MODALITY_TO_BASE_CLASS_MAPPING"): + processing_utils.MODALITY_TO_BASE_CLASS_MAPPING["audio_tokenizer"] = "PreTrainedModel" +else: + processing_utils.AUTO_TO_BASE_CLASS_MAPPING["AutoModel"] = "PreTrainedModel" +logger = logging.get_logger(__name__) + +AUDIO_PLACEHOLDER = "<|audio|>" +USER_ROLE_PREFIX = "user\n" +USER_TEMPLATE_REFERENCE_PREFIX = ( + "\n" + "- Reference(s):\n" +) +USER_TEMPLATE_AFTER_REFERENCE_SUFFIX = ( + "\n" + "- Text:\n" +) +USER_TEMPLATE_SUFFIX = "\n" +ASSISTANT_TURN_PREFIX = "\n" +ASSISTANT_ROLE_PREFIX = "assistant\n" +USER_MESSAGE_FIELDS = ( + "text", + "reference", + "instruction", + "tokens", + "quality", + "sound_event", + "ambient_sound", + "language", +) + + +def _normalize_template_value(value: Any) -> str: + if value is None: + return "None" + resolved = str(value).strip() + return resolved or "None" + + +def _render_user_prompt_after_reference( + language_code: object | None = None, + prompt_fields: Optional[Dict[str, Any]] = None, +) -> str: + fields = dict(prompt_fields or {}) + return ( + "\n- Instruction:\n" + + _normalize_template_value(fields.get("instruction")) + + "\n- Tokens:\n" + + _normalize_template_value(fields.get("tokens")) + + "\n- Quality:\n" + + _normalize_template_value(fields.get("quality")) + + "\n- Sound Event:\n" + + _normalize_template_value(fields.get("sound_event")) + + "\n- Ambient Sound:\n" + + _normalize_template_value(fields.get("ambient_sound")) + + "\n- Language:\n" + + _normalize_template_value(fields.get("language", language_code)) + + USER_TEMPLATE_AFTER_REFERENCE_SUFFIX + ) + + +@dataclass +class Message: + def to_dict(self) -> Dict[str, Any]: + raise NotImplementedError + + +@dataclass +class UserMessage(Message): + text: Optional[str] = None + reference: Optional[List[Optional[Union[str, os.PathLike, torch.Tensor]]]] = None + instruction: Optional[str] = None + tokens: Optional[int] = None + quality: Optional[str] = None + sound_event: Optional[str] = None + ambient_sound: Optional[str] = None + language: Optional[str] = None + + def __post_init__(self) -> None: + template = """ +- Reference(s): +{reference} +- Instruction: +{instruction} +- Tokens: +{tokens} +- Quality: +{quality} +- Sound Event: +{sound_event} +- Ambient Sound: +{ambient_sound} +- Language: +{language} +- Text: +{text} +""" + + audio_codes_list: list[Union[str, os.PathLike, torch.Tensor]] = [] + if self.reference is None: + reference = "None" + else: + reference_items: list[str] = [] + for speaker_idx, speaker_reference in enumerate(self.reference): + if speaker_reference is None: + continue + # Keep raw audio placeholders directly under "- Reference(s):". + # Speaker labels such as "[S1]:" change the token sequence and + # can affect voice-clone conditioning. + reference_items.append(AUDIO_PLACEHOLDER) + audio_codes_list.append(speaker_reference) + reference = "\n".join(reference_items) if reference_items else "None" + + self._content = ( + template.replace("{reference}", str(reference)) + .replace("{instruction}", str(self.instruction)) + .replace("{tokens}", str(self.tokens)) + .replace("{quality}", str(self.quality)) + .replace("{sound_event}", str(self.sound_event)) + .replace("{ambient_sound}", str(self.ambient_sound)) + .replace("{language}", str(self.language)) + .replace("{text}", str(self.text)) + ) + self._audio_codes_list = audio_codes_list + + def to_dict(self) -> Dict[str, Any]: + return { + "role": "user", + "content": self._content, + "audio_codes_list": self._audio_codes_list, + "text": self.text, + "instruction": self.instruction, + "tokens": self.tokens, + "quality": self.quality, + "sound_event": self.sound_event, + "ambient_sound": self.ambient_sound, + "language": self.language, + } + + +@dataclass +class AssistantMessage(Message): + audio_codes_list: List[Union[str, os.PathLike, torch.Tensor]] + content: str = AUDIO_PLACEHOLDER + + def to_dict(self) -> Dict[str, Any]: + return { + "role": "assistant", + "content": self.content, + "audio_codes_list": self.audio_codes_list, + } + + +class MossTTSLocalProcessor(ProcessorMixin): + attributes = ["tokenizer"] + tokenizer_class = "AutoTokenizer" + audio_tokenizer_class = "AutoModel" + + tokenizer: PreTrainedTokenizerBase + audio_tokenizer: Any + + def __init__( + self, + tokenizer: PreTrainedTokenizerBase, + audio_tokenizer: Any = None, + model_config: Optional[MossTTSLocalConfig] = None, + **kwargs, + ) -> None: + super().__init__(tokenizer=tokenizer, audio_tokenizer=audio_tokenizer, **kwargs) + self.tokenizer = tokenizer + self.audio_tokenizer = audio_tokenizer + self.model_config = model_config or MossTTSLocalConfig() + + def _id_to_token(token_id: int) -> str: + token = tokenizer.convert_ids_to_tokens(int(token_id)) + if isinstance(token, list): + return token[0] if token else "" + return cast(str, token) + + self.audio_user_slot_token = _id_to_token(self.model_config.audio_user_slot_token_id) + self.audio_assistant_slot_token = _id_to_token(self.model_config.audio_assistant_slot_token_id) + self.audio_start_token = _id_to_token(self.model_config.audio_start_token_id) + self.audio_end_token = _id_to_token(self.model_config.audio_end_token_id) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): + trust_remote_code = kwargs.pop("trust_remote_code", True) + kwargs.pop("_from_auto", None) + codec_path = kwargs.pop("codec_path", None) + codec_weight_dtype = kwargs.pop("codec_weight_dtype", "fp32") + codec_compute_dtype = kwargs.pop("codec_compute_dtype", None) + codec_attention_implementation = kwargs.pop("codec_attention_implementation", None) + + codec_kwargs: Dict[str, Any] = {} + if codec_weight_dtype is not None: + # Default to fp32 codec weights; callers can pass codec_weight_dtype="bf16" to reduce memory. + codec_kwargs["codec_weight_dtype"] = codec_weight_dtype + + model_ref = Path(str(pretrained_model_name_or_path)) + model_ref_or_name = model_ref if model_ref.exists() else pretrained_model_name_or_path + model_config = cast( + MossTTSLocalConfig, + AutoConfig.from_pretrained( + model_ref_or_name, + *args, + trust_remote_code=trust_remote_code, + **kwargs, + ), + ) + + if codec_path is None: + try: + processor_dict, _ = cls.get_processor_dict( + pretrained_model_name_or_path, + **dict(kwargs), + ) + codec_path = processor_dict.get("audio_tokenizer_name_or_path") + audio_tokenizer_dict = processor_dict.get("audio_tokenizer", {}) + if isinstance(audio_tokenizer_dict, dict): + codec_path = audio_tokenizer_dict.get("audio_tokenizer_name_or_path") or codec_path + except Exception: + codec_path = None + if codec_path is None: + codec_path = getattr(model_config, "audio_tokenizer_name_or_path", None) + if codec_path is None: + codec_path = "OpenMOSS-Team/MOSS-Audio-Tokenizer-v2" + + tokenizer = AutoTokenizer.from_pretrained( + model_ref_or_name, + *args, + trust_remote_code=trust_remote_code, + **kwargs, + ) + audio_tokenizer = AutoModel.from_pretrained( + codec_path, + trust_remote_code=trust_remote_code, + **kwargs, + **codec_kwargs, + ) + if codec_compute_dtype is not None and hasattr(audio_tokenizer, "set_compute_dtype"): + audio_tokenizer.set_compute_dtype(codec_compute_dtype) + if codec_attention_implementation is not None and hasattr( + audio_tokenizer, "set_attention_implementation" + ): + audio_tokenizer.set_attention_implementation(codec_attention_implementation) + return cls( + tokenizer=tokenizer, + audio_tokenizer=audio_tokenizer, + model_config=model_config, + **kwargs, + ) + + @staticmethod + def build_user_message( + text: Optional[str] = None, + reference: Optional[List[Optional[Union[str, os.PathLike, torch.Tensor]]]] = None, + instruction: Optional[str] = None, + tokens: Optional[int] = None, + quality: Optional[str] = None, + sound_event: Optional[str] = None, + ambient_sound: Optional[str] = None, + language: Optional[str] = None, + ) -> Dict[str, Any]: + if reference is not None and not isinstance(reference, list): + reference = [reference] + return UserMessage( + text=text, + reference=reference, + instruction=instruction, + tokens=tokens, + quality=quality, + sound_event=sound_event, + ambient_sound=ambient_sound, + language=language, + ).to_dict() + + @staticmethod + def build_assistant_message( + audio_codes_list: List[Union[str, os.PathLike, torch.Tensor]], + content: str = AUDIO_PLACEHOLDER, + ) -> Dict[str, Any]: + return AssistantMessage(audio_codes_list=audio_codes_list, content=content).to_dict() + + def _assert_fixed_nq(self, n_vq: Optional[int]) -> int: + config_nq = int(self.model_config.n_vq) + if n_vq is not None and int(n_vq) != config_nq: + raise ValueError( + "This MOSS-TTS-Local-Transformer-v1.5 release uses the RVQ depth stored in the model config. " + f"Expected n_vq={config_nq}, got {int(n_vq)}." + ) + return config_nq + + def _encode_text(self, text: str) -> list[int]: + try: + return list(self.tokenizer.encode(text, add_special_tokens=False)) + except TypeError: + return list(self.tokenizer.encode(text)) + + def _build_text_rows(self, token_ids: Sequence[int], *, device: Optional[torch.device] = None) -> torch.Tensor: + rows = torch.full( + (len(token_ids), int(self.model_config.n_vq) + 1), + int(self.model_config.audio_pad_token_id), + dtype=torch.long, + device=device, + ) + if token_ids: + rows[:, 0] = torch.tensor([int(token_id) for token_id in token_ids], dtype=torch.long, device=rows.device) + return rows + + def _build_audio_rows(self, audio_tokens: torch.Tensor, slot_token_id: int) -> torch.Tensor: + rows = torch.full( + (int(audio_tokens.shape[0]), int(self.model_config.n_vq) + 1), + int(self.model_config.audio_pad_token_id), + dtype=torch.long, + device=audio_tokens.device, + ) + if rows.shape[0] > 0: + rows[:, 0] = int(slot_token_id) + rows[:, 1:] = audio_tokens.to(dtype=torch.long) + return rows + + def _user_prompt_prefix_ids(self) -> list[int]: + return ( + [int(self.model_config.im_start_token_id)] + + self._encode_text(USER_ROLE_PREFIX) + + self._encode_text(USER_TEMPLATE_REFERENCE_PREFIX) + ) + + def _user_prompt_after_reference_ids( + self, + language_code: object | None, + prompt_fields: Optional[Dict[str, Any]], + ) -> list[int]: + return self._encode_text( + _render_user_prompt_after_reference( + language_code=language_code, + prompt_fields=prompt_fields, + ) + ) + + def _assistant_prompt_prefix_ids(self) -> list[int]: + return ( + self._encode_text(USER_TEMPLATE_SUFFIX) + + [int(self.model_config.im_end_token_id)] + + self._encode_text(ASSISTANT_TURN_PREFIX) + + [int(self.model_config.im_start_token_id)] + + self._encode_text(ASSISTANT_ROLE_PREFIX) + ) + + def _prompt_fields_from_user_message(self, message: Dict[str, Any]) -> dict[str, Any]: + fields = {} + for key in ("instruction", "tokens", "quality", "sound_event", "ambient_sound"): + if key in message and message.get(key) is not None: + fields[key] = message.get(key) + if "language" in message and message.get("language") is not None: + fields["language"] = message.get("language") + return fields + + def _build_generation_or_voice_clone_codes( + self, + message: Dict[str, Any], + n_vq: int, + ) -> torch.Tensor: + if "text" not in message: + raise ValueError("Direct MOSS-TTS-Local-Transformer-v1.5 generation requires messages built by build_user_message(...).") + text = "" if message.get("text") is None else str(message.get("text")) + prompt_fields = self._prompt_fields_from_user_message(message) + language_code = message.get("language") + audio_codes_list = self._resolve_audio_items(message.get("audio_codes_list", []), n_vq) + text_token_ids = self._encode_text(text) + + if audio_codes_list: + parts: list[torch.Tensor] = [self._build_text_rows( + self._user_prompt_prefix_ids(), + device=audio_codes_list[0].device, + )] + for reference_codes in audio_codes_list: + parts.append(self._build_text_rows([int(self.model_config.audio_start_token_id)], device=reference_codes.device)) + parts.append(self._build_audio_rows(reference_codes, int(self.model_config.audio_user_slot_token_id))) + parts.append(self._build_text_rows([int(self.model_config.audio_end_token_id)], device=reference_codes.device)) + parts.append( + self._build_text_rows( + self._user_prompt_after_reference_ids(language_code, prompt_fields) + + text_token_ids + + self._assistant_prompt_prefix_ids() + + [int(self.model_config.audio_start_token_id)], + device=audio_codes_list[0].device, + ) + ) + return torch.cat(parts, dim=0) + + prompt_token_ids = ( + self._user_prompt_prefix_ids() + + self._encode_text("None") + + self._user_prompt_after_reference_ids(language_code, prompt_fields) + + text_token_ids + + self._assistant_prompt_prefix_ids() + + [int(self.model_config.audio_start_token_id)] + ) + return self._build_text_rows(prompt_token_ids) + + def _build_continuation_codes( + self, + conversation: list[Dict[str, Any]], + n_vq: int, + ) -> torch.Tensor: + if len(conversation) < 2: + raise ValueError("continuation mode requires a user message followed by an assistant audio message.") + user_message = conversation[-2] + assistant_message = conversation[-1] + if user_message.get("role") != "user" or assistant_message.get("role") != "assistant": + raise ValueError("continuation mode requires the last two messages to be user, assistant.") + if "text" not in user_message: + raise ValueError("Direct MOSS-TTS-Local-Transformer-v1.5 continuation requires user messages built by build_user_message(...).") + + text = "" if user_message.get("text") is None else str(user_message.get("text")) + prompt_fields = self._prompt_fields_from_user_message(user_message) + language_code = user_message.get("language") + prompt_token_ids = ( + self._user_prompt_prefix_ids() + + self._encode_text("None") + + self._user_prompt_after_reference_ids(language_code, prompt_fields) + + self._encode_text(text) + + self._assistant_prompt_prefix_ids() + + [int(self.model_config.audio_start_token_id)] + ) + audio_codes_list = self._resolve_audio_items(assistant_message.get("audio_codes_list", []), n_vq) + if not audio_codes_list: + return self._build_text_rows(prompt_token_ids) + if len(audio_codes_list) != 1: + raise ValueError("The MOSS-TTS-Local-Transformer-v1.5 continuation path expects exactly one prompt audio item.") + prompt_audio_codes = audio_codes_list[0] + return torch.cat( + [ + self._build_text_rows(prompt_token_ids, device=prompt_audio_codes.device), + self._build_audio_rows(prompt_audio_codes, int(self.model_config.audio_assistant_slot_token_id)), + ], + dim=0, + ) + + def _try_build_direct_codes( + self, + conversation: list[Dict[str, Any]], + mode: str, + n_vq: int, + ) -> Optional[torch.Tensor]: + if mode == "generation" and len(conversation) == 1 and conversation[-1].get("role") == "user": + if "text" in conversation[-1]: + return self._build_generation_or_voice_clone_codes(conversation[-1], n_vq) + return None + if mode == "continuation" and len(conversation) >= 2: + if "text" in conversation[-2]: + return self._build_continuation_codes(conversation, n_vq) + return None + return None + + def __call__(self, *args, **kwargs) -> BatchFeature: + conversations = args[0] if args else kwargs.pop("conversations") + mode: str = kwargs.pop("mode", "generation") + apply_chat_template: bool = kwargs.pop("apply_chat_template", True) + n_vq = self._assert_fixed_nq(kwargs.pop("n_vq", None)) + + kwargs.pop("return_tensors", None) + kwargs.pop("padding", None) + kwargs.pop("truncation", None) + + if mode not in {"generation", "continuation", "computing_loss"}: + raise ValueError(f"Unsupported mode: {mode}") + if isinstance(conversations, (Message, dict)): + conversations = [conversations] + elif isinstance(conversations, list) and conversations and all( + isinstance(item, (Message, dict)) for item in conversations + ): + conversations = [conversations] + + input_ids_list: list[torch.Tensor] = [] + for conversation in conversations: + if isinstance(conversation, (Message, dict)): + conversation = [conversation] + conversation = [self._normalize_message(message) for message in conversation] + + if (mode == "generation") ^ (conversation[-1]["role"] == "user"): + raise ValueError("generation mode must end with a user message.") + if mode == "continuation" and conversation[-1]["role"] != "assistant": + raise ValueError("continuation mode must end with an assistant message.") + + direct_codes = self._try_build_direct_codes(conversation, mode, n_vq) + if direct_codes is not None: + input_ids_list.append(direct_codes) + continue + + unified_parts = [] + for message_idx, message in enumerate(conversation): + content = str(message["content"]) + if apply_chat_template: + add_generation_prompt = mode == "generation" and message_idx == len(conversation) - 1 + try: + content = self.tokenizer.apply_chat_template( + [{"role": message["role"], "content": content}], + add_generation_prompt=add_generation_prompt, + tokenize=False, + ) + except Exception: + logger.warning("apply_chat_template failed; falling back to raw message content.") + + raw_audio_items = message.get("audio_codes_list", []) + audio_codes_list = self._resolve_audio_items(raw_audio_items, n_vq) + unified_parts.append( + self._get_unified_codes( + role=message["role"], + content=content, + audio_codes_list=audio_codes_list, + truncation=(mode == "continuation"), + ) + ) + + unified_codes = torch.cat(unified_parts, dim=0) + if mode == "generation": + audio_start_row = torch.full( + (1, n_vq + 1), + int(self.model_config.audio_pad_token_id), + dtype=unified_codes.dtype, + device=unified_codes.device, + ) + audio_start_row[:, 0] = int(self.model_config.audio_start_token_id) + unified_codes = torch.cat([unified_codes, audio_start_row], dim=0) + input_ids_list.append(unified_codes) + + return BatchFeature(data=self._pad(input_ids_list)) + + def _normalize_message(self, message: Union[Message, Dict[str, Any]]) -> Dict[str, Any]: + if isinstance(message, Message): + return message.to_dict() + if not isinstance(message, dict): + raise TypeError("Each message must be a Message or dict.") + if "content" in message and "audio_codes_list" in message: + return message + role = message.get("role") + if role == "user": + return self.build_user_message(**{key: message.get(key) for key in USER_MESSAGE_FIELDS}) + if role == "assistant": + return self.build_assistant_message( + audio_codes_list=message.get("audio_codes_list", []), + content=message.get("content", AUDIO_PLACEHOLDER), + ) + raise ValueError(f"Unsupported role: {role}") + + def _resolve_audio_items( + self, + raw_audio_items: list[Any], + n_vq: int, + ) -> list[torch.Tensor]: + if not raw_audio_items: + return [] + resolved: list[Optional[torch.Tensor]] = [None] * len(raw_audio_items) + paths: list[str] = [] + path_positions: list[int] = [] + for index, item in enumerate(raw_audio_items): + if isinstance(item, torch.Tensor): + if item.ndim != 2 or int(item.shape[1]) != n_vq: + raise ValueError(f"audio code tensor must have shape [T, {n_vq}], got {tuple(item.shape)}.") + resolved[index] = item.to(dtype=torch.long).cpu() + elif isinstance(item, (str, os.PathLike)): + paths.append(str(item)) + path_positions.append(index) + else: + raise TypeError("Audio items must be tensors or path-like values.") + if paths: + encoded = self.encode_audios_from_path(paths, n_vq=n_vq) + for position, codes in zip(path_positions, encoded): + resolved[position] = codes + return [cast(torch.Tensor, item) for item in resolved] + + def _pad(self, input_ids_list: list[torch.Tensor]) -> Dict[str, torch.Tensor]: + device = input_ids_list[0].device + lengths = torch.tensor([item.shape[0] for item in input_ids_list], device=device) + padded = torch.nn.utils.rnn.pad_sequence( + input_ids_list, + batch_first=True, + padding_value=int(self.model_config.audio_pad_token_id), + padding_side="left", + ) + left_pad_mask = (padded.shape[1] - lengths).unsqueeze(1) > torch.arange( + padded.shape[1], + device=device, + ).unsqueeze(0) + padded[..., 0][left_pad_mask] = int(self.model_config.pad_token_id) + attention_mask = torch.zeros(padded.shape[:2], dtype=torch.bool, device=device) + attention_mask[~left_pad_mask] = True + return {"input_ids": padded, "attention_mask": attention_mask} + + @staticmethod + def _replace_audio_placeholders( + content: str, + lengths: list[int], + slot_token: str, + audio_start_token: str, + audio_end_token: str, + ) -> str: + placeholder_count = content.count(AUDIO_PLACEHOLDER) + if placeholder_count != len(lengths): + raise ValueError( + f"Number of {AUDIO_PLACEHOLDER} ({placeholder_count}) does not match " + f"audio item count ({len(lengths)})." + ) + lengths_iter = iter(lengths) + + def replacer(_: re.Match) -> str: + length = int(next(lengths_iter)) + if length <= 0: + return f"{audio_start_token}{audio_end_token}" + return f"{audio_start_token}{slot_token * length}{audio_end_token}" + + return re.sub(re.escape(AUDIO_PLACEHOLDER), replacer, content) + + def _get_unified_codes( + self, + role: str, + content: str, + audio_codes_list: list[torch.Tensor], + truncation: bool, + ) -> torch.Tensor: + n_vq = int(self.model_config.n_vq) + slot_token = self.audio_user_slot_token if role == "user" else self.audio_assistant_slot_token + content = self._replace_audio_placeholders( + content=content, + lengths=[int(codes.shape[0]) for codes in audio_codes_list], + slot_token=slot_token, + audio_start_token=self.audio_start_token, + audio_end_token=self.audio_end_token, + ) + text_codes = torch.tensor( + self.tokenizer.encode(content), + dtype=torch.long, + device=audio_codes_list[0].device if audio_codes_list else None, + ) + + audio_start_indices = torch.where(text_codes == int(self.model_config.audio_start_token_id))[0] + audio_end_indices = torch.where(text_codes == int(self.model_config.audio_end_token_id))[0] + if len(audio_start_indices) != len(audio_codes_list) or len(audio_end_indices) != len(audio_codes_list): + raise ValueError("Audio placeholders do not match the encoded audio spans.") + + if not audio_codes_list: + audio_codes = torch.full( + (len(text_codes), n_vq), + int(self.model_config.audio_pad_token_id), + dtype=torch.long, + device=text_codes.device, + ) + else: + pieces: list[torch.Tensor] = [] + prefix_idx = 0 + for start_t, end_t, codes in zip(audio_start_indices, audio_end_indices, audio_codes_list): + start_idx = int(start_t.item()) + end_idx = int(end_t.item()) + pad_before = torch.full( + (start_idx - prefix_idx + 1, n_vq), + int(self.model_config.audio_pad_token_id), + dtype=torch.long, + device=codes.device, + ) + pieces.extend([pad_before, codes.to(dtype=torch.long)]) + prefix_idx = end_idx + if truncation: + trailing = torch.zeros( + (0, n_vq), + dtype=torch.long, + device=audio_codes_list[0].device, + ) + else: + last_end = int(audio_end_indices[-1].item()) + trailing = torch.full( + (len(text_codes) - last_end, n_vq), + int(self.model_config.audio_pad_token_id), + dtype=torch.long, + device=audio_codes_list[0].device, + ) + pieces.append(trailing) + audio_codes = torch.cat(pieces, dim=0) + + if text_codes.shape[0] != audio_codes.shape[0]: + min_len = min(text_codes.shape[0], audio_codes.shape[0]) + text_codes = text_codes[:min_len] + audio_codes = audio_codes[:min_len] + return torch.cat([text_codes.unsqueeze(1), audio_codes], dim=1) + + def _parse_text_codes(self, start_length: int, text_codes: torch.LongTensor) -> str: + text = cast(str, self.tokenizer.decode(text_codes)) + prefix = cast(str, self.tokenizer.decode(text_codes[:start_length])) + text = text[len(prefix):] + audio_pattern = re.compile( + rf"(?:{re.escape(self.audio_start_token)})?" + rf"(?:{re.escape(self.audio_assistant_slot_token)})*" + rf"{re.escape(self.audio_end_token)}" + ) + return audio_pattern.sub( + lambda match: AUDIO_PLACEHOLDER if self.audio_assistant_slot_token in match.group(0) else "", + text, + ) + + def _parse_audio_codes( + self, + start_length: int, + audio_codes: torch.LongTensor, + *, + return_stereo: bool = True, + ) -> list[torch.Tensor]: + is_pad = audio_codes.eq(int(self.model_config.audio_pad_token_id)).all(dim=1) + non_pad = ~is_pad + if not bool(non_pad.any().item()): + return [] + idx = torch.nonzero(non_pad).squeeze(1) + breaks = torch.where(idx[1:] != idx[:-1] + 1)[0] + 1 + segment_indices = [idx] if breaks.numel() == 0 else list(torch.tensor_split(idx, breaks.cpu().tolist())) + code_segments = [audio_codes[segment] for segment in segment_indices] + decoded = self.decode_audio_codes(code_segments, return_stereo=return_stereo) + + if start_length > 0 and code_segments and decoded: + first_code_length = int(code_segments[0].shape[0]) + if first_code_length > 0: + trim_ratio = max(0.0, min(float(start_length) / float(first_code_length), 1.0)) + if trim_ratio >= 1.0: + decoded = decoded[1:] + elif trim_ratio > 0.0: + trim_samples = int(decoded[0].shape[-1] * trim_ratio) + decoded[0] = decoded[0][..., trim_samples:] + return decoded + + def decode(self, output: Any, *, return_stereo: bool = True) -> list[Optional[AssistantMessage]]: + generated_messages: list[Optional[AssistantMessage]] = [] + for start_length, generation_ids in output: + content = self._parse_text_codes(int(start_length), generation_ids[:, 0]) + audio_codes_list = self._parse_audio_codes( + int(start_length), + generation_ids[:, 1:], + return_stereo=return_stereo, + ) + if content == "": + generated_messages.append(None) + else: + generated_messages.append( + AssistantMessage( + content=content, + audio_codes_list=cast(list[Union[str, torch.Tensor]], audio_codes_list), + ) + ) + return generated_messages + + @staticmethod + def loudness_normalize( + wav: torch.Tensor, + target_dbfs: float = -20.0, + gain_range: tuple[float, float] = (-3.0, 3.0), + ) -> torch.Tensor: + wav = wav.to(torch.float32) + if wav.numel() == 0: + return wav + current_dbfs = 10.0 * torch.log10(torch.mean(wav**2) + 1e-9) + gain = max(gain_range[0], min(float(target_dbfs - current_dbfs), gain_range[1])) + return wav * (10.0 ** (gain / 20.0)) + + def _get_audio_tokenizer_device(self) -> torch.device: + audio_tokenizer = getattr(self, "audio_tokenizer", None) + if audio_tokenizer is None: + raise RuntimeError("audio_tokenizer is not set.") + try: + return next(audio_tokenizer.parameters()).device + except StopIteration: + return torch.device("cpu") + + def encode_audios_from_wav( + self, + wav_list: Union[torch.Tensor, list[torch.Tensor]], + sampling_rate: int, + n_vq: Optional[int] = None, + ) -> list[torch.Tensor]: + n_vq = self._assert_fixed_nq(n_vq) + if self.audio_tokenizer is None: + raise RuntimeError("audio_tokenizer is not set.") + if isinstance(wav_list, torch.Tensor): + wav_list = [wav_list] + target_sr = int(self.model_config.sampling_rate) + device = self._get_audio_tokenizer_device() + prepared = [] + for wav in wav_list: + if wav.ndim == 1: + wav = wav.unsqueeze(0) + if wav.shape[0] == 1: + wav = wav.repeat(2, 1) + elif wav.shape[0] > 2: + wav = wav[:2] + if int(sampling_rate) != target_sr: + wav = torchaudio.functional.resample(wav, int(sampling_rate), target_sr) + prepared.append(self.loudness_normalize(wav).to(device)) + + if hasattr(self.audio_tokenizer, "batch_encode"): + encoded = self.audio_tokenizer.batch_encode(prepared, num_quantizers=n_vq) + audio_codes = encoded.audio_codes + audio_lengths = encoded.audio_codes_lengths + else: + max_len = max(int(wav.shape[-1]) for wav in prepared) + input_values = torch.zeros(len(prepared), 1, max_len, dtype=torch.float32, device=device) + padding_mask = torch.zeros(len(prepared), max_len, dtype=torch.bool, device=device) + for index, wav in enumerate(prepared): + input_values[index, 0, : wav.shape[-1]] = wav + padding_mask[index, : wav.shape[-1]] = True + encoded = self.audio_tokenizer.encode( + input_values, + padding_mask=padding_mask, + num_quantizers=n_vq, + return_dict=True, + ) + audio_codes = encoded.audio_codes + audio_lengths = encoded.audio_codes_lengths + + if audio_codes is None or audio_lengths is None: + raise RuntimeError("audio_tokenizer did not return audio_codes/audio_codes_lengths.") + result = [] + for index in range(int(audio_codes.shape[1])): + length = int(audio_lengths[index].item()) + result.append(audio_codes[:, index, :length].transpose(0, 1).contiguous().cpu().long()) + return result + + def encode_audios_from_path( + self, + wav_path_list: Union[str, os.PathLike, list[Union[str, os.PathLike]]], + n_vq: Optional[int] = None, + ) -> list[torch.Tensor]: + if isinstance(wav_path_list, (str, os.PathLike)): + wav_path_list = [wav_path_list] + wavs = [] + target_sr = int(self.model_config.sampling_rate) + for wav_path in wav_path_list: + wav, sr = torchaudio.load(str(wav_path)) + if int(sr) != target_sr: + wav = torchaudio.functional.resample(wav, int(sr), target_sr) + wavs.append(wav) + return self.encode_audios_from_wav(wavs, target_sr, n_vq=n_vq) + + def decode_audio_codes( + self, + audio_tokens_list: Union[torch.Tensor, list[torch.Tensor]], + *, + return_stereo: bool = True, + ) -> list[torch.Tensor]: + if self.audio_tokenizer is None: + raise RuntimeError("audio_tokenizer is not set.") + if isinstance(audio_tokens_list, torch.Tensor): + audio_tokens_list = [audio_tokens_list] + if not audio_tokens_list: + return [] + + n_vq = int(self.model_config.n_vq) + device = self._get_audio_tokenizer_device() + codes_list = [ + codes[:, :n_vq].transpose(0, 1).contiguous().to(device=device, dtype=torch.long) + for codes in audio_tokens_list + ] + max_len = max(int(codes.shape[1]) for codes in codes_list) + audio_codes = torch.zeros(n_vq, len(codes_list), max_len, device=device, dtype=torch.long) + padding_mask = torch.zeros(len(codes_list), max_len, device=device, dtype=torch.bool) + for index, codes in enumerate(codes_list): + length = int(codes.shape[1]) + audio_codes[:, index, :length] = codes + padding_mask[index, :length] = True + + decoded = self.audio_tokenizer.decode( + audio_codes, + padding_mask=padding_mask, + num_quantizers=n_vq, + return_dict=True, + chunk_duration=8, + ) + audio = decoded.audio + audio_lengths = decoded.audio_lengths + if audio is None or audio_lengths is None: + raise RuntimeError("audio_tokenizer.decode did not return audio/audio_lengths.") + wavs = [] + for index in range(int(audio.shape[0])): + length = int(audio_lengths[index].item()) + wav = audio[index, :, :length].contiguous().cpu().to(torch.float32) + if not return_stereo: + if wav.shape[0] == 1: + wav = wav.squeeze(0) + else: + wav = wav.mean(dim=0) + wavs.append(wav) + return wavs diff --git a/processor_config.json b/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..afcaf45a8830d9129021cdf16da0ee85d983af47 --- /dev/null +++ b/processor_config.json @@ -0,0 +1,7 @@ +{ + "processor_class": "MossTTSLocalProcessor", + "auto_map": { + "AutoProcessor": "processing_moss_tts.MossTTSLocalProcessor" + }, + "audio_tokenizer_name_or_path": "OpenMOSS-Team/MOSS-Audio-Tokenizer-v2" +} \ No newline at end of file diff --git a/qwen3_decoder.py b/qwen3_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..3756bd7033d88388d51a9e64d2fa0efd71959e84 --- /dev/null +++ b/qwen3_decoder.py @@ -0,0 +1,582 @@ +# coding=utf-8 +from __future__ import annotations + +import json +from pathlib import Path +from typing import Optional + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from safetensors.torch import load_file +from transformers.activations import ACT2FN +from transformers.modeling_outputs import BaseModelOutputWithPast + +from .gpt2_decoder import PackedSequenceMetadata, MossTTSNanoGPT2Model + +try: + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import pad_input, unpad_input + + _FLASH_ATTN_AVAILABLE = True +except Exception: + flash_attn_func = None + flash_attn_varlen_func = None + pad_input = None + unpad_input = None + _FLASH_ATTN_AVAILABLE = False + + +class MossQwen3RMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +class MossQwen3RotaryEmbedding(nn.Module): + def __init__(self, config) -> None: + super().__init__() + head_dim = int(getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)) + rope_theta = getattr(config, "rope_theta", None) + if rope_theta is None: + rope_scaling = getattr(config, "rope_scaling", None) + if isinstance(rope_scaling, dict): + rope_theta = rope_scaling.get("rope_theta") + if rope_theta is None: + rope_theta = 1000000.0 + rope_theta = float(rope_theta) + self.head_dim = head_dim + self.rope_theta = rope_theta + self.register_buffer("inv_freq", self._compute_inv_freq(), persistent=False) + + def _compute_inv_freq(self, device: Optional[torch.device] = None) -> torch.Tensor: + return 1.0 / ( + self.rope_theta ** (torch.arange(0, self.head_dim, 2, device=device, dtype=torch.float32) / self.head_dim) + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: torch.LongTensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + inv_freq = self._compute_inv_freq(device=hidden_states.device) + freqs = torch.einsum( + "bs,d->bsd", + position_ids.to(device=hidden_states.device, dtype=inv_freq.dtype), + inv_freq, + ) + emb = torch.cat((freqs, freqs), dim=-1) + return emb.cos().to(dtype=hidden_states.dtype), emb.sin().to(dtype=hidden_states.dtype) + + +def rotate_half(hidden_states: torch.Tensor) -> torch.Tensor: + first_half = hidden_states[..., : hidden_states.shape[-1] // 2] + second_half = hidden_states[..., hidden_states.shape[-1] // 2 :] + return torch.cat((-second_half, first_half), dim=-1) + + +def apply_rotary_pos_emb( + query: torch.Tensor, + key: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + cos = cos.unsqueeze(-2) + sin = sin.unsqueeze(-2) + query = (query * cos) + (rotate_half(query) * sin) + key = (key * cos) + (rotate_half(key) * sin) + return query, key + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + if n_rep == 1: + return hidden_states + batch, seq_len, num_key_value_heads, head_dim = hidden_states.shape + hidden_states = hidden_states[:, :, :, None, :].expand(batch, seq_len, num_key_value_heads, n_rep, head_dim) + return hidden_states.reshape(batch, seq_len, num_key_value_heads * n_rep, head_dim) + + +class MossQwen3MLP(nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) + self.act_fn = ACT2FN[getattr(config, "hidden_act", "silu")] + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states)) + + +class MossQwen3Attention(nn.Module): + def __init__(self, config, layer_idx: int) -> None: + super().__init__() + self.config = config + self.layer_idx = int(layer_idx) + self.hidden_size = int(config.hidden_size) + self.num_heads = int(config.num_attention_heads) + self.num_key_value_heads = int(config.num_key_value_heads) + self.head_dim = int(getattr(config, "head_dim", self.hidden_size // self.num_heads)) + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.scaling = self.head_dim ** -0.5 + self.attention_dropout = float(getattr(config, "attention_dropout", 0.0)) + self.attn_implementation = str(getattr(config, "_attn_implementation", "eager")) + + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=bool(config.attention_bias)) + self.k_proj = nn.Linear( + self.hidden_size, + self.num_key_value_heads * self.head_dim, + bias=bool(config.attention_bias), + ) + self.v_proj = nn.Linear( + self.hidden_size, + self.num_key_value_heads * self.head_dim, + bias=bool(config.attention_bias), + ) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=bool(config.attention_bias)) + self.q_norm = MossQwen3RMSNorm(self.head_dim, eps=float(config.rms_norm_eps)) + self.k_norm = MossQwen3RMSNorm(self.head_dim, eps=float(config.rms_norm_eps)) + + def _causal_attention_mask( + self, + attention_mask: Optional[torch.Tensor], + query_length: int, + key_length: int, + device: torch.device, + ) -> torch.Tensor: + query_positions = torch.arange(query_length, device=device, dtype=torch.long) + query_positions = query_positions + max(key_length - query_length, 0) + key_positions = torch.arange(key_length, device=device, dtype=torch.long) + causal = key_positions.unsqueeze(0) <= query_positions.unsqueeze(1) + causal = causal.unsqueeze(0).unsqueeze(0) + if attention_mask is None: + return causal + key_mask = attention_mask[:, None, None, :].to(dtype=torch.bool) + return causal & key_mask + + def _eager_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + key = repeat_kv(key, self.num_key_value_groups) + value = repeat_kv(value, self.num_key_value_groups) + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + scores = torch.matmul(query, key.transpose(-1, -2)) * self.scaling + mask = self._causal_attention_mask( + attention_mask=attention_mask, + query_length=query.shape[-2], + key_length=key.shape[-2], + device=query.device, + ) + scores = scores.masked_fill(~mask, torch.finfo(scores.dtype).min) + probs = torch.softmax(scores, dim=-1) + if self.training and self.attention_dropout > 0: + probs = torch.dropout(probs, self.attention_dropout, train=True) + output = torch.matmul(probs, value) + return output.transpose(1, 2).contiguous() + + def _sdpa_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + key = repeat_kv(key, self.num_key_value_groups) + value = repeat_kv(value, self.num_key_value_groups) + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + mask = None + if attention_mask is not None or query.shape[-2] != key.shape[-2]: + mask = self._causal_attention_mask( + attention_mask=attention_mask, + query_length=query.shape[-2], + key_length=key.shape[-2], + device=query.device, + ) + output = torch.nn.functional.scaled_dot_product_attention( + query, + key, + value, + attn_mask=mask, + dropout_p=self.attention_dropout if self.training else 0.0, + is_causal=mask is None, + scale=self.scaling, + ) + return output.transpose(1, 2).contiguous() + + def _flash_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + packed_metadata: Optional[PackedSequenceMetadata], + ) -> torch.Tensor: + if not _FLASH_ATTN_AVAILABLE: + raise ImportError("flash_attn is not installed, but attn_implementation='flash_attention_2' was requested.") + if query.device.type != "cuda": + raise ValueError("flash_attention_2 requires CUDA tensors.") + if query.dtype not in (torch.float16, torch.bfloat16): + raise ValueError(f"flash_attention_2 requires fp16/bf16 tensors, got dtype={query.dtype}.") + + dropout_p = self.attention_dropout if self.training else 0.0 + if packed_metadata is not None: + if packed_metadata.indices is not None: + query = query.reshape(-1, self.num_heads, self.head_dim).index_select(0, packed_metadata.indices) + key = key.reshape(-1, self.num_key_value_heads, self.head_dim).index_select(0, packed_metadata.indices) + value = value.reshape(-1, self.num_key_value_heads, self.head_dim).index_select(0, packed_metadata.indices) + output = flash_attn_varlen_func( + query, + key, + value, + packed_metadata.cu_seqlens, + packed_metadata.cu_seqlens, + packed_metadata.max_seqlen, + packed_metadata.max_seqlen, + dropout_p=dropout_p, + causal=True, + ) + if packed_metadata.indices is None: + return output + return pad_input( + output, + packed_metadata.indices, + packed_metadata.batch_size, + packed_metadata.seq_len, + ) + + if attention_mask is None or bool(attention_mask.all()): + return flash_attn_func(query, key, value, dropout_p=dropout_p, causal=True) + + if query.shape[1] != key.shape[1]: + query_attention_mask = attention_mask[:, -query.shape[1] :] + unpadded_query, query_indices, cu_seqlens_q, max_seqlen_q, _ = unpad_input( + query, + query_attention_mask, + ) + unpadded_key, _, cu_seqlens_k, max_seqlen_k, _ = unpad_input(key, attention_mask) + unpadded_value, _, _, _, _ = unpad_input(value, attention_mask) + output = flash_attn_varlen_func( + unpadded_query, + unpadded_key, + unpadded_value, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + dropout_p=dropout_p, + causal=True, + ) + return pad_input(output, query_indices, query.shape[0], query.shape[1]) + + unpadded_query, indices, cu_seqlens, max_seqlen, _ = unpad_input(query, attention_mask) + unpadded_key, _, _, _, _ = unpad_input(key, attention_mask) + unpadded_value, _, _, _, _ = unpad_input(value, attention_mask) + output = flash_attn_varlen_func( + unpadded_query, + unpadded_key, + unpadded_value, + cu_seqlens, + cu_seqlens, + max_seqlen, + max_seqlen, + dropout_p=dropout_p, + causal=True, + ) + return pad_input(output, indices, query.shape[0], query.shape[1]) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + packed_metadata: Optional[PackedSequenceMetadata] = None, + layer_past: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + ) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, torch.Tensor]]]: + input_shape = hidden_states.shape[:-1] + query_states = self.q_norm( + self.q_proj(hidden_states).view(*input_shape, self.num_heads, self.head_dim) + ) + key_states = self.k_norm( + self.k_proj(hidden_states).view(*input_shape, self.num_key_value_heads, self.head_dim) + ) + value_states = self.v_proj(hidden_states).view(*input_shape, self.num_key_value_heads, self.head_dim) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if layer_past is not None: + past_key, past_value = layer_past + key_states = torch.cat([past_key.to(device=key_states.device, dtype=key_states.dtype), key_states], dim=1) + value_states = torch.cat( + [past_value.to(device=value_states.device, dtype=value_states.dtype), value_states], + dim=1, + ) + + present = (key_states, value_states) if use_cache else None + if self.attn_implementation == "flash_attention_2": + attn_output = self._flash_attention( + query=query_states, + key=key_states, + value=value_states, + attention_mask=attention_mask, + packed_metadata=packed_metadata, + ) + elif self.attn_implementation == "sdpa": + attn_output = self._sdpa_attention( + query=query_states, + key=key_states, + value=value_states, + attention_mask=attention_mask, + ) + else: + attn_output = self._eager_attention( + query=query_states, + key=key_states, + value=value_states, + attention_mask=attention_mask, + ) + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + return self.o_proj(attn_output), present + + +class MossQwen3DecoderLayer(nn.Module): + def __init__(self, config, layer_idx: int) -> None: + super().__init__() + self.self_attn = MossQwen3Attention(config=config, layer_idx=layer_idx) + self.mlp = MossQwen3MLP(config) + self.input_layernorm = MossQwen3RMSNorm(config.hidden_size, eps=float(config.rms_norm_eps)) + self.post_attention_layernorm = MossQwen3RMSNorm(config.hidden_size, eps=float(config.rms_norm_eps)) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + packed_metadata: Optional[PackedSequenceMetadata] = None, + layer_past: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + ) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, torch.Tensor]]]: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + attn_output, present = self.self_attn( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + packed_metadata=packed_metadata, + layer_past=layer_past, + use_cache=use_cache, + ) + hidden_states = residual + attn_output + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = residual + self.mlp(hidden_states) + return hidden_states, present + + +class MossQwen3Model(nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.config = config + self.attn_implementation = str(getattr(config, "_attn_implementation", "eager")) + self.padding_idx = getattr(config, "pad_token_id", None) + self.vocab_size = int(config.vocab_size) + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [MossQwen3DecoderLayer(config, layer_idx=index) for index in range(config.num_hidden_layers)] + ) + self.norm = MossQwen3RMSNorm(config.hidden_size, eps=float(config.rms_norm_eps)) + self.rotary_emb = MossQwen3RotaryEmbedding(config) + self.gradient_checkpointing = False + self.gradient_checkpointing_use_reentrant = bool( + getattr(config, "gradient_checkpointing_use_reentrant", False) + ) + self._reset_parameters() + + def _reset_parameters(self) -> None: + init_std = float(getattr(self.config, "initializer_range", 0.02)) + for module in self.modules(): + if isinstance(module, nn.Linear): + nn.init.normal_(module.weight, mean=0.0, std=init_std) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, nn.Embedding): + nn.init.normal_(module.weight, mean=0.0, std=init_std) + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + def load_qwen3_pretrained_weights(self, pretrained_path: str) -> None: + model_dir = Path(pretrained_path) + index_path = model_dir / "model.safetensors.index.json" + if not index_path.exists(): + raise FileNotFoundError(f"Missing Qwen3 safetensors index: {index_path}") + with index_path.open("r", encoding="utf-8") as handle: + index = json.load(handle) + weight_map = index.get("weight_map", {}) + shard_to_keys: dict[str, list[str]] = {} + for key, shard in weight_map.items(): + if not key.startswith("model."): + continue + shard_to_keys.setdefault(str(shard), []).append(key) + + state_dict = self.state_dict() + loaded_state = {} + for shard, keys in sorted(shard_to_keys.items()): + shard_tensors = load_file(str(model_dir / shard), device="cpu") + for key in keys: + target_key = key[len("model.") :] + if target_key not in state_dict: + continue + tensor = shard_tensors[key] + if tuple(tensor.shape) != tuple(state_dict[target_key].shape): + raise ValueError( + f"Shape mismatch while loading Qwen3 weight {key}: " + f"checkpoint={tuple(tensor.shape)} model={tuple(state_dict[target_key].shape)}" + ) + loaded_state[target_key] = tensor + + missing, unexpected = self.load_state_dict(loaded_state, strict=False) + unexpected = [key for key in unexpected if key] + if unexpected: + raise RuntimeError(f"Unexpected Qwen3 pretrained keys after load: {unexpected[:10]}") + missing = [key for key in missing if key not in loaded_state] + if missing: + raise RuntimeError(f"Missing Qwen3 pretrained keys after load: {missing[:10]}") + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[tuple[tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: bool = True, + cu_seqlens: Optional[torch.Tensor] = None, + num_sequences: Optional[torch.Tensor] = None, + ) -> BaseModelOutputWithPast: + del input_ids, output_attentions + if inputs_embeds is None: + raise ValueError("inputs_embeds must be provided.") + + use_cache = bool(use_cache) + if use_cache and cu_seqlens is not None: + raise ValueError("use_cache=True is not supported together with cu_seqlens packing.") + + hidden_states = inputs_embeds + if attention_mask is None: + attention_mask = torch.ones(hidden_states.shape[:2], dtype=torch.bool, device=hidden_states.device) + else: + attention_mask = attention_mask.to(dtype=torch.bool, device=hidden_states.device) + query_attention_mask = attention_mask[:, -hidden_states.shape[1] :] + + packed_metadata = None + if position_ids is None: + if cu_seqlens is not None: + position_ids = MossTTSNanoGPT2Model.build_packed_position_ids( + attention_mask=attention_mask, + cu_seqlens=cu_seqlens.to(device=hidden_states.device), + num_sequences=num_sequences.to(device=hidden_states.device) if num_sequences is not None else None, + sequence_length=hidden_states.shape[1], + ) + elif attention_mask is not None: + position_ids = attention_mask.long().cumsum(dim=-1) - 1 + position_ids = position_ids.masked_fill(~attention_mask, 0) + position_ids = position_ids[:, -hidden_states.shape[1] :] + else: + past_length = 0 + if past_key_values is not None and len(past_key_values) > 0: + past_length = past_key_values[0][0].shape[1] + position_ids = torch.arange(hidden_states.shape[1], device=hidden_states.device, dtype=torch.long) + position_ids = position_ids + past_length + position_ids = position_ids.unsqueeze(0).expand(hidden_states.shape[0], -1) + + if cu_seqlens is not None and self.attn_implementation == "flash_attention_2": + packed_metadata = MossTTSNanoGPT2Model.build_packed_metadata( + hidden_states=hidden_states, + cu_seqlens=cu_seqlens.to(device=hidden_states.device), + num_sequences=num_sequences.to(device=hidden_states.device) if num_sequences is not None else None, + ) + + hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype) + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + all_hidden_states = () if output_hidden_states else None + presents = [] if use_cache else None + for layer_index, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if self.gradient_checkpointing and self.training: + if use_cache: + raise ValueError("use_cache=True is not supported when gradient checkpointing is enabled during training.") + + def custom_forward(*inputs): + output, _ = decoder_layer( + hidden_states=inputs[0], + attention_mask=inputs[1], + packed_metadata=packed_metadata, + layer_past=None, + use_cache=False, + position_embeddings=position_embeddings, + ) + return output + + hidden_states = torch.utils.checkpoint.checkpoint( + custom_forward, + hidden_states, + attention_mask, + use_reentrant=self.gradient_checkpointing_use_reentrant, + ) + present = None + else: + hidden_states, present = decoder_layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + packed_metadata=packed_metadata, + layer_past=None if past_key_values is None else past_key_values[layer_index], + use_cache=use_cache, + position_embeddings=position_embeddings, + ) + + hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype) + if presents is not None: + presents.append(present) + + hidden_states = self.norm(hidden_states) + hidden_states = hidden_states * query_attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype) + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return (hidden_states, tuple(presents) if presents is not None else None, all_hidden_states, None) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=tuple(presents) if presents is not None else None, + hidden_states=all_hidden_states, + attentions=None, + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..88abf351361f054eb36f09c6f71256259b192bc3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +torch>=2.9,<3 +torchaudio>=2.9,<3 +transformers==5.14.1 +safetensors>=0.8,<1 +numpy>=2,<3 +soundfile>=0.14,<1 +accelerate==1.14.0 +huggingface_hub>=1.29,<2 +einops>=0.8 +scipy +lameenc>=1.8 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a2a920bbe1a549d19078491a836432ca805caa22 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,21 @@ +{ + "additional_special_tokens": [ + "<|audio_start|>", + "<|audio_end|>", + "<|audio_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/stats/DATA_READY.json b/stats/DATA_READY.json new file mode 100644 index 0000000000000000000000000000000000000000..45c0acd11a0642eb6c5a512afb8ac54e1434cfeb --- /dev/null +++ b/stats/DATA_READY.json @@ -0,0 +1,218 @@ +{ + "status": "READY", + "pilot_only": false, + "selection_contract": { + "kind": "large13-notebooklm-multispeaker-v1", + "parent": "laion/Humaneness-Voice-Large-1.3-Alpha", + "base_step": 9799, + "base_weight_files": { + "model-00001-of-00003.safetensors": "3a9831cfd1944316e8b7d5fcaea845bf8b49b832824950709ce0eabf56b4702a", + "model-00002-of-00003.safetensors": "1fdaae7bd4f09b867152ef91761106e7bfef986b5f9979f69463466e864018a5", + "model-00003-of-00003.safetensors": "c149c250be927df24cdaa59c5b8ff50c3f0471f664f6483bb58236f07afa31df" + }, + "languages": [ + "en", + "de", + "fr", + "es", + "nl" + ], + "epochs": 3, + "world": 64, + "accumulation": 2, + "peaks": [ + 2e-05, + 4e-05 + ], + "warmup": 0.05, + "cosine_floor": 0.1, + "prefix": "plain-text speaker-labeled references plus strictly past audio context; native audio placeholders; no new special tokens", + "context_max_frames": 750, + "reference_min_frames": 63, + "reference_max_frames": 187, + "target_max_frames": 2250, + "max_context_rows": 32768, + "loss": "only final assistant audio, all 12 trained codebooks", + "split": "episode-disjoint, stratified by publisher language; duplicate audio audited", + "source_filter": "original beginning and middle DNSMOS BAK each >=4.0", + "eval": { + "per_language": { + "two_turn": 5, + "four_turn": 5 + }, + "seeds": "fixed before training", + "heldout_episodes": true + }, + "checkpoints_active_seconds": 900, + "maximum_training_allocations": 3, + "no_lora": true, + "distribution": "FSDP1 root FULL_SHARD; FP32 optimizer parameters/reduction; BF16 compute; full CPU state checkpoints", + "serialization": "Explicit ChatML with byte-identical train/inference prefix; no template fallback or direct-generation shortcut", + "published_scope": "Only the user-requested en/de/fr/es/nl source episodes; other languages remain in original journals and archives" + }, + "base_weights_sha256": "b66725ff44243b7b59c84fba9c201d8f993551e2b3c79211f390042a41f07c06", + "database_sha256": "0e36a157e1810673d15a50fe1d1fb31cf36bafdb45fe09d5ba859dc3abeb6c14", + "presentations": 718464, + "nonpadding_presentations": 718281, + "unique_train_targets": 239427, + "phases": [ + { + "name": "Globally shuffled epoch 1", + "end_position": 239488 + }, + { + "name": "Globally shuffled epoch 2", + "end_position": 478976 + }, + { + "name": "Globally shuffled epoch 3", + "end_position": 718464 + } + ], + "canary_keys": [ + 1, + 372, + 4176, + 16, + 23624, + 222170, + 66970 + ], + "validation_keys": [ + 4470, + 4472, + 4473, + 4474, + 194214, + 194215, + 194217, + 194219, + 294871, + 294872, + 294873, + 294874, + 19459, + 19460, + 19461, + 19464, + 75081, + 75084, + 75089, + 75090 + ], + "evaluation_tasks": 50, + "evaluation_sha256": "1fd7b56db53a096031dedd274aa60ea7c6546181a66d6f0ab5037d1a40a58a0a", + "episode_split_verified": true, + "statistics": [ + { + "language": "de", + "split": "test", + "windows": 93, + "codec_hours": 0.1506222222222222 + }, + { + "language": "de", + "split": "train", + "windows": 4997, + "codec_hours": 9.151177777777777 + }, + { + "language": "de", + "split": "validation", + "windows": 46, + "codec_hours": 0.0971777777777778 + }, + { + "language": "en", + "split": "test", + "windows": 161, + "codec_hours": 0.2633111111111111 + }, + { + "language": "en", + "split": "train", + "windows": 207561, + "codec_hours": 310.92517777777783 + }, + { + "language": "en", + "split": "validation", + "windows": 3902, + "codec_hours": 5.888955555555556 + }, + { + "language": "es", + "split": "test", + "windows": 70, + "codec_hours": 0.10746666666666667 + }, + { + "language": "es", + "split": "train", + "windows": 25380, + "codec_hours": 33.64906666666667 + }, + { + "language": "es", + "split": "validation", + "windows": 368, + "codec_hours": 0.48633333333333334 + }, + { + "language": "fr", + "split": "test", + "windows": 144, + "codec_hours": 0.1645777777777778 + }, + { + "language": "fr", + "split": "train", + "windows": 820, + "codec_hours": 0.6252222222222222 + }, + { + "language": "fr", + "split": "validation", + "windows": 149, + "codec_hours": 0.1634888888888889 + }, + { + "language": "nl", + "split": "test", + "windows": 105, + "codec_hours": 0.22775555555555557 + }, + { + "language": "nl", + "split": "train", + "windows": 669, + "codec_hours": 1.0362 + }, + { + "language": "nl", + "split": "validation", + "windows": 74, + "codec_hours": 0.14373333333333335 + } + ], + "rejections": [ + { + "reason": "boundary_contains_other_words", + "candidates": 677661 + }, + { + "reason": "not_exactly_two_diarized_speakers", + "candidates": 985 + }, + { + "reason": "target_length", + "candidates": 27339 + }, + { + "reason": "two_clean_past_references_unavailable", + "candidates": 95293 + } + ], + "annotation_sha256": "ffe152f39d0d651625fa7747b8d132b587642773b48196e44d5923cfd432e1fd", + "time": 1793631377.1572342 +} diff --git a/stats/fullft-DONE.json b/stats/fullft-DONE.json new file mode 100644 index 0000000000000000000000000000000000000000..4c2c878a3dd6a37a2d6cd492613bd2f0263edef2 --- /dev/null +++ b/stats/fullft-DONE.json @@ -0,0 +1,6 @@ +{ + "status": "COMPLETE", + "step": 5613, + "contract_sha256": "795b57ea6cfca9f0b264e20f2bd5da797d0e9a00e998fb2fc3c5c25a681e818d", + "time": 1793645723.802151 +} diff --git a/stats/fullft-RUN_CONTRACT.json b/stats/fullft-RUN_CONTRACT.json new file mode 100644 index 0000000000000000000000000000000000000000..715f0d39157eb448c76d29fc2dc0a66e3e3371f0 --- /dev/null +++ b/stats/fullft-RUN_CONTRACT.json @@ -0,0 +1,93 @@ +{ + "kind": "large13-notebooklm-multispeaker-v1", + "model": "large", + "base": "/e/scratch/laionize/schuhmann1/hvs_large_1_3_20261022/release_1_3/model", + "base_weights_sha256": "b66725ff44243b7b59c84fba9c201d8f993551e2b3c79211f390042a41f07c06", + "database_sha256": "0e36a157e1810673d15a50fe1d1fb31cf36bafdb45fe09d5ba859dc3abeb6c14", + "presentations": 718464, + "actual_presentations": 718281, + "phases": [ + { + "name": "Globally shuffled epoch 1", + "end_position": 239488 + }, + { + "name": "Globally shuffled epoch 2", + "end_position": 478976 + }, + { + "name": "Globally shuffled epoch 3", + "end_position": 718464 + } + ], + "selection_contract": { + "kind": "large13-notebooklm-multispeaker-v1", + "parent": "laion/Humaneness-Voice-Large-1.3-Alpha", + "base_step": 9799, + "base_weight_files": { + "model-00001-of-00003.safetensors": "3a9831cfd1944316e8b7d5fcaea845bf8b49b832824950709ce0eabf56b4702a", + "model-00002-of-00003.safetensors": "1fdaae7bd4f09b867152ef91761106e7bfef986b5f9979f69463466e864018a5", + "model-00003-of-00003.safetensors": "c149c250be927df24cdaa59c5b8ff50c3f0471f664f6483bb58236f07afa31df" + }, + "languages": [ + "en", + "de", + "fr", + "es", + "nl" + ], + "epochs": 3, + "world": 64, + "accumulation": 2, + "peaks": [ + 2e-05, + 4e-05 + ], + "warmup": 0.05, + "cosine_floor": 0.1, + "prefix": "plain-text speaker-labeled references plus strictly past audio context; native audio placeholders; no new special tokens", + "context_max_frames": 750, + "reference_min_frames": 63, + "reference_max_frames": 187, + "target_max_frames": 2250, + "max_context_rows": 32768, + "loss": "only final assistant audio, all 12 trained codebooks", + "split": "episode-disjoint, stratified by publisher language; duplicate audio audited", + "source_filter": "original beginning and middle DNSMOS BAK each >=4.0", + "eval": { + "per_language": { + "two_turn": 5, + "four_turn": 5 + }, + "seeds": "fixed before training", + "heldout_episodes": true + }, + "checkpoints_active_seconds": 900, + "maximum_training_allocations": 3, + "no_lora": true, + "distribution": "FSDP1 root FULL_SHARD; FP32 optimizer parameters/reduction; BF16 compute; full CPU state checkpoints", + "serialization": "Explicit ChatML with byte-identical train/inference prefix; no template fallback or direct-generation shortcut", + "published_scope": "Only the user-requested en/de/fr/es/nl source episodes; other languages remain in original journals and archives" + }, + "world": 64, + "accumulation": 2, + "effective_batch": 128, + "seed": 20261102, + "peak_semantic": 2e-05, + "peak_talker": 4e-05, + "warmup_fraction": 0.05, + "cosine_floor": 0.1, + "lr_restarts": 0, + "total_steps": 5613, + "checkpoints_seconds": 900, + "keep_all_checkpoints": true, + "optimizer_rng_scheduler_sampler_saved": true, + "pilot": false, + "distribution": "FSDP1 root FULL_SHARD, original FP32 optimizer parameters, BF16 compute, FP32 reduction, full CPU checkpoint", + "training_code_sha256": "118513d7a48bc36782c5b8ad7333013ad9fcbf2e40814e7f41ab8cb2a60f75e1", + "parent_train_fullft_sha256": "4e5f84b948afd3eabd28f762349d1a2c7429232ffc5bd6524f891c82c6a613d3", + "parent_attention_sha256": "98e46fbe74b8b4e8d48ff12e91a40ce79aab59ea2a970202088a5f91d53bb2cf", + "snapshot_sha256": "600c0ad3c0d72f1adbfa49cb097f1c983cfc17acf519ad7dc729c341bd5ba45d", + "variant": "podcast", + "contract_sha256": "795b57ea6cfca9f0b264e20f2bd5da797d0e9a00e998fb2fc3c5c25a681e818d" +} diff --git a/stats/fullft-STATUS.json b/stats/fullft-STATUS.json new file mode 100644 index 0000000000000000000000000000000000000000..8c25a8378a73814a2989533977a22f6063d5e153 --- /dev/null +++ b/stats/fullft-STATUS.json @@ -0,0 +1,11 @@ +{ + "state": "COMPLETE", + "step": 5613, + "total_steps": 5613, + "loss": 4.5207825703407405, + "job_id": "2650924", + "gpus": 64, + "elapsed_this_job_s": 12179.607034354936, + "peak_gpu_gb": 34.143688192, + "updated_unix": 1793645723.7996464 +} diff --git a/stats/fullft-steps.jsonl b/stats/fullft-steps.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..56abd6a8636ea2046db6fa8246383659ed1252cf --- /dev/null +++ b/stats/fullft-steps.jsonl @@ -0,0 +1,284 @@ +{"step": 1, "total_steps": 5613, "loss": 5.587478797882795, "lr_semantic": 7.117437722419929e-08, "lr_talker": 1.4234875444839858e-07, "elapsed_s": 11.013597084907815, "job_id": "2650924"} +{"step": 2, "total_steps": 5613, "loss": 5.525776091963053, "lr_semantic": 1.4234875444839858e-07, "lr_talker": 2.8469750889679715e-07, "elapsed_s": 16.60024484898895, "job_id": "2650924"} +{"step": 3, "total_steps": 5613, "loss": 5.623797632753849, "lr_semantic": 2.135231316725979e-07, "lr_talker": 4.270462633451958e-07, "elapsed_s": 24.13340073195286, "job_id": "2650924"} +{"step": 20, "total_steps": 5613, "loss": 5.518706202507019, "lr_semantic": 1.423487544483986e-06, "lr_talker": 2.846975088967972e-06, "elapsed_s": 74.78572288993746, "job_id": "2650924"} +{"step": 40, "total_steps": 5613, "loss": 5.090944893658161, "lr_semantic": 2.846975088967972e-06, "lr_talker": 5.693950177935944e-06, "elapsed_s": 120.8007314610295, "job_id": "2650924"} +{"step": 60, "total_steps": 5613, "loss": 4.929759006947279, "lr_semantic": 4.270462633451958e-06, "lr_talker": 8.540925266903915e-06, "elapsed_s": 170.59753852407448, "job_id": "2650924"} +{"step": 80, "total_steps": 5613, "loss": 4.938163060694933, "lr_semantic": 5.693950177935944e-06, "lr_talker": 1.1387900355871889e-05, "elapsed_s": 232.3358086280059, "job_id": "2650924"} +{"step": 100, "total_steps": 5613, "loss": 4.852787749841809, "lr_semantic": 7.11743772241993e-06, "lr_talker": 1.423487544483986e-05, "elapsed_s": 273.8830103040673, "job_id": "2650924"} +{"step": 120, "total_steps": 5613, "loss": 4.8588650450110435, "lr_semantic": 8.540925266903915e-06, "lr_talker": 1.708185053380783e-05, "elapsed_s": 313.6654968110379, "job_id": "2650924"} +{"step": 140, "total_steps": 5613, "loss": 4.8637976460158825, "lr_semantic": 9.964412811387902e-06, "lr_talker": 1.9928825622775804e-05, "elapsed_s": 353.53465432301164, "job_id": "2650924"} +{"step": 160, "total_steps": 5613, "loss": 4.863433074206114, "lr_semantic": 1.1387900355871889e-05, "lr_talker": 2.2775800711743777e-05, "elapsed_s": 393.17336230608635, "job_id": "2650924"} +{"step": 180, "total_steps": 5613, "loss": 4.836579687893391, "lr_semantic": 1.2811387900355874e-05, "lr_talker": 2.5622775800711747e-05, "elapsed_s": 432.5499518690631, "job_id": "2650924"} +{"step": 200, "total_steps": 5613, "loss": 4.824142433702946, "lr_semantic": 1.423487544483986e-05, "lr_talker": 2.846975088967972e-05, "elapsed_s": 472.52124926890247, "job_id": "2650924"} +{"step": 220, "total_steps": 5613, "loss": 4.832095634192228, "lr_semantic": 1.5658362989323845e-05, "lr_talker": 3.131672597864769e-05, "elapsed_s": 512.6217995649204, "job_id": "2650924"} +{"step": 240, "total_steps": 5613, "loss": 4.821955591440201, "lr_semantic": 1.708185053380783e-05, "lr_talker": 3.416370106761566e-05, "elapsed_s": 553.1844750219025, "job_id": "2650924"} +{"step": 260, "total_steps": 5613, "loss": 4.816163405776024, "lr_semantic": 1.8505338078291815e-05, "lr_talker": 3.701067615658363e-05, "elapsed_s": 593.9156070020981, "job_id": "2650924"} +{"step": 280, "total_steps": 5613, "loss": 4.817517191171646, "lr_semantic": 1.9928825622775804e-05, "lr_talker": 3.985765124555161e-05, "elapsed_s": 635.7070532739162, "job_id": "2650924"} +{"step": 300, "total_steps": 5613, "loss": 4.843491647392511, "lr_semantic": 1.9999436057735316e-05, "lr_talker": 3.999887211547063e-05, "elapsed_s": 676.0006452200469, "job_id": "2650924"} +{"step": 320, "total_steps": 5613, "loss": 4.812273908406496, "lr_semantic": 1.9997624023821825e-05, "lr_talker": 3.999524804764365e-05, "elapsed_s": 715.7607630849816, "job_id": "2650924"} +{"step": 340, "total_steps": 5613, "loss": 4.772692650556564, "lr_semantic": 1.9994562587693766e-05, "lr_talker": 3.998912517538753e-05, "elapsed_s": 755.3094924690668, "job_id": "2650924"} +{"step": 360, "total_steps": 5613, "loss": 4.824440844357014, "lr_semantic": 1.9990252174459486e-05, "lr_talker": 3.998050434891897e-05, "elapsed_s": 795.2929087539669, "job_id": "2650924"} +{"step": 380, "total_steps": 5613, "loss": 4.802605751901865, "lr_semantic": 1.9984693382659205e-05, "lr_talker": 3.996938676531841e-05, "elapsed_s": 834.7539963200688, "job_id": "2650924"} +{"step": 400, "total_steps": 5613, "loss": 4.753821540623903, "lr_semantic": 1.99778869841819e-05, "lr_talker": 3.99557739683638e-05, "elapsed_s": 874.9783155990299, "job_id": "2650924"} +{"step": 420, "total_steps": 5613, "loss": 4.8291470147669315, "lr_semantic": 1.996983392415811e-05, "lr_talker": 3.993966784831622e-05, "elapsed_s": 942.8503442539368, "job_id": "2650924"} +{"step": 440, "total_steps": 5613, "loss": 4.769236523658037, "lr_semantic": 1.9960535320828727e-05, "lr_talker": 3.9921070641657454e-05, "elapsed_s": 983.7482710410841, "job_id": "2650924"} +{"step": 460, "total_steps": 5613, "loss": 4.784003414213657, "lr_semantic": 1.9949992465389688e-05, "lr_talker": 3.9899984930779376e-05, "elapsed_s": 1023.9067495749332, "job_id": "2650924"} +{"step": 480, "total_steps": 5613, "loss": 4.800135903060436, "lr_semantic": 1.993820682181269e-05, "lr_talker": 3.987641364362538e-05, "elapsed_s": 1065.0519276841078, "job_id": "2650924"} +{"step": 500, "total_steps": 5613, "loss": 4.788406662642956, "lr_semantic": 1.9925180026641902e-05, "lr_talker": 3.9850360053283805e-05, "elapsed_s": 1119.3691214669961, "job_id": "2650924"} +{"step": 520, "total_steps": 5613, "loss": 4.823740411549807, "lr_semantic": 1.991091388876674e-05, "lr_talker": 3.982182777753348e-05, "elapsed_s": 1159.5253813730087, "job_id": "2650924"} +{"step": 540, "total_steps": 5613, "loss": 4.778647020459175, "lr_semantic": 1.989541038917064e-05, "lr_talker": 3.979082077834128e-05, "elapsed_s": 1199.1351977949962, "job_id": "2650924"} +{"step": 560, "total_steps": 5613, "loss": 4.772263254970312, "lr_semantic": 1.9878671680656022e-05, "lr_talker": 3.9757343361312045e-05, "elapsed_s": 1240.0329527850263, "job_id": "2650924"} +{"step": 580, "total_steps": 5613, "loss": 4.764200121164322, "lr_semantic": 1.986070008754533e-05, "lr_talker": 3.972140017509066e-05, "elapsed_s": 1280.5460716281086, "job_id": "2650924"} +{"step": 600, "total_steps": 5613, "loss": 4.794390715658665, "lr_semantic": 1.984149810535829e-05, "lr_talker": 3.968299621071658e-05, "elapsed_s": 1320.8885340450797, "job_id": "2650924"} +{"step": 620, "total_steps": 5613, "loss": 4.754822490736842, "lr_semantic": 1.9821068400465378e-05, "lr_talker": 3.9642136800930756e-05, "elapsed_s": 1361.7153889329638, "job_id": "2650924"} +{"step": 640, "total_steps": 5613, "loss": 4.773165099322796, "lr_semantic": 1.979941380971758e-05, "lr_talker": 3.959882761943516e-05, "elapsed_s": 1402.0059591881, "job_id": "2650924"} +{"step": 660, "total_steps": 5613, "loss": 4.765598749741912, "lr_semantic": 1.9776537340052457e-05, "lr_talker": 3.955307468010491e-05, "elapsed_s": 1443.30640967004, "job_id": "2650924"} +{"step": 680, "total_steps": 5613, "loss": 4.755720004439354, "lr_semantic": 1.97524421680766e-05, "lr_talker": 3.95048843361532e-05, "elapsed_s": 1492.9528597129975, "job_id": "2650924"} +{"step": 700, "total_steps": 5613, "loss": 4.7521882597357035, "lr_semantic": 1.9727131639624558e-05, "lr_talker": 3.9454263279249115e-05, "elapsed_s": 1533.5034493610729, "job_id": "2650924"} +{"step": 720, "total_steps": 5613, "loss": 4.788029182702303, "lr_semantic": 1.97006092692942e-05, "lr_talker": 3.94012185385884e-05, "elapsed_s": 1574.881404126063, "job_id": "2650924"} +{"step": 740, "total_steps": 5613, "loss": 4.762650026008487, "lr_semantic": 1.967287873995872e-05, "lr_talker": 3.934575747991744e-05, "elapsed_s": 1614.4268972349819, "job_id": "2650924"} +{"step": 760, "total_steps": 5613, "loss": 4.774876531213522, "lr_semantic": 1.9643943902255194e-05, "lr_talker": 3.928788780451039e-05, "elapsed_s": 1654.7846474519465, "job_id": "2650924"} +{"step": 780, "total_steps": 5613, "loss": 4.775570642203093, "lr_semantic": 1.9613808774049926e-05, "lr_talker": 3.922761754809985e-05, "elapsed_s": 1694.818456322886, "job_id": "2650924"} +{"step": 800, "total_steps": 5613, "loss": 4.761255389079452, "lr_semantic": 1.95824775398805e-05, "lr_talker": 3.9164955079761e-05, "elapsed_s": 1734.431733086938, "job_id": "2650924"} +{"step": 820, "total_steps": 5613, "loss": 4.793417455628514, "lr_semantic": 1.9549954550374726e-05, "lr_talker": 3.909990910074945e-05, "elapsed_s": 1774.7855006279424, "job_id": "2650924"} +{"step": 840, "total_steps": 5613, "loss": 4.769841007888317, "lr_semantic": 1.9516244321646533e-05, "lr_talker": 3.9032488643293066e-05, "elapsed_s": 1814.8339103450999, "job_id": "2650924"} +{"step": 860, "total_steps": 5613, "loss": 4.799153067171574, "lr_semantic": 1.9481351534668842e-05, "lr_talker": 3.8962703069337684e-05, "elapsed_s": 1882.465703408001, "job_id": "2650924"} +{"step": 880, "total_steps": 5613, "loss": 4.769105590879917, "lr_semantic": 1.944528103462358e-05, "lr_talker": 3.889056206924716e-05, "elapsed_s": 1945.0641413300764, "job_id": "2650924"} +{"step": 900, "total_steps": 5613, "loss": 4.804544523358345, "lr_semantic": 1.9408037830228883e-05, "lr_talker": 3.8816075660457767e-05, "elapsed_s": 2000.0417511779815, "job_id": "2650924"} +{"step": 920, "total_steps": 5613, "loss": 4.7401402592659, "lr_semantic": 1.9369627093043593e-05, "lr_talker": 3.873925418608719e-05, "elapsed_s": 2039.9671033429913, "job_id": "2650924"} +{"step": 940, "total_steps": 5613, "loss": 4.748336724936962, "lr_semantic": 1.9330054156749133e-05, "lr_talker": 3.8660108313498266e-05, "elapsed_s": 2080.364146329928, "job_id": "2650924"} +{"step": 960, "total_steps": 5613, "loss": 4.739069011062384, "lr_semantic": 1.9289324516408877e-05, "lr_talker": 3.8578649032817754e-05, "elapsed_s": 2120.9013028850313, "job_id": "2650924"} +{"step": 980, "total_steps": 5613, "loss": 4.76051914319396, "lr_semantic": 1.924744382770511e-05, "lr_talker": 3.849488765541022e-05, "elapsed_s": 2160.1376525240485, "job_id": "2650924"} +{"step": 1000, "total_steps": 5613, "loss": 4.767393801361322, "lr_semantic": 1.92044179061537e-05, "lr_talker": 3.84088358123074e-05, "elapsed_s": 2200.0085443970747, "job_id": "2650924"} +{"step": 1020, "total_steps": 5613, "loss": 4.76765163987875, "lr_semantic": 1.916025272629654e-05, "lr_talker": 3.832050545259308e-05, "elapsed_s": 2241.2521952099632, "job_id": "2650924"} +{"step": 1040, "total_steps": 5613, "loss": 4.7280378099530935, "lr_semantic": 1.911495442087193e-05, "lr_talker": 3.822990884174386e-05, "elapsed_s": 2282.928931066999, "job_id": "2650924"} +{"step": 1060, "total_steps": 5613, "loss": 4.730967672541738, "lr_semantic": 1.906852927996301e-05, "lr_talker": 3.813705855992602e-05, "elapsed_s": 2321.671146604931, "job_id": "2650924"} +{"step": 1080, "total_steps": 5613, "loss": 4.730238527059555, "lr_semantic": 1.9020983750124297e-05, "lr_talker": 3.8041967500248594e-05, "elapsed_s": 2362.0779010460246, "job_id": "2650924"} +{"step": 1100, "total_steps": 5613, "loss": 4.72141157835722, "lr_semantic": 1.8972324433486553e-05, "lr_talker": 3.7944648866973106e-05, "elapsed_s": 2416.252424715087, "job_id": "2650924"} +{"step": 1120, "total_steps": 5613, "loss": 4.736902706325054, "lr_semantic": 1.892255808683999e-05, "lr_talker": 3.784511617367998e-05, "elapsed_s": 2459.3653775320854, "job_id": "2650924"} +{"step": 1140, "total_steps": 5613, "loss": 4.764671867713332, "lr_semantic": 1.8871691620696043e-05, "lr_talker": 3.7743383241392086e-05, "elapsed_s": 2500.5573995329905, "job_id": "2650924"} +{"step": 1160, "total_steps": 5613, "loss": 4.721237454563379, "lr_semantic": 1.881973209832778e-05, "lr_talker": 3.763946419665556e-05, "elapsed_s": 2549.46644647792, "job_id": "2650924"} +{"step": 1180, "total_steps": 5613, "loss": 4.731536339968443, "lr_semantic": 1.87666867347891e-05, "lr_talker": 3.75333734695782e-05, "elapsed_s": 2589.767932805, "job_id": "2650924"} +{"step": 1200, "total_steps": 5613, "loss": 4.735003147274256, "lr_semantic": 1.8712562895912868e-05, "lr_talker": 3.7425125791825735e-05, "elapsed_s": 2635.105361911934, "job_id": "2650924"} +{"step": 1220, "total_steps": 5613, "loss": 4.734358720481396, "lr_semantic": 1.865736809728808e-05, "lr_talker": 3.731473619457616e-05, "elapsed_s": 2675.131558994064, "job_id": "2650924"} +{"step": 1240, "total_steps": 5613, "loss": 4.756669394671917, "lr_semantic": 1.8601110003216273e-05, "lr_talker": 3.720222000643255e-05, "elapsed_s": 2715.594623999903, "job_id": "2650924"} +{"step": 1260, "total_steps": 5613, "loss": 4.7002569902688265, "lr_semantic": 1.8543796425647266e-05, "lr_talker": 3.708759285129453e-05, "elapsed_s": 2756.165475001093, "job_id": "2650924"} +{"step": 1280, "total_steps": 5613, "loss": 4.731663167476654, "lr_semantic": 1.8485435323094385e-05, "lr_talker": 3.697087064618877e-05, "elapsed_s": 2835.434735188959, "job_id": "2650924"} +{"step": 1300, "total_steps": 5613, "loss": 4.719036556780338, "lr_semantic": 1.8426034799529363e-05, "lr_talker": 3.685206959905873e-05, "elapsed_s": 2876.0257180531044, "job_id": "2650924"} +{"step": 1320, "total_steps": 5613, "loss": 4.743280284106731, "lr_semantic": 1.8365603103257037e-05, "lr_talker": 3.6731206206514074e-05, "elapsed_s": 2929.1522962970193, "job_id": "2650924"} +{"step": 1340, "total_steps": 5613, "loss": 4.7402151226997375, "lr_semantic": 1.8304148625769972e-05, "lr_talker": 3.6608297251539944e-05, "elapsed_s": 2980.258687512018, "job_id": "2650924"} +{"step": 1360, "total_steps": 5613, "loss": 4.711447082459927, "lr_semantic": 1.8241679900583252e-05, "lr_talker": 3.6483359801166504e-05, "elapsed_s": 3022.2461457950994, "job_id": "2650924"} +{"step": 1380, "total_steps": 5613, "loss": 4.722232082858682, "lr_semantic": 1.8178205602049493e-05, "lr_talker": 3.6356411204098985e-05, "elapsed_s": 3062.724447991932, "job_id": "2650924"} +{"step": 1400, "total_steps": 5613, "loss": 4.741885308176279, "lr_semantic": 1.811373454415437e-05, "lr_talker": 3.622746908830874e-05, "elapsed_s": 3103.5578841960523, "job_id": "2650924"} +{"step": 1420, "total_steps": 5613, "loss": 4.7331896759569645, "lr_semantic": 1.8048275679292674e-05, "lr_talker": 3.609655135858535e-05, "elapsed_s": 3145.3643633520696, "job_id": "2650924"} +{"step": 1440, "total_steps": 5613, "loss": 4.734523702412844, "lr_semantic": 1.798183809702522e-05, "lr_talker": 3.596367619405044e-05, "elapsed_s": 3195.941620595986, "job_id": "2650924"} +{"step": 1460, "total_steps": 5613, "loss": 4.7298208605498075, "lr_semantic": 1.7914431022816666e-05, "lr_talker": 3.582886204563333e-05, "elapsed_s": 3244.4972031789366, "job_id": "2650924"} +{"step": 1480, "total_steps": 5613, "loss": 4.717615306377411, "lr_semantic": 1.7846063816754468e-05, "lr_talker": 3.5692127633508935e-05, "elapsed_s": 3284.8206422769, "job_id": "2650924"} +{"step": 1500, "total_steps": 5613, "loss": 4.71048554033041, "lr_semantic": 1.7776745972249152e-05, "lr_talker": 3.5553491944498304e-05, "elapsed_s": 3328.4186289079953, "job_id": "2650924"} +{"step": 1520, "total_steps": 5613, "loss": 4.734640143811703, "lr_semantic": 1.7706487114716083e-05, "lr_talker": 3.5412974229432166e-05, "elapsed_s": 3386.097419199068, "job_id": "2650924"} +{"step": 1540, "total_steps": 5613, "loss": 4.74219785630703, "lr_semantic": 1.7635297000238855e-05, "lr_talker": 3.527059400047771e-05, "elapsed_s": 3427.683724239003, "job_id": "2650924"} +{"step": 1560, "total_steps": 5613, "loss": 4.720453768968582, "lr_semantic": 1.7563185514214604e-05, "lr_talker": 3.512637102842921e-05, "elapsed_s": 3468.931013341993, "job_id": "2650924"} +{"step": 1580, "total_steps": 5613, "loss": 4.726194063201547, "lr_semantic": 1.7490162669981296e-05, "lr_talker": 3.498032533996259e-05, "elapsed_s": 3508.7784433478955, "job_id": "2650924"} +{"step": 1600, "total_steps": 5613, "loss": 4.761078203096986, "lr_semantic": 1.7416238607427315e-05, "lr_talker": 3.483247721485463e-05, "elapsed_s": 3549.4732208650094, "job_id": "2650924"} +{"step": 1620, "total_steps": 5613, "loss": 4.743052579462528, "lr_semantic": 1.7341423591583436e-05, "lr_talker": 3.468284718316687e-05, "elapsed_s": 3590.2429941399023, "job_id": "2650924"} +{"step": 1640, "total_steps": 5613, "loss": 4.746201619505882, "lr_semantic": 1.7265728011197426e-05, "lr_talker": 3.453145602239485e-05, "elapsed_s": 3660.068397742929, "job_id": "2650924"} +{"step": 1660, "total_steps": 5613, "loss": 4.726677805185318, "lr_semantic": 1.7189162377291467e-05, "lr_talker": 3.4378324754582935e-05, "elapsed_s": 3737.718020984903, "job_id": "2650924"} +{"step": 1680, "total_steps": 5613, "loss": 4.733044046908617, "lr_semantic": 1.7111737321702624e-05, "lr_talker": 3.422347464340525e-05, "elapsed_s": 3783.821527886903, "job_id": "2650924"} +{"step": 1700, "total_steps": 5613, "loss": 4.732866659760475, "lr_semantic": 1.703346359560651e-05, "lr_talker": 3.406692719121302e-05, "elapsed_s": 3824.358500377042, "job_id": "2650924"} +{"step": 1720, "total_steps": 5613, "loss": 4.724854338914156, "lr_semantic": 1.6954352068024366e-05, "lr_talker": 3.390870413604873e-05, "elapsed_s": 3867.4716033579316, "job_id": "2650924"} +{"step": 1740, "total_steps": 5613, "loss": 4.7217960227280855, "lr_semantic": 1.687441372431383e-05, "lr_talker": 3.374882744862766e-05, "elapsed_s": 3908.567238708958, "job_id": "2650924"} +{"step": 1760, "total_steps": 5613, "loss": 4.7270301431417465, "lr_semantic": 1.67936596646435e-05, "lr_talker": 3.3587319329287e-05, "elapsed_s": 3949.0184991960414, "job_id": "2650924"} +{"step": 1780, "total_steps": 5613, "loss": 4.7064435333013535, "lr_semantic": 1.6712101102451584e-05, "lr_talker": 3.342420220490317e-05, "elapsed_s": 3988.7838503790554, "job_id": "2650924"} +{"step": 1800, "total_steps": 5613, "loss": 4.758328091353178, "lr_semantic": 1.6629749362888808e-05, "lr_talker": 3.3259498725777616e-05, "elapsed_s": 4028.887228704989, "job_id": "2650924"} +{"step": 1820, "total_steps": 5613, "loss": 4.739157922565937, "lr_semantic": 1.654661588124581e-05, "lr_talker": 3.309323176249162e-05, "elapsed_s": 4069.656256943941, "job_id": "2650924"} +{"step": 1840, "total_steps": 5613, "loss": 4.708026945590973, "lr_semantic": 1.6462712201365248e-05, "lr_talker": 3.2925424402730496e-05, "elapsed_s": 4110.507350655971, "job_id": "2650924"} +{"step": 1860, "total_steps": 5613, "loss": 4.730698335915804, "lr_semantic": 1.637804997403884e-05, "lr_talker": 3.275609994807768e-05, "elapsed_s": 4150.359022886958, "job_id": "2650924"} +{"step": 1880, "total_steps": 5613, "loss": 4.625305764377117, "lr_semantic": 1.6292640955389545e-05, "lr_talker": 3.258528191077909e-05, "elapsed_s": 4215.603769697947, "job_id": "2650924"} +{"step": 1900, "total_steps": 5613, "loss": 4.610010115429759, "lr_semantic": 1.6206497005239105e-05, "lr_talker": 3.241299401047821e-05, "elapsed_s": 4255.96354693803, "job_id": "2650924"} +{"step": 1920, "total_steps": 5613, "loss": 4.655441582202911, "lr_semantic": 1.6119630085461212e-05, "lr_talker": 3.2239260170922425e-05, "elapsed_s": 4296.034290221054, "job_id": "2650924"} +{"step": 1940, "total_steps": 5613, "loss": 4.625532982870936, "lr_semantic": 1.6032052258320482e-05, "lr_talker": 3.2064104516640964e-05, "elapsed_s": 4336.216298351064, "job_id": "2650924"} +{"step": 1960, "total_steps": 5613, "loss": 4.633244961500168, "lr_semantic": 1.5943775684797518e-05, "lr_talker": 3.1887551369595036e-05, "elapsed_s": 4376.645100472961, "job_id": "2650924"} +{"step": 1980, "total_steps": 5613, "loss": 4.6431789100170135, "lr_semantic": 1.5854812622900225e-05, "lr_talker": 3.170962524580045e-05, "elapsed_s": 4416.419933855068, "job_id": "2650924"} +{"step": 2000, "total_steps": 5613, "loss": 4.610240997746587, "lr_semantic": 1.5765175425961696e-05, "lr_talker": 3.153035085192339e-05, "elapsed_s": 4470.1600845619105, "job_id": "2650924"} +{"step": 2020, "total_steps": 5613, "loss": 4.615652056410909, "lr_semantic": 1.5674876540924814e-05, "lr_talker": 3.134975308184963e-05, "elapsed_s": 4509.586936654989, "job_id": "2650924"} +{"step": 2040, "total_steps": 5613, "loss": 4.6264883652329445, "lr_semantic": 1.5583928506613907e-05, "lr_talker": 3.1167857013227815e-05, "elapsed_s": 4550.146989380941, "job_id": "2650924"} +{"step": 2060, "total_steps": 5613, "loss": 4.645322745665908, "lr_semantic": 1.5492343951993585e-05, "lr_talker": 3.098468790398717e-05, "elapsed_s": 4591.555366650922, "job_id": "2650924"} +{"step": 2080, "total_steps": 5613, "loss": 4.631776470690966, "lr_semantic": 1.5400135594415136e-05, "lr_talker": 3.080027118883027e-05, "elapsed_s": 4631.77258078591, "job_id": "2650924"} +{"step": 2100, "total_steps": 5613, "loss": 4.5987372770905495, "lr_semantic": 1.5307316237850564e-05, "lr_talker": 3.061463247570113e-05, "elapsed_s": 4671.6307264589705, "job_id": "2650924"} +{"step": 2120, "total_steps": 5613, "loss": 4.652963560074568, "lr_semantic": 1.5213898771114663e-05, "lr_talker": 3.0427797542229326e-05, "elapsed_s": 4712.762785148108, "job_id": "2650924"} +{"step": 2140, "total_steps": 5613, "loss": 4.626855904236436, "lr_semantic": 1.5119896166075293e-05, "lr_talker": 3.0239792332150587e-05, "elapsed_s": 4755.9194285268895, "job_id": "2650924"} +{"step": 2160, "total_steps": 5613, "loss": 4.651434026658535, "lr_semantic": 1.5025321475852096e-05, "lr_talker": 3.005064295170419e-05, "elapsed_s": 4795.616878828965, "job_id": "2650924"} +{"step": 2180, "total_steps": 5613, "loss": 4.620770875364542, "lr_semantic": 1.4930187833003962e-05, "lr_talker": 2.9860375666007923e-05, "elapsed_s": 4835.597036266001, "job_id": "2650924"} +{"step": 2200, "total_steps": 5613, "loss": 4.610138909891248, "lr_semantic": 1.4834508447705466e-05, "lr_talker": 2.9669016895410933e-05, "elapsed_s": 4875.969234729884, "job_id": "2650924"} +{"step": 2220, "total_steps": 5613, "loss": 4.630080381408334, "lr_semantic": 1.4738296605912493e-05, "lr_talker": 2.9476593211824986e-05, "elapsed_s": 4915.884354220005, "job_id": "2650924"} +{"step": 2240, "total_steps": 5613, "loss": 4.613454734906554, "lr_semantic": 1.4641565667517379e-05, "lr_talker": 2.9283131335034757e-05, "elapsed_s": 4955.729310723022, "job_id": "2650924"} +{"step": 2260, "total_steps": 5613, "loss": 4.627314366400242, "lr_semantic": 1.454432906449375e-05, "lr_talker": 2.90886581289875e-05, "elapsed_s": 4995.154172322946, "job_id": "2650924"} +{"step": 2280, "total_steps": 5613, "loss": 4.6216220539063215, "lr_semantic": 1.4446600299031396e-05, "lr_talker": 2.889320059806279e-05, "elapsed_s": 5035.364102754975, "job_id": "2650924"} +{"step": 2300, "total_steps": 5613, "loss": 4.6380799021571875, "lr_semantic": 1.4348392941661333e-05, "lr_talker": 2.8696785883322666e-05, "elapsed_s": 5075.395715229912, "job_id": "2650924"} +{"step": 2320, "total_steps": 5613, "loss": 4.595997866243124, "lr_semantic": 1.424972062937146e-05, "lr_talker": 2.849944125874292e-05, "elapsed_s": 5145.3028938879725, "job_id": "2650924"} +{"step": 2340, "total_steps": 5613, "loss": 4.63494280539453, "lr_semantic": 1.4150597063712881e-05, "lr_talker": 2.8301194127425763e-05, "elapsed_s": 5185.5523059929255, "job_id": "2650924"} +{"step": 2360, "total_steps": 5613, "loss": 4.6449728310108185, "lr_semantic": 1.4051036008897368e-05, "lr_talker": 2.8102072017794736e-05, "elapsed_s": 5225.750247712946, "job_id": "2650924"} +{"step": 2380, "total_steps": 5613, "loss": 4.655777344480157, "lr_semantic": 1.3951051289886044e-05, "lr_talker": 2.7902102579772088e-05, "elapsed_s": 5272.4179692131, "job_id": "2650924"} +{"step": 2400, "total_steps": 5613, "loss": 4.625490026548505, "lr_semantic": 1.385065679046968e-05, "lr_talker": 2.770131358093936e-05, "elapsed_s": 5319.016970624914, "job_id": "2650924"} +{"step": 2420, "total_steps": 5613, "loss": 4.636262411251664, "lr_semantic": 1.3749866451340795e-05, "lr_talker": 2.749973290268159e-05, "elapsed_s": 5359.171833059983, "job_id": "2650924"} +{"step": 2440, "total_steps": 5613, "loss": 4.626359686255455, "lr_semantic": 1.3648694268157858e-05, "lr_talker": 2.7297388536315717e-05, "elapsed_s": 5399.3266346929595, "job_id": "2650924"} +{"step": 2460, "total_steps": 5613, "loss": 4.611562255769968, "lr_semantic": 1.3547154289601876e-05, "lr_talker": 2.7094308579203753e-05, "elapsed_s": 5440.610315886093, "job_id": "2650924"} +{"step": 2480, "total_steps": 5613, "loss": 4.644266894087195, "lr_semantic": 1.3445260615425607e-05, "lr_talker": 2.6890521230851213e-05, "elapsed_s": 5479.786678743083, "job_id": "2650924"} +{"step": 2500, "total_steps": 5613, "loss": 4.623125057667494, "lr_semantic": 1.3343027394495654e-05, "lr_talker": 2.6686054788991308e-05, "elapsed_s": 5519.988826805027, "job_id": "2650924"} +{"step": 2520, "total_steps": 5613, "loss": 4.619581380859017, "lr_semantic": 1.3240468822827806e-05, "lr_talker": 2.648093764565561e-05, "elapsed_s": 5560.519715071889, "job_id": "2650924"} +{"step": 2540, "total_steps": 5613, "loss": 4.620337029919028, "lr_semantic": 1.3137599141615759e-05, "lr_talker": 2.6275198283231517e-05, "elapsed_s": 5603.863650236977, "job_id": "2650924"} +{"step": 2560, "total_steps": 5613, "loss": 4.655339788645506, "lr_semantic": 1.3034432635253615e-05, "lr_talker": 2.606886527050723e-05, "elapsed_s": 5643.633498328971, "job_id": "2650924"} +{"step": 2580, "total_steps": 5613, "loss": 4.640034716576338, "lr_semantic": 1.2930983629352353e-05, "lr_talker": 2.5861967258704706e-05, "elapsed_s": 5683.5464796489105, "job_id": "2650924"} +{"step": 2600, "total_steps": 5613, "loss": 4.628433585166931, "lr_semantic": 1.2827266488750601e-05, "lr_talker": 2.5654532977501202e-05, "elapsed_s": 5723.099614835111, "job_id": "2650924"} +{"step": 2620, "total_steps": 5613, "loss": 4.6318860817700624, "lr_semantic": 1.2723295615519917e-05, "lr_talker": 2.5446591231039834e-05, "elapsed_s": 5769.974594871048, "job_id": "2650924"} +{"step": 2640, "total_steps": 5613, "loss": 4.611671743914485, "lr_semantic": 1.2619085446964971e-05, "lr_talker": 2.5238170893929942e-05, "elapsed_s": 5812.693918016972, "job_id": "2650924"} +{"step": 2660, "total_steps": 5613, "loss": 4.646994363516569, "lr_semantic": 1.2514650453618761e-05, "lr_talker": 2.5029300907237523e-05, "elapsed_s": 5852.106621605111, "job_id": "2650924"} +{"step": 2680, "total_steps": 5613, "loss": 4.643543936312199, "lr_semantic": 1.241000513723327e-05, "lr_talker": 2.482001027446654e-05, "elapsed_s": 5895.532369503053, "job_id": "2650924"} +{"step": 2700, "total_steps": 5613, "loss": 4.665915217250586, "lr_semantic": 1.2305164028765737e-05, "lr_talker": 2.4610328057531474e-05, "elapsed_s": 5935.712447670056, "job_id": "2650924"} +{"step": 2720, "total_steps": 5613, "loss": 4.630682196468115, "lr_semantic": 1.2200141686360935e-05, "lr_talker": 2.440028337272187e-05, "elapsed_s": 5975.7568022529595, "job_id": "2650924"} +{"step": 2740, "total_steps": 5613, "loss": 4.66122155636549, "lr_semantic": 1.2094952693329617e-05, "lr_talker": 2.4189905386659235e-05, "elapsed_s": 6015.790169884916, "job_id": "2650924"} +{"step": 2760, "total_steps": 5613, "loss": 4.595508266240358, "lr_semantic": 1.19896116561235e-05, "lr_talker": 2.3979223312247e-05, "elapsed_s": 6098.267138675088, "job_id": "2650924"} +{"step": 2780, "total_steps": 5613, "loss": 4.6194125432521105, "lr_semantic": 1.1884133202307006e-05, "lr_talker": 2.3768266404614013e-05, "elapsed_s": 6138.280159062939, "job_id": "2650924"} +{"step": 2800, "total_steps": 5613, "loss": 4.6435114070773125, "lr_semantic": 1.1778531978526136e-05, "lr_talker": 2.3557063957052272e-05, "elapsed_s": 6178.649523121072, "job_id": "2650924"} +{"step": 2820, "total_steps": 5613, "loss": 4.632688293233514, "lr_semantic": 1.1672822648474594e-05, "lr_talker": 2.3345645296949187e-05, "elapsed_s": 6218.725610052003, "job_id": "2650924"} +{"step": 2840, "total_steps": 5613, "loss": 4.620898652821779, "lr_semantic": 1.1567019890857644e-05, "lr_talker": 2.3134039781715288e-05, "elapsed_s": 6257.902763654944, "job_id": "2650924"} +{"step": 2860, "total_steps": 5613, "loss": 4.625769730657339, "lr_semantic": 1.1461138397353813e-05, "lr_talker": 2.2922276794707626e-05, "elapsed_s": 6297.234707331052, "job_id": "2650924"} +{"step": 2880, "total_steps": 5613, "loss": 4.595105350017548, "lr_semantic": 1.1355192870574828e-05, "lr_talker": 2.2710385741149656e-05, "elapsed_s": 6336.9683743950445, "job_id": "2650924"} +{"step": 2900, "total_steps": 5613, "loss": 4.603524900972843, "lr_semantic": 1.124919802202401e-05, "lr_talker": 2.249839604404802e-05, "elapsed_s": 6377.163062052103, "job_id": "2650924"} +{"step": 2920, "total_steps": 5613, "loss": 4.6121920589357615, "lr_semantic": 1.1143168570053474e-05, "lr_talker": 2.228633714010695e-05, "elapsed_s": 6417.040328416042, "job_id": "2650924"} +{"step": 2940, "total_steps": 5613, "loss": 4.605383405461907, "lr_semantic": 1.1037119237820326e-05, "lr_talker": 2.2074238475640652e-05, "elapsed_s": 6458.799037778983, "job_id": "2650924"} +{"step": 2960, "total_steps": 5613, "loss": 4.624900868162513, "lr_semantic": 1.093106475124223e-05, "lr_talker": 2.186212950248446e-05, "elapsed_s": 6499.928429676918, "job_id": "2650924"} +{"step": 2980, "total_steps": 5613, "loss": 4.616175290197134, "lr_semantic": 1.0825019836952577e-05, "lr_talker": 2.1650039673905153e-05, "elapsed_s": 6542.408533280017, "job_id": "2650924"} +{"step": 3000, "total_steps": 5613, "loss": 4.639580648392439, "lr_semantic": 1.0718999220255559e-05, "lr_talker": 2.1437998440511117e-05, "elapsed_s": 6581.880402375013, "job_id": "2650924"} +{"step": 3020, "total_steps": 5613, "loss": 4.603767963126302, "lr_semantic": 1.0613017623081425e-05, "lr_talker": 2.122603524616285e-05, "elapsed_s": 6625.251445908099, "job_id": "2650924"} +{"step": 3040, "total_steps": 5613, "loss": 4.6375779788941145, "lr_semantic": 1.0507089761942209e-05, "lr_talker": 2.1014179523884418e-05, "elapsed_s": 6669.651214292971, "job_id": "2650924"} +{"step": 3060, "total_steps": 5613, "loss": 4.590810565277934, "lr_semantic": 1.0401230345888205e-05, "lr_talker": 2.080246069177641e-05, "elapsed_s": 6708.853607652942, "job_id": "2650924"} +{"step": 3080, "total_steps": 5613, "loss": 4.607462951913476, "lr_semantic": 1.0295454074465483e-05, "lr_talker": 2.0590908148930965e-05, "elapsed_s": 6749.735308313044, "job_id": "2650924"} +{"step": 3100, "total_steps": 5613, "loss": 4.623072760179639, "lr_semantic": 1.0189775635674717e-05, "lr_talker": 2.0379551271349434e-05, "elapsed_s": 6791.6384207950905, "job_id": "2650924"} +{"step": 3120, "total_steps": 5613, "loss": 4.6187803372740746, "lr_semantic": 1.0084209703931636e-05, "lr_talker": 2.016841940786327e-05, "elapsed_s": 6832.483076035045, "job_id": "2650924"} +{"step": 3140, "total_steps": 5613, "loss": 4.590356990695, "lr_semantic": 9.978770938029332e-06, "lr_talker": 1.9957541876058664e-05, "elapsed_s": 6873.765201058006, "job_id": "2650924"} +{"step": 3160, "total_steps": 5613, "loss": 4.622335035353899, "lr_semantic": 9.873473979102764e-06, "lr_talker": 1.9746947958205528e-05, "elapsed_s": 6950.473682532087, "job_id": "2650924"} +{"step": 3180, "total_steps": 5613, "loss": 4.616848412901163, "lr_semantic": 9.768333448595686e-06, "lr_talker": 1.953666689719137e-05, "elapsed_s": 7027.935233348981, "job_id": "2650924"} +{"step": 3200, "total_steps": 5613, "loss": 4.60705847106874, "lr_semantic": 9.663363946230362e-06, "lr_talker": 1.9326727892460723e-05, "elapsed_s": 7068.292897902895, "job_id": "2650924"} +{"step": 3220, "total_steps": 5613, "loss": 4.6382788103073835, "lr_semantic": 9.558580047980217e-06, "lr_talker": 1.9117160095960435e-05, "elapsed_s": 7114.814613187918, "job_id": "2650924"} +{"step": 3240, "total_steps": 5613, "loss": 4.5957195814698935, "lr_semantic": 9.453996304045853e-06, "lr_talker": 1.8907992608091706e-05, "elapsed_s": 7175.236491824966, "job_id": "2650924"} +{"step": 3260, "total_steps": 5613, "loss": 4.632833460345864, "lr_semantic": 9.349627236834623e-06, "lr_talker": 1.8699254473669247e-05, "elapsed_s": 7215.2017654269, "job_id": "2650924"} +{"step": 3280, "total_steps": 5613, "loss": 4.62228437513113, "lr_semantic": 9.245487338944068e-06, "lr_talker": 1.8490974677888137e-05, "elapsed_s": 7254.26144577493, "job_id": "2650924"} +{"step": 3300, "total_steps": 5613, "loss": 4.634927546605468, "lr_semantic": 9.141591071149456e-06, "lr_talker": 1.828318214229891e-05, "elapsed_s": 7295.908539799973, "job_id": "2650924"} +{"step": 3320, "total_steps": 5613, "loss": 4.62289790250361, "lr_semantic": 9.037952860395806e-06, "lr_talker": 1.8075905720791612e-05, "elapsed_s": 7335.5786440589, "job_id": "2650924"} +{"step": 3340, "total_steps": 5613, "loss": 4.631784085184336, "lr_semantic": 8.934587097794555e-06, "lr_talker": 1.786917419558911e-05, "elapsed_s": 7374.582625098992, "job_id": "2650924"} +{"step": 3360, "total_steps": 5613, "loss": 4.603944197297096, "lr_semantic": 8.831508136625242e-06, "lr_talker": 1.7663016273250485e-05, "elapsed_s": 7415.573455769103, "job_id": "2650924"} +{"step": 3380, "total_steps": 5613, "loss": 4.604606613516808, "lr_semantic": 8.728730290342388e-06, "lr_talker": 1.7457460580684776e-05, "elapsed_s": 7454.984159963904, "job_id": "2650924"} +{"step": 3400, "total_steps": 5613, "loss": 4.6076365280896425, "lr_semantic": 8.62626783058798e-06, "lr_talker": 1.725253566117596e-05, "elapsed_s": 7494.683540343074, "job_id": "2650924"} +{"step": 3420, "total_steps": 5613, "loss": 4.607591565698385, "lr_semantic": 8.524134985209698e-06, "lr_talker": 1.7048269970419396e-05, "elapsed_s": 7533.725569159025, "job_id": "2650924"} +{"step": 3440, "total_steps": 5613, "loss": 4.63418716751039, "lr_semantic": 8.422345936285268e-06, "lr_talker": 1.6844691872570537e-05, "elapsed_s": 7572.850217720959, "job_id": "2650924"} +{"step": 3460, "total_steps": 5613, "loss": 4.61283715069294, "lr_semantic": 8.32091481815313e-06, "lr_talker": 1.664182963630626e-05, "elapsed_s": 7637.1775283101015, "job_id": "2650924"} +{"step": 3480, "total_steps": 5613, "loss": 4.619726676493883, "lr_semantic": 8.21985571544977e-06, "lr_talker": 1.643971143089954e-05, "elapsed_s": 7676.934100366896, "job_id": "2650924"} +{"step": 3500, "total_steps": 5613, "loss": 4.607974525541067, "lr_semantic": 8.119182661153934e-06, "lr_talker": 1.623836532230787e-05, "elapsed_s": 7719.310810083989, "job_id": "2650924"} +{"step": 3520, "total_steps": 5613, "loss": 4.624380446970463, "lr_semantic": 8.018909634638035e-06, "lr_talker": 1.603781926927607e-05, "elapsed_s": 7759.803617738886, "job_id": "2650924"} +{"step": 3540, "total_steps": 5613, "loss": 4.585276786237955, "lr_semantic": 7.919050559726964e-06, "lr_talker": 1.583810111945393e-05, "elapsed_s": 7800.0703570400365, "job_id": "2650924"} +{"step": 3560, "total_steps": 5613, "loss": 4.628160396590829, "lr_semantic": 7.819619302764659e-06, "lr_talker": 1.5639238605529317e-05, "elapsed_s": 7840.456648838939, "job_id": "2650924"} +{"step": 3580, "total_steps": 5613, "loss": 4.606885613873601, "lr_semantic": 7.72062967068863e-06, "lr_talker": 1.544125934137726e-05, "elapsed_s": 7880.857028143015, "job_id": "2650924"} +{"step": 3600, "total_steps": 5613, "loss": 4.604534378275275, "lr_semantic": 7.6220954091127486e-06, "lr_talker": 1.5244190818225497e-05, "elapsed_s": 7948.6377753249835, "job_id": "2650924"} +{"step": 3620, "total_steps": 5613, "loss": 4.642087118700147, "lr_semantic": 7.524030200418524e-06, "lr_talker": 1.5048060400837048e-05, "elapsed_s": 7989.182672624011, "job_id": "2650924"} +{"step": 3640, "total_steps": 5613, "loss": 4.6124731246382, "lr_semantic": 7.426447661855198e-06, "lr_talker": 1.4852895323710396e-05, "elapsed_s": 8037.746763360919, "job_id": "2650924"} +{"step": 3660, "total_steps": 5613, "loss": 4.610769663006067, "lr_semantic": 7.32936134364886e-06, "lr_talker": 1.465872268729772e-05, "elapsed_s": 8077.404462696984, "job_id": "2650924"} +{"step": 3680, "total_steps": 5613, "loss": 4.59444822743535, "lr_semantic": 7.2327847271208765e-06, "lr_talker": 1.4465569454241753e-05, "elapsed_s": 8116.995957107982, "job_id": "2650924"} +{"step": 3700, "total_steps": 5613, "loss": 4.599885921925306, "lr_semantic": 7.136731222815878e-06, "lr_talker": 1.4273462445631756e-05, "elapsed_s": 8156.510635749903, "job_id": "2650924"} +{"step": 3720, "total_steps": 5613, "loss": 4.619421795010567, "lr_semantic": 7.041214168639587e-06, "lr_talker": 1.4082428337279174e-05, "elapsed_s": 8196.255844068015, "job_id": "2650924"} +{"step": 3740, "total_steps": 5613, "loss": 4.6178270019590855, "lr_semantic": 6.946246828006742e-06, "lr_talker": 1.3892493656013484e-05, "elapsed_s": 8236.720981243998, "job_id": "2650924"} +{"step": 3760, "total_steps": 5613, "loss": 4.5287997759878635, "lr_semantic": 6.851842387999329e-06, "lr_talker": 1.3703684775998659e-05, "elapsed_s": 8307.158814005088, "job_id": "2650924"} +{"step": 3780, "total_steps": 5613, "loss": 4.5157803148031235, "lr_semantic": 6.7580139575354435e-06, "lr_talker": 1.3516027915070887e-05, "elapsed_s": 8347.439263015985, "job_id": "2650924"} +{"step": 3800, "total_steps": 5613, "loss": 4.492436217144132, "lr_semantic": 6.664774565548999e-06, "lr_talker": 1.3329549131097999e-05, "elapsed_s": 8388.475140315946, "job_id": "2650924"} +{"step": 3820, "total_steps": 5613, "loss": 4.514497401192784, "lr_semantic": 6.5721371591805535e-06, "lr_talker": 1.3144274318361107e-05, "elapsed_s": 8428.67107571708, "job_id": "2650924"} +{"step": 3840, "total_steps": 5613, "loss": 4.46937383338809, "lr_semantic": 6.48011460197945e-06, "lr_talker": 1.29602292039589e-05, "elapsed_s": 8469.715263373917, "job_id": "2650924"} +{"step": 3860, "total_steps": 5613, "loss": 4.508008701726794, "lr_semantic": 6.388719672117623e-06, "lr_talker": 1.2777439344235246e-05, "elapsed_s": 8509.82451614691, "job_id": "2650924"} +{"step": 3880, "total_steps": 5613, "loss": 4.498665811493993, "lr_semantic": 6.2979650606152165e-06, "lr_talker": 1.2595930121230433e-05, "elapsed_s": 8550.05722452607, "job_id": "2650924"} +{"step": 3900, "total_steps": 5613, "loss": 4.44620075263083, "lr_semantic": 6.207863369578337e-06, "lr_talker": 1.2415726739156674e-05, "elapsed_s": 8590.212388169952, "job_id": "2650924"} +{"step": 3920, "total_steps": 5613, "loss": 4.501476686447859, "lr_semantic": 6.118427110449115e-06, "lr_talker": 1.223685422089823e-05, "elapsed_s": 8630.740923413076, "job_id": "2650924"} +{"step": 3940, "total_steps": 5613, "loss": 4.569125657901168, "lr_semantic": 6.029668702268385e-06, "lr_talker": 1.205933740453677e-05, "elapsed_s": 8669.870596830035, "job_id": "2650924"} +{"step": 3960, "total_steps": 5613, "loss": 4.461178727447987, "lr_semantic": 5.9416004699512094e-06, "lr_talker": 1.1883200939902419e-05, "elapsed_s": 8710.126772614894, "job_id": "2650924"} +{"step": 3980, "total_steps": 5613, "loss": 4.515434131026268, "lr_semantic": 5.854234642575411e-06, "lr_talker": 1.1708469285150823e-05, "elapsed_s": 8750.327735577943, "job_id": "2650924"} +{"step": 4000, "total_steps": 5613, "loss": 4.514987234026194, "lr_semantic": 5.76758335168349e-06, "lr_talker": 1.153516670336698e-05, "elapsed_s": 8790.649440234993, "job_id": "2650924"} +{"step": 4020, "total_steps": 5613, "loss": 4.504336902871728, "lr_semantic": 5.681658629598021e-06, "lr_talker": 1.1363317259196043e-05, "elapsed_s": 8830.257627018029, "job_id": "2650924"} +{"step": 4040, "total_steps": 5613, "loss": 4.487777344882488, "lr_semantic": 5.596472407750888e-06, "lr_talker": 1.1192944815501776e-05, "elapsed_s": 8870.364806099096, "job_id": "2650924"} +{"step": 4060, "total_steps": 5613, "loss": 4.495494468137622, "lr_semantic": 5.512036515026457e-06, "lr_talker": 1.1024073030052915e-05, "elapsed_s": 8910.096186374081, "job_id": "2650924"} +{"step": 4080, "total_steps": 5613, "loss": 4.483938731253147, "lr_semantic": 5.428362676119063e-06, "lr_talker": 1.0856725352238127e-05, "elapsed_s": 8950.116008858895, "job_id": "2650924"} +{"step": 4100, "total_steps": 5613, "loss": 4.494588997215033, "lr_semantic": 5.345462509904901e-06, "lr_talker": 1.0690925019809801e-05, "elapsed_s": 8990.26555560599, "job_id": "2650924"} +{"step": 4120, "total_steps": 5613, "loss": 4.494730705395341, "lr_semantic": 5.263347527828662e-06, "lr_talker": 1.0526695055657324e-05, "elapsed_s": 9030.054463834036, "job_id": "2650924"} +{"step": 4140, "total_steps": 5613, "loss": 4.53859705477953, "lr_semantic": 5.182029132305044e-06, "lr_talker": 1.0364058264610089e-05, "elapsed_s": 9070.23684858391, "job_id": "2650924"} +{"step": 4160, "total_steps": 5613, "loss": 4.527783302590251, "lr_semantic": 5.101518615135439e-06, "lr_talker": 1.0203037230270879e-05, "elapsed_s": 9110.06045971089, "job_id": "2650924"} +{"step": 4180, "total_steps": 5613, "loss": 4.503189731389284, "lr_semantic": 5.021827155939951e-06, "lr_talker": 1.0043654311879902e-05, "elapsed_s": 9150.62700702995, "job_id": "2650924"} +{"step": 4200, "total_steps": 5613, "loss": 4.496820015832782, "lr_semantic": 4.942965820605014e-06, "lr_talker": 9.885931641210027e-06, "elapsed_s": 9216.700601124903, "job_id": "2650924"} +{"step": 4220, "total_steps": 5613, "loss": 4.473198881372809, "lr_semantic": 4.864945559746785e-06, "lr_talker": 9.72989111949357e-06, "elapsed_s": 9256.489080969943, "job_id": "2650924"} +{"step": 4240, "total_steps": 5613, "loss": 4.493655517697334, "lr_semantic": 4.787777207190554e-06, "lr_talker": 9.575554414381108e-06, "elapsed_s": 9297.0773801459, "job_id": "2650924"} +{"step": 4260, "total_steps": 5613, "loss": 4.497149819508195, "lr_semantic": 4.7114714784663626e-06, "lr_talker": 9.422942956932725e-06, "elapsed_s": 9337.289414040046, "job_id": "2650924"} +{"step": 4280, "total_steps": 5613, "loss": 4.501642098650336, "lr_semantic": 4.636038969321073e-06, "lr_talker": 9.272077938642147e-06, "elapsed_s": 9377.615158363013, "job_id": "2650924"} +{"step": 4300, "total_steps": 5613, "loss": 4.520987527444959, "lr_semantic": 4.56149015424703e-06, "lr_talker": 9.12298030849406e-06, "elapsed_s": 9417.534220606089, "job_id": "2650924"} +{"step": 4320, "total_steps": 5613, "loss": 4.527938229963183, "lr_semantic": 4.487835385027589e-06, "lr_talker": 8.975670770055178e-06, "elapsed_s": 9457.697311616037, "job_id": "2650924"} +{"step": 4340, "total_steps": 5613, "loss": 4.470086505636573, "lr_semantic": 4.415084889299689e-06, "lr_talker": 8.830169778599377e-06, "elapsed_s": 9497.87012825394, "job_id": "2650924"} +{"step": 4360, "total_steps": 5613, "loss": 4.477155515924096, "lr_semantic": 4.343248769133634e-06, "lr_talker": 8.686497538267268e-06, "elapsed_s": 9538.255287026055, "job_id": "2650924"} +{"step": 4380, "total_steps": 5613, "loss": 4.554677220061421, "lr_semantic": 4.272336999630336e-06, "lr_talker": 8.544673999260673e-06, "elapsed_s": 9578.239094093908, "job_id": "2650924"} +{"step": 4400, "total_steps": 5613, "loss": 4.504549885168672, "lr_semantic": 4.202359427536182e-06, "lr_talker": 8.404718855072364e-06, "elapsed_s": 9618.372801762074, "job_id": "2650924"} +{"step": 4420, "total_steps": 5613, "loss": 4.468563262373209, "lr_semantic": 4.133325769875727e-06, "lr_talker": 8.266651539751453e-06, "elapsed_s": 9659.369848165894, "job_id": "2650924"} +{"step": 4440, "total_steps": 5613, "loss": 4.50471817702055, "lr_semantic": 4.065245612602387e-06, "lr_talker": 8.130491225204775e-06, "elapsed_s": 9698.31437128107, "job_id": "2650924"} +{"step": 4460, "total_steps": 5613, "loss": 4.508229313418269, "lr_semantic": 3.998128409267349e-06, "lr_talker": 7.996256818534698e-06, "elapsed_s": 9738.533771286951, "job_id": "2650924"} +{"step": 4480, "total_steps": 5613, "loss": 4.4808026403188705, "lr_semantic": 3.931983479706852e-06, "lr_talker": 7.863966959413704e-06, "elapsed_s": 9777.471496402053, "job_id": "2650924"} +{"step": 4500, "total_steps": 5613, "loss": 4.475344978272915, "lr_semantic": 3.866820008748059e-06, "lr_talker": 7.733640017496119e-06, "elapsed_s": 9817.360064570094, "job_id": "2650924"} +{"step": 4520, "total_steps": 5613, "loss": 4.47369010746479, "lr_semantic": 3.802647044933639e-06, "lr_talker": 7.605294089867278e-06, "elapsed_s": 9857.07974260603, "job_id": "2650924"} +{"step": 4540, "total_steps": 5613, "loss": 4.481849290430546, "lr_semantic": 3.739473499265307e-06, "lr_talker": 7.478946998530614e-06, "elapsed_s": 9896.727220227942, "job_id": "2650924"} +{"step": 4560, "total_steps": 5613, "loss": 4.489246996119618, "lr_semantic": 3.6773081439664425e-06, "lr_talker": 7.354616287932885e-06, "elapsed_s": 9935.44392012991, "job_id": "2650924"} +{"step": 4580, "total_steps": 5613, "loss": 4.507993955165148, "lr_semantic": 3.6161596112640023e-06, "lr_talker": 7.232319222528005e-06, "elapsed_s": 9975.47193939588, "job_id": "2650924"} +{"step": 4600, "total_steps": 5613, "loss": 4.501274397596717, "lr_semantic": 3.556036392189834e-06, "lr_talker": 7.112072784379668e-06, "elapsed_s": 10015.058491972042, "job_id": "2650924"} +{"step": 4620, "total_steps": 5613, "loss": 4.511318193748593, "lr_semantic": 3.4969468354016343e-06, "lr_talker": 6.993893670803269e-06, "elapsed_s": 10055.509547737893, "job_id": "2650924"} +{"step": 4640, "total_steps": 5613, "loss": 4.528082717210054, "lr_semantic": 3.4388991460236567e-06, "lr_talker": 6.877798292047313e-06, "elapsed_s": 10095.575066080084, "job_id": "2650924"} +{"step": 4660, "total_steps": 5613, "loss": 4.532296687364578, "lr_semantic": 3.3819013845073606e-06, "lr_talker": 6.763802769014721e-06, "elapsed_s": 10163.833614460891, "job_id": "2650924"} +{"step": 4680, "total_steps": 5613, "loss": 4.509938716888428, "lr_semantic": 3.325961465512132e-06, "lr_talker": 6.651922931024264e-06, "elapsed_s": 10204.811694727046, "job_id": "2650924"} +{"step": 4700, "total_steps": 5613, "loss": 4.50479955598712, "lr_semantic": 3.2710871568062707e-06, "lr_talker": 6.542174313612541e-06, "elapsed_s": 10247.901896313066, "job_id": "2650924"} +{"step": 4720, "total_steps": 5613, "loss": 4.486642941832542, "lr_semantic": 3.217286078188368e-06, "lr_talker": 6.434572156376736e-06, "elapsed_s": 10287.350444924086, "job_id": "2650924"} +{"step": 4740, "total_steps": 5613, "loss": 4.51556740142405, "lr_semantic": 3.164565700429215e-06, "lr_talker": 6.32913140085843e-06, "elapsed_s": 10327.08104562806, "job_id": "2650924"} +{"step": 4760, "total_steps": 5613, "loss": 4.509736044332385, "lr_semantic": 3.1129333442344288e-06, "lr_talker": 6.2258666884688576e-06, "elapsed_s": 10370.734344020952, "job_id": "2650924"} +{"step": 4780, "total_steps": 5613, "loss": 4.491977885365486, "lr_semantic": 3.0623961792278984e-06, "lr_talker": 6.124792358455797e-06, "elapsed_s": 10410.598235925892, "job_id": "2650924"} +{"step": 4800, "total_steps": 5613, "loss": 4.490510620176792, "lr_semantic": 3.012961222956229e-06, "lr_talker": 6.025922445912458e-06, "elapsed_s": 10449.997615390923, "job_id": "2650924"} +{"step": 4820, "total_steps": 5613, "loss": 4.508950624614954, "lr_semantic": 2.9646353399142746e-06, "lr_talker": 5.929270679828549e-06, "elapsed_s": 10489.11248591193, "job_id": "2650924"} +{"step": 4840, "total_steps": 5613, "loss": 4.514956993982196, "lr_semantic": 2.9174252405919544e-06, "lr_talker": 5.834850481183909e-06, "elapsed_s": 10528.85751005006, "job_id": "2650924"} +{"step": 4860, "total_steps": 5613, "loss": 4.498611994087696, "lr_semantic": 2.871337480542431e-06, "lr_talker": 5.742674961084862e-06, "elapsed_s": 10568.632050517015, "job_id": "2650924"} +{"step": 4880, "total_steps": 5613, "loss": 4.55120425671339, "lr_semantic": 2.8263784594718203e-06, "lr_talker": 5.6527569189436406e-06, "elapsed_s": 10609.146117511904, "job_id": "2650924"} +{"step": 4900, "total_steps": 5613, "loss": 4.476833371445537, "lr_semantic": 2.7825544203505256e-06, "lr_talker": 5.565108840701051e-06, "elapsed_s": 10648.69591521495, "job_id": "2650924"} +{"step": 4920, "total_steps": 5613, "loss": 4.547092678025365, "lr_semantic": 2.739871448546349e-06, "lr_talker": 5.479742897092698e-06, "elapsed_s": 10688.280809924938, "job_id": "2650924"} +{"step": 4940, "total_steps": 5613, "loss": 4.518194805830717, "lr_semantic": 2.6983354709794795e-06, "lr_talker": 5.396670941958959e-06, "elapsed_s": 10728.113615673967, "job_id": "2650924"} +{"step": 4960, "total_steps": 5613, "loss": 4.499944154173136, "lr_semantic": 2.6579522552994934e-06, "lr_talker": 5.315904510598987e-06, "elapsed_s": 10767.352765566902, "job_id": "2650924"} +{"step": 4980, "total_steps": 5613, "loss": 4.5239817798137665, "lr_semantic": 2.6187274090844515e-06, "lr_talker": 5.237454818168903e-06, "elapsed_s": 10807.99079614901, "job_id": "2650924"} +{"step": 5000, "total_steps": 5613, "loss": 4.521790949627757, "lr_semantic": 2.5806663790622466e-06, "lr_talker": 5.161332758124493e-06, "elapsed_s": 10847.802871332038, "job_id": "2650924"} +{"step": 5020, "total_steps": 5613, "loss": 4.479529803618789, "lr_semantic": 2.543774450354269e-06, "lr_talker": 5.087548900708538e-06, "elapsed_s": 10886.892361076083, "job_id": "2650924"} +{"step": 5040, "total_steps": 5613, "loss": 4.53331470862031, "lr_semantic": 2.508056745741524e-06, "lr_talker": 5.016113491483048e-06, "elapsed_s": 10926.426077683922, "job_id": "2650924"} +{"step": 5060, "total_steps": 5613, "loss": 4.475990153849125, "lr_semantic": 2.473518224953281e-06, "lr_talker": 4.947036449906562e-06, "elapsed_s": 10966.475102063967, "job_id": "2650924"} +{"step": 5080, "total_steps": 5613, "loss": 4.487173097208142, "lr_semantic": 2.4401636839783747e-06, "lr_talker": 4.8803273679567495e-06, "elapsed_s": 11005.521174782887, "job_id": "2650924"} +{"step": 5100, "total_steps": 5613, "loss": 4.518084799870849, "lr_semantic": 2.40799775439924e-06, "lr_talker": 4.81599550879848e-06, "elapsed_s": 11070.073245553067, "job_id": "2650924"} +{"step": 5120, "total_steps": 5613, "loss": 4.458881165832281, "lr_semantic": 2.377024902748767e-06, "lr_talker": 4.754049805497534e-06, "elapsed_s": 11116.48473715689, "job_id": "2650924"} +{"step": 5140, "total_steps": 5613, "loss": 4.502075454220176, "lr_semantic": 2.3472494298900887e-06, "lr_talker": 4.694498859780177e-06, "elapsed_s": 11175.641847911058, "job_id": "2650924"} +{"step": 5160, "total_steps": 5613, "loss": 4.50912244617939, "lr_semantic": 2.318675470419362e-06, "lr_talker": 4.637350940838724e-06, "elapsed_s": 11215.239849477075, "job_id": "2650924"} +{"step": 5180, "total_steps": 5613, "loss": 4.4947781171649694, "lr_semantic": 2.291306992091649e-06, "lr_talker": 4.582613984183298e-06, "elapsed_s": 11255.565913405968, "job_id": "2650924"} +{"step": 5200, "total_steps": 5613, "loss": 4.493680028244853, "lr_semantic": 2.265147795269948e-06, "lr_talker": 4.530295590539896e-06, "elapsed_s": 11295.986628266051, "job_id": "2650924"} +{"step": 5220, "total_steps": 5613, "loss": 4.495013313367963, "lr_semantic": 2.2402015123974873e-06, "lr_talker": 4.480403024794975e-06, "elapsed_s": 11336.742370306049, "job_id": "2650924"} +{"step": 5240, "total_steps": 5613, "loss": 4.492965376004577, "lr_semantic": 2.21647160749332e-06, "lr_talker": 4.43294321498664e-06, "elapsed_s": 11376.87850391306, "job_id": "2650924"} +{"step": 5260, "total_steps": 5613, "loss": 4.506780857220292, "lr_semantic": 2.19396137567132e-06, "lr_talker": 4.38792275134264e-06, "elapsed_s": 11416.976068895077, "job_id": "2650924"} +{"step": 5280, "total_steps": 5613, "loss": 4.499888459220529, "lr_semantic": 2.172673942682619e-06, "lr_talker": 4.345347885365238e-06, "elapsed_s": 11456.664269140922, "job_id": "2650924"} +{"step": 5300, "total_steps": 5613, "loss": 4.518908817321062, "lr_semantic": 2.152612264481573e-06, "lr_talker": 4.305224528963146e-06, "elapsed_s": 11496.755048733903, "job_id": "2650924"} +{"step": 5320, "total_steps": 5613, "loss": 4.489227641373873, "lr_semantic": 2.133779126815298e-06, "lr_talker": 4.267558253630596e-06, "elapsed_s": 11537.315214381088, "job_id": "2650924"} +{"step": 5340, "total_steps": 5613, "loss": 4.488119645044208, "lr_semantic": 2.116177144836843e-06, "lr_talker": 4.232354289673686e-06, "elapsed_s": 11577.329608292086, "job_id": "2650924"} +{"step": 5360, "total_steps": 5613, "loss": 4.507138418033719, "lr_semantic": 2.0998087627420547e-06, "lr_talker": 4.199617525484109e-06, "elapsed_s": 11617.551594171906, "job_id": "2650924"} +{"step": 5380, "total_steps": 5613, "loss": 4.483428550884128, "lr_semantic": 2.084676253430176e-06, "lr_talker": 4.169352506860352e-06, "elapsed_s": 11657.543182423105, "job_id": "2650924"} +{"step": 5400, "total_steps": 5613, "loss": 4.507531909272075, "lr_semantic": 2.0707817181882362e-06, "lr_talker": 4.1415634363764725e-06, "elapsed_s": 11697.616981758969, "job_id": "2650924"} +{"step": 5420, "total_steps": 5613, "loss": 4.5199560560286045, "lr_semantic": 2.0581270863992647e-06, "lr_talker": 4.116254172798529e-06, "elapsed_s": 11737.684882604051, "job_id": "2650924"} +{"step": 5440, "total_steps": 5613, "loss": 4.516109714284539, "lr_semantic": 2.046714115274381e-06, "lr_talker": 4.093428230548762e-06, "elapsed_s": 11777.653627076885, "job_id": "2650924"} +{"step": 5460, "total_steps": 5613, "loss": 4.533677160739899, "lr_semantic": 2.0365443896087896e-06, "lr_talker": 4.073088779217579e-06, "elapsed_s": 11817.504115035059, "job_id": "2650924"} +{"step": 5480, "total_steps": 5613, "loss": 4.511550577357411, "lr_semantic": 2.027619321561713e-06, "lr_talker": 4.055238643123426e-06, "elapsed_s": 11858.314438190078, "job_id": "2650924"} +{"step": 5500, "total_steps": 5613, "loss": 4.508249634876847, "lr_semantic": 2.0199401504603083e-06, "lr_talker": 4.0398803009206166e-06, "elapsed_s": 11898.535210394999, "job_id": "2650924"} +{"step": 5520, "total_steps": 5613, "loss": 4.4926785957068205, "lr_semantic": 2.0135079426275685e-06, "lr_talker": 4.027015885255137e-06, "elapsed_s": 11938.88822698011, "job_id": "2650924"} +{"step": 5540, "total_steps": 5613, "loss": 4.5294253770262, "lr_semantic": 2.0083235912342546e-06, "lr_talker": 4.016647182468509e-06, "elapsed_s": 12006.199359528022, "job_id": "2650924"} +{"step": 5560, "total_steps": 5613, "loss": 4.465063022449613, "lr_semantic": 2.0043878161748764e-06, "lr_talker": 4.008775632349753e-06, "elapsed_s": 12046.419657595921, "job_id": "2650924"} +{"step": 5580, "total_steps": 5613, "loss": 4.495070664212108, "lr_semantic": 2.00170116396772e-06, "lr_talker": 4.00340232793544e-06, "elapsed_s": 12086.452257984085, "job_id": "2650924"} +{"step": 5600, "total_steps": 5613, "loss": 4.527787942439318, "lr_semantic": 2.000264007678965e-06, "lr_talker": 4.00052801535793e-06, "elapsed_s": 12125.804159580963, "job_id": "2650924"} +{"step": 5613, "total_steps": 5613, "loss": 4.5207825703407405, "lr_semantic": 2.0000000000000003e-06, "lr_talker": 4.000000000000001e-06, "elapsed_s": 12151.526500405977, "job_id": "2650924"} diff --git a/stats/fullft-validation.jsonl b/stats/fullft-validation.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..627b20f9932bfe3e7c9d20fdc2e73b3fc5eea326 --- /dev/null +++ b/stats/fullft-validation.jsonl @@ -0,0 +1 @@ +{"step": 0, "mean_loss": 5.40239987373352, "cases": 20, "time": 1793633547.4968805}\n{"step": 413, "mean_loss": 4.707755017280578, "cases": 20, "time": 1793634445.4615188}\n{"step": 847, "mean_loss": 4.679264855384827, "cases": 20, "time": 1793635373.4987214}\n{"step": 1260, "mean_loss": 4.667382431030274, "cases": 20, "time": 1793636300.7190757}\n{"step": 1652, "mean_loss": 4.656465816497803, "cases": 20, "time": 1793637235.9532719}\n{"step": 1871, "mean_loss": 4.651724743843078, "cases": 20, "time": 1793637716.785558}\n{"step": 2311, "mean_loss": 4.658494067192078, "cases": 20, "time": 1793638642.6502783}\n{"step": 2744, "mean_loss": 4.639594459533692, "cases": 20, "time": 1793639568.245913}\n{"step": 3160, "mean_loss": 4.634997391700745, "cases": 20, "time": 1793640494.9953086}\n{"step": 3581, "mean_loss": 4.620176243782043, "cases": 20, "time": 1793641427.281522}\n{"step": 3742, "mean_loss": 4.615711975097656, "cases": 20, "time": 1793641785.2390747}\n{"step": 4190, "mean_loss": 4.643639993667603, "cases": 20, "time": 1793642715.493736}\n{"step": 4641, "mean_loss": 4.642439317703247, "cases": 20, "time": 1793643642.0362904}\n{"step": 5089, "mean_loss": 4.637552738189697, "cases": 20, "time": 1793644568.1563737}\n{"step": 5525, "mean_loss": 4.63270890712738, "cases": 20, "time": 1793645493.5265963}\n{"step": 5613, "mean_loss": 4.6320901870727536, "cases": 20, "time": 1793645696.0330918}\n \ No newline at end of file diff --git a/stats/rank64-CONTRACT.json b/stats/rank64-CONTRACT.json new file mode 100644 index 0000000000000000000000000000000000000000..cddeaab932a46bf7cd36f5414d497d3ce699b413 --- /dev/null +++ b/stats/rank64-CONTRACT.json @@ -0,0 +1,27 @@ +{ + "variant": "podcast", + "parent": "/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/large/training/checkpoints/step-00005613", + "parent_ready_sha256": "99c2578128c510e2dce04250a78bca465bf4e2ad1f7302eeb94a3a83cf0ec169", + "parent_bytes": 18201842943, + "ladder_sha256": "35492447edcf91fa5aa15a54394cb0355c609655ac53076ee09c29e0a8480f39", + "training_code_sha256": "22df229f2c5ea25c1273091fdf875dc53f79ec55b57fc96831c6ac2302d73214", + "schedule_sha256": "b070ddd84b67c8333bb9ad727aa3a594da55f768a7b8849a0c0edbe0e6dad674", + "packer_sha256": "d379eefc2128df8578da394e782aa64ca73dd9b69a76a5ab488d34f6588ad0a7", + "reader_sha256": "eb45d1a325eb5858d32fcbe2be0f76cd58eee54fc9e7fd8adc4b0da1a6fbb230", + "adapter_sha256": "f7eaf7ac6b6da344653f72dee77d498bc7019a3bc32fe715695d9cd2a994e579", + "world": 4, + "rank": 64, + "alpha": 64, + "seed": 20261102, + "peak": 3e-05, + "phase_steps": [ + 183, + 160, + 132, + 91, + 43, + 43 + ], + "total_steps": 652, + "reference": "two strictly past 5\u201315s speaker references, strictly past context" +} diff --git a/stats/rank64-FINISHED.json b/stats/rank64-FINISHED.json new file mode 100644 index 0000000000000000000000000000000000000000..2e323ceda867364cd68d68100ba2fb46c45acce7 --- /dev/null +++ b/stats/rank64-FINISHED.json @@ -0,0 +1,5 @@ +{ + "step": 652, + "contract_sha256": "88d1284099df084d941f8274e953612a864c607e9ae4380af738280ef9dfc517", + "time": 1793657711.2167256 +} diff --git a/stats/rank64-STATUS.json b/stats/rank64-STATUS.json new file mode 100644 index 0000000000000000000000000000000000000000..04e69a74267f076c4768495a6b149aa3b6349652 --- /dev/null +++ b/stats/rank64-STATUS.json @@ -0,0 +1,9 @@ +{ + "step": 652, + "total_steps": 652, + "loss": 4.351958274841309, + "lr": 3e-06, + "grad_norm": 0.8629185557365417, + "job": "2657062", + "time": 1793657710.508457 +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8da72c77edd838e0a08d37d2b3005f25b9e9b60c --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06902d1fb775216338802205886a24bc715ccc606bd872a892e3d3c83ca1b9e2 +size 11423220 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5426bc4f185cb5a3f45b0c7db3d3379eb6019fed --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,253 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|audio_start|>", + "<|audio_end|>", + "<|audio_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training/dialogues.py b/training/dialogues.py new file mode 100644 index 0000000000000000000000000000000000000000..84770f84e1bc57f85e9b84e8a4327cfe132ee3d8 --- /dev/null +++ b/training/dialogues.py @@ -0,0 +1,102 @@ +"""Ten new topics, each tested at two, six and eight alternating turns.""" +TOPICS=[ +('en','coffee',[ + 'I tried making coffee without looking at my phone this morning. It felt oddly peaceful, actually.', + 'That sounds lovely. Although I would probably spend the whole time wondering whether the kettle was taking longer.', + 'Exactly! At first I kept reaching for a notification that was not even there. Then I noticed the rain.', + 'We fill every little gap, don’t we? Sometimes the nicest part of a morning is that nothing happens.', + 'And the coffee tasted better. Not because I made it differently, but because I finally paid attention.', + 'Well, now you have turned a cup of coffee into a small philosophy lesson. I am surprisingly convinced.', + 'Let’s not get carried away. Tomorrow I might forget the water and just stare thoughtfully at the empty cup.', + 'Honestly, that would still be a very committed experiment. We should report back next week.']), +('en','walking',[ + 'Do you ever take a different route home just because the usual one has started to feel invisible?', + 'Yes, and then I discover a tiny shop that has apparently been there for about twenty years.', + 'That happened yesterday. There was a bookshop with a sleeping dog right beside the door. I almost missed it.', + 'A sleeping dog is an excellent recommendation. It suggests the atmosphere is relaxed and the customers are trustworthy.', + 'The owner asked what I liked reading, and I realised I had not answered that question in ages.', + 'What did you say? Please tell me you did not just point to the nearest shelf and panic.', + 'Almost. Then I said I wanted something that would surprise me without making the whole week feel heavier.', + 'That is a wonderful request. Perhaps taking the longer route is sometimes the quickest way to find something new.']), +('en','learning',[ + 'I finally played that difficult passage on the piano without stopping. Only once, but I am counting it.', + 'You absolutely should. One good attempt means your hands have discovered that the thing is possible.', + 'The funny part is that I had been practising it faster and faster, which mostly made it sound worse.', + 'Then somebody told you to slow down, and you were deeply offended by this very sensible advice?', + 'A little! But after a few quiet minutes, I could actually hear where I was rushing ahead.', + 'That is the awkward secret of learning. Sometimes trying harder just means repeating the mistake with more confidence.', + 'So tomorrow I will be patient, start slowly, and resist the urge to perform a heroic concert for the neighbours.', + 'A thoughtful decision. Although I think the neighbours would appreciate a short victory fanfare at the end.']), +('de','cooking',[ + 'Ich habe gestern endlich das Brot selbst gebacken. Die Küche sah danach allerdings ziemlich abenteuerlich aus.', + 'Das gehört dazu. Die entscheidende Frage ist doch: Hat es gut geschmeckt, oder war es eher ein Türstopper?', + 'Überraschend gut! Die Kruste war ein bisschen dunkel, aber innen war es weich. Ich war richtig stolz darauf.', + 'Dann war das keine dunkle Kruste, sondern ein bewusst gewähltes Röstaroma. So würde ich es jedenfalls verkaufen.', + 'Genau das habe ich auch gesagt. Beim zweiten Versuch möchte ich trotzdem vorher schauen, wie heiß der Ofen wirklich ist.', + 'Vernünftig. Ich finde sowieso schön, wenn man etwas macht, bei dem man das Ergebnis hinterher gemeinsam essen kann.', + 'Und wenn es schiefgeht, hat man wenigstens eine Geschichte. Vielleicht brauche ich beim nächsten Mal weniger Mehl auf dem Boden.', + 'Oder mehr Gäste zum Aufräumen. Du lädst zum Brotessen ein und erwähnst den kleinen Arbeitseinsatz erst später.']), +('de','train',[ + 'Im Zug hat mich heute ein Kind gefragt, warum die Wolken eigentlich nicht einfach vom Himmel fallen.', + 'Das ist eine ausgezeichnete Frage. Hast du wissenschaftlich geantwortet oder erst einmal sehr lange aus dem Fenster geschaut?', + 'Beides. Ich wollte mit winzigen Wassertröpfchen anfangen, aber dann fragte es sofort, warum die nicht alle zusammenkleben.', + 'Kinder haben dieses Talent, jede vermeintlich einfache Erklärung genau an der spannendsten Stelle auseinanderzunehmen.', + 'Ja, und plötzlich saßen drei Erwachsene da und überlegten gemeinsam. Das war viel netter als auf das Handy zu schauen.', + 'Dann war die eigentliche Entdeckung vielleicht gar nicht das Wetter, sondern dass ein fremder Mensch ein Gespräch anfangen kann.', + 'Am Ende hat das Kind gesagt: Vielleicht bleiben die Wolken oben, weil sie noch irgendwo hinwollen. Das fand ich wunderschön.', + 'Das gefällt mir auch. Die Erklärung würde ich behalten, selbst wenn du später noch die genaue Physik nachliest.']), +('de','garden',[ + 'Auf meinem Balkon wächst jetzt tatsächlich etwas Grünes. Ich hätte nicht gedacht, dass mich so ein kleiner Trieb so freut.', + 'Was hast du gepflanzt? Und bitte sag mir, dass du diesmal noch weißt, in welchen Topf die Samen gekommen sind.', + 'Basilikum, glaube ich. Die Beschriftung hat der Regen mitgenommen. Es könnte also auch eine sehr überzeugende andere Pflanze sein.', + 'Dann betrachten wir das vorerst als botanisches Rätsel. Du musst nichts überstürzen und solltest vielleicht noch nicht daran knabbern.', + 'Keine Sorge. Ich finde es einfach angenehm, morgens kurz nachzusehen, ob wieder ein neues Blatt dazugekommen ist.', + 'Das verstehe ich. Es ist eine kleine Veränderung, die nicht drängelt und trotzdem zeigt, dass etwas vorangeht.', + 'Genau. Und ich merke, dass ich dabei geduldiger werde. Obwohl ich immer noch dreimal täglich nach dem angeblichen Basilikum schaue.', + 'Geduld mit regelmäßiger Kontrolle. Das klingt nach einem guten Anfang. Jetzt fehlt nur noch eine wetterfeste Beschriftung.']), +('fr','jardin',[ + 'Ce matin, j’ai découvert une petite fleur sur mon balcon. Je pensais vraiment que cette plante avait abandonné.', + 'Comme quoi, elle préparait peut-être sa surprise en silence. Tu lui as donné un peu plus de lumière ?', + 'Oui, et surtout un peu moins d’eau. Apparemment, mon enthousiasme dépassait largement ses besoins. Je voulais trop bien faire.', + 'C’est assez humain, finalement. On croit aider en faisant davantage, alors que parfois il faut simplement laisser respirer.', + 'Tu as raison. Maintenant, je regarde les nouvelles feuilles avant de prendre mon téléphone. Cela change le début de ma journée.', + 'Et tu ne peux pas accélérer la prochaine fleur en actualisant la page. C’est peut-être exactement ce qui fait du bien.', + 'Voilà. Même si je vérifie encore beaucoup trop souvent. J’appelle cela de la patience, mais une patience très attentive.', + 'C’est une jolie manière de le dire. Garde cette attention, et pense seulement à poser l’arrosoir de temps en temps.']), +('fr','chance',[ + 'J’ai retrouvé un vieux carnet dans une boîte. Il était rempli de projets que j’avais complètement oubliés.', + 'Et alors, tu as eu envie de les reprendre, ou surtout de remercier ton ancien toi de ne pas avoir tout essayé ?', + 'Un peu des deux. Il y avait une idée de voyage magnifique, puis une recette de soupe franchement inquiétante.', + 'C’est rassurant. Un bon carnet devrait contenir de grandes ambitions et quelques décisions culinaires discutables.', + 'Ce qui m’a touché, c’est de voir combien je pouvais être enthousiaste. Pas forcément réaliste, mais vraiment curieux.', + 'Peut-être que tu peux garder cette curiosité sans reprendre chaque projet. On a le droit de changer de direction.', + 'Oui. Je vais choisir une seule petite chose, quelque chose que je pourrais faire ce week-end, sans organisation compliquée.', + 'Très bonne idée. Et pour la soupe, nous attendrons prudemment une deuxième opinion avant de rallumer le feu.']), +('es','radio',[ + 'Esta mañana escuché una canción que no oía desde el colegio. De repente recordé hasta el camino de vuelta a casa.', + 'Es increíble cómo pasa eso. A veces una melodía guarda recuerdos que ni siquiera sabíamos que seguían allí.', + 'Sí, y no era una canción especialmente importante. Creo que simplemente sonaba siempre en la pequeña tienda de la esquina.', + 'Entonces también recuerdas la tienda, el ruido de la calle, quizá alguna conversación. Todo aparece junto, sin pedir permiso.', + 'Exactamente. Me dieron ganas de escribirle a un amigo de aquella época, aunque llevamos bastante tiempo sin hablar.', + 'Hazlo. No necesitas un mensaje perfecto. Puedes empezar diciendo que escuchaste algo y te acordaste de él.', + 'Eso voy a hacer. A veces complico demasiado estas cosas, como si retomar una amistad necesitara una gran explicación.', + 'Y muchas veces basta con una frase sincera. Bueno, y con no enviarle veinte canciones antes de que conteste.']), +('es','friends',[ + 'Ayer cociné para unos amigos y olvidé poner sal. Me di cuenta cuando todos estaban intentando ser demasiado amables.', + 'Ese silencio tan educado lo dice todo. ¿Alguien pidió el salero o decidiste presentar el plato como una experiencia delicada?', + 'Al final me reí y lo dije yo. Fue un alivio, porque en cuanto lo reconocí todos empezaron a contar sus propios desastres.', + 'Eso suele mejorar una cena más que una receta perfecta. De repente nadie tiene que fingir que siempre le sale todo bien.', + 'Una amiga contó que una vez horneó un pastel sin harina. Todavía no entiendo muy bien qué fue lo que terminó sirviendo.', + 'Quizá inventó un postre nuevo. Hay errores que suenan terribles hasta que alguien les pone un nombre elegante.', + 'En cualquier caso, terminamos hablando durante horas. La próxima vez pondré sal, pero espero que la conversación sea igual de buena.', + 'Seguro que sí. Y si no, siempre puedes olvidar las cucharas. Aunque yo preferiría conservar la comida y buscar otra anécdota.'])] + +def tasks(): + result=[] + for group,n,seconds in [('short',2,18.),('medium',6,38.4),('long',8,59.2)]: + for i,(lang,topic,lines) in enumerate(TOPICS): + pause=.24;per=(seconds-pause*(n-1))/n + turns=[dict(speaker='S1' if j%2==0 else 'S2',text=text,seconds=round(per,2),direction='warm, curious, relaxed conversational delivery' if j%2==0 else 'engaged, friendly, spontaneous conversational reply') for j,text in enumerate(lines[:n])] + result.append(dict(task_id=group+'-'+lang+'-'+topic,group=group,language=lang,topic=topic,seed=20261103+i,pause_s=pause, + general='A natural two-person podcast conversation. Two distinct stable voices take turns, listening and responding warmly. Clear studio speech, organic rhythm and small natural pauses. Read only the words in SCRIPT.',turns=turns)) + return result diff --git a/training/fullft/annotation_worker.py b/training/fullft/annotation_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..2fe36f841d96dcfd406b0a272bbbdf8c290f426e --- /dev/null +++ b/training/fullft/annotation_worker.py @@ -0,0 +1,365 @@ +"""Resumable episode annotation. Neural execution is restricted to Slurm GPUs. + +Original archives are immutable. Each of 64 partitions has a transactional +SQLite journal; codec blocks, raw ASR, full scores and speaker posteriors survive +preemption. Output TARs are committed only after episode/schema validation. +""" +import argparse +import gzip +import hashlib +import io +import json +import math +import os +from pathlib import Path +import sqlite3 +import subprocess +import sys +import tarfile +import tempfile +import time +import traceback + +HERE=Path(__file__).resolve().parent +ROOT=Path('/e/scratch/laionize/schuhmann1/notebooklm_podcast_preparation_20261101') +sys.path.insert(0,'/e/home/jusers/schuhmann1/jupiter/notebooklm_quality_20261101') +from media_runtime import FF,environment,preflight +from timeline import words_from_asr,attribute,sentences_from_words,examples + +def packed(value): + return gzip.compress(json.dumps(value,ensure_ascii=False,allow_nan=False,separators=(',',':')).encode(),mtime=0) + +def unpacked(value):return json.loads(gzip.decompress(value)) + +def array_blob(array): + import numpy as np + buf=io.BytesIO();np.save(buf,array,allow_pickle=False);return buf.getvalue() + +class Journal: + def __init__(self,path): + self.db=sqlite3.connect(path,timeout=120) + self.db.execute('PRAGMA journal_mode=WAL') + self.db.execute('PRAGMA synchronous=FULL') + self.db.executescript('''CREATE TABLE IF NOT EXISTS stages(uid TEXT,stage TEXT,value BLOB,PRIMARY KEY(uid,stage)); + CREATE TABLE IF NOT EXISTS errors(uid TEXT,time REAL,trace TEXT); + CREATE TABLE IF NOT EXISTS done(uid TEXT PRIMARY KEY,hours REAL,words INTEGER,sentences INTEGER,examples INTEGER,elapsed_s REAL);''') + def get(self,uid,stage): + row=self.db.execute('SELECT value FROM stages WHERE uid=? AND stage=?',(uid,stage)).fetchone() + return row[0] if row else None + def put(self,uid,stage,value): + self.db.execute('INSERT OR REPLACE INTO stages VALUES(?,?,?)',(uid,stage,value));self.db.commit() + def stage(self,uid,name,fn): + value=self.get(uid,name) + if value is not None:return unpacked(value) + print(json.dumps(dict(event='stage',uid=uid,stage=name,time=time.time())),flush=True) + value=fn();self.put(uid,name,packed(value));return value + +class Models: + def quality(self,uid): + if not hasattr(self,'quality_windows'): + self.quality_windows=json.loads((ROOT/'quality_windows.json').read_text()) + return self.quality_windows[uid] + + def asr(self,audio,path,language): + import torch + if language in {'ja','zh','no'}: + if not hasattr(self,'whisper'): + from transformers import AutoProcessor,WhisperForConditionalGeneration,pipeline + p='/e/scratch/laionize/schuhmann1/hvs_real_2500h_annotation_20261017/models/whisper-large-v3-turbo' + proc=AutoProcessor.from_pretrained(p,local_files_only=True) + model=WhisperForConditionalGeneration.from_pretrained(p,local_files_only=True,dtype=torch.bfloat16,attn_implementation='eager').cuda().eval() + self.whisper=pipeline('automatic-speech-recognition',model=model,tokenizer=proc.tokenizer, + feature_extractor=proc.feature_extractor,device=0,chunk_length_s=30,stride_length_s=5) + # CJK requires Unicode-token alignment, not whitespace grouping. + # Use tokenizer's language fallback for word collation. Current + # upstream return_language=True prepends an init token without a + # corresponding timestamp and can fail with an IndexError, so it + # must not be used as a substitute for this tokenizer setting. + from transformers.models.whisper.tokenization_whisper import LANGUAGES + self.whisper.tokenizer.set_prefix_tokens(language=LANGUAGES[language],task='transcribe') + # Use short, silence-seeking NON-overlapping clips. The upstream + # long-file overlap merger can repeat CJK units and move their + # times backwards by a whole stride. Explicit offsets avoid that + # failure without sorting a transcript or inventing alignments. + import numpy as np + boundaries=[0] + while len(audio)-boundaries[-1]>30*16000: + lo=boundaries[-1]+20*16000;hi=boundaries[-1]+25*16000 + step=800;starts=list(range(lo,hi-step+1,step)) + cut=min(starts,key=lambda i:float(np.mean(audio[i:i+step]**2)))+step//2 + boundaries.append(cut) + boundaries.append(len(audio)) + inputs=[{'raw':audio[a:b],'sampling_rate':16000} for a,b in zip(boundaries,boundaries[1:])] + results=list(self.whisper(inputs,batch_size=8,return_timestamps='word', + generate_kwargs={'language':language,'task':'transcribe','do_sample':False,'max_new_tokens':440})) + segments=[] + for index,(a,b,raw) in enumerate(zip(boundaries,boundaries[1:],results)): + offset=a/16000;duration=(b-a)/16000;words=[] + for chunk in raw['chunks']: + local_a,local_b=chunk['timestamp'] + if local_a is None or local_b is None:raise ValueError('Fallback returned an untimed word; quarantine, never invent an endpoint') + if local_a<-.05 or local_b>duration+.05 or local_b480000: + # Preserve unusual long ASR intervals without guessing word + # boundaries or truncating the score window at 30 seconds. + preds[i]=he.predict_waveform(c,include_frames=False) + for i,(s,pred) in enumerate(zip(batch,preds)): + pred=add_reference_z(pred,self.baseline) + pred['event_timestamp_scope']='score-window-local; add sentence.score_window_start_s for episode time' + detail=caption_detail(caption_view(pred,self.baseline,self.settings),self.baseline, + k_voicenet=min(3,self.settings['kv']),k_emonet=self.settings['ke'],always_on=[], + synonym_seed=stable_hash(uid)%(1<<30)+start+i+1,template=self.settings['template']) + s.update(direction_en=render_caption(detail),caption_detail=detail,humaneness_ears_medium=pred) + events=[] + for e in bursts: + top=e['top3_classes'];prob=top[0]['softmax_probability'] if top else 0 + events.append(dict(e,written=top[0]['label'] if prob>=.5 else 'Vocal Burst',class_probability=prob)) + return dict(global_prediction=global_pred,sentences=sentences,events=events,caption_settings=self.settings) + + def codec(self,wave,journal,uid): + import numpy as np + import torch + if not hasattr(self,'codec_model'): + from transformers import AutoModel + p='/e/scratch/reformo/schuhmann1_moss/models/moss_codec_v2' + self.codec_model=AutoModel.from_pretrained(p,trust_remote_code=True,local_files_only=True,codec_weight_dtype='fp32').cuda().eval() + self.codec_model.set_compute_dtype('bf16');self.codec_model.set_attention_implementation('sdpa') + blob=journal.get(uid,'codec-continuous') + # One streaming session per COMPLETE episode. batch_encode internally + # processes 4.8s chunks with state carried across all boundaries. Do not + # concatenate independently encoded clips and call that continuous. + if blob is None: + frames=math.ceil(len(wave)/3840) + w=np.pad(wave,(0,frames*3840-len(wave))) + w=torch.from_numpy(np.repeat(w[None,:],2,axis=0)).cuda() + with torch.inference_mode():out=self.codec_model.batch_encode([w],num_quantizers=32,chunk_duration=4.8) + arr=out.audio_codes[:,0,:int(out.audio_codes_lengths[0])].T.cpu().numpy().astype('=1024:raise ValueError('MOSS token out of vocabulary') + if arr.shape!=(math.ceil(len(wave)/3840),32):raise ValueError('Continuous codec timeline lost frames') + if blob is None:journal.put(uid,'codec-continuous',array_blob(arr)) + if getattr(self,'pilot',False) and journal.get(uid,'pilot-codec-checks') is None: + prefix=torch.from_numpy(np.repeat(np.asarray(wave[:460800])[None,:],2,axis=0).copy()).cuda() + with torch.inference_mode():old=self.codec_model.batch_encode([prefix],num_quantizers=12,chunk_duration=4.8) + oldcodes=old.audio_codes[:,0,:60].T.cpu().numpy() + if not np.array_equal(arr[:60,:12],oldcodes):raise ValueError('12/32 quantizer prefix parity failed') + checks=[] + for nq in [12,32]: + code=torch.from_numpy(arr[:120].T.astype(np.int64)).cuda() + with torch.inference_mode():decoded=self.codec_model.batch_decode([code],num_quantizers=nq,chunk_duration=4.8) + count=int(decoded.audio_lengths[0]);decoded_audio=decoded.audio[0,:,:count].float().mean(0).cpu().numpy() + if count!=120*3840 or not np.isfinite(decoded_audio).all():raise ValueError('Codec decode smoke failed') + rms=float(np.sqrt(np.mean(decoded_audio**2))) + if rms<1e-5:raise ValueError('Unexpected silent codec reconstruction') + mp3=subprocess.run([str(FF),'-v','error','-f','f32le','-ar','48000','-ac','1','-i','pipe:0', + '-b:a','128k','-f','mp3','pipe:1'],input=decoded_audio.astype('.25:raise ValueError('Decoded duration differs from source inventory') + asr=journal.stage(uid,'asr',lambda:models.asr(audio,tmp/'asr.wav',row['language'])) + words=words_from_asr(asr,duration) + import re + import unicodedata + def normalized(value):return ''.join(c for c in unicodedata.normalize('NFKC',value).casefold() if c.isalnum()) + text_parity=normalized(asr.get('text',''))==normalized(''.join(w['text'] for w in words)) + if not text_parity:raise ValueError('ASR full transcript does not match all timed units; do not silently drop words') + def diarize(): + turns,probs=models.diar(audio) + journal.put(uid,'speaker-posteriors',array_blob(probs)) + return turns + turns=journal.stage(uid,'diarization',diarize) + if not journal.get(uid,'speaker-posteriors'):raise ValueError('Missing full speaker activity scores') + speakers,turns=attribute(words,turns) + if not speakers:raise ValueError('Diarization produced no speakers') + # Detect a silently truncated long ASR request before packaging it. + last_speech=max(t['end_s'] for t in turns) + if words[-1]['end_s']=4.0',background_start=row['background_start'],background_middle=row['background_middle'], + original_scored_windows=models.quality(uid),transcript_word_parity=text_parity, + alignment_unit=asr.get('alignment_unit','word'), + publisher_language=row['language'],language_is_independently_verified=False), + provenance=dict(asr_model=asr['model'],diarization_model='nvidia/Nemotron-3-Diarization', + diarization_revision='f667ed73aee57d40cc39428eb768b4fd87a0a29e',alignment_method=asr['timestamp_policy'], + created_at=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),procedural_revision='7ebd4a9b7da96df989af8fc6ad0ce34f4e8aa3ed', + sentence_builder='v2-point-attribution-no-word-loss', + transformers_source='ca8c871ede62b521a75a2ddfc5e1968ac9c8e097',data_only_multispeaker_contract=True)) + for speaker in speakers: + if speaker['reference']:speaker['reference']['audio_uri']=record['audio']['audio_uri'] + prompts(record) + if not record['examples']:raise ValueError('No unambiguous timed training examples') + jsonschema.validate(record,json.loads((HERE/'podcast_episode_v1.schema.json').read_text())) + journal.put(uid,'record',packed(record));journal.put(uid,'codec-final',array_blob(codes)) + journal.db.execute('INSERT OR REPLACE INTO done VALUES(?,?,?,?,?,?)',(uid,duration/3600,len(words),len(sentences),len(record['examples']),time.monotonic()-began));journal.db.commit() + print(json.dumps(dict(event='done',uid=uid,duration_s=duration,words=len(words),sentences=len(sentences),speakers=len(speakers),examples=len(record['examples']),elapsed_s=time.monotonic()-began)),flush=True) + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--pilot',action='store_true');args=parser.parse_args() + if not os.environ.get('SLURM_JOB_ID'):raise RuntimeError('Neural work requires an allocated Slurm compute node') + import torch + if not torch.cuda.is_available():raise RuntimeError('No allocated CUDA GPU') + preflight();torch.set_num_threads(4) + rank=int(os.environ.get('SLURM_PROCID','0'));world=int(os.environ.get('SLURM_NTASKS','1')) + name='pilot' if args.pilot else 'full';state=ROOT/'state'/name;state.mkdir(parents=True,exist_ok=True) + selection=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/SELECTED_EPISODES.json') + rows=json.loads((ROOT/'pilot.json' if args.pilot else selection).read_text()) + models=Models();models.pilot=args.pilot;failed=0 + deadline=time.monotonic()+9*3600 + for partition in range(64): + if partition%world!=rank:continue + selected=[r for r in rows if r['partition']==partition] + if not selected:continue + journal=Journal(state/f'part-{partition:03d}.sqlite') + for row in selected: + if time.monotonic()>deadline: + (state/f'drained-{os.environ["SLURM_JOB_ID"]}-{rank}.json').write_text(json.dumps(dict(job=os.environ['SLURM_JOB_ID'],rank=rank,world=world,next_uid=row['uid'],reason='9h safe drain before 10h allocation limit'))+'\n') + print('SAFE_WALLTIME_STOP',flush=True);return + try:annotate(row,journal,models) + except Exception: + failed+=1;trace=traceback.format_exc();print(trace,flush=True) + journal.db.execute('INSERT INTO errors VALUES(?,?,?)',(row['uid'],time.time(),trace));journal.db.commit() + if args.pilot or failed>=3:raise + journal.db.execute('PRAGMA wal_checkpoint(TRUNCATE)');journal.db.close() + if failed:raise RuntimeError(f'{failed} episodes quarantined; see journal, do not mark complete') + +if __name__=='__main__':main() diff --git a/training/fullft/common.py b/training/fullft/common.py new file mode 100644 index 0000000000000000000000000000000000000000..84f720bc5037cfc2b2a1d429dd91c15080d1511e --- /dev/null +++ b/training/fullft/common.py @@ -0,0 +1,64 @@ +"""Five-language NotebookLM full FT from final HV Large 1.3, not SFT3.""" +from pathlib import Path +import hashlib,json,math,os,re,unicodedata +HOME=Path('/e/home/jusers/schuhmann1/jupiter') +CODE=Path(__file__).resolve().parent +ROOT=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102') +RUN=ROOT +ANNOTATION=Path('/e/scratch/laionize/schuhmann1/notebooklm_podcast_preparation_20261101') +BASE=Path('/e/scratch/laionize/schuhmann1/acting_challenges_finetune') +PARENT_ENGINE=HOME/'hvs_multilingual_fullft_20261013' +PARENT_RELEASE=Path('/e/scratch/laionize/schuhmann1/hvs_large_1_3_20261022/release_1_3') +SFT3=PARENT_RELEASE/'model' # Compatibility variable, NOT the original SFT3. +BASE_WEIGHTS={'model-00001-of-00003.safetensors':'3a9831cfd1944316e8b7d5fcaea845bf8b49b832824950709ce0eabf56b4702a', + 'model-00002-of-00003.safetensors':'1fdaae7bd4f09b867152ef91761106e7bfef986b5f9979f69463466e864018a5', + 'model-00003-of-00003.safetensors':'c149c250be927df24cdaa59c5b8ff50c3f0471f664f6483bb58236f07afa31df'} +BASE_SHA=hashlib.sha256(json.dumps(BASE_WEIGHTS,sort_keys=True).encode()).hexdigest() +WORLD,ACCUM,SEED,EPOCHS=64,2,20261102,3 +BATCH=WORLD*ACCUM +PEAK=(2e-5,4e-5) +WARMUP,FLOOR=.05,.10 +VARIANT='podcast' +LANGUAGES={'en':'English','de':'German','fr':'French','es':'Spanish','nl':'Dutch'} +MONITOR=HOME/'HUMANENESS_LARGE13_NOTEBOOKLM_TRAINING_2026-11-02.html' +DATASET_REPO='laion/NotebookLM-Podcasts-Humaneness-Prepared' +SPACE_REPO='laion/humaneness-voice-large-podcast-listening' +CONTRACT={'kind':'large13-notebooklm-multispeaker-v1','parent':'laion/Humaneness-Voice-Large-1.3-Alpha', + 'base_step':9799,'base_weight_files':BASE_WEIGHTS,'languages':list(LANGUAGES),'epochs':EPOCHS, + 'world':WORLD,'accumulation':ACCUM,'peaks':list(PEAK),'warmup':WARMUP,'cosine_floor':FLOOR, + 'prefix':'plain-text speaker-labeled references plus strictly past audio context; native audio placeholders; no new special tokens', + 'context_max_frames':750,'reference_min_frames':63,'reference_max_frames':187, + 'target_max_frames':2250,'max_context_rows':32768,'loss':'only final assistant audio, all 12 trained codebooks', + 'split':'episode-disjoint, stratified by publisher language; duplicate audio audited', + 'source_filter':'original beginning and middle DNSMOS BAK each >=4.0', + 'eval':{'per_language':{'two_turn':5,'four_turn':5},'seeds':'fixed before training','heldout_episodes':True}, + 'checkpoints_active_seconds':900,'maximum_training_allocations':3,'no_lora':True} +CONTRACT['distribution']='FSDP1 root FULL_SHARD; FP32 optimizer parameters/reduction; BF16 compute; full CPU state checkpoints' +CONTRACT['serialization']='Explicit ChatML with byte-identical train/inference prefix; no template fallback or direct-generation shortcut' +CONTRACT['published_scope']='Only the user-requested en/de/fr/es/nl source episodes; other languages remain in original journals and archives' + +def save(path,value): + path=Path(path);path.parent.mkdir(parents=True,exist_ok=True);temp=path.with_name(path.name+'.tmp-'+str(os.getpid())) + temp.write_text(json.dumps(value,ensure_ascii=False,indent=2,allow_nan=False)+'\n');os.replace(temp,path) +def sha(path): + h=hashlib.sha256() + with Path(path).open('rb') as f: + for b in iter(lambda:f.read(8<<20),b''):h.update(b) + return h.hexdigest() +def signature(x):return hashlib.sha256(json.dumps(x,sort_keys=True,ensure_ascii=False).encode()).hexdigest() +def norm(x):return ' '.join(re.findall(r'\w+',unicodedata.normalize('NFKC',x or '').casefold())) +def factor(step,total): + warm=max(1,round(total*WARMUP)) + if step<=warm:return step/warm + p=min(1.,(step-warm)/max(1,total-warm));return FLOOR+(1-FLOOR)*.5*(1+math.cos(math.pi*p)) +def verify_parent(): + proof=json.loads((PARENT_RELEASE/'EXPORT_READY.json').read_text()) + if not proof['passed'] or proof['step']!=9799 or not proof['full_model_not_adapter']:raise RuntimeError('Wrong parent export') + for name,digest in BASE_WEIGHTS.items(): + if proof['weights'][name]['sha256']!=digest or sha(SFT3/name)!=digest:raise RuntimeError('Large 1.3 parent shard mismatch') + return BASE_SHA +def require_dataset(): + d=json.loads((RUN/'large/DATA_READY.json').read_text()) + if d['status']!='READY' or d['pilot_only'] or d['selection_contract']!=CONTRACT:raise RuntimeError('Full podcast data contract not ready') + if d['evaluation_tasks']!=50 or not d['episode_split_verified']:raise RuntimeError('Podcast split/evaluation gate failed') + return d diff --git a/training/fullft/data.py b/training/fullft/data.py new file mode 100644 index 0000000000000000000000000000000000000000..66f44296d3f06a562c88a27e8400c143104c6b8b --- /dev/null +++ b/training/fullft/data.py @@ -0,0 +1,57 @@ +"""Random access to 32-book continuous timelines; model consumes first twelve.""" +from collections import OrderedDict +import hashlib,io,json,sqlite3 +import numpy as np +from common import * + +class Data: + def __init__(self,pilot=False,path=None): + if pilot:raise RuntimeError('No partial-corpus production mode') + self.root=RUN/'large';self.ready=json.loads((self.root/'DATA_READY.json').read_text()) + self.db=sqlite3.connect('file:'+str(path or self.root/'mix.sqlite')+'?mode=ro&immutable=1',uri=True) + self.db.row_factory=sqlite3.Row;self.files=OrderedDict();self.cache=OrderedDict() + self.count=self.db.execute('SELECT count(*) FROM presentations').fetchone()[0] + def episode(self,uid): + if uid not in self.cache: + r=self.db.execute('SELECT * FROM episodes WHERE uid=?',(uid,)).fetchone() + if not r:raise KeyError(uid) + if r['tar'] not in self.files: + self.files[r['tar']]=open(r['tar'],'rb') + if len(self.files)>16:self.files.popitem(last=False)[1].close() + f=self.files[r['tar']];f.seek(r['code_offset']);blob=f.read(r['code_size']) + if hashlib.sha256(blob).hexdigest()!=r['code_sha']:raise RuntimeError('Continuous codec timeline hash changed') + codes=np.load(io.BytesIO(blob),allow_pickle=False) + if codes.ndim!=2 or codes.shape[1]!=32 or codes.dtype.kind not in 'iu' or codes.min()<0 or codes.max()>1023:raise ValueError('Invalid podcast codec array') + self.cache[uid]=codes + if len(self.cache)>8:self.cache.popitem(last=False) + self.cache.move_to_end(uid);return self.cache[uid] + def row(self,key,condition='reference'): + r=self.db.execute('SELECT * FROM samples WHERE key=?',(int(key),)).fetchone() + if not r:raise KeyError(key) + ex=json.loads(r['example_json']);codes=self.episode(r['episode']) + a,b=ex['target_frame_start'],ex['target_frame_end'];ca,cb=ex['context_frame_start'],ex['context_frame_end'] + if not 0<=ca<=cb<=a750:raise ValueError('Future context or invalid boundaries') + refs={} + for speaker,q in ex['speaker_references'].items(): + ra,rb=q['start_frame'],q['end_frame'] + if not 0<=ra=? AND pos') + audio.append(torch.as_tensor(np.asarray(codes),dtype=torch.long)) + if context is not None and len(context): + if context.ndim!=2 or context.shape[1]!=12 or len(context)>750:raise ValueError('Past context exceeds 60s or has wrong codebook layout') + blocks.append('Previous conversation, chronological and strictly before target:\n'+row['context_description']+'\n<|audio|>') + audio.append(torch.as_tensor(np.asarray(context),dtype=torch.long)) + replacement='\n'.join(blocks) if blocks else 'None' + marker='- Reference(s):\nNone\n' + if user['content'].count(marker)!=1:raise RuntimeError('Native MOSS user wrapper changed') + user['content']=user['content'].replace(marker,'- Reference(s):\n'+replacement+'\n',1) + user['audio_codes_list']=audio + # The parent processor's direct generation shortcut reconstructs its prompt + # from `text` and ignores modified content. Disable that shortcut explicitly. + # Use deterministic ChatML in BOTH train and infer, not a missing-template + # fallback or an assistant-only prefix. + user.pop('text',None) + user['content']='<|im_start|>user\n'+user['content']+'<|im_end|>\n' + if user['content'].count('<|audio|>')!=len(audio):raise RuntimeError('Reference/context routing lost an audio placeholder') + return user + +def generation_inputs(proc,row): + user=build_user(proc,row) + user['content']+='<|im_start|>assistant\n' + return proc([[user]],mode='generation',apply_chat_template=False) + +def pack_example(proc,config,row): + user=build_user(proc,row) + assistant=proc.build_assistant_message(audio_codes_list=[torch.as_tensor(np.asarray(row['target_codes']),dtype=torch.long)]) + assistant['content']='<|im_start|>assistant\n'+assistant['content']+'<|im_end|>\n' + ids=proc([[user,assistant]],mode='computing_loss',apply_chat_template=False)['input_ids'][0].long() + if ids.ndim!=2 or ids.shape[1]!=13 or len(ids)>CONTRACT['max_context_rows']:raise ValueError('Invalid full packed context; no truncation allowed') + labels=torch.full_like(ids,-100);labels[:-1]=ids[1:] + # Count reference frames first; never infer the last assistant boundary from + # mere audio token count. Only AUDIO_ASSISTANT slots follow the user message. + assist,aend=int(config.audio_assistant_slot_token_id),int(config.audio_end_token_id) + first=ids[:,0].eq(assist).nonzero() + if not first.numel():raise ValueError('Missing assistant audio') + supervised=labels[:,0].eq(assist)|labels[:,0].eq(aend) + supervised[:max(0,int(first[0])-1)]=False + labels[~supervised]=-100 + audio_labels=labels[:,1:];audio_labels[(audio_labels>=int(config.audio_pad_token_id))|(audio_labels<0)]=-100 + if int((audio_labels>=0).sum())!=row['frames']*12 or int(labels[:,0].eq(aend).sum())!=1: + raise ValueError('Assistant target-only supervision mismatch') + return {'input_ids':ids[None],'labels':labels[None],'attention_mask':torch.ones(1,len(ids),dtype=torch.long)} diff --git a/training/fullft/prepare_training.py b/training/fullft/prepare_training.py new file mode 100644 index 0000000000000000000000000000000000000000..e6862cc5daeea5a50864ed95c631b14b41e44e21 --- /dev/null +++ b/training/fullft/prepare_training.py @@ -0,0 +1,245 @@ +"""Episode-disjoint training windows and a frozen 50-task, two/four-turn test.""" +import collections,concurrent.futures,gzip,io,itertools,json,math,os,random,sqlite3,tarfile,time +import numpy as np +from common import * +from annotation_worker import prompts + +SCHEMA='''CREATE TABLE episodes(uid TEXT PRIMARY KEY,language TEXT,tar TEXT,code_offset INTEGER,code_size INTEGER,code_sha TEXT, +audio_member TEXT,audio_sha TEXT,record_offset INTEGER,record_size INTEGER,split TEXT,stats TEXT); +CREATE TABLE samples(key INTEGER PRIMARY KEY,uid TEXT UNIQUE,episode TEXT,source TEXT,language TEXT,split TEXT,kind TEXT, +frames INTEGER,general TEXT,inline TEXT,plain TEXT,example_json TEXT,context_bound INTEGER); +CREATE TABLE presentations(pos INTEGER PRIMARY KEY,sample_key INTEGER,phase INTEGER,conditioning TEXT,repeat INTEGER); +CREATE TABLE finished_shards(path TEXT PRIMARY KEY,sha256 TEXT); +CREATE TABLE rejected(episode TEXT,reason TEXT,count INTEGER);''' + +def frame_start(a): + n=math.floor(a*12.5) + while n/12.5>a:n-=1 + return n + +def clean_runs(record): + runs=[];run=[] + for s in record['sentences']: + if s['speaker_id'] not in {'S1','S2'} or s['overlap'] or s['end_s']<=s['start_s']: + if run:runs.append(run);run=[] + continue + if run and s['start_s']-run[-1]['end_s']>1.5:runs.append(run);run=[] + run.append(s) + if run:runs.append(run) + return runs + +def turns(group): + out=[] + for s in group: + if out and out[-1][0]['speaker_id']==s['speaker_id']:out[-1].append(s) + else:out.append([s]) + return out + +def references(record,start,literal): + if '_clean_refs' not in record: + pools={'S1':[],'S2':[]} + for t in record['diarization']: + speaker=t['speaker_id'] + if speaker not in pools or len(pools[speaker])>=8:continue + a=math.ceil(t['start_s']*12.5-1e-9);b=min(math.floor(t['end_s']*12.5+1e-9),a+187) + if b-a<63:continue + if any(o['speaker_id']!=speaker and max(a/12.5,o['start_s'])ex['target_start_s'] or any(v['end_frame']>start for v in refs.values()):raise ValueError('future_leakage') + return ex + +def windows(record): + if {s['speaker_id'] for s in record['speakers']}!={'S1','S2'}:return [],{'not_exactly_two_diarized_speakers':1} + record['_by_word']={w['word_id']:w for w in record['words']} + result=[];reasons=collections.Counter();ordinal=0 + for run in clean_runs(record): + groups=[];pending=[] + for s in run: + proposed=pending+[s] + if pending and (s['end_s']-pending[0]['start_s']>60 or len(turns(proposed))>4):groups.append(pending);pending=[] + pending.append(s) + if pending:groups.append(pending) + candidates=[('train-window',g) for g in groups] + ts=turns(run) + for n in [2,4]: + # Reservoir-like deterministic ordering covers the episode rather than + # over-selecting its opening. Only 12 candidates per kind are retained. + possible=[sum(ts[i:i+n],[]) for i in range(len(ts)-n+1) if ts[i+n-1][-1]['end_s']-ts[i][0]['start_s']<=180] + possible.sort(key=lambda g:signature([SEED,record['episode_id'],n,g[0]['sentence_id']])) + kept=0 + for g in possible: + if kept>=12:break + try:ex=make_example(record,g,'probe') + except ValueError:continue + candidates.append((f'eval{n}',g));kept+=1 + for kind,g in candidates: + ordinal+=1;uid=record['episode_id']+f'-{kind}-{ordinal:05d}' + try:ex=make_example(record,g,uid) + except ValueError as exc:reasons[str(exc)]+=1;continue + result.append(dict(uid=uid,episode=record['episode_id'],source='NotebookLM',language=record['quality']['publisher_language'], + split='pending',kind=kind,frames=ex['target_frame_end']-ex['target_frame_start'],general=ex['general'],inline=ex['script'], + plain=ex['literal_text'],example_json=json.dumps(ex,ensure_ascii=False),context_bound=None)) + return result,dict(reasons) + +def scan_shard(shard): + path=Path(shard['tar']);pending={};episodes=[];samples=[];rejected=[] + with tarfile.open(path,'r:') as tar: + for m in tar: + if m.name.startswith('_'):continue + if m.name.endswith('.speakers.npy'):continue + suffix='.moss.npy' if m.name.endswith('.moss.npy') else '.json' if m.name.endswith('.json') else None + uid=m.name[:-len(suffix)] if suffix else Path(m.name).stem + entry=pending.setdefault(uid,{}) + if suffix=='.moss.npy': + blob=tar.extractfile(m).read();codes=np.load(io.BytesIO(blob),allow_pickle=False) + if codes.ndim!=2 or codes.shape[1]!=32 or codes.dtype!=np.dtype('1023:raise ValueError('Invalid stored 32-book timeline') + entry['codec']=(m.offset_data,m.size,hashlib.sha256(blob).hexdigest(),len(codes)) + elif suffix=='.json': + entry['record']=json.load(tar.extractfile(m));entry['record_pointer']=(m.offset_data,m.size) + else: + entry['audio']=(m.name,hashlib.file_digest(tar.extractfile(m),'sha256').hexdigest()) + if not all(k in entry for k in ['record','record_pointer','codec','audio']):continue + pending.pop(uid);record=entry['record'];lang=record['quality']['publisher_language'] + if lang not in LANGUAGES:continue + if record['episode_id']!=uid or min(record['quality']['background_start'],record['quality']['background_middle'])<4:raise ValueError('Wrong clean cohort') + rows,reasons=windows(record);samples.extend(rows) + for reason,count in reasons.items():rejected.append((uid,reason,count)) + offset,size,digest,nframes=entry['codec'];ao,ah=entry['audio'];jo,js=entry['record_pointer'] + if nframes!=record['codec']['num_frames']:raise ValueError('Lost codec frames') + episodes.append((uid,lang,str(path),offset,size,digest,ao,ah,jo,js,'pending',json.dumps(dict(duration_s=record['audio']['duration_s'],sentences=len(record['sentences']),windows=len(rows))))) + if pending:raise ValueError('Incomplete packed episode') + return str(path),episodes,samples,rejected + +def main(): + if not os.environ.get('SLURM_JOB_ID'):raise RuntimeError('Bulk training preparation requires allocated compute') + verify_parent();annotation=json.loads((ROOT/'ANNOTATION_READY.json').read_text()) + root=RUN/'large';root.mkdir(parents=True,exist_ok=True) + if (root/'DATA_READY.json').exists():require_dataset();return + path=root/'mix.building.sqlite';fresh=not path.exists();db=sqlite3.connect(path);db.row_factory=sqlite3.Row + if fresh:db.executescript(SCHEMA) + done={r[0] for r in db.execute('SELECT path FROM finished_shards')};todo=[s for s in annotation['shards'] if s['tar'] not in done] + with concurrent.futures.ProcessPoolExecutor(max_workers=8) as pool: + for p,episodes,samples,rejected in pool.map(scan_shard,todo): + db.executemany('INSERT INTO episodes VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',episodes) + fields=['uid','episode','source','language','split','kind','frames','general','inline','plain','example_json','context_bound'] + db.executemany('INSERT INTO samples('+','.join(fields)+') VALUES('+','.join('?' for _ in fields)+')',([r[f] for f in fields] for r in samples)) + db.executemany('INSERT INTO rejected VALUES(?,?,?)',rejected) + proof=next(s for s in annotation['shards'] if s['tar']==p);db.execute('INSERT INTO finished_shards VALUES(?,?)',(p,proof['sha256']));db.commit() + print(json.dumps(dict(shard=p,episodes=len(episodes),windows=len(samples))),flush=True) + # Reject duplicate audio before assigning heldouts, not after training. + db.execute('CREATE INDEX IF NOT EXISTS sample_episode ON samples(episode,kind)') + duplicates=db.execute('SELECT audio_sha FROM episodes GROUP BY audio_sha HAVING count(*)>1').fetchall() + for row in duplicates: + ids=[r[0] for r in db.execute('SELECT uid FROM episodes WHERE audio_sha=? ORDER BY uid',(row[0],))] + for uid in ids[1:]:db.execute('DELETE FROM samples WHERE episode=?',(uid,));db.execute('UPDATE episodes SET split="duplicate_audio_excluded" WHERE uid=?',(uid,)) + from transformers import AutoTokenizer + tokenizer=AutoTokenizer.from_pretrained(str(SFT3),local_files_only=True,trust_remote_code=True) + cursor=db.execute('SELECT key,general,inline,plain,frames,example_json FROM samples') + while batch:=list(itertools.islice(cursor,128)): + texts=[] + for r in batch: + ex=json.loads(r['example_json']);description=json.dumps(ex['context_speaker_intervals'],ensure_ascii=False) + texts.append('GENERAL: '+r['general']+'\nSCRIPT:\n'+r['inline']+'\n'+r['plain']+'\n'+description) + lengths=tokenizer(texts,add_special_tokens=False,return_length=True,truncation=False)['length'] + for r,n in zip(batch,lengths): + bound=int(n)+r['frames']+750+2*187+512 + if bound>CONTRACT['max_context_rows']:db.execute('DELETE FROM samples WHERE key=?',(r['key'],)) + else:db.execute('UPDATE samples SET context_bound=? WHERE key=?',(bound,r['key'])) + eval_tasks=[];val_keys=[];canaries=[];phases=[];presentations=[] + for lang in LANGUAGES: + eligible=[r[0] for r in db.execute('SELECT DISTINCT episode FROM samples WHERE language=? AND kind="train-window"',(lang,))] + eligible.sort(key=lambda uid:signature([SEED,'episode-split',lang,uid])) + if len(eligible)<6:raise RuntimeError('Too few valid two-speaker episodes in '+lang) + goodtest=[];kind_counts={} + for uid in eligible: + kinds={r[0]:r[1] for r in db.execute('SELECT kind,count(*) FROM samples WHERE episode=? GROUP BY kind',(uid,))} + kind_counts[uid]=kinds + if kinds.get('eval2',0) and kinds.get('eval4',0):goodtest.append(uid) + # Need five tasks of EACH kind across at least two heldout episodes, not + # an arbitrary three four-turn windows in every individual episode. + goodtest.sort(key=lambda uid:(-min(5,kind_counts[uid].get('eval4',0)),signature([SEED,'test-episode',uid]))) + test=set();coverage={2:0,4:0} + for uid in goodtest: + if len(test)>=2 and min(coverage.values())>=5:break + if len(test)>=len(eligible)-4:break + test.add(uid) + for n in [2,4]:coverage[n]+=kind_counts[uid].get(f'eval{n}',0) + if len(test)<2 or min(coverage.values())<5:raise RuntimeError('Insufficient disjoint real 2/4-turn heldouts in '+lang) + remaining=[u for u in eligible if u not in test] + validation=set(remaining[:max(2,len(eligible)//50)]);train=set(remaining)-validation + for split,uids in [('train',train),('validation',validation),('test',test)]: + db.executemany('UPDATE episodes SET split=? WHERE uid=?',((split,u) for u in uids)) + db.executemany('UPDATE samples SET split=? WHERE episode=?',((split,u) for u in uids)) + for n in [2,4]: + candidates=list(db.execute('SELECT key,episode FROM samples WHERE language=? AND split="test" AND kind=? ORDER BY key',(lang,f'eval{n}'))) + candidates.sort(key=lambda r:signature([SEED,'test-choice',n,r['episode'],r['key']])) + chosen=[];seen_geometry=set() + for r in candidates: + sample=db.execute('SELECT * FROM samples WHERE key=?',(r['key'],)).fetchone();ex=json.loads(sample['example_json']) + geometry=(sample['episode'],ex['target_frame_start'],ex['target_frame_end']) + if geometry in seen_geometry:continue + seen_geometry.add(geometry);chosen.append(dict(task_id=f'{lang}-{n}turn-{len(chosen):02d}',key=r['key'], + episode=sample['episode'],language=lang,turns=n,seed=SEED+len(eval_tasks)+len(chosen),text=sample['plain'],prompt='GENERAL: '+sample['general']+'\nSCRIPT:\n'+sample['inline'], + frame_budget=sample['frames'],example=ex)) + if len(chosen)==5:break + if len(chosen)!=5:raise RuntimeError('Exactly five eval tasks per turn condition are required') + eval_tasks.extend(chosen) + canaries.append(db.execute('SELECT key FROM samples WHERE language=? AND split="train" AND kind="train-window" ORDER BY key LIMIT 1',(lang,)).fetchone()[0]) + val_keys.extend(r[0] for r in db.execute('SELECT key FROM samples WHERE language=? AND split="validation" AND kind="train-window" ORDER BY key LIMIT 4',(lang,))) + keys=[r[0] for r in db.execute('SELECT key FROM samples WHERE split="train" AND kind="train-window" ORDER BY key')] + for epoch in range(EPOCHS): + order=list(keys);random.Random(SEED+epoch).shuffle(order) + presentations.extend((k,epoch,'reference',epoch) for k in order) + padding=(-len(presentations))%BATCH;presentations.extend((None,epoch,'reference',epoch) for _ in range(padding)) + phases.append(dict(name=f'Globally shuffled epoch {epoch+1}',end_position=len(presentations))) + db.executemany('INSERT INTO presentations VALUES(?,?,?,?,?)',((i,*r) for i,r in enumerate(presentations))) + for field in ['frames','context_bound','length(general)+length(inline)+frames']: + canaries.append(db.execute('SELECT key FROM samples WHERE split="train" AND kind="train-window" ORDER BY '+field+' DESC LIMIT 1').fetchone()[0]) + stats=[dict(language=r[0],split=r[1],windows=r[2],codec_hours=r[3]) for r in db.execute('SELECT language,split,count(*),sum(frames)*.08/3600 FROM samples WHERE kind="train-window" GROUP BY language,split')] + rejected=[dict(reason=r[0],candidates=r[1]) for r in db.execute('SELECT reason,sum(count) FROM rejected GROUP BY reason')] + db.commit();db.close();os.replace(path,root/'mix.sqlite') + save(ROOT/'EVALUATION_TASKS.json',eval_tasks) + ready=dict(status='READY',pilot_only=False,selection_contract=CONTRACT,base_weights_sha256=BASE_SHA, + database_sha256=sha(root/'mix.sqlite'),presentations=len(presentations),nonpadding_presentations=len(keys)*EPOCHS, + unique_train_targets=len(keys),phases=phases,canary_keys=list(dict.fromkeys(canaries)),validation_keys=val_keys, + evaluation_tasks=len(eval_tasks),evaluation_sha256=sha(ROOT/'EVALUATION_TASKS.json'),episode_split_verified=True, + statistics=stats,rejections=rejected,annotation_sha256=sha(ROOT/'ANNOTATION_READY.json'),time=time.time()) + save(root/'DATA_READY.json',ready);print(json.dumps(ready),flush=True) + +if __name__=='__main__': + main() diff --git a/training/fullft/tests.py b/training/fullft/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..6d286c8598b3d694cd270f2e7a329f2892537ece --- /dev/null +++ b/training/fullft/tests.py @@ -0,0 +1,56 @@ +"""CPU serialization/timeline fixtures; no model inference or training here.""" +import copy,sys,unittest +import numpy as np +from common import * +from prepare_training import make_example,turns + +def record_fixture(): + words=[];sentences=[];diar=[] + specifications=[('S1',0.,6.,'The first speaker reference.'),('S2',6.,12.,'The other speaker reference.'), + ('S1',14.,15.,'Hello.'),('S2',16.,17.,'Welcome.'),('S1',18.,19.,'Ready?'),('S2',20.,21.,'Absolutely.')] + for i,(s,a,b,text) in enumerate(specifications): + wid=f'w{i}';words.append(dict(word_id=wid,text=text,start_s=a+.08,end_s=b-.08)) + sentences.append(dict(sentence_id=f's{i}',speaker_id=s,start_s=a+.08,end_s=b-.08,text=text,word_ids=[wid],direction_en='warm conversational delivery',overlap=False)) + diar.append(dict(speaker_id=s,start_s=a,end_s=b)) + return dict(episode_id='fixture',speakers=[{'speaker_id':'S1'},{'speaker_id':'S2'}],sentences=sentences,words=words, + diarization=diar,burst_events=[],audio={'duration_s':22.},quality={'publisher_language':'en'},_by_word={w['word_id']:w for w in words}) + +class Tests(unittest.TestCase): + def test_two_four_turns_and_no_future(self): + r=record_fixture();original=copy.deepcopy(r['words']) + for n in [2,4]: + ex=make_example(r,r['sentences'][2:2+n],'fixture') + self.assertEqual(ex['turn_count'],n) + self.assertEqual(ex['context_frame_end'],ex['target_frame_start']) + self.assertLessEqual(ex['context_frame_end']-ex['context_frame_start'],750) + for q in ex['speaker_references'].values(): + self.assertLessEqual(q['end_frame'],ex['target_frame_start']);self.assertTrue(63<=q['end_frame']-q['start_frame']<=187) + from prompt_format import verify_timed + self.assertFalse(verify_timed(ex['script'],ex['literal_text'],ex['target_end_s']-ex['target_start_s'])) + self.assertEqual(r['words'],original) + with self.assertRaises(ValueError):make_example(r,r['sentences'][:2],'future') + + def test_native_processor_twenty_rows(self): + import torch + sys.path.insert(0,str(BASE));import train_fullft as ft + ft.SFT3=SFT3;proc,config=ft.load_processor('large') + sys.path.insert(0,str(CODE));from packing import pack_example,build_user,generation_inputs + for i in range(20): + row=dict(plain='Hello. Welcome.',instruction='GENERAL: Natural dialogue.\nSCRIPT:\n(speaker=S1) [0.24 seconds duration] Hello.\n(speaker=S2) [0.24 seconds duration] Welcome.', + frames=6,language='en',speaker_references={'S1':np.full((63,12),1,dtype=np.int16),'S2':np.full((64,12),2,dtype=np.int16)}, + context_codes=np.full((100,12),3,dtype=np.int16),context_description='S1 then S2, earlier turns.',target_codes=np.full((6,12),7,dtype=np.int16)) + user=build_user(proc,row);self.assertEqual(len(user['audio_codes_list']),3) + packed=pack_example(proc,config,row);active=packed['labels'][:,:,1:];active=active[active>=0] + inference=generation_inputs(proc,row)['input_ids'][0].long() + self.assertTrue(torch.equal(inference,packed['input_ids'][0,:len(inference)]),'Training/generation prefix mismatch') + self.assertNotIn('text',user,'Native shortcut would discard speaker labels') + self.assertEqual(len(active),72);self.assertTrue(bool(torch.all(active==7))) + swapped=dict(row,speaker_references={'S1':row['speaker_references']['S2'],'S2':row['speaker_references']['S1']}) + other=pack_example(proc,config,swapped) + self.assertFalse(torch.equal(packed['input_ids'],other['input_ids'])) + self.assertTrue(torch.equal(packed['labels'],other['labels'])) + + def test_warmup(self): + self.assertAlmostEqual(factor(50,1000),1.);self.assertAlmostEqual(factor(1000,1000),.1) + +if __name__=='__main__':unittest.main() diff --git a/training/fullft/timeline.py b/training/fullft/timeline.py new file mode 100644 index 0000000000000000000000000000000000000000..a61d12931f52898492b408674fb1fa9b39d3a0c4 --- /dev/null +++ b/training/fullft/timeline.py @@ -0,0 +1,104 @@ +"""Episode-global words, diarization attribution, sentence and prompt construction.""" +import math +import re + +def words_from_asr(result,duration): + words=[] + for segment_index,segment in enumerate(result.get('segments',[])): + for word in segment.get('words',[]): + text=word.get('word',word.get('text','')) + a=word.get('start');b=word.get('end') + if not text.strip():continue + if a is None or b is None:raise ValueError('ASR word without timestamp') + a,b=float(a),float(b) + if not math.isfinite(a+b) or a<-.05 or bduration+.2:raise ValueError('ASR word outside global episode timeline') + if words and aa] + while indexa:active.append(ordered[index]) + index+=1 + coverage={} + for turn in active: + value=max(0,min(b,turn['end_s'])-max(a,turn['start_s']))/span + coverage[turn['speaker_id']]=min(1.,coverage.get(turn['speaker_id'],0.)+value) + ranked=sorted(coverage.items(),key=lambda x:(-x[1],x[0])) + overlap=[s for s,v in ranked if v>=.2] + confident=bool(ranked and ranked[0][1]>=.6 and (len(ranked)==1 or ranked[0][1]-ranked[1][1]>=.2)) + word.update(speaker_id=ranked[0][0] if confident else None,speaker_coverage=coverage, + speaker_confidence=None,overlap=len(overlap)>1,overlapping_speaker_ids=overlap, + diarization_query_start_s=a,diarization_query_end_s=b, + source_start_sample=round(original_a*48000),source_end_sample=round(original_b*48000)) + return speakers,ordered + +def sentences_from_words(words,language): + groups=[];cur=[] + for word in words: + if cur and (word['speaker_id']!=cur[-1]['speaker_id'] or word['start_s']-cur[-1]['end_s']>=.8 or word['end_s']-cur[0]['start_s']>25): + groups.append(cur);cur=[] + cur.append(word) + if re.search(r'[.!?。！？]["\'”’]*$',word['text'].strip()):groups.append(cur);cur=[] + if cur:groups.append(cur) + result=[] + for group in groups: + a,b=group[0]['start_s'],group[-1]['end_s'] + text=('' if language in ('ja','zh') else ' ').join(x['text'].strip() for x in group) + result.append(dict(sentence_id=f'sent{len(result):06d}',speaker_id=group[0]['speaker_id'],start_s=a,end_s=b, + source_start_sample=round(a*48000),source_end_sample=round(b*48000),text=text, + word_ids=[w['word_id'] for w in group],direction_en=None,overlap=any(x['overlap'] for x in group), + alignment_confidence=None,speaker_confidence=None,zero_duration_alignment=b<=a)) + return result + +def examples(sentences,speakers): + output=[];pending=[] + def add(group): + if not group:return + a,b=group[0]['start_s'],group[-1]['end_s'];tf=math.floor(a*12.5);te=math.ceil(b*12.5) + # Binary rounding can make (floor(a*fps)/fps) exceed a by an ulp + # at exact frame boundaries. Move back a frame rather than permitting + # a context frame that extends into the target. + while tf/12.5>a:tf-=1 + ce=tf;cs=max(0,ce-750) + script='\n'.join(f"(speaker={s['speaker_id']}, {s['direction_en'] or 'clear conversational delivery'}) [{s['end_s']-s['start_s']:.2f} seconds duration] {s['text']}" for s in group) + output.append(dict(example_id=f'example{len(output):06d}',context_start_s=cs/12.5,context_end_s=ce/12.5, + target_start_s=a,target_end_s=b,context_frame_start=cs,context_frame_end=ce,target_frame_start=tf,target_frame_end=te, + target_sentence_ids=[s['sentence_id'] for s in group],general='',script=script,literal_text=' '.join(s['text'] for s in group), + speaker_reference_ids=[s['reference']['reference_id'] for s in speakers if s.get('reference') and s['reference']['end_s']<=a], + prompt='GENERAL:\nSCRIPT:\n'+script)) + for sent in sentences: + if sent['overlap'] or sent['speaker_id'] is None or sent['end_s']<=sent['start_s']: + add(pending);pending=[];continue + if pending and sent['end_s']-pending[0]['start_s']>30:add(pending);pending=[] + pending.append(sent) + add(pending) + for x in output: + assert x['context_frame_end']<=x['target_frame_start'] and x['context_end_s']<=x['target_start_s'] and x['context_end_s']-x['context_start_s']<=60.0001 + return output diff --git a/training/fullft/train.py b/training/fullft/train.py new file mode 100644 index 0000000000000000000000000000000000000000..edfcd5064eb27baea120cdf39aff4c25a334ba90 --- /dev/null +++ b/training/fullft/train.py @@ -0,0 +1,257 @@ +"""Large 1.3 Alpha, three-epoch multi-speaker podcast full FT.""" +import argparse +import json +import math +import os +from pathlib import Path +import signal +import sys +import time +from common import * +from data import Data + +sys.path.insert(0,str(BASE)) +import train_fullft as ft +ft.SFT3 = SFT3 # Explicit final Large 1.3 Alpha, step 9799. +sys.path.insert(0,str(CODE)) +from packing import pack_example +import torch +import torch.distributed as dist + +STOP=False +def stop(sig,frame): + global STOP + STOP=True + +def main(): + p=argparse.ArgumentParser();p.add_argument('--pilot',action='store_true') + p.add_argument('--smoke-steps',type=int,default=0);p.add_argument('--max-runtime-s',type=int,default=32400) + p.add_argument('--verify-smoke-resume',action='store_true') + args=p.parse_args() + if args.pilot and not args.smoke_steps:raise RuntimeError('Pilot data may never start production training') + if args.verify_smoke_resume and not args.smoke_steps:raise RuntimeError('Resume verifier is smoke-only') + if not args.pilot:require_dataset() + from accelerate import Accelerator,FullyShardedDataParallelPlugin + from torch.distributed.fsdp import MixedPrecision,ShardingStrategy,StateDictType,FullStateDictConfig,FullOptimStateDictConfig + from accelerate.utils import set_seed + plugin=FullyShardedDataParallelPlugin(fsdp_version=1,reshard_after_forward=ShardingStrategy.FULL_SHARD, + auto_wrap_policy='NO_WRAP',use_orig_params=True,sync_module_states=True,cpu_ram_efficient_loading=False, + state_dict_type=StateDictType.FULL_STATE_DICT, + state_dict_config=FullStateDictConfig(offload_to_cpu=True,rank0_only=True), + optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=True,rank0_only=True), + mixed_precision_policy=MixedPrecision(param_dtype=torch.bfloat16,reduce_dtype=torch.float32,buffer_dtype=torch.bfloat16), + limit_all_gathers=True,forward_prefetch=False,activation_checkpointing=False) + acc=Accelerator(gradient_accumulation_steps=ACCUM,mixed_precision='bf16',fsdp_plugin=plugin) + if str(acc.distributed_type)!='DistributedType.FSDP':raise RuntimeError('FSDP sharding was not activated') + world,rank=acc.num_processes,acc.process_index + if world!=(4 if args.smoke_steps else WORLD):raise RuntimeError(('Wrong GPU world size',world)) + set_seed(SEED,device_specific=True);torch.set_num_threads(4) + signal.signal(signal.SIGUSR1,stop);signal.signal(signal.SIGTERM,stop) + root=RUN/('pilot' if args.pilot else 'large');ready=json.loads((root/'DATA_READY.json').read_text()) + if ready['status']!='READY' or bool(ready['pilot_only'])!=args.pilot:raise RuntimeError('Prepared data/pilot contract mismatch') + data=Data(args.pilot);n=data.count + if n!=ready['presentations'] or n%BATCH:raise RuntimeError('Presentation/block alignment mismatch') + total=math.ceil(n/(world*ACCUM)) + out=root/('smoke' if args.smoke_steps else 'training');out.mkdir(parents=True,exist_ok=True) + contract={'kind':'large13-notebooklm-multispeaker-v1','model':'large','base':str(SFT3), + 'base_weights_sha256':ready['base_weights_sha256'],'database_sha256':ready['database_sha256'], + 'presentations':n,'actual_presentations':ready['nonpadding_presentations'],'phases':ready['phases'], + 'selection_contract':ready.get('selection_contract'), + 'world':world,'accumulation':ACCUM,'effective_batch':world*ACCUM,'seed':SEED, + 'peak_semantic':PEAK[0],'peak_talker':PEAK[1],'warmup_fraction':WARMUP,'cosine_floor':FLOOR, + 'lr_restarts':0,'total_steps':total,'checkpoints_seconds':900,'keep_all_checkpoints':True, + 'optimizer_rng_scheduler_sampler_saved':True,'pilot':args.pilot, + 'distribution':'FSDP1 root FULL_SHARD, original FP32 optimizer parameters, BF16 compute, FP32 reduction, full CPU checkpoint', + 'training_code_sha256':sha(Path(__file__)),'parent_train_fullft_sha256':sha(BASE/'train_fullft.py'), + 'parent_attention_sha256':sha(PARENT_ENGINE/'memory_attention.py'), + 'snapshot_sha256':signature(json.loads((CODE/'LOCK.json').read_text())), + 'variant':VARIANT} + digest=signature(contract) + existing=out/'RUN_CONTRACT.json' + if existing.exists() and json.loads(existing.read_text()).get('contract_sha256')!=digest:raise RuntimeError('Resume contract changed') + if acc.is_main_process:save(existing,{**contract,'contract_sha256':digest}) + if acc.is_main_process and verify_parent()!=ready['base_weights_sha256']: + raise RuntimeError('Large 1.3 parent changed after data preparation') + acc.wait_for_everyone() + # New full-FT optimizer initialized from released Large 1.3, not SFT3. + model,config=ft.load_model('large') + for name,param in model.named_parameters():param.requires_grad_(not name.startswith('score_conditioner.')) + sys.path.insert(0,str(PARENT_ENGINE)) + from memory_attention import verify,install + attention=verify(acc.device);attention['layers']=install(model) + checkpointed=ft.enable_grad_ckpt(model);proc,_=ft.load_processor('large') + canaries=data.canaries();val_keys=data.validation_keys() + if not canaries or not val_keys:raise RuntimeError('No training/validation canaries') + if acc.is_main_process: + cases=[] + for key in canaries: + for requested in ['reference']: + row=data.row(key,requested);packed=pack_example(proc,config,row) + if int((packed['labels'][:,:,1:]>=0).sum())!=12*row['frames']:raise RuntimeError('Codec label mask mismatch') + if row['reference_codes'] is not None and not 63<=len(row['reference_codes'])<=187:raise RuntimeError('Reference truncation regression') + cases.append({'uid':row['uid'],'source':row['source'],'language':row['language'], + 'requested_condition':requested,'actual_condition':row['conditioning'],'target_frames':row['frames'], + 'reference_frames':{speaker:len(codes) for speaker,codes in row['speaker_references'].items()}, + 'context_frames':len(row['context_codes']), + 'sequence_rows':packed['input_ids'].shape[1]}) + save(out/'PACKING_PREFLIGHT.json',{'passed':True,'cases':cases,'languages':sorted({c['language'] for c in cases})}) + save(out/'ATTENTION_PREFLIGHT.json',attention) + acc.wait_for_everyone() + groups={'semantic':[],'talker':[]} + for name,param in model.named_parameters(): + if param.requires_grad:groups['semantic' if name.startswith(('transformer.','text_lm_head.')) else 'talker'].append(param) + if not all(groups.values()):raise RuntimeError('Missing active semantic/Talker parameters') + optimizer=torch.optim.AdamW([{'name':'semantic','params':groups['semantic'],'lr':PEAK[0]}, + {'name':'talker','params':groups['talker'],'lr':PEAK[1]}],betas=(.9,.95),eps=1e-8,weight_decay=.01,foreach=False) + parameter_counts={name:sum(p.numel() for p in params) for name,params in groups.items()} + wrapper,optimizer=acc.prepare(ft.TrainStep(model),optimizer) + step=0;latest=sorted((out/'checkpoints').glob('step-*/READY.json')) + if latest: + meta=json.loads(latest[-1].read_text()) + if meta['contract_sha256']!=digest:raise RuntimeError('Checkpoint contract mismatch') + acc.load_state(str(latest[-1].parent),map_location='cpu');step=int(meta['step']) + sampler=json.loads((latest[-1].parent/'sampler_state.json').read_text()) + schedule=json.loads((latest[-1].parent/'scheduler_state.json').read_text()) + if sampler != {'next_position':step*world*ACCUM,'map_sha256':ready['database_sha256']}: + raise RuntimeError('Resume sampler does not match next exact presentation') + if schedule['step']!=step or schedule['total']!=total or schedule['peaks']!=list(PEAK): + raise RuntimeError('Resume scheduler contract mismatch') + rng=list(latest[-1].parent.glob('random_states_*.pkl')) + if len(rng)!=world:raise RuntimeError('Missing per-rank RNG states') + acc.wait_for_everyone() + restored_step=step + if args.verify_smoke_resume: + proof=json.loads((out/'SMOKE_SAVED.json').read_text()) + if not step or proof['contract_sha256']!=digest or proof['step']!=step: + raise RuntimeError('Fresh-process smoke requires its own saved state') + if not optimizer.state:raise RuntimeError('AdamW state was not restored') + started=time.monotonic();last_save=started;recent_loss=None + + def status(state,current,**extra): + if not acc.is_main_process:return + record={'state':state,'step':current,'total_steps':total,'loss':recent_loss, + 'job_id':os.environ.get('SLURM_JOB_ID'),'gpus':world,'elapsed_this_job_s':time.monotonic()-started, + 'peak_gpu_gb':torch.cuda.max_memory_allocated()/1e9,'updated_unix':time.time(),**extra} + save(out/'STATUS.json',record) + if not args.smoke_steps: + try: + from monitor import render + render() + except Exception as exc: + # Monitoring is auxiliary; a broken Home mirror must never + # terminate an otherwise healthy optimizer update. + print('Monitor refresh warning:',str(exc),flush=True) + + def validate(current): + # FSDP forward is collective: all ranks execute the same number of + # calls, with masked duplicate padding where there are fewer cases. + # Never unwrap and bypass parameter all-gather during validation. + wrapper.eval();sums=torch.zeros(2,dtype=torch.float64,device=acc.device) + with torch.no_grad(),acc.autocast(): + for slot in range(math.ceil(len(val_keys)/world)): + index=slot*world+rank;active=indexCONTRACT['max_context_rows']: + raise RuntimeError('Full packed example exceeds context budget; never truncate silently') + if not bool(packed['attention_mask'].all()) or packed['input_ids'].shape[0]!=1:raise ValueError('Attention patch requires a single unpadded clip') + batch={k:v.to(acc.device,non_blocking=True) for k,v in packed.items()} + with acc.accumulate(wrapper): + loss=wrapper(batch['input_ids'],batch['attention_mask'],batch['labels']) + if not torch.isfinite(loss).item():raise FloatingPointError('Nonfinite training loss: '+row['uid']) + acc.backward(loss*(scale if active else 0.)) + if acc.sync_gradients: + norm=acc.clip_grad_norm_(wrapper.parameters(),1.) + if not torch.isfinite(norm).item():raise FloatingPointError('Nonfinite gradient norm') + optimizer.step();optimizer.zero_grad(set_to_none=True) + if active:losses[0]+=loss.detach().double();losses[1]+=1 + if current<=3 or current%20==0 or current==total: + gathered=acc.gather(losses[None]).sum(0);recent_loss=float((gathered[0]/gathered[1]).item()) + if acc.is_main_process: + line={'step':current,'total_steps':total,'loss':recent_loss,'lr_semantic':optimizer.param_groups[0]['lr'], + 'lr_talker':optimizer.param_groups[1]['lr'],'elapsed_s':time.monotonic()-started,'job_id':os.environ.get('SLURM_JOB_ID')} + with (out/'steps.jsonl').open('a') as f:f.write(json.dumps(line)+'\n') + print(json.dumps(line),flush=True) + status('RUNNING',current) + if args.verify_smoke_resume: + gathered=acc.gather(losses[None]).sum(0) + measured=float((gathered[0]/gathered[1]).item()) + if not math.isfinite(measured):raise FloatingPointError('Nonfinite post-resume update') + if acc.is_main_process: + save(out/'SMOKE_READY.json',{'passed':True,'optimizer_resume_tested':True,'step':current, + 'restored_step':restored_step,'fresh_process_resume':True,'post_resume_update_loss':measured, + 'map_sha256':ready['database_sha256'],'contract_sha256':digest,'pilot_only':args.pilot,'time':time.time()}) + acc.wait_for_everyone();data.close();dist.destroy_process_group();return + reasons=[None,'smoke','final','phase_boundary','periodic','runtime','signal'] + reason=('smoke' if args.smoke_steps and current>=args.smoke_steps else 'final' if current==total else + 'runtime' if time.monotonic()-started>args.max_runtime_s else 'signal' if STOP else + 'phase_boundary' if current in boundaries else 'periodic' if time.monotonic()-last_save>=900 else None) if acc.is_main_process else None + flag=torch.tensor([reasons.index(reason)],device=acc.device);dist.broadcast(flag,src=0);reason=reasons[int(flag.item())] + if reason:path=checkpoint(current,reason);last_save=time.monotonic() + if reason=='smoke': + # An actual continuation starts in a NEW Slurm process. In-process + # reload retains DDP/autograd allocations and caused a misleading + # 95-GiB transient peak. The next independent srun restores this + # checkpoint AND executes a finite optimizer update before PASS. + if acc.is_main_process:save(out/'SMOKE_SAVED.json',{'step':current, + 'map_sha256':ready['database_sha256'],'contract_sha256':digest,'pilot_only':args.pilot,'time':time.time()}) + acc.wait_for_everyone();data.close();dist.destroy_process_group();return + if reason in ('runtime','signal'): + status('RESUME_REQUIRED',current);acc.wait_for_everyone();data.close();dist.destroy_process_group();return + status('COMPLETE',total) + if acc.is_main_process:save(out/'DONE.json',{'status':'COMPLETE','step':total,'contract_sha256':digest,'time':time.time()}) + data.close();acc.wait_for_everyone();dist.destroy_process_group() + +if __name__=='__main__':main() diff --git a/training/fullft/train.sbatch b/training/fullft/train.sbatch new file mode 100644 index 0000000000000000000000000000000000000000..8bf84317a7bbb02bbeb0972d2c5f3f39382d6291 --- /dev/null +++ b/training/fullft/train.sbatch @@ -0,0 +1,27 @@ +#!/bin/bash +#SBATCH --account=laionize +#SBATCH --partition=booster +#SBATCH --nodes=16 +#SBATCH --ntasks-per-node=4 +#SBATCH --gres=gpu:4 +#SBATCH --cpus-per-task=16 +#SBATCH --time=10:00:00 +#SBATCH --signal=USR1@600 +#SBATCH --job-name=hv13-podcast-fullft +#SBATCH --output=/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/logs/train-%j.log +set -euo pipefail +ulimit -c 0 +C=${HV_PODCAST_CODE:?Frozen code required} +/usr/bin/python3 -B "$C/launch.py" verify +srun --kill-on-bad-exit=1 --gpus-per-task=1 --gpu-bind=single:1 bash "$C/env.sh" "$C/gpu_preflight.py" +/usr/bin/python3 -B "$C/check_gpu_preflight.py" +export MASTER_ADDR +MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n1) +export MASTER_PORT=$((23000+SLURM_JOB_ID%17000)) +export NCCL_IB_TIMEOUT=22 TORCH_NCCL_ASYNC_ERROR_HANDLING=1 +srun --kill-on-bad-exit=1 --gpus-per-task=1 --gpu-bind=single:1 --cpu-bind=none \ + --output=/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/logs/train-%j-rank-%t.log \ + --error=/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/logs/train-%j-rank-%t.err \ + bash -c 'export RANK=$SLURM_PROCID LOCAL_RANK=0 WORLD_SIZE=$SLURM_NTASKS + exec bash "$HV_PODCAST_CODE/env.sh" "$HV_PODCAST_CODE/train.py" --max-runtime-s 32400' +/usr/bin/python3 -B "$C/launch.py" finished --parent "$SLURM_JOB_ID" diff --git a/training/rank64/PLAN.md b/training/rank64/PLAN.md new file mode 100644 index 0000000000000000000000000000000000000000..8c0d39b430b5974743d87d3eea5fad1e3ed8c729 --- /dev/null +++ b/training/rank64/PLAN.md @@ -0,0 +1,29 @@ +# NotebookLM high-CE rank-64 LoRA: preparation, 2026-11-02 + +## User-authorized submission at 22:24 + +The user explicitly requested submission after being told the valid target data are small. Fresh rank-64 two-parent training uses `ladder-past-only.json`: 732 unique examples; 50.56 + 43.23 + 35.87 + 25.46 + 11.66 + 11.66 target minutes across six stages (about 2.97 hours of presentations). No held-out episodes, language changes or later references were introduced. The four-GPU batches give phase lengths 183, 160, 132, 91, 43 and 43 optimizer updates, 652 total, peak 3e-5 at the end of phase 1 and one cosine through phases 2–6. + +NotebookLM base: smoke 2657048; production 2657062 depends on its success. Acting High-8654 base: smoke 2657051; production 2657065 depends on its success. Each smoke checks actual native train/inference prefix parity and restores optimizer/adapter/RNG into a fresh process for a matching third update. GPU success is not yet established. The original parent weights are not changed; every successful production checkpoint preserves adapter, optimizer, rank RNG and cursor. A bounded same-contract continuation is submitted on safe stop. The high-level TTS/listening evaluation still needs a separately validated follow-up; no unmeasured improvement is claimed. + +## Updated six-stage ladder, 2026-11-02 + +Both requested rank-64 runs use the same nested English, German, French and Spanish quality ladder: one pass each over the top 5, 4, 3, 2, 1 and 1 hour(s) of scored windows per language; warm up during the first pass and use one cosine schedule across the remaining five. This supersedes the earlier five-epoch proposal below. German has only 138 minutes and French 23 minutes of eligible scored windows in total; therefore the requested nominal caps cannot be reached in these languages. Even before excluding heldout episodes, reference failures or quantized boundaries, the six passes contain at most about 45.2 hours of presented audio. Do not call them 50–100 hours of unique speech. Repetition is intentional and is separately counted. + +`build_ladder.py` reconstructs sentence groups contained in the original scored windows on an allocated node, using the original podcast prompt builder, the original train split and the original strict past-reference rule. `prepare_ladder.sbatch` job 2655776 was submitted as preparation only and completed. A policy change permitting later, non-overlapping identity references still requires the user's response; the submitted runs retain strict past references. + +## Previous two-run proposal + +- Run A: full-fine-tuned NotebookLM checkpoint `large/training/checkpoints/step-00005613`, rank 64 LoRA, five passes over up to one hour of scored source windows per EN/DE/FR/ES language. +- Run B: acting high-LR checkpoint `high/large/training/checkpoints/step-00008654`, rank 64 LoRA, five passes over up to two hours of scored source windows per EN/DE/FR/ES language. This is a provisional development winner, not a proven best model. +- Exclude NL and every other language; four languages are explicitly named. Keep source episodes with begin AND middle predicted `dnsmos:BAK >= 4.05`; rank the **actual 30 s scored windows** by each window's EARS `audiobox:CE`, as on the approved listening page. Never label unscored whole episodes as the top scored audio. +- One schedule per run: linear warmup across first full epoch, peak at the end of that epoch, cosine decay over the following four epochs. Candidate peak 3e-5 (three times the historical rank-16 real-speech 1e-5); floor 10% of peak. This is a proposal requiring smoke and validation, not an already tested optimum. +- Fixed source `mix.sqlite`, original episode-disjoint test and validation splits. The exact existing podcast packer retains two speaker references, past context and target-only labels, and must be used consistently in training/inference. Preserve all 32 source codec books; train the model on its first 12. Monitor catastrophic forgetting and WER/turn fidelity on held-out podcasts and ordinary acting samples. Periodic and epoch checkpoints must contain adapter, optimizer, RNG, cursor and schedule for exact same-contract resume. + +## Critical data blocker + +`prepare_candidates.py` has materialized auditable candidate lists under Scratch `high_ce_lora_20261102`. The score windows contain only 23 minutes for French even before split and reference checks. The existing clean past-reference training examples strictly contained within the selected windows total only 9.75 minutes for the one-hour cap and 21.35 minutes for the two-hour cap across all four languages; these are not multi-hour training sets. Beginning windows often precede either of the two clean 5–15-second identity references. Episode-level held-out material is excluded. Do not relabel a short dataset as several hours, train on heldout episodes, copy missing references from the target, silently use later audio as conversation history, or silently extend scores from the first/middle 30 seconds to entire episodes. + +The exact allowed policy for identity references from later non-overlapping parts of the same episode is awaiting user input. Independently, sentence-level reconstruction inside the scored windows may recover some valid samples, but requires exact alignment and GPU-smoke verification; the duration cannot exceed the selected scored audio and French remains short. If later identity references are permitted, they must have different transcript and no overlap with the target; conversation context remains strictly past and never contains future speech. The new reference policy must be logged as a changed conditioning contract, not an exact continuation of the earlier full-FT data loader. + +The user subsequently authorized submission on the smaller strict-past-reference corpus. Do not describe the stored candidates as the originally intended multi-hour corpus. Both parent checkpoints must be loaded and checked and a fresh-process LoRA optimizer/RNG/prompt-parity smoke must pass before either dependent production job starts. The candidate files are selection manifests, not copies of the audio or model weights. diff --git a/training/rank64/adapters.py b/training/rank64/adapters.py new file mode 100644 index 0000000000000000000000000000000000000000..8c6f228a3b788e434fe720b1c0c084ab20d8800d --- /dev/null +++ b/training/rank64/adapters.py @@ -0,0 +1,67 @@ +"""FP32 rank-128 adapters on frozen BF16 semantic and local attention.""" +from contextlib import contextmanager +import math +import torch +from torch import nn + + +class LoRALinear(nn.Module): + def __init__(self, base, rank=128): + super().__init__() + self.base, self.rank, self.enabled = base, rank, True + self.a = nn.Parameter(torch.empty(rank, base.in_features, device=base.weight.device, dtype=torch.float32)) + self.b = nn.Parameter(torch.zeros(base.out_features, rank, device=base.weight.device, dtype=torch.float32)) + nn.init.kaiming_uniform_(self.a, a=math.sqrt(5)) + # alpha=rank => scaling=1; do NOT add unrelated merge scaling later. + def forward(self, x): + result = self.base(x) + if self.enabled: + # Disable outer autocast so optimizer-backed adapter storage and + # arithmetic remain FP32. Cast only the residual into base output. + with torch.autocast(x.device.type, enabled=False): + residual = torch.nn.functional.linear(torch.nn.functional.linear(x.float(), self.a), self.b) + result = result + residual.to(result.dtype) + return result + + +def attach(model, rank=128): + model.requires_grad_(False) + chosen = [] + for name, module in list(model.named_modules()): + semantic = name.startswith("transformer.") and name.rsplit(".", 1)[-1] in {"q_proj", "k_proj", "v_proj", "o_proj"} + local = name.startswith("local_transformer.") and ".attn." in name and name.rsplit(".", 1)[-1] in {"c_attn", "c_proj"} + if semantic or local: + if not isinstance(module, nn.Linear): + raise TypeError(f"Expected Linear: {name}: {type(module)}") + parent, leaf = name.rsplit(".", 1) + setattr(model.get_submodule(parent), leaf, LoRALinear(module, rank)) + chosen.append(name) + if not any(n.startswith("transformer.") for n in chosen) or not any(n.startswith("local_transformer.") for n in chosen): + raise RuntimeError("Both semantic and Talker attention adapters must be present") + return chosen + + +@contextmanager +def disabled(model): + modules = [m for m in model.modules() if isinstance(m, LoRALinear)] + states = [m.enabled for m in modules] + try: + for m in modules: + m.enabled = False + yield + finally: + for m, state in zip(modules, states): + m.enabled = state + + +def adapter_state(model): + return {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad} + + +def restore(model, state): + params = {n: p for n, p in model.named_parameters() if p.requires_grad} + if set(params) != set(state): + raise ValueError("Adapter parameter names changed") + with torch.no_grad(): + for name, value in state.items(): + params[name].copy_(value) diff --git a/training/rank64/build_ladder.py b/training/rank64/build_ladder.py new file mode 100644 index 0000000000000000000000000000000000000000..7584ad34e07974240e828f2bc7ad4e466ad0b5be --- /dev/null +++ b/training/rank64/build_ladder.py @@ -0,0 +1,65 @@ +"""Build a bounded, exact-window, past-reference-only podcast LoRA candidate ladder.""" +import collections,hashlib,json,os,sqlite3,sys,time +from pathlib import Path + +SNAP=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/code_snapshot_v4') +sys.path.insert(0,str(SNAP)) +from prepare_training import clean_runs,make_example +SRC=Path('/e/scratch/laionize/schuhmann1/notebooklm_podcast_preparation_20261101') +ROOT=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102') +OUT=ROOT/'high_ce_lora_20261102' +LANGS=('en','de','fr','es');TIERS=(5,4,3,2,1,1) + +def main(): + if not os.environ.get('SLURM_JOB_ID'):raise RuntimeError('Episode reconstruction requires allocated compute') + eps=json.loads((SRC/'accepted.json').read_text());quality=json.loads((SRC/'quality_windows.json').read_text()) + groups=collections.defaultdict(list) + for e in eps: + if e['language'] not in LANGS:continue + q=quality[e['uid']] + if min(q['begin']['scores']['dnsmos:BAK'],q['middle']['scores']['dnsmos:BAK'])<4.05:continue + for kind in ('begin','middle'): + x=q[kind];groups[e['language']].append(dict(episode=e['uid'],lang=e['language'],window=kind,start=x['start_s'],duration=x['segment_s'],ce=x['scores']['audiobox:CE'])) + selected={};pool={} + for hours in set(TIERS): + selected[hours]=[] + for lang in LANGS: + seconds=0 + for w in sorted(groups[lang],key=lambda x:(-x['ce'],x['episode'],x['window'])): + if seconds+w['duration']<=hours*3600+1e-6: + selected[hours].append(w);seconds+=w['duration'];pool[(w['episode'],w['window'])]=w + db=sqlite3.connect('file:'+str(ROOT/'large/mix.sqlite')+'?mode=ro&immutable=1',uri=True);db.row_factory=sqlite3.Row + cache={};reasons=collections.Counter();examples={};bywindow=collections.defaultdict(list) + for w in pool.values(): + row=db.execute('SELECT * FROM episodes WHERE uid=?',(w['episode'],)).fetchone() + if row is None:reasons['episode_missing']+=1;continue + if row['split']!='train':reasons['episode_not_training_split']+=1;continue + if row['uid'] not in cache: + with open(row['tar'],'rb') as f:f.seek(row['record_offset']);record=json.loads(f.read(row['record_size'])) + record['_by_word']={x['word_id']:x for x in record['words']};cache[row['uid']]=record + record=cache[row['uid']];lo=w['start'];hi=lo+w['duration'] + for run in clean_runs(record): + segment=[s for s in run if s['start_s']>=lo and s['end_s']<=hi] + if not segment:continue + uid=f"{row['uid']}-{w['window']}-{segment[0]['sentence_id']}-{segment[-1]['sentence_id']}" + try: + ex=make_example(record,segment,uid) + if ex['target_frame_start']/12.5hi+1e-7: + raise ValueError('quantized_target_leaves_scored_window') + if not ex['literal_text'].strip():raise ValueError('empty_text') + except ValueError as error:reasons[str(error)]+=1;continue + key=(row['uid'],ex['target_frame_start'],ex['target_frame_end']) + keystr=hashlib.sha256(json.dumps(key).encode()).hexdigest() + examples[keystr]=dict(uid=uid,episode=row['uid'],language=w['lang'],frames=ex['target_frame_end']-ex['target_frame_start'],example=ex,window=w) + bywindow[w['episode'],w['window']].append(keystr) + stats=[];epochs=[] + for epoch,hours in enumerate(TIERS,1): + keys=list(dict.fromkeys(k for w in selected[hours] for k in bywindow[w['episode'],w['window']])) + langs={lang:dict(examples=sum(examples[k]['language']==lang for k in keys),target_minutes=sum(examples[k]['frames'] for k in keys if examples[k]['language']==lang)/12.5/60) for lang in LANGS} + epochs.append(dict(epoch=epoch,hours_cap=hours,keys=keys));stats.append(dict(epoch=epoch,hours_cap=hours,examples=len(keys),languages=langs)) + result=dict(status='PREPARATION_ONLY_NO_TRAINING',language_policy=list(LANGS),selection='BAK >=4.05 for both episodes windows; CE ranked original scored windows',reference_policy='two clean 5–15s references and context strictly before target',episode_split='train only',epochs=epochs,examples=examples,statistics=stats,rejections=dict(reasons),created=time.time()) + OUT.mkdir(parents=True,exist_ok=True); path=OUT/'ladder-past-only.json';tmp=path.with_suffix('.tmp');tmp.write_text(json.dumps(result,ensure_ascii=False,allow_nan=False));os.replace(tmp,path) + proof=dict(path=str(path),sha256=hashlib.file_digest(path.open('rb'),'sha256').hexdigest(),unique_examples=len(examples),statistics=stats,rejections=dict(reasons)) + (OUT/'ladder-past-only-summary.json').write_text(json.dumps(proof,indent=2)+'\n');print(json.dumps(proof,indent=2),flush=True) + +if __name__=='__main__':main() diff --git a/training/rank64/prepare_candidates.py b/training/rank64/prepare_candidates.py new file mode 100644 index 0000000000000000000000000000000000000000..1ed4affc15a5b2fb11f1a96bb8e724d882e60e3b --- /dev/null +++ b/training/rank64/prepare_candidates.py @@ -0,0 +1,45 @@ +"""Select only existing clean train targets contained in the scored listening windows.""" +import hashlib,json,sqlite3 +from collections import defaultdict +from pathlib import Path + +SOURCE=Path('/e/scratch/laionize/schuhmann1/notebooklm_podcast_preparation_20261101') +DATABASE=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/large/mix.sqlite') +OUT=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/high_ce_lora_20261102') +LANGS=('en','de','fr','es') +def main(): + episodes=json.loads((SOURCE/'accepted.json').read_text());quality=json.loads((SOURCE/'quality_windows.json').read_text());bylang=defaultdict(list) + for e in episodes: + if e['language'] not in LANGS:continue + q=quality[e['uid']];a=q['begin']['scores']['dnsmos:BAK'];b=q['middle']['scores']['dnsmos:BAK'] + if min(a,b)<4.05:continue + for window in ('begin','middle'): + x=q[window];bylang[e['language']].append({'uid':e['uid'],'window':window,'start_s':x['start_s'],'duration_s':x['segment_s'],'ce':x['scores']['audiobox:CE'],'background_begin':a,'background_middle':b}) + db=sqlite3.connect('file:'+str(DATABASE)+'?mode=ro&immutable=1',uri=True);db.row_factory=sqlite3.Row + OUT.mkdir(parents=True,exist_ok=True) + for seconds in (3600,7200): + selected=[];targets={};stats={} + for lang in LANGS: + rows=sorted(bylang[lang],key=lambda x:(-x['ce'],x['uid'],x['window']));remaining=seconds;current=[] + for x in rows: + if x['duration_s']<=remaining+1e-6:current.append(x);remaining-=x['duration_s'] + counts=defaultdict(int);frames=0 + for x in current: + e=db.execute('SELECT split FROM episodes WHERE uid=?',(x['uid'],)).fetchone() + if not e:counts['missing_episode']+=1;continue + if e['split']!='train':counts['non_train_windows']+=1;continue + counts['train_windows']+=1 + for row in db.execute('SELECT key,uid,episode,frames,example_json FROM samples WHERE episode=? AND split=? AND kind=?',(x['uid'],'train','train-window')): + ex=json.loads(row['example_json']);start=ex['target_frame_start']/12.5;end=ex['target_frame_end']/12.5 + if start+1e-7>=x['start_s'] and end<=x['start_s']+x['duration_s']+1e-7: + if row['key'] not in targets: + item=dict(key=row['key'],uid=row['uid'],episode=row['episode'],frames=row['frames'],language=lang,scored_window=x) + targets[row['key']]=item;frames+=row['frames'] + selected.extend(current) + stats[lang]=dict(eligible_windows=len(rows),selected_windows=len(current),selected_minutes=(seconds-remaining)/60,training_target_minutes=frames/12.5/60,counts=dict(counts)) + result={'requested_seconds_per_language':seconds,'languages':LANGS,'reference_policy':'original two speaker references strictly before target','train_only':True, + 'data_source':str(DATABASE),'filter':'both episode-window dnsmos:BAK >=4.05; top CE of scored 30 s windows', + 'targets':list(targets.values()),'selection':selected,'statistics':stats} + raw=json.dumps(result,ensure_ascii=False,indent=2).encode();(OUT/f'candidates-{seconds//3600}h.json').write_bytes(raw) + print(json.dumps({'limit_hours':seconds//3600,'targets':len(targets),'sha256':hashlib.sha256(raw).hexdigest(),'statistics':stats}),flush=True) +if __name__=='__main__':main() diff --git a/training/rank64/schedule.py b/training/rank64/schedule.py new file mode 100644 index 0000000000000000000000000000000000000000..f44473cae07f76446bf84159b8d4bdc25bce48d7 --- /dev/null +++ b/training/rank64/schedule.py @@ -0,0 +1,24 @@ +"""A single warm-up plus cosine schedule over unequal ladder stages.""" +import math + +TIER_HOURS=(5,4,3,2,1,1) +PEAK_LR=3e-5 +FLOOR_FRACTION=.1 + +def factor(update,phase_steps): + if len(phase_steps)!=len(TIER_HOURS) or any(int(x)!=x or x<=0 for x in phase_steps): + raise ValueError('Six positive phase lengths are required') + total=sum(phase_steps) + if not 1<=update<=total:raise ValueError('Update outside schedule') + warm=phase_steps[0] + if update<=warm:return update/warm + progress=(update-warm)/(total-warm) + return FLOOR_FRACTION+(1-FLOOR_FRACTION)*.5*(1+math.cos(math.pi*progress)) + +if __name__=='__main__': + steps=(100,80,60,40,20,20) + assert factor(1,steps)==.01 and factor(100,steps)==1. + assert abs(factor(sum(steps),steps)-FLOOR_FRACTION)<1e-12 + values=[factor(i,steps) for i in range(100,sum(steps)+1)] + assert all(a>=b for a,b in zip(values,values[1:])) + print('PASS: 6 phases, first-stage warmup, one monotonic cosine, correct endpoint') diff --git a/training/rank64/train_ladder.py b/training/rank64/train_ladder.py new file mode 100644 index 0000000000000000000000000000000000000000..dcea98419ba11855c00fd077576074b58d710992 --- /dev/null +++ b/training/rank64/train_ladder.py @@ -0,0 +1,154 @@ +"""Four-GPU rank-64 podcast LoRA. Strict past references; six-stage schedule.""" +import argparse, hashlib, importlib.util, json, math, os, random, signal, sys, time +from pathlib import Path +import numpy as np +import torch +import torch.distributed as dist +from safetensors.torch import load_file,save_file +from schedule import factor,PEAK_LR,TIER_HOURS + +HOME=Path('/e/home/jusers/schuhmann1/jupiter') +SNAP=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/code_snapshot_v4') +ROOT=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/high_ce_lora_20261102') +MANIFEST=ROOT/'ladder-past-only.json' +BASE=Path('/e/scratch/laionize/schuhmann1/hvs_large_1_3_20261022/release_1_3/model') +PARENTS={'podcast':Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/large/training/checkpoints/step-00005613'), + 'acting':Path('/e/scratch/laionize/schuhmann1/hvs_large13_extreme_topups_20261102/high/large/training/checkpoints/step-00008654')} +ADAPTER=Path('/e/scratch/laionize/schuhmann1/hvs_large_1_3_20261022/rl_ce33_emo33_genuine34_20261022/code_v2_duration_20261022/adapters.py') +SEED=20261102;WORLD=4;STOP=False +def on_signal(*_): + global STOP;STOP=True +def sha(path): + h=hashlib.sha256() + with open(path,'rb') as f: + for b in iter(lambda:f.read(8<<20),b''):h.update(b) + return h.hexdigest() +def write(path,obj): + tmp=path.with_name(path.name+'.tmp-'+str(os.getpid()));tmp.write_text(json.dumps(obj,indent=2,allow_nan=False)+'\n');os.replace(tmp,path) +def loadmodule(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m +def canonical(n):return n.replace('._checkpoint_wrapped_module.','.') +def state(model):return {canonical(n):p.detach().cpu().clone().contiguous() for n,p in model.named_parameters() if p.requires_grad} +def restore(model,values): + ps={canonical(n):p for n,p in model.named_parameters() if p.requires_grad} + if ps.keys()!=values.keys():raise ValueError('Adapter parameter mismatch') + with torch.no_grad(): + for n,p in ps.items():p.copy_(values[n]) +def modelrow(data,item): + ex=item['example'];codes=data.episode(item['episode']);a,b=ex['target_frame_start'],ex['target_frame_end'];ca,cb=ex['context_frame_start'],ex['context_frame_end'] + if not 0<=ca<=cb<=a750:raise ValueError('Context/target invalid') + refs={} + for speaker,r in ex['speaker_references'].items(): + ra,rb=r['start_frame'],r['end_frame'] + if not 0<=ra1e-6 or not all(torch.allclose(actual[k],v,atol=1e-7,rtol=1e-5) for k,v in expected.items()):raise RuntimeError('Fresh-process replay mismatch') + write(ROOT/a.variant/'SMOKE_READY.json',dict(passed=True,contract_sha256=digest,modules=len(chosen),time=time.time())) + dist.barrier();dist.destroy_process_group();return + flag=torch.tensor([int(STOP or time.monotonic()-started>34000)],device='cuda:0');dist.all_reduce(flag,op=dist.ReduceOp.MAX) + ends={sum(steps[:i]) for i in range(1,len(steps)+1)} + if a.mode=='production' and (current in ends or current%25==0 or time.monotonic()-last>900 or flag.item()):checkpoint_now(current,'final' if current==total else 'periodic');last=time.monotonic() + if flag.item(): + if rank==0:write(out/'RESUME_REQUIRED.json',dict(step=current,time=time.time())) + dist.barrier();dist.destroy_process_group();return + if rank==0:write(out/'FINISHED.json',dict(step=total,contract_sha256=digest,time=time.time())) + data.close();dist.barrier();dist.destroy_process_group() +if __name__=='__main__':main() diff --git a/training/rank64/train_ladder.sbatch b/training/rank64/train_ladder.sbatch new file mode 100644 index 0000000000000000000000000000000000000000..4529e52e64dc0868d6bd6774487c9127dfa35333 --- /dev/null +++ b/training/rank64/train_ladder.sbatch @@ -0,0 +1,27 @@ +#!/bin/bash +#SBATCH --account=laionize +#SBATCH --partition=booster +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=4 +#SBATCH --gres=gpu:4 +#SBATCH --cpus-per-task=16 +#SBATCH --time=10:00:00 +#SBATCH --signal=USR1@300 +#SBATCH --job-name=podcast-r64-ladder +#SBATCH --output=/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/high_ce_lora_20261102/train-%j.log +set -euo pipefail +ulimit -c 0 +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=$((15000 + SLURM_JOB_ID % 40000)) +C=/e/home/jusers/schuhmann1/jupiter/hv_large13_notebooklm_20261102/high_ce_lora +E=/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/code_snapshot_v4/env.sh +srun --kill-on-bad-exit=1 --gpus-per-task=1 --gpu-bind=single:1 --cpu-bind=none bash "$E" "$C/train_ladder.py" --variant "$PODCAST_VARIANT" --mode production +R=/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102/high_ce_lora_20261102/$PODCAST_VARIANT/training +if [[ -f "$R/FINISHED.json" ]]; then exit 0; fi +if [[ -f "$R/RESUME_REQUIRED.json" && ${PODCAST_CHAIN:-0} -lt 3 ]]; then + next=$(( ${PODCAST_CHAIN:-0} + 1 )) + sbatch --parsable --dependency="afterok:$SLURM_JOB_ID" --export="ALL,PODCAST_VARIANT=$PODCAST_VARIANT,PODCAST_CHAIN=$next" "$C/train_ladder.sbatch" > "$R/continuation-$next.txt" +else + echo 'No finished checkpoint or safe continuation; inspect logs.' >&2 + exit 1 +fi diff --git a/training/release_and_eval.py b/training/release_and_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..78b431981185a0a2bf851d99812419916f1981ae --- /dev/null +++ b/training/release_and_eval.py @@ -0,0 +1,229 @@ +"""Allocated-node HQ reference preparation, checked merge and 30-dialogue evaluation.""" +import argparse,base64,hashlib,html,importlib.util,io,json,math,os,shutil,sqlite3,subprocess,sys,tarfile,time +from pathlib import Path +import numpy as np + +HOME=Path('/e/home/jusers/schuhmann1/jupiter');CODE=Path(__file__).resolve().parent +ROOT=Path('/e/scratch/laionize/schuhmann1/hv_notebooklm_edition_20261103') +POD=Path('/e/scratch/laionize/schuhmann1/hv_large13_notebooklm_20261102');LADDER=POD/'high_ce_lora_20261102' +PARENT=POD/'large/training/checkpoints/step-00005613';ADAPTER=LADDER/'podcast/training/checkpoints/step-00000652' +BASE=Path('/e/scratch/laionize/schuhmann1/hvs_large_1_3_20261022/release_1_3/model') +ADAPTER_CODE=Path('/e/scratch/laionize/schuhmann1/hvs_large_1_3_20261022/rl_ce33_emo33_genuine34_20261022/code_v2_duration_20261022/adapters.py') +ASSETS=Path('/e/scratch/laionize/schuhmann1/hvs15_acting_search_20261024/assets') +PAGE=HOME/'HUMANENESS_NOTEBOOKLM_EDITION_HQ_REFERENCES_2026-11-03.html' +MODEL_REPO='laion/Humaneness-Voice-Large-NotebookLM-Edition';SPACE='laion/humaneness-voice-large-podcast-listening' +DATASET='laion/NotebookLM-Podcasts-Humaneness-Prepared' + +def write(path,obj): + path=Path(path);path.parent.mkdir(parents=True,exist_ok=True);temp=path.with_name(path.name+'.tmp-'+str(os.getpid()));temp.write_text(json.dumps(obj,ensure_ascii=False,indent=2,allow_nan=False)+'\n');os.replace(temp,path) +def sha(path): + with open(path,'rb') as f:return hashlib.file_digest(f,'sha256').hexdigest() +def module(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m +def require_gpu(): + if not os.environ.get('SLURM_JOB_ID'):raise RuntimeError('Neural and bulk conversion work requires allocated compute') + import torch + torch.cuda.set_device(0);torch.set_num_threads(4) +def judges(): + sys.path.insert(0,'/e/scratch/laionize/schuhmann1/hvs_large_1_5_followup_20261024/rl_rank32/code_v1') + from judges import Judges + return Judges(ASSETS,'cuda:0',with_asr=True) +def assess(judge,wave,sr): + from scipy.signal import resample_poly + g=math.gcd(sr,16000);x=resample_poly(wave,16000//g,sr//g).astype(np.float32) + cuts=[x[i:i+480000] for i in range(0,len(x),480000)];raw=[];hyp=[];weights=[] + for cut in cuts: + weights.append(len(cut));z=cut if len(cut)>=1600 else np.pad(cut,(0,1600-len(cut))) + raw.extend(judge.ears_scores([z]));hyp.extend(judge.transcribe([z])) + w=np.asarray(weights)/sum(weights);scores={k:float(sum(a*b[k] for a,b in zip(w,raw))) for k in raw[0]} + return scores,' '.join(hyp) +def mp3(wave,sr): + import lameenc + e=lameenc.Encoder();e.set_bit_rate(128);e.set_in_sample_rate(sr);e.set_channels(1);e.set_quality(2) + return e.encode((np.clip(wave,-1,1)*32767).astype('=8:break + if len(pools[lang])<2:raise ValueError('Fewer than two clean HQ identity references for '+lang) + # Prefer one source episode's two different speakers; otherwise two distinct + # source identities, still from the exact HQ training-window pool. + selected={};arrays={};media=tarfile.open(ROOT/'reference_audio.tar.tmp','w');proof=[] + for lang,pool in pools.items(): + pairs=[(a,b) for i,a in enumerate(pool) for b in pool[i+1:] if (a['episode'],a['original_speaker'])!=(b['episode'],b['original_speaker'])] + def rank(pair): + same=pair[0]['episode']==pair[1]['episode'];quality=min(c['scores']['dnsmos:BAK'] for c in pair) + return (quality>=4.,same,sum(c['scores']['audiobox:CE']+.3*c['scores']['audiobox:PQ'] for c in pair)) + pair=max(pairs,key=rank);selected[lang]={} + for speaker,c in zip(['S1','S2'],pair): + row,record=episodes[c['episode']] + with open(row['tar'],'rb') as f:f.seek(row['code_offset']);codes=np.load(io.BytesIO(f.read(row['code_size'])),allow_pickle=False) + arrays[lang+'_'+speaker]=codes[c['start_frame']:c['end_frame'],:12].copy() + wave=c.pop('_wave');name=lang+'-'+speaker;add_tar(media,name+'.mp3',mp3(wave,48000)) + import soundfile as sf + wav=io.BytesIO();sf.write(wav,wave,48000,format='WAV',subtype='PCM_16');add_tar(media,name+'.wav',wav.getvalue()) + c.update(speaker=speaker,seconds=len(wave)/48000,media_member=name+'.mp3',wav_member=name+'.wav',source_split='train',source_audio='Original media crop, not a codec reconstruction') + selected[lang][speaker]=c;proof.append({'language':lang,'speaker':speaker,'CE':c['scores']['audiobox:CE'],'PQ':c['scores']['audiobox:PQ'],'BAK':c['scores']['dnsmos:BAK']}) + media.close();os.replace(ROOT/'reference_audio.tar.tmp',ROOT/'reference_audio.tar');np.savez(ROOT/'references.npz',**arrays) + write(ROOT/'references.json',selected);write(ROOT/'tasks.json',tasks());write(ROOT/'REFERENCES_READY.json',{'passed':True,'references':8,'tasks':30,'reference_npz_sha256':sha(ROOT/'references.npz'),'tasks_sha256':sha(ROOT/'tasks.json'),'selection':'Actual HQ training windows; clean single-speaker original-media crops. Cold-start dialogue, no past context.','quality':proof}) + print(json.dumps(proof),flush=True) + +def export(): + require_gpu() + if (ROOT/'EXPORT_READY.json').exists():return + import torch + from safetensors.torch import load_file + import base_runtime + adapter=load_file(str(ADAPTER/'adapter.safetensors'),device='cpu');ready=json.loads((ADAPTER/'READY.json').read_text()) + if ready['step']!=652 or sha(ADAPTER/'adapter.safetensors')!=ready['adapter_sha256']:raise ValueError('Wrong adapter') + if json.loads((PARENT/'READY.json').read_text())['step']!=5613:raise ValueError('Wrong NotebookLM parent') + model,proc=base_runtime.load_model(BASE) + state=torch.load(PARENT/'pytorch_model_fsdp.bin',map_location='cpu',weights_only=True,mmap=True) + if not all(k.startswith('base.') for k in state):raise ValueError('Parent wrapper mismatch') + missing,extra=model.load_state_dict({k[5:]:v for k,v in state.items()},strict=False) + allowed={'text_lm_head.weight'}|{f'audio_lm_heads.{i}.weight' for i in range(12)} + if extra or set(missing)-allowed:raise ValueError('Parent keys mismatch') + model.tie_weights();del state + ad=module('edition_adapters',ADAPTER_CODE);names=ad.attach(model,64);ad.restore(model,adapter) + if len(names)!=146:raise ValueError('Expected 146 adapted modules') + # Verify every merged Linear against the actual trained residual definition. + errors=[] + with torch.no_grad(),torch.autocast('cuda',dtype=torch.bfloat16): + for name in names: + layer=model.get_submodule(name);x=torch.randn(3,layer.base.in_features,device='cuda',dtype=torch.bfloat16)*.1 + before=layer(x).float() + with torch.autocast('cuda',enabled=False):merged=(layer.base.weight.float()+layer.b@layer.a).to(layer.base.weight.dtype) + after=torch.nn.functional.linear(x,merged,layer.base.bias).float() + scale=max(.01,float(before.abs().max()));error=float((before-after).abs().max())/scale + if not math.isfinite(error) or error>.05:raise ValueError('Merge linear parity failed: '+name) + errors.append(error);layer.base.weight.copy_(merged);parent,leaf=name.rsplit('.',1);setattr(model.get_submodule(parent),leaf,layer.base) + model.tie_weights();target=ROOT/'model';target.mkdir(exist_ok=True) + for p in BASE.iterdir(): + if p.name in ['config.json','generation_config.json','tokenizer.json','tokenizer_config.json','added_tokens.json','special_tokens_map.json','processor_config.json','configuration_moss_tts.py','modeling_moss_tts.py','processing_moss_tts.py','qwen3_decoder.py','gpt2_decoder.py','__init__.py','requirements.txt']:shutil.copy2(p,target/p.name) + model.config._name_or_path=MODEL_REPO + model.save_pretrained(target,safe_serialization=True,max_shard_size='4GB') + # Reload the exported files on the allocated GPU and compare the packed + # semantic prefix. This verifies portable loading, not just saved file names. + import podcast_infer + refs=np.load(ROOT/'references.npz');t=json.loads((ROOT/'tasks.json').read_text())[0] + row=podcast_infer.row_from_dialogue(t,{s:refs[t['language']+'_'+s] for s in ['S1','S2']});batch=podcast_infer.generation_inputs(proc,row) + with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16):expected=model(input_ids=batch['input_ids'].to('cuda'),attention_mask=batch['attention_mask'].to('cuda')).last_hidden_state[:,-1].float().cpu() + del model;torch.cuda.empty_cache();restored,_=base_runtime.load_model(target) + with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16):actual=restored(input_ids=batch['input_ids'].to('cuda'),attention_mask=batch['attention_mask'].to('cuda')).last_hidden_state[:,-1].float().cpu() + if not torch.equal(expected,actual):raise ValueError('Export/reload semantic prefix changed') + for n in ['podcast_infer.py','base_runtime.py']:shutil.copy2(CODE/n,target/n) + (target/'adapter').mkdir(exist_ok=True);shutil.copy2(ADAPTER/'adapter.safetensors',target/'adapter/adapter.safetensors');shutil.copy2(ADAPTER/'READY.json',target/'adapter/READY.json') + (target/'adapter/README.txt').write_text('Rank 64, alpha 64, scale 1, final update 652. Already merged in root weights. Do not apply twice. Original parent is NotebookLM full FT step 5613, not unmodified Large 1.3.\n') + weights={f.name:{'bytes':f.stat().st_size,'sha256':sha(f)} for f in target.glob('*.safetensors')} + write(ROOT/'EXPORT_READY.json',{'passed':True,'parent_step':5613,'adapter_step':652,'rank':64,'alpha':64,'scale':1.,'modules':len(names),'merged':True,'linear_max_relative_error':max(errors),'export_reload_semantic_prefix_exact':True,'weights':weights,'adapter_sha256':sha(ADAPTER/'adapter.safetensors'),'time':time.time()}) + +def worker(): + require_gpu() + import torch + import base_runtime,podcast_infer + sys.path.insert(0,'/e/scratch/laionize/schuhmann1/hvs_large_1_5_followup_20261024/rl_rank32/code_v1') + from reward_text import word_error + rank=int(os.environ['SLURM_PROCID']);world=int(os.environ['SLURM_NTASKS']);out=ROOT/'evaluation';out.mkdir(exist_ok=True) + if not json.loads((ROOT/'EXPORT_READY.json').read_text())['passed']:raise ValueError('Export not validated') + model,proc=base_runtime.load_model(ROOT/'model');base_runtime.load_codec(proc,ASSETS/'codec');judge=judges() + ref=np.load(ROOT/'references.npz');data=json.loads((ROOT/'tasks.json').read_text());contract=sha(ROOT/'tasks.json')+sha(ROOT/'EXPORT_READY.json') + db=sqlite3.connect(out/f'rank-{rank}.sqlite');db.execute('CREATE TABLE IF NOT EXISTS results(task TEXT PRIMARY KEY,metadata TEXT,mp3 BLOB,codes BLOB)') + for t in data[rank::world]: + previous=db.execute('SELECT metadata FROM results WHERE task=?',(t['task_id'],)).fetchone() + if previous: + if json.loads(previous[0])['contract']!=contract:raise ValueError('Eval contract changed') + continue + row=podcast_infer.row_from_dialogue(t,{s:ref[t['language']+'_'+s] for s in ['S1','S2']}) + started=time.time();codes,meta=podcast_infer.generate(model,proc,row,t['seed']);wave,sr=base_runtime.decode(proc,codes);scores,hyp=assess(judge,wave,sr) + r={**meta,'task_id':t['task_id'],'group':t['group'],'language':t['language'],'turns':len(t['turns']),'duration_s':len(wave)/sr,'contract':contract, + 'ce':scores['audiobox:CE'],'pq':scores['audiobox:PQ'],'genuineness':scores['genuineness_0_6'],'blend':scores['blend_0_10'],'wer':word_error(row['plain'],hyp),'hypothesis':hyp,'scores':scores,'elapsed_s':time.time()-started} + blob=io.BytesIO();np.save(blob,codes.numpy().astype('' + p=['Humaneness Voice Large — NotebookLM Edition: HQ references

Humaneness Voice Large — NotebookLM Edition

', + '

Selected model: NotebookLM + rank 64 · final 652, merged at scale 1. Thirty new dialogue scripts test two, six and eight alternating turns. Ten short exchanges, ten approximately 30–40-second conversations, and ten up to one minute. EN/DE/FR/ES are included; unreliable Dutch source labels are excluded. Each conversation is generated jointly as one waveform, not assembled from separately generated turns.

', + '

Both identity references come from the actual high-quality training-window pool: original beginning/middle background scores each ≥4.05, ranked by Content Enjoyment, with clean single-speaker 5–15-second crops. The reference players contain original-media crops, not the codec reconstructions used by the previous grid. The model still receives the corresponding twelve-codebook MOSS representation. References are reused across length conditions, and every script is newly written. These are new-text, in-domain voice demonstrations, not a held-out-speaker benchmark. No past conversation is supplied in this cold-start test.

', + '

Controls: speaker identity is routed through S1/S2 reference blocks; each SCRIPT turn includes its speaker, delivery and requested duration. Sampling uses temperature 1, top-p .95, top-k 50, CFG 1, one fixed seed per topic. Durations and turn boundaries are instructions, not forced edits. EARS quality predictions and ASR WER describe complete outputs in windows up to 30 seconds. Listen to judge speaker switches; requested labels alone do not establish correct diarization.

', + '

Merged model, code and podcast prompting guide · Earlier four-model held-out comparison

'] + for r in summaries:p.append(''+''.join(''%r[k] for k in ['ce','pq','genuineness','wer','duration_s'])+'') + p.append('
Length groupNCE ↑PQ ↑Genuineness ↑WER ↓Actual secondsFrame caps
'+r['group']+''+str(r['n'])+'%.3f'+str(r['frame_caps'])+'

') + for t in tasks: + r=next(r for r in records if r['task_id']==t['task_id']);language=t['language'] + p.append('

'+esc(t['task_id'])+'

'+str(r['turns'])+' requested turns · target %.2f s · generated %.2f s · CE %.2f · PQ %.2f · WER %.1f%%

'%(r['requested_seconds'],r['duration_s'],r['ce'],r['pq'],100*r['wer'])+'
') + for speaker in ['S1','S2']: + q=refs[language][speaker];p.append('
'+speaker+' · original HQ identity reference'+player(refmedia[q['media_member']],q['media_member'])+'%.2f s · reference CE %.2f · PQ %.2f · BAK %.2f
'%(q['seconds'],q['scores']['audiobox:CE'],q['scores']['audiobox:PQ'],q['scores']['dnsmos:BAK'])) + p.append('
NEW · NotebookLM + rank 64 · final 652'+player(audio[t['task_id']],t['task_id']+'.mp3')+'
Dialogue, exact directions and ASR

'+esc(t['general'])+'

') + for turn in t['turns']:p.append('

'+turn['speaker']+' ('+esc(turn['direction'])+') [%.2f seconds] '%turn['seconds']+esc(turn['text'])+'

') + p.append('

ASR: '+esc(r['hypothesis'])+'

') + p.append('');return ''.join(p) + PAGE.write_text(page(True));(ROOT/'evaluation/streaming.html').write_text(page(False)) + write(ROOT/'evaluation/HTML_READY.json',{'complete':True,'tasks':30,'players':90,'local_html':str(PAGE),'time':time.time()}) + print(PAGE,flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('mode',choices=['references','export','worker','render']);a=p.parse_args();globals()[a.mode]()