Spaces:
Sleeping
Sleeping
Download models/content_encoder/content_encoder.py from Spider-8251/UniFlow-Audio: direct link, hf CLI and curl.
- Browser
- Download file 14 kB
-
https://huggingface.co/spaces/Spider-8251/UniFlow-Audio/resolve/main/models/content_encoder/content_encoder.py
- Command line
-
hf download hf://spaces/Spider-8251/UniFlow-Audio/models/content_encoder/content_encoder.py
-
curl -L -o content_encoder.py https://huggingface.co/spaces/Spider-8251/UniFlow-Audio/resolve/main/models/content_encoder/content_encoder.py
14 kB
| from typing import Any | |
| import torch | |
| import torch.nn as nn | |
| class ContentEncoder(nn.Module): | |
| def __init__( | |
| self, | |
| embed_dim: int, | |
| text_encoder: nn.Module = None, | |
| video_encoder: nn.Module = None, | |
| midi_encoder: nn.Module = None, | |
| phoneme_encoder: nn.Module = None, | |
| pitch_encoder: nn.Module = None, | |
| audio_encoder: nn.Module = None, | |
| speaker_encoder: nn.Module = None, | |
| singer_encoder: nn.Module = None, | |
| ): | |
| super().__init__() | |
| self.embed_dim = embed_dim | |
| self.text_encoder = text_encoder | |
| self.midi_encoder = midi_encoder | |
| self.phoneme_encoder = phoneme_encoder | |
| self.pitch_encoder = pitch_encoder | |
| self.audio_encoder = audio_encoder | |
| self.video_encoder = video_encoder | |
| self.speaker_encoder = speaker_encoder | |
| self.singer_encoder = singer_encoder | |
| def encode_content( | |
| self, batch_content: list[Any], batch_task: list[str], | |
| device: str | torch.device | |
| ): | |
| batch_ta_content_output = [] | |
| batch_ta_content_mask = [] | |
| batch_nta_content_output = [] | |
| batch_nta_content_mask = [] | |
| batch_la_content_output = [] | |
| dummy_ta_content = torch.zeros(1, self.embed_dim, device=device) | |
| dummy_ta_content_mask = torch.tensor([True], | |
| dtype=torch.bool, | |
| device=device) | |
| dummy_nta_content = torch.zeros(1, self.embed_dim, device=device) | |
| dummy_nta_content_mask = torch.tensor([True], | |
| dtype=torch.bool, | |
| device=device) | |
| zero_la_content = torch.zeros(1, 1, self.embed_dim, device=device) | |
| for content, task in zip(batch_content, batch_task): | |
| if task == "audio_super_resolution" or task == "speech_enhancement": | |
| content_dict = { | |
| "waveform": torch.as_tensor(content).float(), | |
| "waveform_lengths": torch.as_tensor(content.shape[0]), | |
| } | |
| for key in list(content_dict.keys()): | |
| content_dict[key] = content_dict[key].unsqueeze(0).to( | |
| device | |
| ) | |
| out_dict = self.audio_encoder(**content_dict) | |
| la_content_output_dict = { | |
| "output": zero_la_content, | |
| } | |
| ta_content = out_dict["output"][0] | |
| ta_content_mask = out_dict["mask"][0] | |
| nta_content = dummy_nta_content | |
| nta_content_mask = dummy_nta_content_mask | |
| elif task == "text_to_audio" or task == "text_to_music": | |
| out_dict = self.text_encoder([content]) | |
| ta_content = dummy_ta_content | |
| ta_content_mask = dummy_ta_content_mask | |
| nta_content = out_dict["output"][0] | |
| nta_content_mask = out_dict["mask"][0] | |
| la_content_output_dict = { | |
| "output": zero_la_content, | |
| } | |
| elif task == "video_to_audio": | |
| content_dict = { | |
| "clip": torch.as_tensor(content["clip"]).float(), | |
| "sync": torch.as_tensor(content["sync"]).float(), | |
| "duration": torch.as_tensor([content["duration"]]).float(), | |
| "clip_lengths": torch.as_tensor(content["clip"].shape[0]).long(), | |
| "sync_lengths": torch.as_tensor(content["sync"].shape[0]).long(), | |
| } | |
| for key in list(content_dict.keys()): | |
| content_dict[key] = content_dict[key].unsqueeze(0).to( | |
| device | |
| ) | |
| out_dict = self.video_encoder(**content_dict) | |
| ta_content = out_dict["ta_output"][0] | |
| ta_content_mask = out_dict["ta_mask"][0] | |
| if out_dict["nta_output"] is None: | |
| nta_content = dummy_nta_content | |
| nta_content_mask = dummy_nta_content_mask | |
| else: | |
| nta_content = out_dict["nta_output"][0] | |
| nta_content_mask = out_dict["nta_mask"][0] | |
| la_content_output_dict = { | |
| "output": zero_la_content, | |
| } | |
| elif task == "singing_voice_synthesis": | |
| content_dict = { | |
| "phoneme": | |
| torch.as_tensor(content["phoneme"]).long(), | |
| "midi": | |
| torch.as_tensor(content["midi"]).long(), | |
| "midi_duration": | |
| torch.as_tensor(content["midi_duration"]).float(), | |
| "is_slur": | |
| torch.as_tensor(content["is_slur"]).long() | |
| } | |
| for key in list(content_dict.keys()): | |
| content_dict[key] = content_dict[key].unsqueeze(0).to( | |
| device | |
| ) | |
| content_dict["lengths"] = torch.as_tensor([ | |
| len(content["phoneme"]) | |
| ]) | |
| out_dict = self.midi_encoder(**content_dict) | |
| ta_content = out_dict["output"][0] | |
| ta_content_mask = out_dict["mask"][0] | |
| out_dict = self.singer_encoder( | |
| speaker_id=torch.as_tensor(content["spk"]).long().unsqueeze(0).to(device), | |
| ) | |
| nta_content = out_dict["output"][0] | |
| nta_content_mask = out_dict["mask"][0] | |
| la_content_output_dict = {"output": zero_la_content} | |
| elif task == "text_to_speech": | |
| content_dict = { | |
| "phoneme": torch.as_tensor(content["phoneme"]).long(), | |
| } | |
| for key in list(content_dict.keys()): | |
| content_dict[key] = content_dict[key].unsqueeze(0).to( | |
| device | |
| ) | |
| content_dict["phoneme_lengths"] = torch.as_tensor([ | |
| len(content["phoneme"]) | |
| ]) | |
| out_dict = self.phoneme_encoder(**content_dict) | |
| ta_content = out_dict["output"][0] | |
| ta_content_mask = out_dict["mask"][0] | |
| prompt_waveform = torch.as_tensor( | |
| content["prompt_waveform"] | |
| ).float().to(device).unsqueeze(0) | |
| prompt_waveform_lengths = torch.as_tensor([ | |
| len(content["prompt_waveform"]) | |
| ]) | |
| out_dict = self.speaker_encoder( | |
| **{ | |
| "waveform": prompt_waveform, | |
| "waveform_lengths": prompt_waveform_lengths, | |
| } | |
| ) | |
| nta_content = out_dict["output"][0] | |
| nta_content_mask = out_dict["mask"][0] | |
| la_content_output_dict = {"output": zero_la_content} | |
| else: | |
| raise ValueError(f"Unsupported task: {task}") | |
| batch_ta_content_output.append(ta_content) | |
| batch_ta_content_mask.append(ta_content_mask) | |
| batch_nta_content_output.append(nta_content) | |
| batch_nta_content_mask.append(nta_content_mask) | |
| batch_la_content_output.append(la_content_output_dict["output"][0]) | |
| batch_ta_content_output = nn.utils.rnn.pad_sequence( | |
| batch_ta_content_output, batch_first=True, padding_value=0 | |
| ) | |
| batch_ta_content_mask = nn.utils.rnn.pad_sequence( | |
| batch_ta_content_mask, batch_first=True, padding_value=False | |
| ) | |
| batch_nta_content_output = nn.utils.rnn.pad_sequence( | |
| batch_nta_content_output, batch_first=True, padding_value=0 | |
| ) | |
| batch_nta_content_mask = nn.utils.rnn.pad_sequence( | |
| batch_nta_content_mask, batch_first=True, padding_value=False | |
| ) | |
| batch_la_content_output = nn.utils.rnn.pad_sequence( | |
| batch_la_content_output, batch_first=True, padding_value=0 | |
| ) | |
| return { | |
| "ta_content": batch_ta_content_output, | |
| "ta_content_mask": batch_ta_content_mask, | |
| "nta_content": batch_nta_content_output, | |
| "nta_content_mask": batch_nta_content_mask, | |
| "la_content": batch_la_content_output, | |
| } | |
| class BatchedContentEncoder(ContentEncoder): | |
| def encode_content( | |
| self, batch_content: list | dict, batch_task: list[str], | |
| device: str | torch.device | |
| ): | |
| task = batch_task[0] | |
| batch_size = len(batch_task) | |
| dummy_ta_content = torch.zeros(1, 1, self.embed_dim, device=device) | |
| dummy_ta_content_mask = torch.tensor([[True]], | |
| dtype=torch.bool, | |
| device=device) | |
| dummy_nta_content = torch.zeros(1, 1, self.embed_dim, device=device) | |
| dummy_nta_content_mask = torch.tensor([[True]], | |
| dtype=torch.bool, | |
| device=device) | |
| zero_la_content = torch.zeros(1, 1, self.embed_dim, device=device) | |
| if task == "audio_super_resolution" or task == "speech_enhancement": | |
| content_dict = { | |
| "waveform": | |
| batch_content["content"].unsqueeze(1).float().to(device), | |
| "waveform_lengths": | |
| batch_content["content_lengths"].long().to(device), | |
| } | |
| out_dict = self.audio_encoder(**content_dict) | |
| ta_content_output = out_dict["output"] | |
| ta_content_mask = out_dict["mask"] | |
| nta_content_output = dummy_nta_content.expand(batch_size, -1, -1) | |
| nta_content_mask = dummy_nta_content_mask.expand(batch_size, -1) | |
| la_content_output = zero_la_content | |
| elif task == "text_to_audio" or task == "text_to_music": | |
| out_dict = self.text_encoder(batch_content) | |
| ta_content_output = dummy_ta_content.expand(batch_size, -1, -1) | |
| ta_content_mask = dummy_ta_content_mask.expand(batch_size, -1) | |
| nta_content_output = out_dict["output"] | |
| nta_content_mask = out_dict["mask"] | |
| la_content_output = zero_la_content | |
| elif task == "video_to_audio": | |
| out_dict = self.video_encoder(**batch_content) | |
| ta_content_output = out_dict["ta_output"] | |
| ta_content_mask = out_dict["ta_mask"] | |
| nta_content_output = out_dict["nta_output"] | |
| nta_content_mask = out_dict["nta_mask"] | |
| if nta_content_output is None: | |
| nta_content_output = dummy_nta_content.expand( | |
| batch_size, -1, -1 | |
| ) | |
| nta_content_mask = dummy_nta_content_mask.expand( | |
| batch_size, -1 | |
| ) | |
| la_content_output = zero_la_content | |
| elif task == "singing_voice_synthesis": | |
| content_dict = { | |
| "phoneme": | |
| batch_content["phoneme"].long().to(device), | |
| "midi": | |
| batch_content["midi"].long().to(device), | |
| "midi_duration": | |
| batch_content["midi_duration"].float().to(device), | |
| "is_slur": | |
| batch_content["is_slur"].long().to(device), | |
| "lengths": | |
| batch_content["phoneme_lengths"].long().cpu(), | |
| } | |
| out_dict = self.midi_encoder(**content_dict) | |
| ta_content_output = out_dict["output"] | |
| ta_content_mask = out_dict["mask"] | |
| out_dict = self.singer_encoder( | |
| speaker_id=batch_content["spk"].long().to(device) | |
| ) | |
| nta_content_output = out_dict["output"] | |
| nta_content_mask = out_dict["mask"] | |
| la_content_output = zero_la_content | |
| elif task == "text_to_speech": | |
| content_dict = { | |
| "phoneme": | |
| batch_content["phoneme"].long().to(device), | |
| "phoneme_lengths": | |
| batch_content["phoneme_lengths"].long().cpu(), | |
| } | |
| out_dict = self.phoneme_encoder(**content_dict) | |
| ta_content_output = out_dict["output"] | |
| ta_content_mask = out_dict["mask"] | |
| out_dict = self.speaker_encoder( | |
| waveform=batch_content["prompt_waveform"].float().to(device), | |
| waveform_lengths=batch_content["prompt_waveform_lengths"].long( | |
| ), | |
| ) | |
| nta_content_output = out_dict["output"] | |
| nta_content_mask = out_dict["mask"] | |
| la_content_output = zero_la_content | |
| elif task == "singing_acoustic_modeling": | |
| content_dict = { | |
| "phoneme": batch_content["phoneme"].long().to(device), | |
| "lengths": batch_content["phoneme_lengths"].long().to(device), | |
| } | |
| out_dict = self.pitch_encoder(**content_dict) | |
| ta_content_output = out_dict["output"] | |
| ta_content_mask = out_dict["mask"] | |
| nta_content_output = dummy_nta_content.expand(batch_size, -1, -1) | |
| nta_content_mask = dummy_nta_content_mask.expand(batch_size, -1) | |
| content_dict = { | |
| "f0": batch_content["f0"].float().to(device), | |
| "uv": batch_content["uv"].float().to(device), | |
| } | |
| la_content_output = self.pitch_encoder.encode_pitch(**content_dict) | |
| else: | |
| raise ValueError(f"Unsupported task: {task}") | |
| return { | |
| "ta_content": ta_content_output, | |
| "ta_content_mask": ta_content_mask, | |
| "nta_content": nta_content_output, | |
| "nta_content_mask": nta_content_mask, | |
| "la_content": la_content_output, | |
| } | |