--- library_name: transformers datasets: - mythicinfinity/libritts - ylacombe/cml-tts base_model: - facebook/hubert-base-ls960 --- Multilingual speaker-invariant speech content encoder based on [SPIN method](https://arxiv.org/abs/2305.11072) taken from [FreeSVC](https://huggingface.co/alefiury/free-svc), converted to transformers library format. Usage: ```py from transformers import AutoProcessor, HubertModel feature_extractor = AutoFeatureExtractor.from_pretrained("hr16/multilingual_spin") model = HubertModel.from_pretrained("hr16/multilingual_spin") audio_input = torch.randn(16000) # 1 second of audio at 16kHz inputs = feature_extractor(audio_input, sampling_rate=16000, return_tensors="pt") with torch.no_grad(): outputs = model(**inputs) print(outputs.last_hidden_state.shape) ``` Conversion code: ```py import re import torch def convert_hubert_s3prl_to_transformers(model, checkpoint_path: str): # 1. Load checkpoint state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=False) if "model" in state_dict: state_dict = state_dict["model"] elif "state_dict" in state_dict: state_dict = state_dict["state_dict"] hf_state_dict = {} # 2. Map weights for key, value in state_dict.items(): # Skip unused pretraining components if ( key.startswith("pred_head") or key.startswith("loss_module") or "label_embs_concat" in key ): continue # Mask embedding if key == "encoder.model.mask_emb": hf_state_dict["masked_spec_embed"] = value continue # Feature extractor group norm / conv layers if "feature_extractor.conv_layers" in key: # e.g.: encoder.model.feature_extractor.conv_layers.0.0.weight -> feature_extractor.conv_layers.0.conv.weight # e.g.: encoder.model.feature_extractor.conv_layers.0.2.weight -> feature_extractor.conv_layers.0.layer_norm.weight match = re.search(r"conv_layers\.(\d+)\.(\d+)\.(weight|bias)", key) if match: layer_idx, sub_idx, param_name = match.groups() if sub_idx == "0": hf_key = f"feature_extractor.conv_layers.{layer_idx}.conv.{param_name}" elif sub_idx == "2": hf_key = f"feature_extractor.conv_layers.{layer_idx}.layer_norm.{param_name}" hf_state_dict[hf_key] = value continue # Feature projection layer norm if key in ("encoder.model.layer_norm.weight", "encoder.model.layer_norm.bias"): param_name = key.split(".")[-1] hf_state_dict[f"feature_projection.layer_norm.{param_name}"] = value continue # Feature projection linear layer if "post_extract_proj" in key: param_name = key.split(".")[-1] hf_state_dict[f"feature_projection.projection.{param_name}"] = value continue # Positional Convolution Embedding if "pos_conv.0" in key: param_name = key.split(".")[-1] # Handles weight_v, weight_g, bias hf_state_dict[f"encoder.pos_conv_embed.conv.{param_name}"] = value continue # Encoder pre-LayerNorm if key in ( "encoder.model.encoder.layer_norm.weight", "encoder.model.encoder.layer_norm.bias", ): param_name = key.split(".")[-1] hf_state_dict[f"encoder.layer_norm.{param_name}"] = value continue # Encoder Transformer Layers if "encoder.model.encoder.layers." in key: # Map layer components m = re.match( r"encoder\.model\.encoder\.layers\.(\d+)\.(.+)", key ) if m: layer_idx, rest = m.groups() prefix = f"encoder.layers.{layer_idx}" if "self_attn." in rest: # self_attn.{q,k,v,out}_proj.{weight,bias} attn_component = rest.replace("self_attn.", "attention.") hf_state_dict[f"{prefix}.{attn_component}"] = value elif "self_attn_layer_norm." in rest: param_name = rest.split(".")[-1] hf_state_dict[f"{prefix}.layer_norm.{param_name}"] = value elif "fc1." in rest: param_name = rest.split(".")[-1] hf_state_dict[ f"{prefix}.feed_forward.intermediate_dense.{param_name}" ] = value elif "fc2." in rest: param_name = rest.split(".")[-1] hf_state_dict[ f"{prefix}.feed_forward.output_dense.{param_name}" ] = value elif "final_layer_norm." in rest: param_name = rest.split(".")[-1] hf_state_dict[f"{prefix}.final_layer_norm.{param_name}"] = value continue print(f"Unmapped key: {key}") model.load_state_dict(hf_state_dict) return model from transformers import HubertModel orig_hubert = HubertModel.from_pretrained("facebook/hubert-base-ls960") spin = convert_hubert_s3prl_to_transformers(HubertModel.from_pretrained("facebook/hubert-base-ls960"), "spin.ckpt") spin.push_to_hub("hr16/multilingual_spin") ``` State dict mean-square differences: ``` masked_spec_embed diff: 0.0 feature_extractor.conv_layers.0.conv.weight diff: 0.0 feature_extractor.conv_layers.0.layer_norm.weight diff: 0.0 feature_extractor.conv_layers.0.layer_norm.bias diff: 0.0 feature_extractor.conv_layers.1.conv.weight diff: 0.0 feature_extractor.conv_layers.2.conv.weight diff: 0.0 feature_extractor.conv_layers.3.conv.weight diff: 0.0 feature_extractor.conv_layers.4.conv.weight diff: 0.0 feature_extractor.conv_layers.5.conv.weight diff: 0.0 feature_extractor.conv_layers.6.conv.weight diff: 0.0 feature_projection.layer_norm.weight diff: 0.0 feature_projection.layer_norm.bias diff: 0.0 feature_projection.projection.weight diff: 0.0 feature_projection.projection.bias diff: 0.0 encoder.pos_conv_embed.conv.bias diff: 0.0 encoder.pos_conv_embed.conv.parametrizations.weight.original0 diff: 0.0 encoder.pos_conv_embed.conv.parametrizations.weight.original1 diff: 0.0 encoder.layer_norm.weight diff: 0.0 encoder.layer_norm.bias diff: 0.0 encoder.layers.0.attention.k_proj.weight diff: 0.0 encoder.layers.0.attention.k_proj.bias diff: 0.0 encoder.layers.0.attention.v_proj.weight diff: 0.0 encoder.layers.0.attention.v_proj.bias diff: 0.0 encoder.layers.0.attention.q_proj.weight diff: 0.0 encoder.layers.0.attention.q_proj.bias diff: 0.0 encoder.layers.0.attention.out_proj.weight diff: 0.0 encoder.layers.0.attention.out_proj.bias diff: 0.0 encoder.layers.0.layer_norm.weight diff: 0.0 encoder.layers.0.layer_norm.bias diff: 0.0 encoder.layers.0.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.0.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.0.feed_forward.output_dense.weight diff: 0.0 encoder.layers.0.feed_forward.output_dense.bias diff: 0.0 encoder.layers.0.final_layer_norm.weight diff: 0.0 encoder.layers.0.final_layer_norm.bias diff: 0.0 encoder.layers.1.attention.k_proj.weight diff: 0.0 encoder.layers.1.attention.k_proj.bias diff: 0.0 encoder.layers.1.attention.v_proj.weight diff: 0.0 encoder.layers.1.attention.v_proj.bias diff: 0.0 encoder.layers.1.attention.q_proj.weight diff: 0.0 encoder.layers.1.attention.q_proj.bias diff: 0.0 encoder.layers.1.attention.out_proj.weight diff: 0.0 encoder.layers.1.attention.out_proj.bias diff: 0.0 encoder.layers.1.layer_norm.weight diff: 0.0 encoder.layers.1.layer_norm.bias diff: 0.0 encoder.layers.1.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.1.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.1.feed_forward.output_dense.weight diff: 0.0 encoder.layers.1.feed_forward.output_dense.bias diff: 0.0 encoder.layers.1.final_layer_norm.weight diff: 0.0 encoder.layers.1.final_layer_norm.bias diff: 0.0 encoder.layers.2.attention.k_proj.weight diff: 0.0 encoder.layers.2.attention.k_proj.bias diff: 0.0 encoder.layers.2.attention.v_proj.weight diff: 0.0 encoder.layers.2.attention.v_proj.bias diff: 0.0 encoder.layers.2.attention.q_proj.weight diff: 0.0 encoder.layers.2.attention.q_proj.bias diff: 0.0 encoder.layers.2.attention.out_proj.weight diff: 0.0 encoder.layers.2.attention.out_proj.bias diff: 0.0 encoder.layers.2.layer_norm.weight diff: 0.0 encoder.layers.2.layer_norm.bias diff: 0.0 encoder.layers.2.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.2.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.2.feed_forward.output_dense.weight diff: 0.0 encoder.layers.2.feed_forward.output_dense.bias diff: 0.0 encoder.layers.2.final_layer_norm.weight diff: 0.0 encoder.layers.2.final_layer_norm.bias diff: 0.0 encoder.layers.3.attention.k_proj.weight diff: 0.0 encoder.layers.3.attention.k_proj.bias diff: 0.0 encoder.layers.3.attention.v_proj.weight diff: 0.0 encoder.layers.3.attention.v_proj.bias diff: 0.0 encoder.layers.3.attention.q_proj.weight diff: 0.0 encoder.layers.3.attention.q_proj.bias diff: 0.0 encoder.layers.3.attention.out_proj.weight diff: 0.0 encoder.layers.3.attention.out_proj.bias diff: 0.0 encoder.layers.3.layer_norm.weight diff: 0.0 encoder.layers.3.layer_norm.bias diff: 0.0 encoder.layers.3.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.3.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.3.feed_forward.output_dense.weight diff: 0.0 encoder.layers.3.feed_forward.output_dense.bias diff: 0.0 encoder.layers.3.final_layer_norm.weight diff: 0.0 encoder.layers.3.final_layer_norm.bias diff: 0.0 encoder.layers.4.attention.k_proj.weight diff: 0.0 encoder.layers.4.attention.k_proj.bias diff: 0.0 encoder.layers.4.attention.v_proj.weight diff: 0.0 encoder.layers.4.attention.v_proj.bias diff: 0.0 encoder.layers.4.attention.q_proj.weight diff: 0.0 encoder.layers.4.attention.q_proj.bias diff: 0.0 encoder.layers.4.attention.out_proj.weight diff: 0.0 encoder.layers.4.attention.out_proj.bias diff: 0.0 encoder.layers.4.layer_norm.weight diff: 0.0 encoder.layers.4.layer_norm.bias diff: 0.0 encoder.layers.4.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.4.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.4.feed_forward.output_dense.weight diff: 0.0 encoder.layers.4.feed_forward.output_dense.bias diff: 0.0 encoder.layers.4.final_layer_norm.weight diff: 0.0 encoder.layers.4.final_layer_norm.bias diff: 0.0 encoder.layers.5.attention.k_proj.weight diff: 0.0 encoder.layers.5.attention.k_proj.bias diff: 0.0 encoder.layers.5.attention.v_proj.weight diff: 0.0 encoder.layers.5.attention.v_proj.bias diff: 0.0 encoder.layers.5.attention.q_proj.weight diff: 0.0 encoder.layers.5.attention.q_proj.bias diff: 0.0 encoder.layers.5.attention.out_proj.weight diff: 0.0 encoder.layers.5.attention.out_proj.bias diff: 0.0 encoder.layers.5.layer_norm.weight diff: 0.0 encoder.layers.5.layer_norm.bias diff: 0.0 encoder.layers.5.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.5.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.5.feed_forward.output_dense.weight diff: 0.0 encoder.layers.5.feed_forward.output_dense.bias diff: 0.0 encoder.layers.5.final_layer_norm.weight diff: 0.0 encoder.layers.5.final_layer_norm.bias diff: 0.0 encoder.layers.6.attention.k_proj.weight diff: 0.0 encoder.layers.6.attention.k_proj.bias diff: 0.0 encoder.layers.6.attention.v_proj.weight diff: 0.0 encoder.layers.6.attention.v_proj.bias diff: 0.0 encoder.layers.6.attention.q_proj.weight diff: 0.0 encoder.layers.6.attention.q_proj.bias diff: 0.0 encoder.layers.6.attention.out_proj.weight diff: 0.0 encoder.layers.6.attention.out_proj.bias diff: 0.0 encoder.layers.6.layer_norm.weight diff: 0.0 encoder.layers.6.layer_norm.bias diff: 0.0 encoder.layers.6.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.6.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.6.feed_forward.output_dense.weight diff: 0.0 encoder.layers.6.feed_forward.output_dense.bias diff: 0.0 encoder.layers.6.final_layer_norm.weight diff: 0.0 encoder.layers.6.final_layer_norm.bias diff: 0.0 encoder.layers.7.attention.k_proj.weight diff: 0.0 encoder.layers.7.attention.k_proj.bias diff: 0.0 encoder.layers.7.attention.v_proj.weight diff: 0.0 encoder.layers.7.attention.v_proj.bias diff: 0.0 encoder.layers.7.attention.q_proj.weight diff: 0.0 encoder.layers.7.attention.q_proj.bias diff: 0.0 encoder.layers.7.attention.out_proj.weight diff: 0.0 encoder.layers.7.attention.out_proj.bias diff: 0.0 encoder.layers.7.layer_norm.weight diff: 0.0 encoder.layers.7.layer_norm.bias diff: 0.0 encoder.layers.7.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.7.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.7.feed_forward.output_dense.weight diff: 0.0 encoder.layers.7.feed_forward.output_dense.bias diff: 0.0 encoder.layers.7.final_layer_norm.weight diff: 0.0 encoder.layers.7.final_layer_norm.bias diff: 0.0 encoder.layers.8.attention.k_proj.weight diff: 0.0 encoder.layers.8.attention.k_proj.bias diff: 0.0 encoder.layers.8.attention.v_proj.weight diff: 0.0 encoder.layers.8.attention.v_proj.bias diff: 0.0 encoder.layers.8.attention.q_proj.weight diff: 0.0 encoder.layers.8.attention.q_proj.bias diff: 0.0 encoder.layers.8.attention.out_proj.weight diff: 0.0 encoder.layers.8.attention.out_proj.bias diff: 0.0 encoder.layers.8.layer_norm.weight diff: 0.0 encoder.layers.8.layer_norm.bias diff: 0.0 encoder.layers.8.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.8.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.8.feed_forward.output_dense.weight diff: 0.0 encoder.layers.8.feed_forward.output_dense.bias diff: 0.0 encoder.layers.8.final_layer_norm.weight diff: 0.0 encoder.layers.8.final_layer_norm.bias diff: 0.0 encoder.layers.9.attention.k_proj.weight diff: 0.0 encoder.layers.9.attention.k_proj.bias diff: 0.0 encoder.layers.9.attention.v_proj.weight diff: 0.0 encoder.layers.9.attention.v_proj.bias diff: 0.0 encoder.layers.9.attention.q_proj.weight diff: 0.0 encoder.layers.9.attention.q_proj.bias diff: 0.0 encoder.layers.9.attention.out_proj.weight diff: 0.0 encoder.layers.9.attention.out_proj.bias diff: 0.0 encoder.layers.9.layer_norm.weight diff: 0.0 encoder.layers.9.layer_norm.bias diff: 0.0 encoder.layers.9.feed_forward.intermediate_dense.weight diff: 0.0 encoder.layers.9.feed_forward.intermediate_dense.bias diff: 0.0 encoder.layers.9.feed_forward.output_dense.weight diff: 0.0 encoder.layers.9.feed_forward.output_dense.bias diff: 0.0 encoder.layers.9.final_layer_norm.weight diff: 0.0 encoder.layers.9.final_layer_norm.bias diff: 0.0 encoder.layers.10.attention.k_proj.weight diff: 0.0026198839768767357 encoder.layers.10.attention.k_proj.bias diff: 0.0010964443208649755 encoder.layers.10.attention.v_proj.weight diff: 0.001084218267351389 encoder.layers.10.attention.v_proj.bias diff: 8.861951209837571e-05 encoder.layers.10.attention.q_proj.weight diff: 0.002644478576257825 encoder.layers.10.attention.q_proj.bias diff: 0.0009331119363196194 encoder.layers.10.attention.out_proj.weight diff: 0.0009169431868940592 encoder.layers.10.attention.out_proj.bias diff: 0.00011753084982046857 encoder.layers.10.layer_norm.weight diff: 0.004704533610492945 encoder.layers.10.layer_norm.bias diff: 0.00040463244658894837 encoder.layers.10.feed_forward.intermediate_dense.weight diff: 0.002256407169625163 encoder.layers.10.feed_forward.intermediate_dense.bias diff: 8.606249321019277e-05 encoder.layers.10.feed_forward.output_dense.weight diff: 0.0017423474928364158 encoder.layers.10.feed_forward.output_dense.bias diff: 7.794254634063691e-05 encoder.layers.10.final_layer_norm.weight diff: 0.003364397445693612 encoder.layers.10.final_layer_norm.bias diff: 0.000663460697978735 encoder.layers.11.attention.k_proj.weight diff: 0.0019817741122096777 encoder.layers.11.attention.k_proj.bias diff: 0.032148730009794235 encoder.layers.11.attention.v_proj.weight diff: 0.0010857016313821077 encoder.layers.11.attention.v_proj.bias diff: 0.00020938993839081377 encoder.layers.11.attention.q_proj.weight diff: 0.002357392804697156 encoder.layers.11.attention.q_proj.bias diff: 0.0008044671267271042 encoder.layers.11.attention.out_proj.weight diff: 0.0011331378482282162 encoder.layers.11.attention.out_proj.bias diff: 0.00014901244139764458 encoder.layers.11.layer_norm.weight diff: 0.0027753293979912996 encoder.layers.11.layer_norm.bias diff: 0.0056587993167340755 encoder.layers.11.feed_forward.intermediate_dense.weight diff: 0.0032594590447843075 encoder.layers.11.feed_forward.intermediate_dense.bias diff: 0.0002475830551702529 encoder.layers.11.feed_forward.output_dense.weight diff: 0.002621536375954747 encoder.layers.11.feed_forward.output_dense.bias diff: 2.9405724490061402e-05 encoder.layers.11.final_layer_norm.weight diff: 0.002755063585937023 encoder.layers.11.final_layer_norm.bias diff: 0.00021274278697092086 ```