| --- |
| tags: |
| - espnet |
| - audio |
| - classification |
| datasets: |
| - nsynth |
| license: cc-by-4.0 |
| --- |
| |
| ## ESPnet2 CLS model |
|
|
| ### `espnet/OpenBEATS-Large-NsynthInstrument` |
|
|
| This model was trained by Shikhar Bharadwaj using nsynth recipe in [espnet](https://github.com/espnet/espnet/). |
|
|
| ## CLS config |
|
|
| <details><summary>expand</summary> |
|
|
| ``` |
| config: /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/earlarge2/conf/ear_large/nsynth_instrument.yaml |
| print_config: false |
| log_level: INFO |
| drop_last_iter: false |
| dry_run: false |
| iterator_type: sequence |
| valid_iterator_type: null |
| output_dir: /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/nsynth_instrument/cls_earlarge2 |
| ngpu: 1 |
| seed: 0 |
| num_workers: 2 |
| num_att_plot: 0 |
| dist_backend: nccl |
| dist_init_method: env:// |
| dist_world_size: null |
| dist_rank: null |
| local_rank: 0 |
| dist_master_addr: null |
| dist_master_port: null |
| dist_launcher: null |
| multiprocessing_distributed: false |
| unused_parameters: true |
| sharded_ddp: false |
| use_deepspeed: false |
| deepspeed_config: null |
| gradient_as_bucket_view: true |
| ddp_comm_hook: null |
| cudnn_enabled: true |
| cudnn_benchmark: false |
| cudnn_deterministic: true |
| use_tf32: false |
| collect_stats: false |
| write_collected_feats: false |
| max_epoch: 30 |
| patience: null |
| val_scheduler_criterion: |
| - valid |
| - loss |
| early_stopping_criterion: |
| - valid |
| - loss |
| - min |
| best_model_criterion: |
| - - valid |
| - acc |
| - max |
| keep_nbest_models: 1 |
| nbest_averaging_interval: 0 |
| grad_clip: 1 |
| grad_clip_type: 2.0 |
| grad_noise: false |
| accum_grad: 1 |
| no_forward_run: false |
| resume: true |
| train_dtype: float32 |
| use_amp: false |
| log_interval: null |
| use_matplotlib: true |
| use_tensorboard: true |
| create_graph_in_tensorboard: false |
| use_wandb: true |
| wandb_project: audioverse |
| wandb_id: null |
| wandb_entity: shikhar |
| wandb_name: nsynth_instrument.earlarge2 |
| wandb_model_log_interval: -1 |
| detect_anomaly: false |
| use_adapter: false |
| adapter: lora |
| save_strategy: all |
| adapter_conf: {} |
| pretrain_path: null |
| init_param: [] |
| ignore_init_mismatch: false |
| freeze_param: [] |
| num_iters_per_epoch: null |
| batch_size: 32 |
| valid_batch_size: 16 |
| batch_bins: 1000000 |
| valid_batch_bins: null |
| category_sample_size: 10 |
| train_shape_file: |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/nsynth_instrument/cls_stats_16k/train/speech_shape |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/nsynth_instrument/cls_stats_16k/train/label_shape |
| valid_shape_file: |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/nsynth_instrument/cls_stats_16k/valid/speech_shape |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/nsynth_instrument/cls_stats_16k/valid/label_shape |
| batch_type: folded |
| valid_batch_type: null |
| fold_length: |
| - 480000 |
| - 600 |
| sort_in_batch: descending |
| shuffle_within_batch: false |
| sort_batch: descending |
| multiple_iterator: false |
| utt2weight_file: null |
| chunk_length: 500 |
| chunk_shift_ratio: 0.5 |
| num_cache_chunks: 1024 |
| chunk_excluded_key_prefixes: [] |
| chunk_default_fs: null |
| chunk_max_abs_length: null |
| chunk_discard_short_samples: true |
| train_data_path_and_name_and_type: |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/nsynth_instrument/train/wav.scp |
| - speech |
| - sound |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/nsynth_instrument/train/text |
| - label |
| - text |
| valid_data_path_and_name_and_type: |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/nsynth_instrument/valid/wav.scp |
| - speech |
| - sound |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/nsynth_instrument/valid/text |
| - label |
| - text |
| multi_task_dataset: false |
| allow_variable_data_keys: false |
| max_cache_size: 0.0 |
| max_cache_fd: 32 |
| allow_multi_rates: false |
| valid_max_cache_size: null |
| exclude_weight_decay: false |
| exclude_weight_decay_conf: {} |
| optim: adamw |
| optim_conf: |
| lr: 3.0e-05 |
| weight_decay: 0.01 |
| betas: |
| - 0.9 |
| - 0.98 |
| scheduler: cosineannealingwarmuprestarts |
| scheduler_conf: |
| first_cycle_steps: 50000 |
| warmup_steps: 5000 |
| max_lr: 3.0e-05 |
| min_lr: 5.0e-06 |
| lightning_conf: {} |
| token_list: |
| - bass |
| - keyboard |
| - organ |
| - mallet |
| - guitar |
| - string |
| - reed |
| - brass |
| - vocal |
| - flute |
| - synth_lead |
| - <blank> |
| - <unk> |
| text_token_list: null |
| text_bpemodel: null |
| init: xavier_normal |
| input_size: 1 |
| use_preprocessor: true |
| frontend: null |
| frontend_conf: {} |
| specaug: null |
| specaug_conf: {} |
| normalize: null |
| normalize_conf: {} |
| preencoder: null |
| preencoder_conf: {} |
| encoder: beats |
| encoder_conf: |
| beats_ckpt_path: /work/nvme/bbjs/sbharadwaj/7Msounds/exp/beats_iter1_large1.tune_lr1.0e-4_warmup40000_bins1600000_totalsteps400000/epoch_latest.pt |
| beats_config: |
| layer_wise_gradient_decay_ratio: 0.3 |
| encoder_layerdrop: 0.1 |
| dropout: 0.0 |
| use_weighted_representation: false |
| specaug_config: |
| apply_time_warp: true |
| apply_freq_mask: false |
| apply_time_mask: true |
| time_mask_width_ratio_range: |
| - 0 |
| - 0.06 |
| num_time_mask: 1 |
| roll_augment: true |
| roll_interval: 1 |
| text_encoder: null |
| text_encoder_conf: {} |
| embedding_fusion: null |
| embedding_fusion_conf: {} |
| decoder: linear |
| decoder_conf: {} |
| model: espnet |
| model_conf: |
| classification_type: multi-class |
| lsm_weight: 0.1 |
| required: |
| - output_dir |
| - token_list |
| version: '202412' |
| distributed: false |
| ``` |
|
|
| </details> |
|
|
| ### Citations |
|
|
| ```BibTex |
| |
| @article{bharadwaj2025openbeats, |
| title={OpenBEATs: A Fully Open-Source General-Purpose Audio Encoder}, |
| author={Bharadwaj, Shikhar and Cornell, Samuele and Choi, Kwanghee and Fukayama, Satoru and Shim, Hye-jin and Deshmukh, Soham and Watanabe, Shinji}, |
| journal={arXiv preprint arXiv:2507.14129}, |
| year={2025} |
| } |
| |
| @inproceedings{watanabe2018espnet, |
| author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai}, |
| title={{ESPnet}: End-to-End Speech Processing Toolkit}, |
| year={2018}, |
| booktitle={Proceedings of Interspeech}, |
| pages={2207--2211}, |
| doi={10.21437/Interspeech.2018-1456}, |
| url={http://dx.doi.org/10.21437/Interspeech.2018-1456} |
| } |
| |
| |
| |
| |
| |
| |
| ``` |
|
|
