decoder-only fine-tune (middle)

config.json

@@ -0,0 +1,197 @@
+{
+  "architectures": [
+    "CohereAsrForConditionalGeneration"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "batch_size": 128,
+  "bos_token_id": 4,
+  "decoder_start_token_id": 13764,
+  "decoding": {
+    "beam": {
+      "beam_size": 1,
+      "len_pen": 0.0,
+      "max_generation_delta": -256
+    },
+    "return_best_hypothesis": true,
+    "strategy": "beam"
+  },
+  "dtype": "bfloat16",
+  "encoder": {
+    "att_context_size": [
+      -1,
+      -1
+    ],
+    "causal_downsampling": false,
+    "conv_context_size": null,
+    "conv_kernel_size": 9,
+    "conv_norm_type": "batch_norm",
+    "d_model": 1280,
+    "dropout": 0,
+    "dropout_att": 0,
+    "dropout_emb": 0,
+    "dropout_pre_encoder": 0,
+    "feat_in": 128,
+    "feat_out": -1,
+    "ff_expansion_factor": 4,
+    "n_heads": 8,
+    "n_layers": 48,
+    "pos_emb_max_len": 5000,
+    "reduction": null,
+    "reduction_factor": 1,
+    "reduction_position": null,
+    "self_attention_model": "rel_pos",
+    "subsampling": "dw_striding",
+    "subsampling_conv_channels": 256,
+    "subsampling_factor": 8,
+    "untie_biases": true,
+    "xscaling": false
+  },
+  "encoder_config": {
+    "activation_dropout": 0.0,
+    "attention_bias": true,
+    "attention_dropout": 0.0,
+    "conv_kernel_size": 9,
+    "convolution_bias": true,
+    "dropout": 0.0,
+    "dropout_positions": 0.0,
+    "dtype": "bfloat16",
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "initializer_range": 0.02,
+    "intermediate_size": 5120,
+    "layerdrop": 0.0,
+    "max_position_embeddings": 5000,
+    "model_type": "parakeet_encoder",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 48,
+    "num_key_value_heads": 8,
+    "num_mel_bins": 128,
+    "scale_input": false,
+    "subsampling_conv_channels": 256,
+    "subsampling_conv_kernel_size": 3,
+    "subsampling_conv_stride": 2,
+    "subsampling_factor": 8
+  },
+  "eos_token_id": 3,
+  "head": {
+    "activation": "relu",
+    "dropout": 0,
+    "hidden_size": 1024,
+    "log_softmax": true,
+    "num_classes": 16384,
+    "num_layers": 1,
+    "use_transformer_init": true
+  },
+  "head_dim": 128,
+  "hidden_act": "relu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "is_encoder_decoder": true,
+  "log_batch_stats": false,
+  "log_prediction": true,
+  "max_audio_clip_s": 35,
+  "max_position_embeddings": 1024,
+  "max_seq_len": 1024,
+  "model_defaults": {
+    "asr_enc_hidden": 1280,
+    "lm_dec_hidden": 1024,
+    "lm_enc_hidden": 1024
+  },
+  "model_type": "cohere_asr",
+  "multitask_metrics_cfg": {
+    "log_predictions": true,
+    "metrics": {
+      "wer": {
+        "constraint": ".source_lang==.target_lang"
+      }
+    }
+  },
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "overlap_chunk_second": 5.0,
+  "pad_token_id": 2,
+  "preprocessor": {
+    "dither": 1e-05,
+    "features": 128,
+    "frame_splicing": 1,
+    "log": true,
+    "n_fft": 512,
+    "normalize": "per_feature",
+    "pad_to": 0,
+    "pad_value": 0.0,
+    "sample_rate": 16000,
+    "window": "hann",
+    "window_size": 0.025,
+    "window_stride": 0.01
+  },
+  "prompt_defaults": [
+    {
+      "role": "user",
+      "slots": {
+        "decodercontext": "",
+        "diarize": "<|nodiarize|>",
+        "emotion": "<|emo:undefined|>",
+        "itn": "<|noitn|>",
+        "pnc": "<|pnc|>",
+        "source_lang": "<|en|>",
+        "target_lang": "<|en|>",
+        "timestamp": "<|notimestamp|>"
+      }
+    },
+    {
+      "role": "user_partial",
+      "slots": {
+        "decodercontext": ""
+      }
+    }
+  ],
+  "prompt_format": "cohere_asr",
+  "sample_rate": 16000,
+  "supported_languages": [
+    "en",
+    "ar"
+  ],
+  "tie_word_embeddings": false,
+  "transf_decoder": {
+    "config_dict": {
+      "attn_layer_dropout": 0,
+      "attn_score_dropout": 0,
+      "embedding_dropout": 0,
+      "ffn_dropout": 0,
+      "hidden_act": "relu",
+      "hidden_size": 1024,
+      "inner_size": 4096,
+      "learn_positional_encodings": false,
+      "lm_dec_hidden": 1280,
+      "max_sequence_length": 1024,
+      "num_attention_heads": 8,
+      "num_layers": 8,
+      "num_token_types": 0,
+      "pre_ln": true,
+      "vocab_size": "None"
+    },
+    "encoder": null,
+    "model_name": null,
+    "pre_ln_final_layer_norm": true,
+    "pretrained": false
+  },
+  "transf_encoder": {
+    "attn_layer_dropout": 0,
+    "attn_score_dropout": 0,
+    "ffn_dropout": 0,
+    "hidden_size": 1024,
+    "inner_size": 4096,
+    "mask_future": false,
+    "num_attention_heads": 8,
+    "num_layers": 0,
+    "pre_ln": true,
+    "pre_ln_final_layer_norm": true
+  },
+  "transformers_version": "5.17.0",
+  "use_cache": false,
+  "use_loss_mask_for_prompt": false,
+  "vocab_size": 16384
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 4,
+  "decoder_start_token_id": 13764,
+  "eos_token_id": 3,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 2,
+  "transformers_version": "5.17.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aca7ba9fa08c8b45922c7e25081c1d669bdb765cb485e2e13cfc98474ba4204e
+size 4131809192

processor_config.json

@@ -0,0 +1,22 @@
+{
+  "feature_extractor": {
+    "auto_map": {
+      "AutoProcessor": "processing_cohere_asr.CohereAsrProcessor"
+    },
+    "dither": 1e-05,
+    "feature_extractor_type": "CohereAsrFeatureExtractor",
+    "feature_size": 128,
+    "hop_length": 160,
+    "max_audio_clip_s": 35.0,
+    "min_energy_window_samples": 1600,
+    "n_fft": 512,
+    "overlap_chunk_second": 5.0,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "preemphasis": 0.97,
+    "return_attention_mask": true,
+    "sampling_rate": 16000,
+    "win_length": 400
+  },
+  "processor_class": "CohereAsrProcessor"
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftranscript|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "CohereAsrProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

train_meta.json

@@ -0,0 +1,50 @@
+{
+  "base_model": "CohereLabs/cohere-transcribe-arabic-07-2026",
+  "variant": "middle",
+  "method": "decoder_only_finetune",
+  "target_column": "english",
+  "dataset": "Ahmed1/cohere-asr-cs",
+  "effective_batch_size": 32,
+  "seed": 42,
+  "bf16": true,
+  "train_examples": 2780,
+  "val_examples": 454,
+  "metrics": {
+    "train_runtime": 248.8494,
+    "train_samples_per_second": 22.343,
+    "train_steps_per_second": 0.699,
+    "total_flos": 2.618070162836816e+19,
+    "train_loss": 0.19591110910492382,
+    "epoch": 2.0
+  },
+  "freeze": {
+    "variant": "middle",
+    "decoder_module": "model.decoder",
+    "decoder_layers_total": 8,
+    "decoder_layers_trained": [
+      3,
+      4
+    ],
+    "train_decoder_io": true,
+    "output_head": "proj_out",
+    "io_trainable_params": 35935232,
+    "layer_trainable_params": 33593344,
+    "encoder_module": "model.encoder",
+    "encoder_trainable_params": 0,
+    "total_params": 2065647872,
+    "trainable_params": 69528576,
+    "trainable_pct": 3.366
+  },
+  "hyperparameters": {
+    "learning_rate": 1e-05,
+    "num_train_epochs": 2,
+    "lr_scheduler_type": "cosine",
+    "warmup_ratio": 0.03,
+    "per_device_train_batch_size": 16,
+    "gradient_accumulation_steps": 2,
+    "optim": "adamw_bnb_8bit",
+    "weight_decay": 0.01,
+    "max_grad_norm": 1.0,
+    "save_total_limit": 1
+  }
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d140316592d314a599dc4f7ad225343b290d08a80b32787018e9a4300ae0e793
+size 5393