Hooshaai/svd-linear-attention-roberta-sst2-replace

SVD Linear Attention: roberta-sst2 with replace

This repository hosts a validated factorized/compressed checkpoint of roberta-sst2 using the replace module evaluated on sst2.

Evaluation Summary

• Accuracy: 92.89%
• F1 Score: 0.9292
• Precision: 0.929
• Recall: 0.9291
• Validation Loss: 0.2026
• Verification Status: PASS
• Date: 2026-09-24

Efficiency & Compression Profile

• Original Parameters: 124647170
• Compressed Parameters: 124647170
• Compression Rate: 1.0x
• Compression Ratio: 0.0%
• FLOPs Reduction: 0.0%
• Factorized Layers: 0
• Fallback Layers: 0
• Avg Reconstruction Error: None
• Peak VRAM: 335.13 MB
• Inference Throughput: 14415.31 tokens/sec

Benchmark Configuration

{
  "model": "models/roberta_sst2",
  "dataset_dir": "D:\\SVD_Project\\Codes\\framework\\data\\glue_data\\sst2",
  "dataset": "sst2",
  "output_dir": "results/test_replace_fixed/",
  "parallel": 2,
  "max_steps": 1,
  "finetune_steps": 200,
  "finetune_mode": "head",
  "lora_r": 16,
  "lora_alpha": 32.0,
  "lora_dropout": 0.05,
  "lora_target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj",
  "recovery_steps": 0,
  "recovery_modules": "",
  "optimizer": "adamw",
  "use_muon": false,
  "muon_lr": 0.02,
  "adam_lr": 5e-05,
  "weight_decay": 0.01,
  "batch_size": 8,
  "learning_rate": 0.0002,
  "precision": "fp16",
  "freeze_clf": false,
  "seq_len": 128,
  "kl_distill": false,
  "kl_alpha": 0.5,
  "kl_temperature": 2.0,
  "timeout": 3600,
  "seed": 42,
  "gradient_checkpointing": false,
  "strict_compat": false,
  "model_family": "roberta",
  "fresh": false,
  "no_skip": false,
  "dry_run": false,
  "debug_module": false,
  "verbose_debug": false,
  "rank": 0,
  "force_rank": false,
  "lora_rank": 32,
  "asvd_dynamic_rank": true,
  "asvd_energy_threshold": 0.95,
  "asvd_error_threshold": 0.2,
  "aasvd_max_error": 0.85,
  "aasvd_target_attention_only": true,
  "aasvd_ema_beta": 0.9,
  "aasvd_fallback_to_original": true,
  "group_size": 3,
  "defer_export": false,
  "max_error": 0.4,
  "a3_target_attention_only": true,
  "span_lambda": 0.0,
  "adaptive_ranks_max_error": 0.85,
  "adaptive_ranks_energy_threshold": 0.92,
  "adaptive_ranks_target_attention_only": true,
  "adasvd_max_error": 0.7,
  "adasvd_target_attention_only": true,
  "adasvd_min_rank": 16,
  "admm_sparsity": 0.5,
  "admm_quant_bits": 4,
  "admm_structured_24": true,
  "admm_max_error": 0.45,
  "admm_target_attention_only": true,
  "admm_rho1": 0.1,
  "admm_rho2": 0.1,
  "admm_max_iter": 8,
  "admm_pruner_sparsity": 0.5,
  "admm_pruner_structured_24": true,
  "admm_pruner_max_error": 0.45,
  "admm_pruner_target_attention_only": true,
  "admm_pruner_lambda_distill": 0.1,
  "admm_pruner_distill_batch_size": 32,
  "admm_pruner_rho": 0.1,
  "admm_pruner_max_iter": 6,
  "armor_rank": 16,
  "armor_sparsity": 0.5,
  "armor_structured_24": true,
  "armor_max_error": 0.35,
  "armor_target_attention_only": true,
  "armor_decouple": true,
  "armor_lora_alpha": 1.0,
  "armor_skip_attention_qkv": false,
  "astro_bits": 8,
  "astro_symmetric": true,
  "astro_per_channel": false,
  "astro_use_ste": false,
  "astro_max_error": 0.2,
  "astro_target_attention_only": true,
  "astro_use_fisher_adaptive_bits": false,
  "astro_target_avg_bits": 6.0,
  "astro_min_bits": 2,
  "astro_max_bits": 8,
  "astro_fisher_beta": 0.5,
  "cachenotes_task": "qa",
  "cachenotes_keep_ratio": 0.5,
  "cachenotes_max_error": 0.35,
  "cachenotes_target_attention_only": true,
  "cachenotes_lora_r": 16,
  "cachenotes_lora_alpha": 1.0,
  "cachenotes_skip_attention_qkv": true,
  "come_sparsity": 0.3,
  "come_alpha_init": 0.1,
  "come_lora_r": 16,
  "come_lora_alpha": 1.0,
  "come_skip_attention_qkv": true,
  "come_target_attention_only": false,
  "coopq_bits": 4,
  "coopq_min_bits": 4,
  "coopq_max_bits": 8,
  "coopq_symmetric": false,
  "coopq_per_channel": true,
  "coopq_use_ste": true,
  "coopq_target_attention_only": true,
  "cr_rank": 32,
  "cr_max_error": 0.35,
  "cr_target_attention_only": true,
  "cur_rank": 32,
  "cur_min_rank": 8,
  "cur_method": "leverage",
  "cur_max_error": 0.35,
  "cur_target_attention_only": true,
  "cur_moe_rank": 32,
  "cur_moe_target_attention_only": true,
  "delta_kv_rank": 16,
  "delta_kv_sparsity": 0.29,
  "delta_kv_max_error": 0.35,
  "delta_kv_target_attention_only": true,
  "direct_svd_rank": 128,
  "direct_svd_max_error": 0.35,
  "direct_svd_target_attention_only": true,
  "dobi_init_rank_ratio": 0.5,
  "dobi_init_gate_val": 2.5,
  "dobi_max_error": 0.35,
  "dobi_target_attention_only": true,
  "dynamic_min_rank": 16,
  "dynamic_max_rank": 256,
  "dynamic_target_param_ratio": 0.5,
  "dynamic_max_error": 0.35,
  "dynamic_target_attention_only": true,
  "enlighten_rank": 128,
  "enlighten_sparsity": 0.5,
  "enlighten_structured_24": true,
  "enlighten_max_error": 0.35,
  "enlighten_target_attention_only": true,
  "entropy_budget_total_budget_ratio": 0.7,
  "entropy_budget_max_error": 0.35,
  "entropy_budget_target_attention_only": true,
  "erc_allowed_error": 0.2,
  "erc_max_error": 0.35,
  "erc_target_attention_only": true,
  "faq_bits": 8,
  "faq_alpha": 0.5,
  "faq_symmetric": true,
  "faq_use_ste": true,
  "faq_lora_r": 16,
  "faq_max_error": 0.35,
  "faq_target_attention_only": true,
  "flap_sparsity": 0.2,
  "flap_metric_type": "WIFV",
  "flap_structured_24": false,
  "flap_max_error": 0.35,
  "flap_target_attention_only": true,
  "flat_llm_keep_ratio": 0.7,
  "flat_llm_damping": 0.0001,
  "flat_llm_structured_24": false,
  "flat_llm_max_error": 0.35,
  "flat_llm_target_attention_only": true,
  "frandi_rank": 128,
  "frandi_lora_r": 16,
  "frandi_max_error": 0.35,
  "frandi_target_attention_only": true,
  "grecodec_rank": 128,
  "grecodec_max_iter": 15,
  "grecodec_lora_r": 16,
  "grecodec_max_error": 0.35,
  "grecodec_target_attention_only": true,
  "hashed_net_buckets": 65536,
  "hashed_net_use_sign": true,
  "hashed_net_lora_r": 16,
  "hashed_net_max_error": 0.35,
  "hashed_net_target_attention_only": true,
  "huffman_quantize_bits": 8,
  "huffman_max_error": 0.2,
  "huffman_target_attention_only": true,
  "hybrid_svd_rank": 128,
  "hybrid_svd_use_full": false,
  "hybrid_svd_lora_r": 16,
  "hybrid_svd_max_error": 0.55,
  "hybrid_svd_target_attention_only": true,
  "hybridkv_budget": 0.3,
  "hybridkv_static_ratio": 0.7,
  "hybridkv_max_cache_len": 128,
  "hybridkv_num_sinks": 4,
  "hybridkv_recent_window": 32,
  "hybridkv_target_attention_only": true,
  "kmeans_clusters": 256,
  "kmeans_lora_r": 16,
  "kmeans_max_error": 0.2,
  "kmeans_target_attention_only": true,
  "kronecker_rank": 128,
  "kronecker_max_error": 0.5,
  "kronecker_target_attention_only": true,
  "tt_ranks": "1,2,2,1",
  "kronecker_tt_lora_max_error": 0.5,
  "kronecker_tt_lora_target_attention_only": true,
  "kvtc_bits": 4,
  "kvtc_pca_energy": 0.9,
  "kvtc_max_error": 0.2,
  "kvtc_target_attention_only": true,
  "kvtc_fallback_to_original": true,
  "liftr_gelu_coefficient": 0.8,
  "liftr_target_attention_only": true,
  "lillama_rank_ratio": 0.5,
  "lillama_max_error": 0.5,
  "lillama_target_attention_only": true,
  "lsc_sparsity": 0.3,
  "lsc_max_error": 0.5,
  "lsc_target_attention_only": true
}

Reproducibility

Trained and factorized via the Unified SVD Linear Attention Framework.