{
  "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.4,
  "internal_ratio_kept": 0.6,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove40/finetune_v/merge.pt",
  "mean_rel_err": 0.524213519213455,
  "max_rel_err": 0.6444101929664612,
  "param_fraction": 0.6003230168269231,
  "ranks": {
    "0.q_proj": 1383,
    "0.k_proj": 581,
    "0.v_proj": 529,
    "0.o_proj": 1263,
    "0.gate_proj": 2016,
    "0.up_proj": 2016,
    "0.down_proj": 2043,
    "1.q_proj": 1388,
    "1.k_proj": 591,
    "1.v_proj": 557,
    "1.o_proj": 1300,
    "1.gate_proj": 2016,
    "1.up_proj": 2016,
    "1.down_proj": 2371,
    "2.q_proj": 1131,
    "2.k_proj": 448,
    "2.v_proj": 439,
    "2.o_proj": 1088,
    "2.gate_proj": 2016,
    "2.up_proj": 2016,
    "2.down_proj": 1622,
    "3.q_proj": 1128,
    "3.k_proj": 446,
    "3.v_proj": 435,
    "3.o_proj": 1077,
    "3.gate_proj": 2016,
    "3.up_proj": 2016,
    "3.down_proj": 1624,
    "4.q_proj": 1126,
    "4.k_proj": 445,
    "4.v_proj": 439,
    "4.o_proj": 1084,
    "4.gate_proj": 2016,
    "4.up_proj": 2016,
    "4.down_proj": 1645,
    "5.q_proj": 1126,
    "5.k_proj": 445,
    "5.v_proj": 439,
    "5.o_proj": 1092,
    "5.gate_proj": 2016,
    "5.up_proj": 2016,
    "5.down_proj": 1662,
    "6.q_proj": 1127,
    "6.k_proj": 446,
    "6.v_proj": 440,
    "6.o_proj": 1101,
    "6.gate_proj": 2016,
    "6.up_proj": 2016,
    "6.down_proj": 1675,
    "7.q_proj": 1126,
    "7.k_proj": 446,
    "7.v_proj": 441,
    "7.o_proj": 1110,
    "7.gate_proj": 2016,
    "7.up_proj": 2016,
    "7.down_proj": 1688,
    "8.q_proj": 1122,
    "8.k_proj": 444,
    "8.v_proj": 441,
    "8.o_proj": 1109,
    "8.gate_proj": 2016,
    "8.up_proj": 2016,
    "8.down_proj": 1685,
    "9.q_proj": 1121,
    "9.k_proj": 444,
    "9.v_proj": 442,
    "9.o_proj": 1109,
    "9.gate_proj": 2016,
    "9.up_proj": 2016,
    "9.down_proj": 1692,
    "10.q_proj": 1122,
    "10.k_proj": 444,
    "10.v_proj": 442,
    "10.o_proj": 1118,
    "10.gate_proj": 2016,
    "10.up_proj": 2016,
    "10.down_proj": 1692,
    "11.q_proj": 1120,
    "11.k_proj": 444,
    "11.v_proj": 443,
    "11.o_proj": 1121,
    "11.gate_proj": 2016,
    "11.up_proj": 2016,
    "11.down_proj": 1695,
    "12.q_proj": 1127,
    "12.k_proj": 447,
    "12.v_proj": 447,
    "12.o_proj": 1129,
    "12.gate_proj": 2016,
    "12.up_proj": 2016,
    "12.down_proj": 1714,
    "13.q_proj": 1123,
    "13.k_proj": 446,
    "13.v_proj": 445,
    "13.o_proj": 1127,
    "13.gate_proj": 2016,
    "13.up_proj": 2016,
    "13.down_proj": 1714,
    "14.q_proj": 1132,
    "14.k_proj": 451,
    "14.v_proj": 453,
    "14.o_proj": 1139,
    "14.gate_proj": 2016,
    "14.up_proj": 2016,
    "14.down_proj": 1742,
    "15.q_proj": 1145,
    "15.k_proj": 458,
    "15.v_proj": 461,
    "15.o_proj": 1157,
    "15.gate_proj": 2016,
    "15.up_proj": 2016,
    "15.down_proj": 1777,
    "16.q_proj": 1151,
    "16.k_proj": 461,
    "16.v_proj": 463,
    "16.o_proj": 1162,
    "16.gate_proj": 2016,
    "16.up_proj": 2016,
    "16.down_proj": 1800,
    "17.q_proj": 1174,
    "17.k_proj": 473,
    "17.v_proj": 476,
    "17.o_proj": 1188,
    "17.gate_proj": 2016,
    "17.up_proj": 2016,
    "17.down_proj": 1852,
    "18.q_proj": 1174,
    "18.k_proj": 472,
    "18.v_proj": 476,
    "18.o_proj": 1191,
    "18.gate_proj": 2016,
    "18.up_proj": 2016,
    "18.down_proj": 1866,
    "19.q_proj": 1158,
    "19.k_proj": 465,
    "19.v_proj": 469,
    "19.o_proj": 1179,
    "19.gate_proj": 2016,
    "19.up_proj": 2016,
    "19.down_proj": 1844,
    "20.q_proj": 1130,
    "20.k_proj": 449,
    "20.v_proj": 454,
    "20.o_proj": 1146,
    "20.gate_proj": 2016,
    "20.up_proj": 2016,
    "20.down_proj": 1786,
    "21.q_proj": 1123,
    "21.k_proj": 444,
    "21.v_proj": 451,
    "21.o_proj": 1137,
    "21.gate_proj": 2016,
    "21.up_proj": 2016,
    "21.down_proj": 1772,
    "22.q_proj": 1112,
    "22.k_proj": 437,
    "22.v_proj": 445,
    "22.o_proj": 1121,
    "22.gate_proj": 2016,
    "22.up_proj": 2016,
    "22.down_proj": 1749,
    "23.q_proj": 1112,
    "23.k_proj": 438,
    "23.v_proj": 444,
    "23.o_proj": 1129,
    "23.gate_proj": 2016,
    "23.up_proj": 2016,
    "23.down_proj": 1752,
    "24.q_proj": 1112,
    "24.k_proj": 437,
    "24.v_proj": 445,
    "24.o_proj": 1133,
    "24.gate_proj": 2016,
    "24.up_proj": 2016,
    "24.down_proj": 1754,
    "25.q_proj": 1114,
    "25.k_proj": 439,
    "25.v_proj": 449,
    "25.o_proj": 1141,
    "25.gate_proj": 2016,
    "25.up_proj": 2016,
    "25.down_proj": 1766,
    "26.q_proj": 1119,
    "26.k_proj": 441,
    "26.v_proj": 452,
    "26.o_proj": 1149,
    "26.gate_proj": 2016,
    "26.up_proj": 2016,
    "26.down_proj": 1780,
    "27.q_proj": 1123,
    "27.k_proj": 444,
    "27.v_proj": 454,
    "27.o_proj": 1149,
    "27.gate_proj": 2016,
    "27.up_proj": 2016,
    "27.down_proj": 1792,
    "28.q_proj": 1139,
    "28.k_proj": 452,
    "28.v_proj": 463,
    "28.o_proj": 1171,
    "28.gate_proj": 2016,
    "28.up_proj": 2016,
    "28.down_proj": 1838,
    "29.q_proj": 1148,
    "29.k_proj": 457,
    "29.v_proj": 469,
    "29.o_proj": 1193,
    "29.gate_proj": 2016,
    "29.up_proj": 2016,
    "29.down_proj": 1861,
    "30.q_proj": 1165,
    "30.k_proj": 466,
    "30.v_proj": 481,
    "30.o_proj": 1215,
    "30.gate_proj": 2016,
    "30.up_proj": 2016,
    "30.down_proj": 1912,
    "31.q_proj": 1263,
    "31.k_proj": 519,
    "31.v_proj": 536,
    "31.o_proj": 1336,
    "31.gate_proj": 2016,
    "31.up_proj": 2016,
    "31.down_proj": 2179
  },
  "rel_err": {
    "0.q_proj": 0.09311120212078094,
    "0.k_proj": 0.08253687620162964,
    "0.v_proj": 0.3433501720428467,
    "0.o_proj": 0.3911000192165375,
    "0.gate_proj": 0.3707573413848877,
    "0.up_proj": 0.41067802906036377,
    "0.down_proj": 0.5313096046447754,
    "1.q_proj": 0.16670213639736176,
    "1.k_proj": 0.10798212885856628,
    "1.v_proj": 0.5118529796600342,
    "1.o_proj": 0.5213813781738281,
    "1.gate_proj": 0.4368397891521454,
    "1.up_proj": 0.4705049991607666,
    "1.down_proj": 0.4971780776977539,
    "2.q_proj": 0.46672654151916504,
    "2.k_proj": 0.46059876680374146,
    "2.v_proj": 0.6164757013320923,
    "2.o_proj": 0.5800544619560242,
    "2.gate_proj": 0.512566328048706,
    "2.up_proj": 0.5365164875984192,
    "2.down_proj": 0.6227588057518005,
    "3.q_proj": 0.4239010214805603,
    "3.k_proj": 0.3750090003013611,
    "3.v_proj": 0.6444101929664612,
    "3.o_proj": 0.6232957243919373,
    "3.gate_proj": 0.5219470858573914,
    "3.up_proj": 0.551502525806427,
    "3.down_proj": 0.6144695281982422,
    "4.q_proj": 0.4298405945301056,
    "4.k_proj": 0.3984021842479706,
    "4.v_proj": 0.6045357584953308,
    "4.o_proj": 0.5685608386993408,
    "4.gate_proj": 0.523536205291748,
    "4.up_proj": 0.5611042976379395,
    "4.down_proj": 0.6211814284324646,
    "5.q_proj": 0.46155083179473877,
    "5.k_proj": 0.4410848319530487,
    "5.v_proj": 0.6360407471656799,
    "5.o_proj": 0.6054348349571228,
    "5.gate_proj": 0.5178220868110657,
    "5.up_proj": 0.5632009506225586,
    "5.down_proj": 0.6081773638725281,
    "6.q_proj": 0.4475008249282837,
    "6.k_proj": 0.4137071967124939,
    "6.v_proj": 0.6230500340461731,
    "6.o_proj": 0.5891891121864319,
    "6.gate_proj": 0.5309041738510132,
    "6.up_proj": 0.5662814378738403,
    "6.down_proj": 0.5977262258529663,
    "7.q_proj": 0.427289217710495,
    "7.k_proj": 0.38710105419158936,
    "7.v_proj": 0.6161573529243469,
    "7.o_proj": 0.5648438930511475,
    "7.gate_proj": 0.5202533602714539,
    "7.up_proj": 0.5593512654304504,
    "7.down_proj": 0.592329740524292,
    "8.q_proj": 0.43415379524230957,
    "8.k_proj": 0.3839348256587982,
    "8.v_proj": 0.576714813709259,
    "8.o_proj": 0.5553375482559204,
    "8.gate_proj": 0.523369312286377,
    "8.up_proj": 0.5563741326332092,
    "8.down_proj": 0.5858190059661865,
    "9.q_proj": 0.42894482612609863,
    "9.k_proj": 0.3965420722961426,
    "9.v_proj": 0.5610228180885315,
    "9.o_proj": 0.5573288202285767,
    "9.gate_proj": 0.5195508599281311,
    "9.up_proj": 0.5543780326843262,
    "9.down_proj": 0.5704837441444397,
    "10.q_proj": 0.42824241518974304,
    "10.k_proj": 0.401052862405777,
    "10.v_proj": 0.5595612525939941,
    "10.o_proj": 0.5597419738769531,
    "10.gate_proj": 0.5190359354019165,
    "10.up_proj": 0.5517367720603943,
    "10.down_proj": 0.5768575668334961,
    "11.q_proj": 0.4241175949573517,
    "11.k_proj": 0.3747548758983612,
    "11.v_proj": 0.5366345643997192,
    "11.o_proj": 0.5581367611885071,
    "11.gate_proj": 0.5184133052825928,
    "11.up_proj": 0.5504465103149414,
    "11.down_proj": 0.5751660466194153,
    "12.q_proj": 0.41172537207603455,
    "12.k_proj": 0.3747641444206238,
    "12.v_proj": 0.5728425979614258,
    "12.o_proj": 0.5404180288314819,
    "12.gate_proj": 0.5148531198501587,
    "12.up_proj": 0.5455449223518372,
    "12.down_proj": 0.5717161893844604,
    "13.q_proj": 0.4387548565864563,
    "13.k_proj": 0.40697500109672546,
    "13.v_proj": 0.5708914399147034,
    "13.o_proj": 0.5474628210067749,
    "13.gate_proj": 0.5218628644943237,
    "13.up_proj": 0.5454148054122925,
    "13.down_proj": 0.5676140189170837,
    "14.q_proj": 0.4094313979148865,
    "14.k_proj": 0.38163262605667114,
    "14.v_proj": 0.5798335075378418,
    "14.o_proj": 0.5494734048843384,
    "14.gate_proj": 0.5187147259712219,
    "14.up_proj": 0.5501715540885925,
    "14.down_proj": 0.593805193901062,
    "15.q_proj": 0.40906858444213867,
    "15.k_proj": 0.3624928295612335,
    "15.v_proj": 0.6080702543258667,
    "15.o_proj": 0.5543991327285767,
    "15.gate_proj": 0.5166524648666382,
    "15.up_proj": 0.5478196144104004,
    "15.down_proj": 0.5904906988143921,
    "16.q_proj": 0.4163205027580261,
    "16.k_proj": 0.3787270486354828,
    "16.v_proj": 0.6027750372886658,
    "16.o_proj": 0.5409226417541504,
    "16.gate_proj": 0.520693838596344,
    "16.up_proj": 0.556447446346283,
    "16.down_proj": 0.5904657244682312,
    "17.q_proj": 0.4439752697944641,
    "17.k_proj": 0.41725632548332214,
    "17.v_proj": 0.5800758004188538,
    "17.o_proj": 0.5581274628639221,
    "17.gate_proj": 0.5324439406394958,
    "17.up_proj": 0.5666811466217041,
    "17.down_proj": 0.5825478434562683,
    "18.q_proj": 0.4187489449977875,
    "18.k_proj": 0.3872785270214081,
    "18.v_proj": 0.6302375793457031,
    "18.o_proj": 0.5343031883239746,
    "18.gate_proj": 0.5309059619903564,
    "18.up_proj": 0.564786434173584,
    "18.down_proj": 0.5998350381851196,
    "19.q_proj": 0.4693866968154907,
    "19.k_proj": 0.40707793831825256,
    "19.v_proj": 0.6421195268630981,
    "19.o_proj": 0.5577782392501831,
    "19.gate_proj": 0.536626398563385,
    "19.up_proj": 0.5659677386283875,
    "19.down_proj": 0.6147719621658325,
    "20.q_proj": 0.4903647303581238,
    "20.k_proj": 0.44427719712257385,
    "20.v_proj": 0.6202300190925598,
    "20.o_proj": 0.5741354823112488,
    "20.gate_proj": 0.5429403185844421,
    "20.up_proj": 0.5663204789161682,
    "20.down_proj": 0.6225840449333191,
    "21.q_proj": 0.5229761004447937,
    "21.k_proj": 0.47562214732170105,
    "21.v_proj": 0.5926244854927063,
    "21.o_proj": 0.611431360244751,
    "21.gate_proj": 0.5432093143463135,
    "21.up_proj": 0.5681016445159912,
    "21.down_proj": 0.6183050274848938,
    "22.q_proj": 0.5626530051231384,
    "22.k_proj": 0.5008488297462463,
    "22.v_proj": 0.5983458757400513,
    "22.o_proj": 0.6302191019058228,
    "22.gate_proj": 0.5491480231285095,
    "22.up_proj": 0.5697852969169617,
    "22.down_proj": 0.6164385676383972,
    "23.q_proj": 0.551480233669281,
    "23.k_proj": 0.5012710690498352,
    "23.v_proj": 0.6238268613815308,
    "23.o_proj": 0.6080763339996338,
    "23.gate_proj": 0.5521300435066223,
    "23.up_proj": 0.5716087222099304,
    "23.down_proj": 0.6099771857261658,
    "24.q_proj": 0.5572198629379272,
    "24.k_proj": 0.5183809399604797,
    "24.v_proj": 0.6330755352973938,
    "24.o_proj": 0.6300069689750671,
    "24.gate_proj": 0.5530964136123657,
    "24.up_proj": 0.5734592080116272,
    "24.down_proj": 0.600332498550415,
    "25.q_proj": 0.5632444620132446,
    "25.k_proj": 0.5026987791061401,
    "25.v_proj": 0.6118474006652832,
    "25.o_proj": 0.6110818982124329,
    "25.gate_proj": 0.5554454922676086,
    "25.up_proj": 0.57483971118927,
    "25.down_proj": 0.588827908039093,
    "26.q_proj": 0.558449923992157,
    "26.k_proj": 0.4919636845588684,
    "26.v_proj": 0.5971678495407104,
    "26.o_proj": 0.5927439332008362,
    "26.gate_proj": 0.5562448501586914,
    "26.up_proj": 0.5749430060386658,
    "26.down_proj": 0.5749134421348572,
    "27.q_proj": 0.5691151022911072,
    "27.k_proj": 0.5340715050697327,
    "27.v_proj": 0.6143185496330261,
    "27.o_proj": 0.5845725536346436,
    "27.gate_proj": 0.5597459673881531,
    "27.up_proj": 0.5763874650001526,
    "27.down_proj": 0.5742335319519043,
    "28.q_proj": 0.5281823873519897,
    "28.k_proj": 0.4590943157672882,
    "28.v_proj": 0.5685400366783142,
    "28.o_proj": 0.5630350112915039,
    "28.gate_proj": 0.5594995617866516,
    "28.up_proj": 0.5738884806632996,
    "28.down_proj": 0.5813770294189453,
    "29.q_proj": 0.5108082890510559,
    "29.k_proj": 0.46057870984077454,
    "29.v_proj": 0.6014853715896606,
    "29.o_proj": 0.5444827079772949,
    "29.gate_proj": 0.5542632937431335,
    "29.up_proj": 0.5680719017982483,
    "29.down_proj": 0.5874636769294739,
    "30.q_proj": 0.4883614480495453,
    "30.k_proj": 0.4195691645145416,
    "30.v_proj": 0.535009503364563,
    "30.o_proj": 0.5329458713531494,
    "30.gate_proj": 0.5540768504142761,
    "30.up_proj": 0.5595806241035461,
    "30.down_proj": 0.5732670426368713,
    "31.q_proj": 0.4402003884315491,
    "31.k_proj": 0.3733157217502594,
    "31.v_proj": 0.48748257756233215,
    "31.o_proj": 0.45593205094337463,
    "31.gate_proj": 0.5609548091888428,
    "31.up_proj": 0.5358024835586548,
    "31.down_proj": 0.5254908800125122
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": "shape",
    "param_fraction": 0.6003230168269231,
    "mean_rel_err": 0.523012746524598,
    "max_rel_err": 0.6436969041824341
  },
  "factorisation": {
    "max_rel_err": 0.001442187582142651,
    "factor_params": 4189847552,
    "dense_params": 6979321856
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}