{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.5,
  "internal_ratio_kept": 0.5,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove50/finetune_v/merge.pt",
  "mean_rel_err": 0.5770199772502694,
  "max_rel_err": 0.6914882659912109,
  "param_fraction": 0.4998633824481865,
  "ranks": {
    "0.q_proj": 1212,
    "0.k_proj": 1202,
    "0.v_proj": 1147,
    "0.o_proj": 1160,
    "0.gate_proj": 1526,
    "0.up_proj": 1527,
    "0.down_proj": 1707,
    "1.q_proj": 1121,
    "1.k_proj": 1118,
    "1.v_proj": 1078,
    "1.o_proj": 1075,
    "1.gate_proj": 1509,
    "1.up_proj": 1509,
    "1.down_proj": 1775,
    "2.q_proj": 1022,
    "2.k_proj": 1022,
    "2.v_proj": 1006,
    "2.o_proj": 983,
    "2.gate_proj": 1487,
    "2.up_proj": 1487,
    "2.down_proj": 1404,
    "3.q_proj": 1029,
    "3.k_proj": 1029,
    "3.v_proj": 1022,
    "3.o_proj": 998,
    "3.gate_proj": 1490,
    "3.up_proj": 1490,
    "3.down_proj": 1433,
    "4.q_proj": 1052,
    "4.k_proj": 1053,
    "4.v_proj": 1044,
    "4.o_proj": 1026,
    "4.gate_proj": 1496,
    "4.up_proj": 1496,
    "4.down_proj": 1492,
    "5.q_proj": 1031,
    "5.k_proj": 1032,
    "5.v_proj": 1024,
    "5.o_proj": 1011,
    "5.gate_proj": 1492,
    "5.up_proj": 1492,
    "5.down_proj": 1461,
    "6.q_proj": 1047,
    "6.k_proj": 1048,
    "6.v_proj": 1045,
    "6.o_proj": 1033,
    "6.gate_proj": 1497,
    "6.up_proj": 1496,
    "6.down_proj": 1500,
    "7.q_proj": 1025,
    "7.k_proj": 1025,
    "7.v_proj": 1023,
    "7.o_proj": 1013,
    "7.gate_proj": 1492,
    "7.up_proj": 1491,
    "7.down_proj": 1465,
    "8.q_proj": 1033,
    "8.k_proj": 1034,
    "8.v_proj": 1033,
    "8.o_proj": 1026,
    "8.gate_proj": 1494,
    "8.up_proj": 1494,
    "8.down_proj": 1486,
    "9.q_proj": 1017,
    "9.k_proj": 1018,
    "9.v_proj": 1017,
    "9.o_proj": 1014,
    "9.gate_proj": 1491,
    "9.up_proj": 1490,
    "9.down_proj": 1459,
    "10.q_proj": 1009,
    "10.k_proj": 1010,
    "10.v_proj": 1009,
    "10.o_proj": 1008,
    "10.gate_proj": 1489,
    "10.up_proj": 1489,
    "10.down_proj": 1445,
    "11.q_proj": 1008,
    "11.k_proj": 1009,
    "11.v_proj": 1011,
    "11.o_proj": 1007,
    "11.gate_proj": 1489,
    "11.up_proj": 1488,
    "11.down_proj": 1445,
    "12.q_proj": 1004,
    "12.k_proj": 1004,
    "12.v_proj": 1006,
    "12.o_proj": 1004,
    "12.gate_proj": 1487,
    "12.up_proj": 1487,
    "12.down_proj": 1441,
    "13.q_proj": 1007,
    "13.k_proj": 1008,
    "13.v_proj": 1010,
    "13.o_proj": 1009,
    "13.gate_proj": 1489,
    "13.up_proj": 1489,
    "13.down_proj": 1450,
    "14.q_proj": 1004,
    "14.k_proj": 1005,
    "14.v_proj": 1006,
    "14.o_proj": 1007,
    "14.gate_proj": 1487,
    "14.up_proj": 1487,
    "14.down_proj": 1446,
    "15.q_proj": 1005,
    "15.k_proj": 1006,
    "15.v_proj": 1007,
    "15.o_proj": 1010,
    "15.gate_proj": 1488,
    "15.up_proj": 1487,
    "15.down_proj": 1452,
    "16.q_proj": 1005,
    "16.k_proj": 1006,
    "16.v_proj": 1008,
    "16.o_proj": 1014,
    "16.gate_proj": 1489,
    "16.up_proj": 1489,
    "16.down_proj": 1459,
    "17.q_proj": 1002,
    "17.k_proj": 1003,
    "17.v_proj": 1006,
    "17.o_proj": 1007,
    "17.gate_proj": 1488,
    "17.up_proj": 1487,
    "17.down_proj": 1456,
    "18.q_proj": 1000,
    "18.k_proj": 1000,
    "18.v_proj": 1005,
    "18.o_proj": 1005,
    "18.gate_proj": 1487,
    "18.up_proj": 1487,
    "18.down_proj": 1456,
    "19.q_proj": 994,
    "19.k_proj": 995,
    "19.v_proj": 1000,
    "19.o_proj": 1000,
    "19.gate_proj": 1486,
    "19.up_proj": 1486,
    "19.down_proj": 1447,
    "20.q_proj": 994,
    "20.k_proj": 994,
    "20.v_proj": 999,
    "20.o_proj": 1003,
    "20.gate_proj": 1486,
    "20.up_proj": 1486,
    "20.down_proj": 1450,
    "21.q_proj": 991,
    "21.k_proj": 991,
    "21.v_proj": 998,
    "21.o_proj": 997,
    "21.gate_proj": 1485,
    "21.up_proj": 1485,
    "21.down_proj": 1446,
    "22.q_proj": 991,
    "22.k_proj": 991,
    "22.v_proj": 998,
    "22.o_proj": 1002,
    "22.gate_proj": 1485,
    "22.up_proj": 1485,
    "22.down_proj": 1447,
    "23.q_proj": 989,
    "23.k_proj": 989,
    "23.v_proj": 999,
    "23.o_proj": 999,
    "23.gate_proj": 1485,
    "23.up_proj": 1485,
    "23.down_proj": 1446,
    "24.q_proj": 988,
    "24.k_proj": 989,
    "24.v_proj": 997,
    "24.o_proj": 1001,
    "24.gate_proj": 1485,
    "24.up_proj": 1485,
    "24.down_proj": 1445,
    "25.q_proj": 989,
    "25.k_proj": 988,
    "25.v_proj": 1000,
    "25.o_proj": 1006,
    "25.gate_proj": 1485,
    "25.up_proj": 1485,
    "25.down_proj": 1449,
    "26.q_proj": 989,
    "26.k_proj": 989,
    "26.v_proj": 1000,
    "26.o_proj": 1007,
    "26.gate_proj": 1485,
    "26.up_proj": 1486,
    "26.down_proj": 1451,
    "27.q_proj": 990,
    "27.k_proj": 990,
    "27.v_proj": 1001,
    "27.o_proj": 1006,
    "27.gate_proj": 1486,
    "27.up_proj": 1486,
    "27.down_proj": 1457,
    "28.q_proj": 991,
    "28.k_proj": 991,
    "28.v_proj": 1003,
    "28.o_proj": 1013,
    "28.gate_proj": 1486,
    "28.up_proj": 1487,
    "28.down_proj": 1462,
    "29.q_proj": 992,
    "29.k_proj": 992,
    "29.v_proj": 1003,
    "29.o_proj": 1017,
    "29.gate_proj": 1487,
    "29.up_proj": 1487,
    "29.down_proj": 1469,
    "30.q_proj": 1003,
    "30.k_proj": 1002,
    "30.v_proj": 1016,
    "30.o_proj": 1026,
    "30.gate_proj": 1491,
    "30.up_proj": 1492,
    "30.down_proj": 1525,
    "31.q_proj": 1234,
    "31.k_proj": 1235,
    "31.v_proj": 1247,
    "31.o_proj": 1281,
    "31.gate_proj": 1545,
    "31.up_proj": 1547,
    "31.down_proj": 2018
  },
  "rel_err": {
    "0.q_proj": 0.1519796997308731,
    "0.k_proj": 0.1192033439874649,
    "0.v_proj": 0.38923344016075134,
    "0.o_proj": 0.3858165740966797,
    "0.gate_proj": 0.500773549079895,
    "0.up_proj": 0.5168083906173706,
    "0.down_proj": 0.5532068014144897,
    "1.q_proj": 0.16660481691360474,
    "1.k_proj": 0.1496836543083191,
    "1.v_proj": 0.5328048467636108,
    "1.o_proj": 0.4690547585487366,
    "1.gate_proj": 0.48617124557495117,
    "1.up_proj": 0.5469284653663635,
    "1.down_proj": 0.5976937413215637,
    "2.q_proj": 0.35211285948753357,
    "2.k_proj": 0.3386683166027069,
    "2.v_proj": 0.5837520956993103,
    "2.o_proj": 0.6466013193130493,
    "2.gate_proj": 0.556005597114563,
    "2.up_proj": 0.5962874293327332,
    "2.down_proj": 0.6686410903930664,
    "3.q_proj": 0.45740288496017456,
    "3.k_proj": 0.45834606885910034,
    "3.v_proj": 0.5986074209213257,
    "3.o_proj": 0.6326659917831421,
    "3.gate_proj": 0.613806426525116,
    "3.up_proj": 0.6380705833435059,
    "3.down_proj": 0.6272472739219666,
    "4.q_proj": 0.4375986158847809,
    "4.k_proj": 0.4429137408733368,
    "4.v_proj": 0.5889207720756531,
    "4.o_proj": 0.6280702352523804,
    "4.gate_proj": 0.6121411919593811,
    "4.up_proj": 0.6420316100120544,
    "4.down_proj": 0.6181107759475708,
    "5.q_proj": 0.4545028805732727,
    "5.k_proj": 0.45424583554267883,
    "5.v_proj": 0.600082278251648,
    "5.o_proj": 0.636687159538269,
    "5.gate_proj": 0.6082773804664612,
    "5.up_proj": 0.6440009474754333,
    "5.down_proj": 0.6173216700553894,
    "6.q_proj": 0.4331511855125427,
    "6.k_proj": 0.44137337803840637,
    "6.v_proj": 0.5765433311462402,
    "6.o_proj": 0.599454939365387,
    "6.gate_proj": 0.6015103459358215,
    "6.up_proj": 0.6436546444892883,
    "6.down_proj": 0.6076619625091553,
    "7.q_proj": 0.4461621642112732,
    "7.k_proj": 0.4536467492580414,
    "7.v_proj": 0.5764104127883911,
    "7.o_proj": 0.6230320930480957,
    "7.gate_proj": 0.599027693271637,
    "7.up_proj": 0.6438522934913635,
    "7.down_proj": 0.6086878180503845,
    "8.q_proj": 0.45493605732917786,
    "8.k_proj": 0.4706614911556244,
    "8.v_proj": 0.5578523874282837,
    "8.o_proj": 0.6163948178291321,
    "8.gate_proj": 0.5954751968383789,
    "8.up_proj": 0.6374110579490662,
    "8.down_proj": 0.5979208946228027,
    "9.q_proj": 0.4763244092464447,
    "9.k_proj": 0.48309847712516785,
    "9.v_proj": 0.5613107085227966,
    "9.o_proj": 0.6197980642318726,
    "9.gate_proj": 0.5919429659843445,
    "9.up_proj": 0.632355809211731,
    "9.down_proj": 0.6026204228401184,
    "10.q_proj": 0.44654256105422974,
    "10.k_proj": 0.459806352853775,
    "10.v_proj": 0.5551357865333557,
    "10.o_proj": 0.5909467339515686,
    "10.gate_proj": 0.5873189568519592,
    "10.up_proj": 0.6278164982795715,
    "10.down_proj": 0.6019299626350403,
    "11.q_proj": 0.4331582188606262,
    "11.k_proj": 0.44750073552131653,
    "11.v_proj": 0.539775550365448,
    "11.o_proj": 0.5791270136833191,
    "11.gate_proj": 0.5880950689315796,
    "11.up_proj": 0.62898188829422,
    "11.down_proj": 0.6085322499275208,
    "12.q_proj": 0.47685113549232483,
    "12.k_proj": 0.47473812103271484,
    "12.v_proj": 0.5630918741226196,
    "12.o_proj": 0.62078458070755,
    "12.gate_proj": 0.5933976769447327,
    "12.up_proj": 0.6293315887451172,
    "12.down_proj": 0.6097376346588135,
    "13.q_proj": 0.4816206395626068,
    "13.k_proj": 0.5003007650375366,
    "13.v_proj": 0.5929780602455139,
    "13.o_proj": 0.60856032371521,
    "13.gate_proj": 0.5962281823158264,
    "13.up_proj": 0.6323710680007935,
    "13.down_proj": 0.6194044947624207,
    "14.q_proj": 0.45993104577064514,
    "14.k_proj": 0.4645041525363922,
    "14.v_proj": 0.5858851671218872,
    "14.o_proj": 0.6054980754852295,
    "14.gate_proj": 0.5965513586997986,
    "14.up_proj": 0.6316479444503784,
    "14.down_proj": 0.6229026913642883,
    "15.q_proj": 0.4835875630378723,
    "15.k_proj": 0.48482608795166016,
    "15.v_proj": 0.59104323387146,
    "15.o_proj": 0.6179000735282898,
    "15.gate_proj": 0.6003255248069763,
    "15.up_proj": 0.6353552341461182,
    "15.down_proj": 0.63531893491745,
    "16.q_proj": 0.4797825813293457,
    "16.k_proj": 0.48336681723594666,
    "16.v_proj": 0.620274007320404,
    "16.o_proj": 0.6427569389343262,
    "16.gate_proj": 0.5996921062469482,
    "16.up_proj": 0.632495105266571,
    "16.down_proj": 0.6475937366485596,
    "17.q_proj": 0.5249422788619995,
    "17.k_proj": 0.5217642784118652,
    "17.v_proj": 0.635169506072998,
    "17.o_proj": 0.6561335921287537,
    "17.gate_proj": 0.6028484106063843,
    "17.up_proj": 0.6339511871337891,
    "17.down_proj": 0.6465868353843689,
    "18.q_proj": 0.5513262748718262,
    "18.k_proj": 0.5563322901725769,
    "18.v_proj": 0.6416341066360474,
    "18.o_proj": 0.6792891025543213,
    "18.gate_proj": 0.6122872233390808,
    "18.up_proj": 0.6402170658111572,
    "18.down_proj": 0.6462416052818298,
    "19.q_proj": 0.5357663035392761,
    "19.k_proj": 0.530946671962738,
    "19.v_proj": 0.6263485550880432,
    "19.o_proj": 0.6681627631187439,
    "19.gate_proj": 0.6159399151802063,
    "19.up_proj": 0.6408361196517944,
    "19.down_proj": 0.6481027603149414,
    "20.q_proj": 0.5267692804336548,
    "20.k_proj": 0.5284019708633423,
    "20.v_proj": 0.6414449214935303,
    "20.o_proj": 0.6711487174034119,
    "20.gate_proj": 0.6140052080154419,
    "20.up_proj": 0.6376996636390686,
    "20.down_proj": 0.6590219140052795,
    "21.q_proj": 0.5438850522041321,
    "21.k_proj": 0.5463284850120544,
    "21.v_proj": 0.6339064836502075,
    "21.o_proj": 0.6861968040466309,
    "21.gate_proj": 0.61533522605896,
    "21.up_proj": 0.6377876400947571,
    "21.down_proj": 0.65590900182724,
    "22.q_proj": 0.5565619468688965,
    "22.k_proj": 0.5612902045249939,
    "22.v_proj": 0.6452638506889343,
    "22.o_proj": 0.6826830506324768,
    "22.gate_proj": 0.6154788732528687,
    "22.up_proj": 0.6370535492897034,
    "22.down_proj": 0.6531917452812195,
    "23.q_proj": 0.5894484519958496,
    "23.k_proj": 0.5889238715171814,
    "23.v_proj": 0.6437049508094788,
    "23.o_proj": 0.6743727326393127,
    "23.gate_proj": 0.6192212700843811,
    "23.up_proj": 0.6390761137008667,
    "23.down_proj": 0.6452059745788574,
    "24.q_proj": 0.5363700985908508,
    "24.k_proj": 0.5404852628707886,
    "24.v_proj": 0.6358501315116882,
    "24.o_proj": 0.6789605617523193,
    "24.gate_proj": 0.6206092238426208,
    "24.up_proj": 0.6403064727783203,
    "24.down_proj": 0.6378594636917114,
    "25.q_proj": 0.578160285949707,
    "25.k_proj": 0.5866066217422485,
    "25.v_proj": 0.6443796753883362,
    "25.o_proj": 0.6698061227798462,
    "25.gate_proj": 0.6242399215698242,
    "25.up_proj": 0.6427105069160461,
    "25.down_proj": 0.6289896965026855,
    "26.q_proj": 0.5467449426651001,
    "26.k_proj": 0.5561713576316833,
    "26.v_proj": 0.6325230002403259,
    "26.o_proj": 0.6914882659912109,
    "26.gate_proj": 0.6260568499565125,
    "26.up_proj": 0.6445198059082031,
    "26.down_proj": 0.6264070868492126,
    "27.q_proj": 0.5962280631065369,
    "27.k_proj": 0.6041812300682068,
    "27.v_proj": 0.6541686058044434,
    "27.o_proj": 0.6721190810203552,
    "27.gate_proj": 0.6289679408073425,
    "27.up_proj": 0.6461732983589172,
    "27.down_proj": 0.622887909412384,
    "28.q_proj": 0.577075719833374,
    "28.k_proj": 0.5881885290145874,
    "28.v_proj": 0.649710476398468,
    "28.o_proj": 0.6659877896308899,
    "28.gate_proj": 0.6287640333175659,
    "28.up_proj": 0.6446463465690613,
    "28.down_proj": 0.6314065456390381,
    "29.q_proj": 0.5415160059928894,
    "29.k_proj": 0.5502261519432068,
    "29.v_proj": 0.6414359211921692,
    "29.o_proj": 0.6688233613967896,
    "29.gate_proj": 0.6273680329322815,
    "29.up_proj": 0.6402621865272522,
    "29.down_proj": 0.6377818584442139,
    "30.q_proj": 0.5716490745544434,
    "30.k_proj": 0.5806272625923157,
    "30.v_proj": 0.650600016117096,
    "30.o_proj": 0.6727860569953918,
    "30.gate_proj": 0.6200348138809204,
    "30.up_proj": 0.6269094944000244,
    "30.down_proj": 0.635309100151062,
    "31.q_proj": 0.4421253800392151,
    "31.k_proj": 0.4497847557067871,
    "31.v_proj": 0.5590528845787048,
    "31.o_proj": 0.5637112855911255,
    "31.gate_proj": 0.5982303023338318,
    "31.up_proj": 0.600795328617096,
    "31.down_proj": 0.5228932499885559
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": null,
    "param_fraction": 0.4998633824481865,
    "mean_rel_err": 0.5765954015909561,
    "max_rel_err": 0.691561222076416
  },
  "factorisation": {
    "max_rel_err": 0.0014744423096999526,
    "factor_params": 3237117952,
    "dense_params": 6476005376
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}