{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.4,
  "internal_ratio_kept": 0.6,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove40/finetune_v/merge.pt",
  "mean_rel_err": 0.5146559384718005,
  "max_rel_err": 0.631554126739502,
  "param_fraction": 0.599840865851684,
  "ranks": {
    "0.q_proj": 1454,
    "0.k_proj": 1441,
    "0.v_proj": 1375,
    "0.o_proj": 1391,
    "0.gate_proj": 1832,
    "0.up_proj": 1833,
    "0.down_proj": 2049,
    "1.q_proj": 1344,
    "1.k_proj": 1341,
    "1.v_proj": 1293,
    "1.o_proj": 1289,
    "1.gate_proj": 1812,
    "1.up_proj": 1811,
    "1.down_proj": 2131,
    "2.q_proj": 1226,
    "2.k_proj": 1226,
    "2.v_proj": 1206,
    "2.o_proj": 1178,
    "2.gate_proj": 1785,
    "2.up_proj": 1785,
    "2.down_proj": 1685,
    "3.q_proj": 1234,
    "3.k_proj": 1234,
    "3.v_proj": 1225,
    "3.o_proj": 1197,
    "3.gate_proj": 1789,
    "3.up_proj": 1788,
    "3.down_proj": 1720,
    "4.q_proj": 1261,
    "4.k_proj": 1262,
    "4.v_proj": 1252,
    "4.o_proj": 1230,
    "4.gate_proj": 1796,
    "4.up_proj": 1796,
    "4.down_proj": 1791,
    "5.q_proj": 1236,
    "5.k_proj": 1237,
    "5.v_proj": 1228,
    "5.o_proj": 1212,
    "5.gate_proj": 1791,
    "5.up_proj": 1791,
    "5.down_proj": 1754,
    "6.q_proj": 1256,
    "6.k_proj": 1256,
    "6.v_proj": 1252,
    "6.o_proj": 1239,
    "6.gate_proj": 1797,
    "6.up_proj": 1796,
    "6.down_proj": 1801,
    "7.q_proj": 1229,
    "7.k_proj": 1229,
    "7.v_proj": 1227,
    "7.o_proj": 1214,
    "7.gate_proj": 1791,
    "7.up_proj": 1790,
    "7.down_proj": 1758,
    "8.q_proj": 1239,
    "8.k_proj": 1240,
    "8.v_proj": 1239,
    "8.o_proj": 1231,
    "8.gate_proj": 1794,
    "8.up_proj": 1793,
    "8.down_proj": 1783,
    "9.q_proj": 1219,
    "9.k_proj": 1221,
    "9.v_proj": 1220,
    "9.o_proj": 1215,
    "9.gate_proj": 1789,
    "9.up_proj": 1789,
    "9.down_proj": 1751,
    "10.q_proj": 1209,
    "10.k_proj": 1211,
    "10.v_proj": 1210,
    "10.o_proj": 1209,
    "10.gate_proj": 1787,
    "10.up_proj": 1786,
    "10.down_proj": 1735,
    "11.q_proj": 1209,
    "11.k_proj": 1210,
    "11.v_proj": 1212,
    "11.o_proj": 1207,
    "11.gate_proj": 1787,
    "11.up_proj": 1786,
    "11.down_proj": 1735,
    "12.q_proj": 1203,
    "12.k_proj": 1204,
    "12.v_proj": 1206,
    "12.o_proj": 1204,
    "12.gate_proj": 1785,
    "12.up_proj": 1785,
    "12.down_proj": 1729,
    "13.q_proj": 1208,
    "13.k_proj": 1209,
    "13.v_proj": 1211,
    "13.o_proj": 1211,
    "13.gate_proj": 1787,
    "13.up_proj": 1786,
    "13.down_proj": 1741,
    "14.q_proj": 1203,
    "14.k_proj": 1204,
    "14.v_proj": 1206,
    "14.o_proj": 1207,
    "14.gate_proj": 1785,
    "14.up_proj": 1785,
    "14.down_proj": 1736,
    "15.q_proj": 1205,
    "15.k_proj": 1205,
    "15.v_proj": 1208,
    "15.o_proj": 1211,
    "15.gate_proj": 1787,
    "15.up_proj": 1786,
    "15.down_proj": 1744,
    "16.q_proj": 1205,
    "16.k_proj": 1207,
    "16.v_proj": 1209,
    "16.o_proj": 1216,
    "16.gate_proj": 1787,
    "16.up_proj": 1786,
    "16.down_proj": 1751,
    "17.q_proj": 1201,
    "17.k_proj": 1202,
    "17.v_proj": 1206,
    "17.o_proj": 1209,
    "17.gate_proj": 1785,
    "17.up_proj": 1785,
    "17.down_proj": 1748,
    "18.q_proj": 1200,
    "18.k_proj": 1200,
    "18.v_proj": 1206,
    "18.o_proj": 1205,
    "18.gate_proj": 1785,
    "18.up_proj": 1785,
    "18.down_proj": 1747,
    "19.q_proj": 1193,
    "19.k_proj": 1194,
    "19.v_proj": 1200,
    "19.o_proj": 1199,
    "19.gate_proj": 1784,
    "19.up_proj": 1784,
    "19.down_proj": 1737,
    "20.q_proj": 1192,
    "20.k_proj": 1192,
    "20.v_proj": 1198,
    "20.o_proj": 1203,
    "20.gate_proj": 1784,
    "20.up_proj": 1784,
    "20.down_proj": 1740,
    "21.q_proj": 1189,
    "21.k_proj": 1189,
    "21.v_proj": 1197,
    "21.o_proj": 1196,
    "21.gate_proj": 1783,
    "21.up_proj": 1783,
    "21.down_proj": 1736,
    "22.q_proj": 1189,
    "22.k_proj": 1189,
    "22.v_proj": 1197,
    "22.o_proj": 1202,
    "22.gate_proj": 1783,
    "22.up_proj": 1783,
    "22.down_proj": 1737,
    "23.q_proj": 1186,
    "23.k_proj": 1186,
    "23.v_proj": 1198,
    "23.o_proj": 1198,
    "23.gate_proj": 1782,
    "23.up_proj": 1783,
    "23.down_proj": 1735,
    "24.q_proj": 1185,
    "24.k_proj": 1186,
    "24.v_proj": 1196,
    "24.o_proj": 1201,
    "24.gate_proj": 1782,
    "24.up_proj": 1783,
    "24.down_proj": 1735,
    "25.q_proj": 1186,
    "25.k_proj": 1185,
    "25.v_proj": 1199,
    "25.o_proj": 1207,
    "25.gate_proj": 1783,
    "25.up_proj": 1783,
    "25.down_proj": 1740,
    "26.q_proj": 1187,
    "26.k_proj": 1187,
    "26.v_proj": 1199,
    "26.o_proj": 1209,
    "26.gate_proj": 1783,
    "26.up_proj": 1783,
    "26.down_proj": 1742,
    "27.q_proj": 1188,
    "27.k_proj": 1188,
    "27.v_proj": 1201,
    "27.o_proj": 1206,
    "27.gate_proj": 1784,
    "27.up_proj": 1784,
    "27.down_proj": 1749,
    "28.q_proj": 1189,
    "28.k_proj": 1189,
    "28.v_proj": 1203,
    "28.o_proj": 1214,
    "28.gate_proj": 1784,
    "28.up_proj": 1785,
    "28.down_proj": 1755,
    "29.q_proj": 1189,
    "29.k_proj": 1190,
    "29.v_proj": 1203,
    "29.o_proj": 1220,
    "29.gate_proj": 1785,
    "29.up_proj": 1786,
    "29.down_proj": 1763,
    "30.q_proj": 1202,
    "30.k_proj": 1201,
    "30.v_proj": 1218,
    "30.o_proj": 1230,
    "30.gate_proj": 1789,
    "30.up_proj": 1791,
    "30.down_proj": 1831,
    "31.q_proj": 1480,
    "31.k_proj": 1481,
    "31.v_proj": 1496,
    "31.o_proj": 1536,
    "31.gate_proj": 1855,
    "31.up_proj": 1858,
    "31.down_proj": 2423
  },
  "rel_err": {
    "0.q_proj": 0.14318764209747314,
    "0.k_proj": 0.11331941187381744,
    "0.v_proj": 0.3313101828098297,
    "0.o_proj": 0.3334074020385742,
    "0.gate_proj": 0.4394051432609558,
    "0.up_proj": 0.45333534479141235,
    "0.down_proj": 0.4774848520755768,
    "1.q_proj": 0.1363212615251541,
    "1.k_proj": 0.12104111164808273,
    "1.v_proj": 0.47113242745399475,
    "1.o_proj": 0.39778363704681396,
    "1.gate_proj": 0.42443057894706726,
    "1.up_proj": 0.4789886474609375,
    "1.down_proj": 0.5252040028572083,
    "2.q_proj": 0.29307156801223755,
    "2.k_proj": 0.2808840572834015,
    "2.v_proj": 0.5237805247306824,
    "2.o_proj": 0.5878390669822693,
    "2.gate_proj": 0.4957614839076996,
    "2.up_proj": 0.5317240357398987,
    "2.down_proj": 0.6058598160743713,
    "3.q_proj": 0.39533179998397827,
    "3.k_proj": 0.39482513070106506,
    "3.v_proj": 0.5370124578475952,
    "3.o_proj": 0.5734050869941711,
    "3.gate_proj": 0.5502407550811768,
    "3.up_proj": 0.5727823972702026,
    "3.down_proj": 0.5635493993759155,
    "4.q_proj": 0.37751975655555725,
    "4.k_proj": 0.38127243518829346,
    "4.v_proj": 0.5265920758247375,
    "4.o_proj": 0.5677705407142639,
    "4.gate_proj": 0.5490803122520447,
    "4.up_proj": 0.5769976377487183,
    "4.down_proj": 0.5522221326828003,
    "5.q_proj": 0.3945859670639038,
    "5.k_proj": 0.3935074508190155,
    "5.v_proj": 0.536858081817627,
    "5.o_proj": 0.5777600407600403,
    "5.gate_proj": 0.5458164215087891,
    "5.up_proj": 0.5791149139404297,
    "5.down_proj": 0.5521062612533569,
    "6.q_proj": 0.375798761844635,
    "6.k_proj": 0.382950097322464,
    "6.v_proj": 0.5131592750549316,
    "6.o_proj": 0.5379549264907837,
    "6.gate_proj": 0.5389353632926941,
    "6.up_proj": 0.5784068703651428,
    "6.down_proj": 0.5407786965370178,
    "7.q_proj": 0.3880448639392853,
    "7.k_proj": 0.39485877752304077,
    "7.v_proj": 0.5132298469543457,
    "7.o_proj": 0.5630982518196106,
    "7.gate_proj": 0.5367270112037659,
    "7.up_proj": 0.5791166424751282,
    "7.down_proj": 0.5435407161712646,
    "8.q_proj": 0.3958355188369751,
    "8.k_proj": 0.40927451848983765,
    "8.v_proj": 0.4954420328140259,
    "8.o_proj": 0.5553522706031799,
    "8.gate_proj": 0.533244252204895,
    "8.up_proj": 0.5733633637428284,
    "8.down_proj": 0.5326294302940369,
    "9.q_proj": 0.4168955981731415,
    "9.k_proj": 0.422296404838562,
    "9.v_proj": 0.4987483322620392,
    "9.o_proj": 0.5611084699630737,
    "9.gate_proj": 0.5303514003753662,
    "9.up_proj": 0.5685946941375732,
    "9.down_proj": 0.5385213494300842,
    "10.q_proj": 0.38947921991348267,
    "10.k_proj": 0.40109699964523315,
    "10.v_proj": 0.4944608509540558,
    "10.o_proj": 0.5274718403816223,
    "10.gate_proj": 0.5256984233856201,
    "10.up_proj": 0.5644442439079285,
    "10.down_proj": 0.5384189486503601,
    "11.q_proj": 0.3771919906139374,
    "11.k_proj": 0.3906274139881134,
    "11.v_proj": 0.47947946190834045,
    "11.o_proj": 0.5166332721710205,
    "11.gate_proj": 0.5260279178619385,
    "11.up_proj": 0.5649887323379517,
    "11.down_proj": 0.5448550581932068,
    "12.q_proj": 0.41809821128845215,
    "12.k_proj": 0.4165879487991333,
    "12.v_proj": 0.5010970234870911,
    "12.o_proj": 0.5586170554161072,
    "12.gate_proj": 0.5309696793556213,
    "12.up_proj": 0.565258800983429,
    "12.down_proj": 0.5461880564689636,
    "13.q_proj": 0.42253896594047546,
    "13.k_proj": 0.4398796558380127,
    "13.v_proj": 0.5294481515884399,
    "13.o_proj": 0.546043336391449,
    "13.gate_proj": 0.5335360765457153,
    "13.up_proj": 0.5683612823486328,
    "13.down_proj": 0.5552169680595398,
    "14.q_proj": 0.403841495513916,
    "14.k_proj": 0.40865302085876465,
    "14.v_proj": 0.52288419008255,
    "14.o_proj": 0.5442802906036377,
    "14.gate_proj": 0.5336455702781677,
    "14.up_proj": 0.5672312378883362,
    "14.down_proj": 0.5583865642547607,
    "15.q_proj": 0.425073504447937,
    "15.k_proj": 0.4273808002471924,
    "15.v_proj": 0.5283194184303284,
    "15.o_proj": 0.5536724925041199,
    "15.gate_proj": 0.537350594997406,
    "15.up_proj": 0.5708211660385132,
    "15.down_proj": 0.5700202584266663,
    "16.q_proj": 0.4221418499946594,
    "16.k_proj": 0.42537611722946167,
    "16.v_proj": 0.5565226674079895,
    "16.o_proj": 0.5793879628181458,
    "16.gate_proj": 0.5364647507667542,
    "16.up_proj": 0.568230152130127,
    "16.down_proj": 0.5813849568367004,
    "17.q_proj": 0.4647054970264435,
    "17.k_proj": 0.4621625244617462,
    "17.v_proj": 0.5715993642807007,
    "17.o_proj": 0.5927662253379822,
    "17.gate_proj": 0.5394077897071838,
    "17.up_proj": 0.5690187811851501,
    "17.down_proj": 0.5798382759094238,
    "18.q_proj": 0.48962292075157166,
    "18.k_proj": 0.4945559501647949,
    "18.v_proj": 0.5778101682662964,
    "18.o_proj": 0.6191045045852661,
    "18.gate_proj": 0.5480973720550537,
    "18.up_proj": 0.5750609636306763,
    "18.down_proj": 0.5799038410186768,
    "19.q_proj": 0.47487759590148926,
    "19.k_proj": 0.4706522524356842,
    "19.v_proj": 0.5633808970451355,
    "19.o_proj": 0.608212411403656,
    "19.gate_proj": 0.5518002510070801,
    "19.up_proj": 0.5758308172225952,
    "19.down_proj": 0.5820042490959167,
    "20.q_proj": 0.4659927785396576,
    "20.k_proj": 0.46770182251930237,
    "20.v_proj": 0.5783191919326782,
    "20.o_proj": 0.6094985008239746,
    "20.gate_proj": 0.5498113036155701,
    "20.up_proj": 0.5726807117462158,
    "20.down_proj": 0.5921808481216431,
    "21.q_proj": 0.48177552223205566,
    "21.k_proj": 0.48396918177604675,
    "21.v_proj": 0.5714066028594971,
    "21.o_proj": 0.6256420016288757,
    "21.gate_proj": 0.5510643124580383,
    "21.up_proj": 0.5723118782043457,
    "21.down_proj": 0.5887574553489685,
    "22.q_proj": 0.49377286434173584,
    "22.k_proj": 0.49821627140045166,
    "22.v_proj": 0.5825276970863342,
    "22.o_proj": 0.6214143633842468,
    "22.gate_proj": 0.5508750677108765,
    "22.up_proj": 0.5714125037193298,
    "22.down_proj": 0.5857972502708435,
    "23.q_proj": 0.5258812308311462,
    "23.k_proj": 0.5253252983093262,
    "23.v_proj": 0.5805611610412598,
    "23.o_proj": 0.6131620407104492,
    "23.gate_proj": 0.5545095801353455,
    "23.up_proj": 0.5731971859931946,
    "23.down_proj": 0.5782360434532166,
    "24.q_proj": 0.4733271896839142,
    "24.k_proj": 0.4773584306240082,
    "24.v_proj": 0.5734611749649048,
    "24.o_proj": 0.617493748664856,
    "24.gate_proj": 0.5553052425384521,
    "24.up_proj": 0.573965311050415,
    "24.down_proj": 0.5709168314933777,
    "25.q_proj": 0.51483154296875,
    "25.k_proj": 0.522794783115387,
    "25.v_proj": 0.5812080502510071,
    "25.o_proj": 0.60832679271698,
    "25.gate_proj": 0.558302640914917,
    "25.up_proj": 0.5760963559150696,
    "25.down_proj": 0.5624425411224365,
    "26.q_proj": 0.484011173248291,
    "26.k_proj": 0.49306154251098633,
    "26.v_proj": 0.5701485872268677,
    "26.o_proj": 0.631554126739502,
    "26.gate_proj": 0.559934675693512,
    "26.up_proj": 0.5781747102737427,
    "26.down_proj": 0.5602895021438599,
    "27.q_proj": 0.5331695675849915,
    "27.k_proj": 0.5409024953842163,
    "27.v_proj": 0.5903238654136658,
    "27.o_proj": 0.6105265617370605,
    "27.gate_proj": 0.5627158284187317,
    "27.up_proj": 0.5795223712921143,
    "27.down_proj": 0.556897759437561,
    "28.q_proj": 0.5141112208366394,
    "28.k_proj": 0.5246506333351135,
    "28.v_proj": 0.5859890580177307,
    "28.o_proj": 0.6046826243400574,
    "28.gate_proj": 0.5626722574234009,
    "28.up_proj": 0.5781853795051575,
    "28.down_proj": 0.5647644996643066,
    "29.q_proj": 0.4800718128681183,
    "29.k_proj": 0.48807916045188904,
    "29.v_proj": 0.5776322484016418,
    "29.o_proj": 0.6057113409042358,
    "29.gate_proj": 0.5614248514175415,
    "29.up_proj": 0.5739281177520752,
    "29.down_proj": 0.570681095123291,
    "30.q_proj": 0.5097528100013733,
    "30.k_proj": 0.5179398655891418,
    "30.v_proj": 0.5861728191375732,
    "30.o_proj": 0.6103938817977905,
    "30.gate_proj": 0.5546872615814209,
    "30.up_proj": 0.5617015361785889,
    "30.down_proj": 0.5654463171958923,
    "31.q_proj": 0.37722048163414,
    "31.k_proj": 0.3839050531387329,
    "31.v_proj": 0.4845984876155853,
    "31.o_proj": 0.48935359716415405,
    "31.gate_proj": 0.5315423011779785,
    "31.up_proj": 0.5343829989433289,
    "31.down_proj": 0.43410614132881165
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": null,
    "param_fraction": 0.599840865851684,
    "mean_rel_err": 0.5142416990295585,
    "max_rel_err": 0.6316028237342834
  },
  "factorisation": {
    "max_rel_err": 0.0014253825647756457,
    "factor_params": 3884572672,
    "dense_params": 6476005376
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}