{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.3,
  "internal_ratio_kept": 0.7,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove30/finetune_v/merge.pt",
  "mean_rel_err": 0.45528166218926863,
  "max_rel_err": 0.5735374689102173,
  "param_fraction": 0.6997563653659327,
  "ranks": {
    "0.q_proj": 1697,
    "0.k_proj": 1682,
    "0.v_proj": 1605,
    "0.o_proj": 1624,
    "0.gate_proj": 2137,
    "0.up_proj": 2137,
    "0.down_proj": 2390,
    "1.q_proj": 1568,
    "1.k_proj": 1564,
    "1.v_proj": 1509,
    "1.o_proj": 1504,
    "1.gate_proj": 2113,
    "1.up_proj": 2112,
    "1.down_proj": 2485,
    "2.q_proj": 1431,
    "2.k_proj": 1430,
    "2.v_proj": 1407,
    "2.o_proj": 1375,
    "2.gate_proj": 2082,
    "2.up_proj": 2082,
    "2.down_proj": 1966,
    "3.q_proj": 1440,
    "3.k_proj": 1440,
    "3.v_proj": 1430,
    "3.o_proj": 1397,
    "3.gate_proj": 2086,
    "3.up_proj": 2086,
    "3.down_proj": 2007,
    "4.q_proj": 1472,
    "4.k_proj": 1473,
    "4.v_proj": 1461,
    "4.o_proj": 1436,
    "4.gate_proj": 2095,
    "4.up_proj": 2094,
    "4.down_proj": 2089,
    "5.q_proj": 1442,
    "5.k_proj": 1444,
    "5.v_proj": 1433,
    "5.o_proj": 1415,
    "5.gate_proj": 2089,
    "5.up_proj": 2089,
    "5.down_proj": 2045,
    "6.q_proj": 1466,
    "6.k_proj": 1466,
    "6.v_proj": 1462,
    "6.o_proj": 1446,
    "6.gate_proj": 2096,
    "6.up_proj": 2094,
    "6.down_proj": 2100,
    "7.q_proj": 1434,
    "7.k_proj": 1434,
    "7.v_proj": 1431,
    "7.o_proj": 1417,
    "7.gate_proj": 2089,
    "7.up_proj": 2088,
    "7.down_proj": 2050,
    "8.q_proj": 1446,
    "8.k_proj": 1447,
    "8.v_proj": 1445,
    "8.o_proj": 1436,
    "8.gate_proj": 2092,
    "8.up_proj": 2091,
    "8.down_proj": 2080,
    "9.q_proj": 1423,
    "9.k_proj": 1424,
    "9.v_proj": 1423,
    "9.o_proj": 1418,
    "9.gate_proj": 2087,
    "9.up_proj": 2086,
    "9.down_proj": 2042,
    "10.q_proj": 1411,
    "10.k_proj": 1413,
    "10.v_proj": 1412,
    "10.o_proj": 1410,
    "10.gate_proj": 2084,
    "10.up_proj": 2084,
    "10.down_proj": 2024,
    "11.q_proj": 1411,
    "11.k_proj": 1412,
    "11.v_proj": 1415,
    "11.o_proj": 1409,
    "11.gate_proj": 2084,
    "11.up_proj": 2084,
    "11.down_proj": 2024,
    "12.q_proj": 1404,
    "12.k_proj": 1405,
    "12.v_proj": 1408,
    "12.o_proj": 1405,
    "12.gate_proj": 2082,
    "12.up_proj": 2082,
    "12.down_proj": 2017,
    "13.q_proj": 1408,
    "13.k_proj": 1410,
    "13.v_proj": 1413,
    "13.o_proj": 1413,
    "13.gate_proj": 2084,
    "13.up_proj": 2084,
    "13.down_proj": 2031,
    "14.q_proj": 1404,
    "14.k_proj": 1407,
    "14.v_proj": 1407,
    "14.o_proj": 1409,
    "14.gate_proj": 2082,
    "14.up_proj": 2082,
    "14.down_proj": 2025,
    "15.q_proj": 1405,
    "15.k_proj": 1407,
    "15.v_proj": 1409,
    "15.o_proj": 1414,
    "15.gate_proj": 2083,
    "15.up_proj": 2083,
    "15.down_proj": 2034,
    "16.q_proj": 1405,
    "16.k_proj": 1408,
    "16.v_proj": 1410,
    "16.o_proj": 1418,
    "16.gate_proj": 2084,
    "16.up_proj": 2084,
    "16.down_proj": 2042,
    "17.q_proj": 1402,
    "17.k_proj": 1403,
    "17.v_proj": 1408,
    "17.o_proj": 1410,
    "17.gate_proj": 2083,
    "17.up_proj": 2083,
    "17.down_proj": 2039,
    "18.q_proj": 1400,
    "18.k_proj": 1400,
    "18.v_proj": 1407,
    "18.o_proj": 1407,
    "18.gate_proj": 2082,
    "18.up_proj": 2082,
    "18.down_proj": 2038,
    "19.q_proj": 1392,
    "19.k_proj": 1393,
    "19.v_proj": 1400,
    "19.o_proj": 1399,
    "19.gate_proj": 2081,
    "19.up_proj": 2081,
    "19.down_proj": 2026,
    "20.q_proj": 1391,
    "20.k_proj": 1392,
    "20.v_proj": 1398,
    "20.o_proj": 1404,
    "20.gate_proj": 2081,
    "20.up_proj": 2081,
    "20.down_proj": 2030,
    "21.q_proj": 1387,
    "21.k_proj": 1388,
    "21.v_proj": 1397,
    "21.o_proj": 1395,
    "21.gate_proj": 2080,
    "21.up_proj": 2080,
    "21.down_proj": 2025,
    "22.q_proj": 1388,
    "22.k_proj": 1387,
    "22.v_proj": 1397,
    "22.o_proj": 1403,
    "22.gate_proj": 2080,
    "22.up_proj": 2080,
    "22.down_proj": 2026,
    "23.q_proj": 1385,
    "23.k_proj": 1384,
    "23.v_proj": 1398,
    "23.o_proj": 1398,
    "23.gate_proj": 2079,
    "23.up_proj": 2079,
    "23.down_proj": 2024,
    "24.q_proj": 1383,
    "24.k_proj": 1384,
    "24.v_proj": 1396,
    "24.o_proj": 1401,
    "24.gate_proj": 2079,
    "24.up_proj": 2079,
    "24.down_proj": 2024,
    "25.q_proj": 1384,
    "25.k_proj": 1383,
    "25.v_proj": 1399,
    "25.o_proj": 1408,
    "25.gate_proj": 2079,
    "25.up_proj": 2080,
    "25.down_proj": 2029,
    "26.q_proj": 1385,
    "26.k_proj": 1385,
    "26.v_proj": 1399,
    "26.o_proj": 1409,
    "26.gate_proj": 2080,
    "26.up_proj": 2080,
    "26.down_proj": 2032,
    "27.q_proj": 1386,
    "27.k_proj": 1386,
    "27.v_proj": 1401,
    "27.o_proj": 1408,
    "27.gate_proj": 2081,
    "27.up_proj": 2081,
    "27.down_proj": 2040,
    "28.q_proj": 1387,
    "28.k_proj": 1387,
    "28.v_proj": 1404,
    "28.o_proj": 1417,
    "28.gate_proj": 2081,
    "28.up_proj": 2082,
    "28.down_proj": 2047,
    "29.q_proj": 1388,
    "29.k_proj": 1388,
    "29.v_proj": 1404,
    "29.o_proj": 1423,
    "29.gate_proj": 2082,
    "29.up_proj": 2083,
    "29.down_proj": 2056,
    "30.q_proj": 1403,
    "30.k_proj": 1402,
    "30.v_proj": 1422,
    "30.o_proj": 1435,
    "30.gate_proj": 2087,
    "30.up_proj": 2088,
    "30.down_proj": 2135,
    "31.q_proj": 1728,
    "31.k_proj": 1728,
    "31.v_proj": 1746,
    "31.o_proj": 1793,
    "31.gate_proj": 2164,
    "31.up_proj": 2167,
    "31.down_proj": 2826
  },
  "rel_err": {
    "0.q_proj": 0.13674421608448029,
    "0.k_proj": 0.10850787907838821,
    "0.v_proj": 0.2810197174549103,
    "0.o_proj": 0.2883068323135376,
    "0.gate_proj": 0.3820878565311432,
    "0.up_proj": 0.3937278091907501,
    "0.down_proj": 0.40569689869880676,
    "1.q_proj": 0.11325579136610031,
    "1.k_proj": 0.10081755369901657,
    "1.v_proj": 0.4143904745578766,
    "1.o_proj": 0.3389497399330139,
    "1.gate_proj": 0.3668021559715271,
    "1.up_proj": 0.4152345657348633,
    "1.down_proj": 0.453666090965271,
    "2.q_proj": 0.2440570592880249,
    "2.k_proj": 0.2325412780046463,
    "2.v_proj": 0.46705976128578186,
    "2.o_proj": 0.5312212705612183,
    "2.gate_proj": 0.4374466836452484,
    "2.up_proj": 0.4691368341445923,
    "2.down_proj": 0.5438665747642517,
    "3.q_proj": 0.33946898579597473,
    "3.k_proj": 0.33782610297203064,
    "3.v_proj": 0.4778972566127777,
    "3.o_proj": 0.5165881514549255,
    "3.gate_proj": 0.48844465613365173,
    "3.up_proj": 0.5085727572441101,
    "3.down_proj": 0.5010958909988403,
    "4.q_proj": 0.32332727313041687,
    "4.k_proj": 0.32587742805480957,
    "4.v_proj": 0.4671332538127899,
    "4.o_proj": 0.5094566345214844,
    "4.gate_proj": 0.48788368701934814,
    "4.up_proj": 0.5130534768104553,
    "4.down_proj": 0.48729878664016724,
    "5.q_proj": 0.3404356837272644,
    "5.k_proj": 0.3388126492500305,
    "5.v_proj": 0.4766134023666382,
    "5.o_proj": 0.5210713148117065,
    "5.gate_proj": 0.4849763810634613,
    "5.up_proj": 0.5152232646942139,
    "5.down_proj": 0.48836052417755127,
    "6.q_proj": 0.32434237003326416,
    "6.k_proj": 0.33024048805236816,
    "6.v_proj": 0.45296233892440796,
    "6.o_proj": 0.4798653721809387,
    "6.gate_proj": 0.47807908058166504,
    "6.up_proj": 0.5142185091972351,
    "6.down_proj": 0.4754852056503296,
    "7.q_proj": 0.33590492606163025,
    "7.k_proj": 0.34187355637550354,
    "7.v_proj": 0.454049676656723,
    "7.o_proj": 0.505239725112915,
    "7.gate_proj": 0.4762929379940033,
    "7.up_proj": 0.5152503848075867,
    "7.down_proj": 0.4799094796180725,
    "8.q_proj": 0.3426366448402405,
    "8.k_proj": 0.3538832366466522,
    "8.v_proj": 0.43721604347229004,
    "8.o_proj": 0.4970349967479706,
    "8.gate_proj": 0.47296127676963806,
    "8.up_proj": 0.5100219249725342,
    "8.down_proj": 0.46890032291412354,
    "9.q_proj": 0.3625483214855194,
    "9.k_proj": 0.3672429919242859,
    "9.v_proj": 0.44050657749176025,
    "9.o_proj": 0.5047832727432251,
    "9.gate_proj": 0.47035032510757446,
    "9.up_proj": 0.505872368812561,
    "9.down_proj": 0.47603756189346313,
    "10.q_proj": 0.3382200598716736,
    "10.k_proj": 0.34823906421661377,
    "10.v_proj": 0.4375532865524292,
    "10.o_proj": 0.46764662861824036,
    "10.gate_proj": 0.46595898270606995,
    "10.up_proj": 0.5017201900482178,
    "10.down_proj": 0.47638988494873047,
    "11.q_proj": 0.32728126645088196,
    "11.k_proj": 0.3394625782966614,
    "11.v_proj": 0.4230077862739563,
    "11.o_proj": 0.4579029679298401,
    "11.gate_proj": 0.4656955301761627,
    "11.up_proj": 0.5019961595535278,
    "11.down_proj": 0.4826474189758301,
    "12.q_proj": 0.36442241072654724,
    "12.k_proj": 0.3636908531188965,
    "12.v_proj": 0.4427216649055481,
    "12.o_proj": 0.49870848655700684,
    "12.gate_proj": 0.4701372981071472,
    "12.up_proj": 0.5024581551551819,
    "12.down_proj": 0.4839555323123932,
    "13.q_proj": 0.3692917227745056,
    "13.k_proj": 0.38481763005256653,
    "13.v_proj": 0.46928614377975464,
    "13.o_proj": 0.48683831095695496,
    "13.gate_proj": 0.47251319885253906,
    "13.up_proj": 0.5052496790885925,
    "13.down_proj": 0.4925529658794403,
    "14.q_proj": 0.35274773836135864,
    "14.k_proj": 0.3569324016571045,
    "14.v_proj": 0.46338796615600586,
    "14.o_proj": 0.4851902723312378,
    "14.gate_proj": 0.47242483496665955,
    "14.up_proj": 0.5040963292121887,
    "14.down_proj": 0.49548181891441345,
    "15.q_proj": 0.37204164266586304,
    "15.k_proj": 0.3740440607070923,
    "15.v_proj": 0.4692009389400482,
    "15.o_proj": 0.49231812357902527,
    "15.gate_proj": 0.4763273298740387,
    "15.up_proj": 0.5079472064971924,
    "15.down_proj": 0.506531834602356,
    "16.q_proj": 0.3697231411933899,
    "16.k_proj": 0.37245628237724304,
    "16.v_proj": 0.4960777759552002,
    "16.o_proj": 0.5187640190124512,
    "16.gate_proj": 0.4750789403915405,
    "16.up_proj": 0.5048896670341492,
    "16.down_proj": 0.516768753528595,
    "17.q_proj": 0.40830114483833313,
    "17.k_proj": 0.4063715934753418,
    "17.v_proj": 0.5102226734161377,
    "17.o_proj": 0.5319307446479797,
    "17.gate_proj": 0.4774886965751648,
    "17.up_proj": 0.505339503288269,
    "17.down_proj": 0.5146914124488831,
    "18.q_proj": 0.4319038391113281,
    "18.k_proj": 0.43669185042381287,
    "18.v_proj": 0.5166221261024475,
    "18.o_proj": 0.5600407719612122,
    "18.gate_proj": 0.48564109206199646,
    "18.up_proj": 0.5114160180091858,
    "18.down_proj": 0.5150787830352783,
    "19.q_proj": 0.4182446599006653,
    "19.k_proj": 0.4145455062389374,
    "19.v_proj": 0.5035737156867981,
    "19.o_proj": 0.5499191284179688,
    "19.gate_proj": 0.4894777834415436,
    "19.up_proj": 0.5121901035308838,
    "19.down_proj": 0.5176525712013245,
    "20.q_proj": 0.40922680497169495,
    "20.k_proj": 0.41069889068603516,
    "20.v_proj": 0.5174776911735535,
    "20.o_proj": 0.5493062138557434,
    "20.gate_proj": 0.48762205243110657,
    "20.up_proj": 0.5089380741119385,
    "20.down_proj": 0.5267716646194458,
    "21.q_proj": 0.4238526523113251,
    "21.k_proj": 0.4254571795463562,
    "21.v_proj": 0.511239767074585,
    "21.o_proj": 0.566460132598877,
    "21.gate_proj": 0.48842716217041016,
    "21.up_proj": 0.5083724856376648,
    "21.down_proj": 0.5235127806663513,
    "22.q_proj": 0.43453019857406616,
    "22.k_proj": 0.438902884721756,
    "22.v_proj": 0.5219279527664185,
    "22.o_proj": 0.5610068440437317,
    "22.gate_proj": 0.48826220631599426,
    "22.up_proj": 0.5073332190513611,
    "22.down_proj": 0.520425021648407,
    "23.q_proj": 0.4653097093105316,
    "23.k_proj": 0.46524491906166077,
    "23.v_proj": 0.5198501944541931,
    "23.o_proj": 0.5531998872756958,
    "23.gate_proj": 0.49149203300476074,
    "23.up_proj": 0.509039580821991,
    "23.down_proj": 0.5131181478500366,
    "24.q_proj": 0.4144188165664673,
    "24.k_proj": 0.4183977246284485,
    "24.v_proj": 0.5133265852928162,
    "24.o_proj": 0.557678759098053,
    "24.gate_proj": 0.4918333888053894,
    "24.up_proj": 0.5094541907310486,
    "24.down_proj": 0.5062175393104553,
    "25.q_proj": 0.4548630118370056,
    "25.k_proj": 0.46258294582366943,
    "25.v_proj": 0.5203204154968262,
    "25.o_proj": 0.548804521560669,
    "25.gate_proj": 0.4945409893989563,
    "25.up_proj": 0.5112118124961853,
    "25.down_proj": 0.498342901468277,
    "26.q_proj": 0.42560890316963196,
    "26.k_proj": 0.4341040253639221,
    "26.v_proj": 0.5099557042121887,
    "26.o_proj": 0.5735374689102173,
    "26.gate_proj": 0.49582305550575256,
    "26.up_proj": 0.5132369995117188,
    "26.down_proj": 0.4963560402393341,
    "27.q_proj": 0.47353821992874146,
    "27.k_proj": 0.4811285436153412,
    "27.v_proj": 0.5287732481956482,
    "27.o_proj": 0.5502170324325562,
    "27.gate_proj": 0.49853190779685974,
    "27.up_proj": 0.5145693421363831,
    "27.down_proj": 0.49317970871925354,
    "28.q_proj": 0.4549473822116852,
    "28.k_proj": 0.46495798230171204,
    "28.v_proj": 0.5245335102081299,
    "28.o_proj": 0.5447710156440735,
    "28.gate_proj": 0.4986145496368408,
    "28.up_proj": 0.513562023639679,
    "28.down_proj": 0.5004502534866333,
    "29.q_proj": 0.42228034138679504,
    "29.k_proj": 0.43027618527412415,
    "29.v_proj": 0.5161402225494385,
    "29.o_proj": 0.5447760224342346,
    "29.gate_proj": 0.49749407172203064,
    "29.up_proj": 0.5095394253730774,
    "29.down_proj": 0.5059615969657898,
    "30.q_proj": 0.45088398456573486,
    "30.k_proj": 0.4586084485054016,
    "30.v_proj": 0.5238063335418701,
    "30.o_proj": 0.5494594573974609,
    "30.gate_proj": 0.4911543130874634,
    "30.up_proj": 0.49852705001831055,
    "30.down_proj": 0.49798041582107544,
    "31.q_proj": 0.3175491690635681,
    "31.k_proj": 0.3234711289405823,
    "31.v_proj": 0.41444289684295654,
    "31.o_proj": 0.41846781969070435,
    "31.gate_proj": 0.4670276939868927,
    "31.up_proj": 0.47030943632125854,
    "31.down_proj": 0.347734272480011
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": null,
    "param_fraction": 0.6997563653659327,
    "mean_rel_err": 0.45483418411043075,
    "max_rel_err": 0.5735054612159729
  },
  "factorisation": {
    "max_rel_err": 0.0013921487843617797,
    "factor_params": 4531625984,
    "dense_params": 6476005376
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}