{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.2,
  "internal_ratio_kept": 0.8,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove20/finetune_v/merge.pt",
  "mean_rel_err": 0.39835471925990923,
  "max_rel_err": 0.5163237452507019,
  "param_fraction": 0.7998957658678757,
  "ranks": {
    "0.q_proj": 1939,
    "0.k_proj": 1922,
    "0.v_proj": 1834,
    "0.o_proj": 1856,
    "0.gate_proj": 2443,
    "0.up_proj": 2444,
    "0.down_proj": 2732,
    "1.q_proj": 1792,
    "1.k_proj": 1788,
    "1.v_proj": 1724,
    "1.o_proj": 1719,
    "1.gate_proj": 2415,
    "1.up_proj": 2414,
    "1.down_proj": 2841,
    "2.q_proj": 1635,
    "2.k_proj": 1635,
    "2.v_proj": 1609,
    "2.o_proj": 1572,
    "2.gate_proj": 2380,
    "2.up_proj": 2380,
    "2.down_proj": 2247,
    "3.q_proj": 1646,
    "3.k_proj": 1646,
    "3.v_proj": 1634,
    "3.o_proj": 1597,
    "3.gate_proj": 2385,
    "3.up_proj": 2385,
    "3.down_proj": 2294,
    "4.q_proj": 1682,
    "4.k_proj": 1684,
    "4.v_proj": 1670,
    "4.o_proj": 1641,
    "4.gate_proj": 2395,
    "4.up_proj": 2394,
    "4.down_proj": 2388,
    "5.q_proj": 1648,
    "5.k_proj": 1650,
    "5.v_proj": 1638,
    "5.o_proj": 1617,
    "5.gate_proj": 2388,
    "5.up_proj": 2387,
    "5.down_proj": 2338,
    "6.q_proj": 1675,
    "6.k_proj": 1675,
    "6.v_proj": 1671,
    "6.o_proj": 1653,
    "6.gate_proj": 2395,
    "6.up_proj": 2394,
    "6.down_proj": 2401,
    "7.q_proj": 1639,
    "7.k_proj": 1639,
    "7.v_proj": 1636,
    "7.o_proj": 1619,
    "7.gate_proj": 2388,
    "7.up_proj": 2386,
    "7.down_proj": 2344,
    "8.q_proj": 1653,
    "8.k_proj": 1654,
    "8.v_proj": 1652,
    "8.o_proj": 1641,
    "8.gate_proj": 2391,
    "8.up_proj": 2390,
    "8.down_proj": 2377,
    "9.q_proj": 1626,
    "9.k_proj": 1628,
    "9.v_proj": 1627,
    "9.o_proj": 1621,
    "9.gate_proj": 2386,
    "9.up_proj": 2385,
    "9.down_proj": 2335,
    "10.q_proj": 1613,
    "10.k_proj": 1615,
    "10.v_proj": 1614,
    "10.o_proj": 1613,
    "10.gate_proj": 2383,
    "10.up_proj": 2382,
    "10.down_proj": 2314,
    "11.q_proj": 1612,
    "11.k_proj": 1613,
    "11.v_proj": 1617,
    "11.o_proj": 1610,
    "11.gate_proj": 2383,
    "11.up_proj": 2382,
    "11.down_proj": 2314,
    "12.q_proj": 1605,
    "12.k_proj": 1607,
    "12.v_proj": 1610,
    "12.o_proj": 1606,
    "12.gate_proj": 2380,
    "12.up_proj": 2380,
    "12.down_proj": 2306,
    "13.q_proj": 1611,
    "13.k_proj": 1612,
    "13.v_proj": 1615,
    "13.o_proj": 1616,
    "13.gate_proj": 2382,
    "13.up_proj": 2382,
    "13.down_proj": 2321,
    "14.q_proj": 1605,
    "14.k_proj": 1607,
    "14.v_proj": 1609,
    "14.o_proj": 1610,
    "14.gate_proj": 2380,
    "14.up_proj": 2380,
    "14.down_proj": 2315,
    "15.q_proj": 1607,
    "15.k_proj": 1608,
    "15.v_proj": 1612,
    "15.o_proj": 1616,
    "15.gate_proj": 2382,
    "15.up_proj": 2382,
    "15.down_proj": 2324,
    "16.q_proj": 1607,
    "16.k_proj": 1610,
    "16.v_proj": 1612,
    "16.o_proj": 1621,
    "16.gate_proj": 2382,
    "16.up_proj": 2382,
    "16.down_proj": 2334,
    "17.q_proj": 1603,
    "17.k_proj": 1604,
    "17.v_proj": 1609,
    "17.o_proj": 1612,
    "17.gate_proj": 2381,
    "17.up_proj": 2381,
    "17.down_proj": 2330,
    "18.q_proj": 1600,
    "18.k_proj": 1601,
    "18.v_proj": 1608,
    "18.o_proj": 1608,
    "18.gate_proj": 2380,
    "18.up_proj": 2380,
    "18.down_proj": 2329,
    "19.q_proj": 1591,
    "19.k_proj": 1592,
    "19.v_proj": 1600,
    "19.o_proj": 1600,
    "19.gate_proj": 2378,
    "19.up_proj": 2378,
    "19.down_proj": 2316,
    "20.q_proj": 1590,
    "20.k_proj": 1591,
    "20.v_proj": 1598,
    "20.o_proj": 1605,
    "20.gate_proj": 2378,
    "20.up_proj": 2378,
    "20.down_proj": 2320,
    "21.q_proj": 1586,
    "21.k_proj": 1586,
    "21.v_proj": 1597,
    "21.o_proj": 1595,
    "21.gate_proj": 2378,
    "21.up_proj": 2378,
    "21.down_proj": 2315,
    "22.q_proj": 1586,
    "22.k_proj": 1586,
    "22.v_proj": 1597,
    "22.o_proj": 1603,
    "22.gate_proj": 2377,
    "22.up_proj": 2378,
    "22.down_proj": 2317,
    "23.q_proj": 1583,
    "23.k_proj": 1582,
    "23.v_proj": 1598,
    "23.o_proj": 1598,
    "23.gate_proj": 2377,
    "23.up_proj": 2377,
    "23.down_proj": 2314,
    "24.q_proj": 1581,
    "24.k_proj": 1582,
    "24.v_proj": 1595,
    "24.o_proj": 1602,
    "24.gate_proj": 2377,
    "24.up_proj": 2377,
    "24.down_proj": 2314,
    "25.q_proj": 1582,
    "25.k_proj": 1581,
    "25.v_proj": 1599,
    "25.o_proj": 1610,
    "25.gate_proj": 2377,
    "25.up_proj": 2377,
    "25.down_proj": 2320,
    "26.q_proj": 1583,
    "26.k_proj": 1583,
    "26.v_proj": 1599,
    "26.o_proj": 1611,
    "26.gate_proj": 2378,
    "26.up_proj": 2378,
    "26.down_proj": 2323,
    "27.q_proj": 1585,
    "27.k_proj": 1584,
    "27.v_proj": 1602,
    "27.o_proj": 1609,
    "27.gate_proj": 2378,
    "27.up_proj": 2379,
    "27.down_proj": 2331,
    "28.q_proj": 1586,
    "28.k_proj": 1586,
    "28.v_proj": 1605,
    "28.o_proj": 1619,
    "28.gate_proj": 2379,
    "28.up_proj": 2380,
    "28.down_proj": 2340,
    "29.q_proj": 1587,
    "29.k_proj": 1587,
    "29.v_proj": 1605,
    "29.o_proj": 1627,
    "29.gate_proj": 2380,
    "29.up_proj": 2382,
    "29.down_proj": 2350,
    "30.q_proj": 1604,
    "30.k_proj": 1603,
    "30.v_proj": 1625,
    "30.o_proj": 1640,
    "30.gate_proj": 2386,
    "30.up_proj": 2387,
    "30.down_proj": 2441,
    "31.q_proj": 1974,
    "31.k_proj": 1975,
    "31.v_proj": 1995,
    "31.o_proj": 2049,
    "31.gate_proj": 2474,
    "31.up_proj": 2477,
    "31.down_proj": 3231
  },
  "rel_err": {
    "0.q_proj": 0.13252992928028107,
    "0.k_proj": 0.10504010319709778,
    "0.v_proj": 0.23756639659404755,
    "0.o_proj": 0.2499486804008484,
    "0.gate_proj": 0.32737889885902405,
    "0.up_proj": 0.336822509765625,
    "0.down_proj": 0.33574700355529785,
    "1.q_proj": 0.09638579189777374,
    "1.k_proj": 0.0869532972574234,
    "1.v_proj": 0.3620476722717285,
    "1.o_proj": 0.2894635498523712,
    "1.gate_proj": 0.31223469972610474,
    "1.up_proj": 0.3540322184562683,
    "1.down_proj": 0.3827917277812958,
    "2.q_proj": 0.20203469693660736,
    "2.k_proj": 0.19166691601276398,
    "2.v_proj": 0.4135085940361023,
    "2.o_proj": 0.4775710105895996,
    "2.gate_proj": 0.38029783964157104,
    "2.up_proj": 0.40832197666168213,
    "2.down_proj": 0.4817647933959961,
    "3.q_proj": 0.28911951184272766,
    "3.k_proj": 0.2867107093334198,
    "3.v_proj": 0.42236825823783875,
    "3.o_proj": 0.4629066586494446,
    "3.gate_proj": 0.4270183742046356,
    "3.up_proj": 0.4446209967136383,
    "3.down_proj": 0.43980148434638977,
    "4.q_proj": 0.2749663293361664,
    "4.k_proj": 0.27662697434425354,
    "4.v_proj": 0.410868376493454,
    "4.o_proj": 0.45410028100013733,
    "4.gate_proj": 0.4263415038585663,
    "4.up_proj": 0.449140727519989,
    "4.down_proj": 0.4236817955970764,
    "5.q_proj": 0.2918066382408142,
    "5.k_proj": 0.2901315689086914,
    "5.v_proj": 0.4196918308734894,
    "5.o_proj": 0.46734270453453064,
    "5.gate_proj": 0.42409467697143555,
    "5.up_proj": 0.4514884650707245,
    "5.down_proj": 0.42588138580322266,
    "6.q_proj": 0.27811768651008606,
    "6.k_proj": 0.2831922173500061,
    "6.v_proj": 0.39679214358329773,
    "6.o_proj": 0.42545533180236816,
    "6.gate_proj": 0.41754117608070374,
    "6.up_proj": 0.44990018010139465,
    "6.down_proj": 0.4116297662258148,
    "7.q_proj": 0.2889671325683594,
    "7.k_proj": 0.2941109836101532,
    "7.v_proj": 0.3983648121356964,
    "7.o_proj": 0.45035600662231445,
    "7.gate_proj": 0.4161233603954315,
    "7.up_proj": 0.4515121281147003,
    "7.down_proj": 0.41752707958221436,
    "8.q_proj": 0.2945907711982727,
    "8.k_proj": 0.30408018827438354,
    "8.v_proj": 0.38267946243286133,
    "8.o_proj": 0.44145527482032776,
    "8.gate_proj": 0.41320326924324036,
    "8.up_proj": 0.4467734396457672,
    "8.down_proj": 0.406986266374588,
    "9.q_proj": 0.3136454224586487,
    "9.k_proj": 0.31721559166908264,
    "9.v_proj": 0.3859423100948334,
    "9.o_proj": 0.4510596990585327,
    "9.gate_proj": 0.4109175205230713,
    "9.up_proj": 0.4431936740875244,
    "9.down_proj": 0.4149797558784485,
    "10.q_proj": 0.29216355085372925,
    "10.k_proj": 0.3005751371383667,
    "10.v_proj": 0.3847027122974396,
    "10.o_proj": 0.4111638367176056,
    "10.gate_proj": 0.4067463278770447,
    "10.up_proj": 0.43945708870887756,
    "10.down_proj": 0.4159848392009735,
    "11.q_proj": 0.2826056480407715,
    "11.k_proj": 0.2936619520187378,
    "11.v_proj": 0.3708671033382416,
    "11.o_proj": 0.40357112884521484,
    "11.gate_proj": 0.40603089332580566,
    "11.up_proj": 0.4393159747123718,
    "11.down_proj": 0.42167046666145325,
    "12.q_proj": 0.31589677929878235,
    "12.k_proj": 0.3152616322040558,
    "12.v_proj": 0.38857823610305786,
    "12.o_proj": 0.4415610432624817,
    "12.gate_proj": 0.4102623462677002,
    "12.up_proj": 0.43980276584625244,
    "12.down_proj": 0.4229101836681366,
    "13.q_proj": 0.320039302110672,
    "13.k_proj": 0.33416274189949036,
    "13.v_proj": 0.41291582584381104,
    "13.o_proj": 0.4305266737937927,
    "13.gate_proj": 0.4123358428478241,
    "13.up_proj": 0.4423108994960785,
    "13.down_proj": 0.43072378635406494,
    "14.q_proj": 0.30624768137931824,
    "14.k_proj": 0.31049633026123047,
    "14.v_proj": 0.4073112905025482,
    "14.o_proj": 0.4292844831943512,
    "14.gate_proj": 0.41197630763053894,
    "14.up_proj": 0.44101130962371826,
    "14.down_proj": 0.4332798719406128,
    "15.q_proj": 0.32311615347862244,
    "15.k_proj": 0.325520783662796,
    "15.v_proj": 0.41313716769218445,
    "15.o_proj": 0.43440884351730347,
    "15.gate_proj": 0.415483683347702,
    "15.up_proj": 0.44479092955589294,
    "15.down_proj": 0.44361311197280884,
    "16.q_proj": 0.32134386897087097,
    "16.k_proj": 0.32373708486557007,
    "16.v_proj": 0.4383922517299652,
    "16.o_proj": 0.460711807012558,
    "16.gate_proj": 0.41429081559181213,
    "16.up_proj": 0.4417015612125397,
    "16.down_proj": 0.4521862864494324,
    "17.q_proj": 0.35595712065696716,
    "17.k_proj": 0.3545474112033844,
    "17.v_proj": 0.4520525336265564,
    "17.o_proj": 0.4732700288295746,
    "17.gate_proj": 0.41633883118629456,
    "17.up_proj": 0.4417755901813507,
    "17.down_proj": 0.4500233232975006,
    "18.q_proj": 0.37810665369033813,
    "18.k_proj": 0.3822422921657562,
    "18.v_proj": 0.45829951763153076,
    "18.o_proj": 0.5030460357666016,
    "18.gate_proj": 0.4236604869365692,
    "18.up_proj": 0.4474699795246124,
    "18.down_proj": 0.450743168592453,
    "19.q_proj": 0.3656962513923645,
    "19.k_proj": 0.3624645471572876,
    "19.v_proj": 0.4467334449291229,
    "19.o_proj": 0.4933334290981293,
    "19.gate_proj": 0.42747756838798523,
    "19.up_proj": 0.44855284690856934,
    "19.down_proj": 0.4535249173641205,
    "20.q_proj": 0.3567904829978943,
    "20.k_proj": 0.35822567343711853,
    "20.v_proj": 0.45949605107307434,
    "20.o_proj": 0.4911755919456482,
    "20.gate_proj": 0.42556217312812805,
    "20.up_proj": 0.44537943601608276,
    "20.down_proj": 0.4616205394268036,
    "21.q_proj": 0.3698049783706665,
    "21.k_proj": 0.371379554271698,
    "21.v_proj": 0.45389649271965027,
    "21.o_proj": 0.5087231397628784,
    "21.gate_proj": 0.42624449729919434,
    "21.up_proj": 0.44437286257743835,
    "21.down_proj": 0.45840954780578613,
    "22.q_proj": 0.37954530119895935,
    "22.k_proj": 0.38333773612976074,
    "22.v_proj": 0.463877409696579,
    "22.o_proj": 0.5026487112045288,
    "22.gate_proj": 0.4261413812637329,
    "22.up_proj": 0.443158358335495,
    "22.down_proj": 0.4550263583660126,
    "23.q_proj": 0.4084702432155609,
    "23.k_proj": 0.40841731429100037,
    "23.v_proj": 0.4616864025592804,
    "23.o_proj": 0.4950319528579712,
    "23.gate_proj": 0.4287131428718567,
    "23.up_proj": 0.444684237241745,
    "23.down_proj": 0.44846802949905396,
    "24.q_proj": 0.36004146933555603,
    "24.k_proj": 0.363835871219635,
    "24.v_proj": 0.45619717240333557,
    "24.o_proj": 0.4993390142917633,
    "24.gate_proj": 0.4288971722126007,
    "24.up_proj": 0.4449731707572937,
    "24.down_proj": 0.44221973419189453,
    "25.q_proj": 0.39867693185806274,
    "25.k_proj": 0.40591371059417725,
    "25.v_proj": 0.4620579481124878,
    "25.o_proj": 0.491049200296402,
    "25.gate_proj": 0.4311861991882324,
    "25.up_proj": 0.44645044207572937,
    "25.down_proj": 0.43493691086769104,
    "26.q_proj": 0.37144312262535095,
    "26.k_proj": 0.37911906838417053,
    "26.v_proj": 0.45244038105010986,
    "26.o_proj": 0.5163237452507019,
    "26.gate_proj": 0.4322204887866974,
    "26.up_proj": 0.44827595353126526,
    "26.down_proj": 0.4333139657974243,
    "27.q_proj": 0.4169563055038452,
    "27.k_proj": 0.4245080351829529,
    "27.v_proj": 0.46958377957344055,
    "27.o_proj": 0.49201327562332153,
    "27.gate_proj": 0.4350823760032654,
    "27.up_proj": 0.44976213574409485,
    "27.down_proj": 0.43056654930114746,
    "28.q_proj": 0.399061918258667,
    "28.k_proj": 0.4083995521068573,
    "28.v_proj": 0.46559157967567444,
    "28.o_proj": 0.4871852695941925,
    "28.gate_proj": 0.4352478086948395,
    "28.up_proj": 0.4488693177700043,
    "28.down_proj": 0.4368627965450287,
    "29.q_proj": 0.36882483959198,
    "29.k_proj": 0.37630507349967957,
    "29.v_proj": 0.4574759304523468,
    "29.o_proj": 0.48539915680885315,
    "29.gate_proj": 0.43431898951530457,
    "29.up_proj": 0.4452069103717804,
    "29.down_proj": 0.4417956471443176,
    "30.q_proj": 0.3955976068973541,
    "30.k_proj": 0.4027300477027893,
    "30.v_proj": 0.4643517732620239,
    "30.o_proj": 0.4904117286205292,
    "30.gate_proj": 0.42837339639663696,
    "30.up_proj": 0.4352115988731384,
    "30.down_proj": 0.430807888507843,
    "31.q_proj": 0.2638401985168457,
    "31.k_proj": 0.26833730936050415,
    "31.v_proj": 0.34875962138175964,
    "31.o_proj": 0.35200828313827515,
    "31.gate_proj": 0.40333712100982666,
    "31.up_proj": 0.4065763056278229,
    "31.down_proj": 0.26060518622398376
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": null,
    "param_fraction": 0.7993212685066183,
    "mean_rel_err": 0.39799320328581544,
    "max_rel_err": 0.516345202922821
  },
  "factorisation": {
    "max_rel_err": 0.0013380160089582205,
    "factor_params": 5180129280,
    "dense_params": 6476005376
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}