{
  "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.3,
  "internal_ratio_kept": 0.7,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove30/finetune_v/merge.pt",
  "mean_rel_err": 0.46171893551945686,
  "max_rel_err": 0.5801312327384949,
  "param_fraction": 0.7003361628605769,
  "ranks": {
    "0.q_proj": 1613,
    "0.k_proj": 679,
    "0.v_proj": 617,
    "0.o_proj": 1473,
    "0.gate_proj": 2352,
    "0.up_proj": 2352,
    "0.down_proj": 2382,
    "1.q_proj": 1619,
    "1.k_proj": 690,
    "1.v_proj": 650,
    "1.o_proj": 1517,
    "1.gate_proj": 2352,
    "1.up_proj": 2352,
    "1.down_proj": 2765,
    "2.q_proj": 1319,
    "2.k_proj": 523,
    "2.v_proj": 513,
    "2.o_proj": 1270,
    "2.gate_proj": 2352,
    "2.up_proj": 2352,
    "2.down_proj": 1892,
    "3.q_proj": 1316,
    "3.k_proj": 520,
    "3.v_proj": 508,
    "3.o_proj": 1257,
    "3.gate_proj": 2352,
    "3.up_proj": 2352,
    "3.down_proj": 1895,
    "4.q_proj": 1314,
    "4.k_proj": 520,
    "4.v_proj": 512,
    "4.o_proj": 1264,
    "4.gate_proj": 2352,
    "4.up_proj": 2352,
    "4.down_proj": 1919,
    "5.q_proj": 1313,
    "5.k_proj": 520,
    "5.v_proj": 513,
    "5.o_proj": 1274,
    "5.gate_proj": 2352,
    "5.up_proj": 2352,
    "5.down_proj": 1938,
    "6.q_proj": 1315,
    "6.k_proj": 522,
    "6.v_proj": 514,
    "6.o_proj": 1285,
    "6.gate_proj": 2352,
    "6.up_proj": 2352,
    "6.down_proj": 1953,
    "7.q_proj": 1313,
    "7.k_proj": 522,
    "7.v_proj": 515,
    "7.o_proj": 1295,
    "7.gate_proj": 2352,
    "7.up_proj": 2352,
    "7.down_proj": 1969,
    "8.q_proj": 1310,
    "8.k_proj": 519,
    "8.v_proj": 515,
    "8.o_proj": 1294,
    "8.gate_proj": 2352,
    "8.up_proj": 2352,
    "8.down_proj": 1965,
    "9.q_proj": 1308,
    "9.k_proj": 518,
    "9.v_proj": 516,
    "9.o_proj": 1294,
    "9.gate_proj": 2352,
    "9.up_proj": 2352,
    "9.down_proj": 1973,
    "10.q_proj": 1309,
    "10.k_proj": 518,
    "10.v_proj": 516,
    "10.o_proj": 1305,
    "10.gate_proj": 2352,
    "10.up_proj": 2352,
    "10.down_proj": 1974,
    "11.q_proj": 1307,
    "11.k_proj": 518,
    "11.v_proj": 518,
    "11.o_proj": 1308,
    "11.gate_proj": 2352,
    "11.up_proj": 2352,
    "11.down_proj": 1977,
    "12.q_proj": 1315,
    "12.k_proj": 523,
    "12.v_proj": 522,
    "12.o_proj": 1318,
    "12.gate_proj": 2352,
    "12.up_proj": 2352,
    "12.down_proj": 1999,
    "13.q_proj": 1311,
    "13.k_proj": 521,
    "13.v_proj": 520,
    "13.o_proj": 1315,
    "13.gate_proj": 2352,
    "13.up_proj": 2352,
    "13.down_proj": 1999,
    "14.q_proj": 1321,
    "14.k_proj": 526,
    "14.v_proj": 529,
    "14.o_proj": 1328,
    "14.gate_proj": 2352,
    "14.up_proj": 2352,
    "14.down_proj": 2032,
    "15.q_proj": 1336,
    "15.k_proj": 535,
    "15.v_proj": 538,
    "15.o_proj": 1350,
    "15.gate_proj": 2352,
    "15.up_proj": 2352,
    "15.down_proj": 2073,
    "16.q_proj": 1343,
    "16.k_proj": 538,
    "16.v_proj": 540,
    "16.o_proj": 1355,
    "16.gate_proj": 2352,
    "16.up_proj": 2352,
    "16.down_proj": 2099,
    "17.q_proj": 1370,
    "17.k_proj": 552,
    "17.v_proj": 555,
    "17.o_proj": 1386,
    "17.gate_proj": 2352,
    "17.up_proj": 2352,
    "17.down_proj": 2160,
    "18.q_proj": 1369,
    "18.k_proj": 551,
    "18.v_proj": 556,
    "18.o_proj": 1390,
    "18.gate_proj": 2352,
    "18.up_proj": 2352,
    "18.down_proj": 2176,
    "19.q_proj": 1351,
    "19.k_proj": 542,
    "19.v_proj": 547,
    "19.o_proj": 1376,
    "19.gate_proj": 2352,
    "19.up_proj": 2352,
    "19.down_proj": 2151,
    "20.q_proj": 1318,
    "20.k_proj": 524,
    "20.v_proj": 530,
    "20.o_proj": 1337,
    "20.gate_proj": 2352,
    "20.up_proj": 2352,
    "20.down_proj": 2083,
    "21.q_proj": 1310,
    "21.k_proj": 518,
    "21.v_proj": 526,
    "21.o_proj": 1325,
    "21.gate_proj": 2352,
    "21.up_proj": 2352,
    "21.down_proj": 2067,
    "22.q_proj": 1297,
    "22.k_proj": 510,
    "22.v_proj": 519,
    "22.o_proj": 1308,
    "22.gate_proj": 2352,
    "22.up_proj": 2352,
    "22.down_proj": 2040,
    "23.q_proj": 1298,
    "23.k_proj": 511,
    "23.v_proj": 519,
    "23.o_proj": 1317,
    "23.gate_proj": 2352,
    "23.up_proj": 2352,
    "23.down_proj": 2044,
    "24.q_proj": 1297,
    "24.k_proj": 510,
    "24.v_proj": 519,
    "24.o_proj": 1321,
    "24.gate_proj": 2352,
    "24.up_proj": 2352,
    "24.down_proj": 2045,
    "25.q_proj": 1300,
    "25.k_proj": 512,
    "25.v_proj": 523,
    "25.o_proj": 1330,
    "25.gate_proj": 2352,
    "25.up_proj": 2352,
    "25.down_proj": 2059,
    "26.q_proj": 1306,
    "26.k_proj": 515,
    "26.v_proj": 528,
    "26.o_proj": 1340,
    "26.gate_proj": 2352,
    "26.up_proj": 2352,
    "26.down_proj": 2076,
    "27.q_proj": 1311,
    "27.k_proj": 518,
    "27.v_proj": 530,
    "27.o_proj": 1341,
    "27.gate_proj": 2352,
    "27.up_proj": 2352,
    "27.down_proj": 2090,
    "28.q_proj": 1328,
    "28.k_proj": 528,
    "28.v_proj": 541,
    "28.o_proj": 1366,
    "28.gate_proj": 2352,
    "28.up_proj": 2352,
    "28.down_proj": 2144,
    "29.q_proj": 1339,
    "29.k_proj": 533,
    "29.v_proj": 548,
    "29.o_proj": 1392,
    "29.gate_proj": 2352,
    "29.up_proj": 2352,
    "29.down_proj": 2170,
    "30.q_proj": 1359,
    "30.k_proj": 543,
    "30.v_proj": 561,
    "30.o_proj": 1418,
    "30.gate_proj": 2352,
    "30.up_proj": 2352,
    "30.down_proj": 2230,
    "31.q_proj": 1473,
    "31.k_proj": 605,
    "31.v_proj": 626,
    "31.o_proj": 1559,
    "31.gate_proj": 2352,
    "31.up_proj": 2352,
    "31.down_proj": 2541
  },
  "rel_err": {
    "0.q_proj": 0.08783026784658432,
    "0.k_proj": 0.076280377805233,
    "0.v_proj": 0.28279638290405273,
    "0.o_proj": 0.3450925052165985,
    "0.gate_proj": 0.3139413893222809,
    "0.up_proj": 0.34659841656684875,
    "0.down_proj": 0.4610687792301178,
    "1.q_proj": 0.14944231510162354,
    "1.k_proj": 0.08638541400432587,
    "1.v_proj": 0.42672011256217957,
    "1.o_proj": 0.4614779055118561,
    "1.gate_proj": 0.37372663617134094,
    "1.up_proj": 0.4030474126338959,
    "1.down_proj": 0.4160686731338501,
    "2.q_proj": 0.40826842188835144,
    "2.k_proj": 0.38183024525642395,
    "2.v_proj": 0.5488172769546509,
    "2.o_proj": 0.5261715054512024,
    "2.gate_proj": 0.44494783878326416,
    "2.up_proj": 0.46598997712135315,
    "2.down_proj": 0.5656055808067322,
    "3.q_proj": 0.37075719237327576,
    "3.k_proj": 0.3103460371494293,
    "3.v_proj": 0.5801312327384949,
    "3.o_proj": 0.570527195930481,
    "3.gate_proj": 0.45506104826927185,
    "3.up_proj": 0.48117780685424805,
    "3.down_proj": 0.5562737584114075,
    "4.q_proj": 0.3778054118156433,
    "4.k_proj": 0.33330559730529785,
    "4.v_proj": 0.5392099618911743,
    "4.o_proj": 0.5162532329559326,
    "4.gate_proj": 0.4575117528438568,
    "4.up_proj": 0.49015605449676514,
    "4.down_proj": 0.5629571676254272,
    "5.q_proj": 0.4073948264122009,
    "5.k_proj": 0.3724959194660187,
    "5.v_proj": 0.5715569853782654,
    "5.o_proj": 0.549954354763031,
    "5.gate_proj": 0.45217984914779663,
    "5.up_proj": 0.49232661724090576,
    "5.down_proj": 0.5492331981658936,
    "6.q_proj": 0.39479202032089233,
    "6.k_proj": 0.3444615304470062,
    "6.v_proj": 0.5562271475791931,
    "6.o_proj": 0.5352969169616699,
    "6.gate_proj": 0.46358194947242737,
    "6.up_proj": 0.49562251567840576,
    "6.down_proj": 0.5394559502601624,
    "7.q_proj": 0.3770744204521179,
    "7.k_proj": 0.32297107577323914,
    "7.v_proj": 0.5488503575325012,
    "7.o_proj": 0.5085676908493042,
    "7.gate_proj": 0.4547872841358185,
    "7.up_proj": 0.48921388387680054,
    "7.down_proj": 0.5321547985076904,
    "8.q_proj": 0.3834538757801056,
    "8.k_proj": 0.32134145498275757,
    "8.v_proj": 0.5088310837745667,
    "8.o_proj": 0.4981762766838074,
    "8.gate_proj": 0.4569913148880005,
    "8.up_proj": 0.48666849732398987,
    "8.down_proj": 0.5265653729438782,
    "9.q_proj": 0.3789723515510559,
    "9.k_proj": 0.3307971656322479,
    "9.v_proj": 0.4929361045360565,
    "9.o_proj": 0.5009838342666626,
    "9.gate_proj": 0.453980952501297,
    "9.up_proj": 0.4856830835342407,
    "9.down_proj": 0.5116001963615417,
    "10.q_proj": 0.3785707354545593,
    "10.k_proj": 0.3373726010322571,
    "10.v_proj": 0.4934564232826233,
    "10.o_proj": 0.5008266568183899,
    "10.gate_proj": 0.4539565443992615,
    "10.up_proj": 0.48363447189331055,
    "10.down_proj": 0.5184407234191895,
    "11.q_proj": 0.3743845522403717,
    "11.k_proj": 0.3172908425331116,
    "11.v_proj": 0.465146541595459,
    "11.o_proj": 0.5011467337608337,
    "11.gate_proj": 0.4532534182071686,
    "11.up_proj": 0.48237940669059753,
    "11.down_proj": 0.516564130783081,
    "12.q_proj": 0.36303064227104187,
    "12.k_proj": 0.30660074949264526,
    "12.v_proj": 0.5052470564842224,
    "12.o_proj": 0.4829132556915283,
    "12.gate_proj": 0.44975727796554565,
    "12.up_proj": 0.47766903042793274,
    "12.down_proj": 0.5123953223228455,
    "13.q_proj": 0.38796108961105347,
    "13.k_proj": 0.3449362516403198,
    "13.v_proj": 0.5052615404129028,
    "13.o_proj": 0.48829060792922974,
    "13.gate_proj": 0.4554385840892792,
    "13.up_proj": 0.47726166248321533,
    "13.down_proj": 0.5091195106506348,
    "14.q_proj": 0.36075472831726074,
    "14.k_proj": 0.32303425669670105,
    "14.v_proj": 0.5109939575195312,
    "14.o_proj": 0.49242928624153137,
    "14.gate_proj": 0.4523519277572632,
    "14.up_proj": 0.48172876238822937,
    "14.down_proj": 0.532478928565979,
    "15.q_proj": 0.35985133051872253,
    "15.k_proj": 0.30060359835624695,
    "15.v_proj": 0.5368859767913818,
    "15.o_proj": 0.49520623683929443,
    "15.gate_proj": 0.44976216554641724,
    "15.up_proj": 0.47884488105773926,
    "15.down_proj": 0.527091383934021,
    "16.q_proj": 0.3671673536300659,
    "16.k_proj": 0.319665789604187,
    "16.v_proj": 0.5345363020896912,
    "16.o_proj": 0.48225852847099304,
    "16.gate_proj": 0.4529358446598053,
    "16.up_proj": 0.486669659614563,
    "16.down_proj": 0.5261300206184387,
    "17.q_proj": 0.39087575674057007,
    "17.k_proj": 0.3505445420742035,
    "17.v_proj": 0.5095366835594177,
    "17.o_proj": 0.4995217025279999,
    "17.gate_proj": 0.46473342180252075,
    "17.up_proj": 0.496961385011673,
    "17.down_proj": 0.5169269442558289,
    "18.q_proj": 0.3686104118824005,
    "18.k_proj": 0.3233947157859802,
    "18.v_proj": 0.5597348213195801,
    "18.o_proj": 0.47679799795150757,
    "18.gate_proj": 0.4632626473903656,
    "18.up_proj": 0.49456414580345154,
    "18.down_proj": 0.5329453349113464,
    "19.q_proj": 0.4161927103996277,
    "19.k_proj": 0.3401549458503723,
    "19.v_proj": 0.5778656601905823,
    "19.o_proj": 0.49888941645622253,
    "19.gate_proj": 0.4682750403881073,
    "19.up_proj": 0.4958019256591797,
    "19.down_proj": 0.548184871673584,
    "20.q_proj": 0.438224732875824,
    "20.k_proj": 0.380808025598526,
    "20.v_proj": 0.5477811098098755,
    "20.o_proj": 0.5176671147346497,
    "20.gate_proj": 0.4731369912624359,
    "20.up_proj": 0.49586376547813416,
    "20.down_proj": 0.5577734112739563,
    "21.q_proj": 0.4684925973415375,
    "21.k_proj": 0.40865975618362427,
    "21.v_proj": 0.522392213344574,
    "21.o_proj": 0.5564069747924805,
    "21.gate_proj": 0.4738660454750061,
    "21.up_proj": 0.49745243787765503,
    "21.down_proj": 0.5537582039833069,
    "22.q_proj": 0.5066864490509033,
    "22.k_proj": 0.4351329803466797,
    "22.v_proj": 0.5337282419204712,
    "22.o_proj": 0.5765807628631592,
    "22.gate_proj": 0.47878557443618774,
    "22.up_proj": 0.4989929795265198,
    "22.down_proj": 0.5528429746627808,
    "23.q_proj": 0.49622949957847595,
    "23.k_proj": 0.4328460097312927,
    "23.v_proj": 0.5585790872573853,
    "23.o_proj": 0.554423451423645,
    "23.gate_proj": 0.48155754804611206,
    "23.up_proj": 0.50072181224823,
    "23.down_proj": 0.5464255213737488,
    "24.q_proj": 0.5014330148696899,
    "24.k_proj": 0.45270365476608276,
    "24.v_proj": 0.569066047668457,
    "24.o_proj": 0.5753594040870667,
    "24.gate_proj": 0.4825673997402191,
    "24.up_proj": 0.5023998618125916,
    "24.down_proj": 0.5373293161392212,
    "25.q_proj": 0.5077980756759644,
    "25.k_proj": 0.4391773045063019,
    "25.v_proj": 0.5462435483932495,
    "25.o_proj": 0.5570780634880066,
    "25.gate_proj": 0.4843604266643524,
    "25.up_proj": 0.5035067796707153,
    "25.down_proj": 0.5260589122772217,
    "26.q_proj": 0.5026710033416748,
    "26.k_proj": 0.4264448285102844,
    "26.v_proj": 0.5314569473266602,
    "26.o_proj": 0.537655234336853,
    "26.gate_proj": 0.4852832555770874,
    "26.up_proj": 0.5033767223358154,
    "26.down_proj": 0.5124109387397766,
    "27.q_proj": 0.5120652914047241,
    "27.k_proj": 0.4639483690261841,
    "27.v_proj": 0.5471333265304565,
    "27.o_proj": 0.5303576588630676,
    "27.gate_proj": 0.4886907935142517,
    "27.up_proj": 0.5048775672912598,
    "27.down_proj": 0.5119288563728333,
    "28.q_proj": 0.4723275899887085,
    "28.k_proj": 0.39252132177352905,
    "28.v_proj": 0.5004826188087463,
    "28.o_proj": 0.5055510997772217,
    "28.gate_proj": 0.4891432821750641,
    "28.up_proj": 0.5027062296867371,
    "28.down_proj": 0.5162467956542969,
    "29.q_proj": 0.45425698161125183,
    "29.k_proj": 0.39296162128448486,
    "29.v_proj": 0.529412031173706,
    "29.o_proj": 0.4873389005661011,
    "29.gate_proj": 0.4842919409275055,
    "29.up_proj": 0.4974598288536072,
    "29.down_proj": 0.5207532644271851,
    "30.q_proj": 0.4333595335483551,
    "30.k_proj": 0.35803964734077454,
    "30.v_proj": 0.4631883203983307,
    "30.o_proj": 0.47360387444496155,
    "30.gate_proj": 0.4840942919254303,
    "30.up_proj": 0.48931118845939636,
    "30.down_proj": 0.5056915283203125,
    "31.q_proj": 0.38473233580589294,
    "31.k_proj": 0.3066442310810089,
    "31.v_proj": 0.4090970456600189,
    "31.o_proj": 0.3971359431743622,
    "31.gate_proj": 0.4917009174823761,
    "31.up_proj": 0.4686800241470337,
    "31.down_proj": 0.4488861858844757
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": "shape",
    "param_fraction": 0.7003361628605769,
    "mean_rel_err": 0.46043376145618303,
    "max_rel_err": 0.5793784856796265
  },
  "factorisation": {
    "max_rel_err": 0.00140141521114856,
    "factor_params": 4887871488,
    "dense_params": 6979321856
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}