{
  "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.5,
  "internal_ratio_kept": 0.5,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove50/finetune_v/merge.pt",
  "mean_rel_err": 0.5888538884464651,
  "max_rel_err": 0.7068923115730286,
  "param_fraction": 0.5002723106971154,
  "ranks": {
    "0.q_proj": 1152,
    "0.k_proj": 485,
    "0.v_proj": 442,
    "0.o_proj": 1052,
    "0.gate_proj": 1680,
    "0.up_proj": 1680,
    "0.down_proj": 1702,
    "1.q_proj": 1157,
    "1.k_proj": 493,
    "1.v_proj": 465,
    "1.o_proj": 1084,
    "1.gate_proj": 1680,
    "1.up_proj": 1680,
    "1.down_proj": 1976,
    "2.q_proj": 942,
    "2.k_proj": 374,
    "2.v_proj": 367,
    "2.o_proj": 907,
    "2.gate_proj": 1680,
    "2.up_proj": 1680,
    "2.down_proj": 1351,
    "3.q_proj": 940,
    "3.k_proj": 372,
    "3.v_proj": 363,
    "3.o_proj": 898,
    "3.gate_proj": 1680,
    "3.up_proj": 1680,
    "3.down_proj": 1353,
    "4.q_proj": 938,
    "4.k_proj": 371,
    "4.v_proj": 366,
    "4.o_proj": 903,
    "4.gate_proj": 1680,
    "4.up_proj": 1680,
    "4.down_proj": 1371,
    "5.q_proj": 938,
    "5.k_proj": 372,
    "5.v_proj": 366,
    "5.o_proj": 910,
    "5.gate_proj": 1680,
    "5.up_proj": 1680,
    "5.down_proj": 1384,
    "6.q_proj": 939,
    "6.k_proj": 373,
    "6.v_proj": 368,
    "6.o_proj": 917,
    "6.gate_proj": 1680,
    "6.up_proj": 1680,
    "6.down_proj": 1395,
    "7.q_proj": 938,
    "7.k_proj": 373,
    "7.v_proj": 368,
    "7.o_proj": 925,
    "7.gate_proj": 1680,
    "7.up_proj": 1680,
    "7.down_proj": 1406,
    "8.q_proj": 935,
    "8.k_proj": 371,
    "8.v_proj": 368,
    "8.o_proj": 924,
    "8.gate_proj": 1680,
    "8.up_proj": 1680,
    "8.down_proj": 1403,
    "9.q_proj": 934,
    "9.k_proj": 370,
    "9.v_proj": 369,
    "9.o_proj": 924,
    "9.gate_proj": 1680,
    "9.up_proj": 1680,
    "9.down_proj": 1409,
    "10.q_proj": 935,
    "10.k_proj": 371,
    "10.v_proj": 369,
    "10.o_proj": 932,
    "10.gate_proj": 1680,
    "10.up_proj": 1680,
    "10.down_proj": 1410,
    "11.q_proj": 933,
    "11.k_proj": 370,
    "11.v_proj": 370,
    "11.o_proj": 934,
    "11.gate_proj": 1680,
    "11.up_proj": 1680,
    "11.down_proj": 1412,
    "12.q_proj": 939,
    "12.k_proj": 374,
    "12.v_proj": 373,
    "12.o_proj": 941,
    "12.gate_proj": 1680,
    "12.up_proj": 1680,
    "12.down_proj": 1428,
    "13.q_proj": 936,
    "13.k_proj": 372,
    "13.v_proj": 372,
    "13.o_proj": 939,
    "13.gate_proj": 1680,
    "13.up_proj": 1680,
    "13.down_proj": 1428,
    "14.q_proj": 944,
    "14.k_proj": 376,
    "14.v_proj": 378,
    "14.o_proj": 949,
    "14.gate_proj": 1680,
    "14.up_proj": 1680,
    "14.down_proj": 1451,
    "15.q_proj": 955,
    "15.k_proj": 383,
    "15.v_proj": 385,
    "15.o_proj": 964,
    "15.gate_proj": 1680,
    "15.up_proj": 1680,
    "15.down_proj": 1481,
    "16.q_proj": 959,
    "16.k_proj": 385,
    "16.v_proj": 387,
    "16.o_proj": 968,
    "16.gate_proj": 1680,
    "16.up_proj": 1680,
    "16.down_proj": 1500,
    "17.q_proj": 978,
    "17.k_proj": 395,
    "17.v_proj": 397,
    "17.o_proj": 990,
    "17.gate_proj": 1680,
    "17.up_proj": 1680,
    "17.down_proj": 1543,
    "18.q_proj": 978,
    "18.k_proj": 394,
    "18.v_proj": 398,
    "18.o_proj": 993,
    "18.gate_proj": 1680,
    "18.up_proj": 1680,
    "18.down_proj": 1555,
    "19.q_proj": 965,
    "19.k_proj": 388,
    "19.v_proj": 391,
    "19.o_proj": 983,
    "19.gate_proj": 1680,
    "19.up_proj": 1680,
    "19.down_proj": 1537,
    "20.q_proj": 942,
    "20.k_proj": 375,
    "20.v_proj": 379,
    "20.o_proj": 955,
    "20.gate_proj": 1680,
    "20.up_proj": 1680,
    "20.down_proj": 1488,
    "21.q_proj": 936,
    "21.k_proj": 371,
    "21.v_proj": 376,
    "21.o_proj": 947,
    "21.gate_proj": 1680,
    "21.up_proj": 1680,
    "21.down_proj": 1477,
    "22.q_proj": 926,
    "22.k_proj": 365,
    "22.v_proj": 371,
    "22.o_proj": 934,
    "22.gate_proj": 1680,
    "22.up_proj": 1680,
    "22.down_proj": 1457,
    "23.q_proj": 927,
    "23.k_proj": 365,
    "23.v_proj": 371,
    "23.o_proj": 940,
    "23.gate_proj": 1680,
    "23.up_proj": 1680,
    "23.down_proj": 1460,
    "24.q_proj": 926,
    "24.k_proj": 365,
    "24.v_proj": 371,
    "24.o_proj": 944,
    "24.gate_proj": 1680,
    "24.up_proj": 1680,
    "24.down_proj": 1461,
    "25.q_proj": 929,
    "25.k_proj": 366,
    "25.v_proj": 374,
    "25.o_proj": 951,
    "25.gate_proj": 1680,
    "25.up_proj": 1680,
    "25.down_proj": 1471,
    "26.q_proj": 933,
    "26.k_proj": 368,
    "26.v_proj": 377,
    "26.o_proj": 957,
    "26.gate_proj": 1680,
    "26.up_proj": 1680,
    "26.down_proj": 1483,
    "27.q_proj": 936,
    "27.k_proj": 370,
    "27.v_proj": 379,
    "27.o_proj": 958,
    "27.gate_proj": 1680,
    "27.up_proj": 1680,
    "27.down_proj": 1493,
    "28.q_proj": 949,
    "28.k_proj": 378,
    "28.v_proj": 387,
    "28.o_proj": 976,
    "28.gate_proj": 1680,
    "28.up_proj": 1680,
    "28.down_proj": 1532,
    "29.q_proj": 957,
    "29.k_proj": 381,
    "29.v_proj": 392,
    "29.o_proj": 994,
    "29.gate_proj": 1680,
    "29.up_proj": 1680,
    "29.down_proj": 1550,
    "30.q_proj": 971,
    "30.k_proj": 389,
    "30.v_proj": 401,
    "30.o_proj": 1013,
    "30.gate_proj": 1680,
    "30.up_proj": 1680,
    "30.down_proj": 1593,
    "31.q_proj": 1053,
    "31.k_proj": 433,
    "31.v_proj": 448,
    "31.o_proj": 1114,
    "31.gate_proj": 1680,
    "31.up_proj": 1680,
    "31.down_proj": 1816
  },
  "rel_err": {
    "0.q_proj": 0.10284276306629181,
    "0.k_proj": 0.09011625498533249,
    "0.v_proj": 0.4146844744682312,
    "0.o_proj": 0.4441743791103363,
    "0.gate_proj": 0.4323641359806061,
    "0.up_proj": 0.4812891185283661,
    "0.down_proj": 0.6020375490188599,
    "1.q_proj": 0.1907663345336914,
    "1.k_proj": 0.13933585584163666,
    "1.v_proj": 0.5992577075958252,
    "1.o_proj": 0.5850644111633301,
    "1.gate_proj": 0.5018978714942932,
    "1.up_proj": 0.5401790142059326,
    "1.down_proj": 0.5769923329353333,
    "2.q_proj": 0.5321782827377319,
    "2.k_proj": 0.5425702929496765,
    "2.v_proj": 0.682027280330658,
    "2.o_proj": 0.6367513537406921,
    "2.gate_proj": 0.5818910598754883,
    "2.up_proj": 0.6077799201011658,
    "2.down_proj": 0.681037187576294,
    "3.q_proj": 0.48435133695602417,
    "3.k_proj": 0.4475176930427551,
    "3.v_proj": 0.7068923115730286,
    "3.o_proj": 0.677703857421875,
    "3.gate_proj": 0.5896062254905701,
    "3.up_proj": 0.6225286722183228,
    "3.down_proj": 0.6739526987075806,
    "4.q_proj": 0.4879390597343445,
    "4.k_proj": 0.4703967571258545,
    "4.v_proj": 0.6710249185562134,
    "4.o_proj": 0.6244789361953735,
    "4.gate_proj": 0.5910874605178833,
    "4.up_proj": 0.6321871876716614,
    "4.down_proj": 0.6802555918693542,
    "5.q_proj": 0.5210317969322205,
    "5.k_proj": 0.5119411945343018,
    "5.v_proj": 0.6989429593086243,
    "5.o_proj": 0.6632655262947083,
    "5.gate_proj": 0.5846965312957764,
    "5.up_proj": 0.6342030763626099,
    "5.down_proj": 0.6687344908714294,
    "6.q_proj": 0.505556583404541,
    "6.k_proj": 0.48692795634269714,
    "6.v_proj": 0.6872745752334595,
    "6.o_proj": 0.646428108215332,
    "6.gate_proj": 0.5995981693267822,
    "6.up_proj": 0.6370501518249512,
    "6.down_proj": 0.6577861309051514,
    "7.q_proj": 0.4832315444946289,
    "7.k_proj": 0.4549109935760498,
    "7.v_proj": 0.6812716722488403,
    "7.o_proj": 0.6252717971801758,
    "7.gate_proj": 0.5871505737304688,
    "7.up_proj": 0.6294843554496765,
    "7.down_proj": 0.654327392578125,
    "8.q_proj": 0.49038824439048767,
    "8.k_proj": 0.4526974856853485,
    "8.v_proj": 0.6452953219413757,
    "8.o_proj": 0.6178920269012451,
    "8.gate_proj": 0.5910043716430664,
    "8.up_proj": 0.6263056993484497,
    "8.down_proj": 0.647315502166748,
    "9.q_proj": 0.48501771688461304,
    "9.k_proj": 0.4705408811569214,
    "9.v_proj": 0.6292407512664795,
    "9.o_proj": 0.6174561977386475,
    "9.gate_proj": 0.5865891575813293,
    "9.up_proj": 0.6233162879943848,
    "9.down_proj": 0.6318148970603943,
    "10.q_proj": 0.4836317300796509,
    "10.k_proj": 0.4697504937648773,
    "10.v_proj": 0.6263429522514343,
    "10.o_proj": 0.6223848462104797,
    "10.gate_proj": 0.5854211449623108,
    "10.up_proj": 0.6197803020477295,
    "10.down_proj": 0.6372787356376648,
    "11.q_proj": 0.4799323081970215,
    "11.k_proj": 0.4386899471282959,
    "11.v_proj": 0.6085149049758911,
    "11.o_proj": 0.6194866895675659,
    "11.gate_proj": 0.585040271282196,
    "11.up_proj": 0.6186556816101074,
    "11.down_proj": 0.6358811259269714,
    "12.q_proj": 0.46628451347351074,
    "12.k_proj": 0.4467337131500244,
    "12.v_proj": 0.6398075222969055,
    "12.o_proj": 0.602120578289032,
    "12.gate_proj": 0.5809478759765625,
    "12.up_proj": 0.6136550307273865,
    "12.down_proj": 0.6333811283111572,
    "13.q_proj": 0.49495774507522583,
    "13.k_proj": 0.4739055931568146,
    "13.v_proj": 0.6350283026695251,
    "13.o_proj": 0.6104384660720825,
    "13.gate_proj": 0.5884830951690674,
    "13.up_proj": 0.6139264702796936,
    "13.down_proj": 0.6283634305000305,
    "14.q_proj": 0.4639151692390442,
    "14.k_proj": 0.44637247920036316,
    "14.v_proj": 0.6475556492805481,
    "14.o_proj": 0.6104862093925476,
    "14.gate_proj": 0.5859494209289551,
    "14.up_proj": 0.6189398169517517,
    "14.down_proj": 0.6564309597015381,
    "15.q_proj": 0.46387919783592224,
    "15.k_proj": 0.42992931604385376,
    "15.v_proj": 0.675900399684906,
    "15.o_proj": 0.6175917387008667,
    "15.gate_proj": 0.5837653279304504,
    "15.up_proj": 0.6173660159111023,
    "15.down_proj": 0.6548573970794678,
    "16.q_proj": 0.4708837568759918,
    "16.k_proj": 0.4432294964790344,
    "16.v_proj": 0.6696433424949646,
    "16.o_proj": 0.6041324138641357,
    "16.gate_proj": 0.5891684293746948,
    "16.up_proj": 0.626923680305481,
    "16.down_proj": 0.656101644039154,
    "17.q_proj": 0.5025116801261902,
    "17.k_proj": 0.48880496621131897,
    "17.v_proj": 0.6514131426811218,
    "17.o_proj": 0.6207662224769592,
    "17.gate_proj": 0.6006857752799988,
    "17.up_proj": 0.6366637349128723,
    "17.down_proj": 0.6493645906448364,
    "18.q_proj": 0.4747740626335144,
    "18.k_proj": 0.4571532607078552,
    "18.v_proj": 0.6963456869125366,
    "18.o_proj": 0.5963596701622009,
    "18.gate_proj": 0.6004551649093628,
    "18.up_proj": 0.6350386142730713,
    "18.down_proj": 0.6672616004943848,
    "19.q_proj": 0.5268400311470032,
    "19.k_proj": 0.47958895564079285,
    "19.v_proj": 0.7042666077613831,
    "19.o_proj": 0.6199771165847778,
    "19.gate_proj": 0.6060331463813782,
    "19.up_proj": 0.6367842555046082,
    "19.down_proj": 0.6811782121658325,
    "20.q_proj": 0.5465222001075745,
    "20.k_proj": 0.5119665265083313,
    "20.v_proj": 0.6899744272232056,
    "20.o_proj": 0.6347531080245972,
    "20.gate_proj": 0.6125325560569763,
    "20.up_proj": 0.636976957321167,
    "20.down_proj": 0.6877227425575256,
    "21.q_proj": 0.5809774398803711,
    "21.k_proj": 0.5469288229942322,
    "21.v_proj": 0.6610574722290039,
    "21.o_proj": 0.6689409017562866,
    "21.gate_proj": 0.6139337420463562,
    "21.up_proj": 0.6391165256500244,
    "21.down_proj": 0.6832785606384277,
    "22.q_proj": 0.6217024922370911,
    "22.k_proj": 0.5693233609199524,
    "22.v_proj": 0.6632723808288574,
    "22.o_proj": 0.6856058835983276,
    "22.gate_proj": 0.6192406415939331,
    "22.up_proj": 0.6405629515647888,
    "22.down_proj": 0.6809287667274475,
    "23.q_proj": 0.6093561053276062,
    "23.k_proj": 0.5737671256065369,
    "23.v_proj": 0.6871448159217834,
    "23.o_proj": 0.6642407178878784,
    "23.gate_proj": 0.6219133138656616,
    "23.up_proj": 0.6424050331115723,
    "23.down_proj": 0.6742445826530457,
    "24.q_proj": 0.6159307360649109,
    "24.k_proj": 0.5868129134178162,
    "24.v_proj": 0.6966824531555176,
    "24.o_proj": 0.6866542100906372,
    "24.gate_proj": 0.6238306760787964,
    "24.up_proj": 0.6446942687034607,
    "24.down_proj": 0.6646919250488281,
    "25.q_proj": 0.6210862994194031,
    "25.k_proj": 0.5705580115318298,
    "25.v_proj": 0.679048478603363,
    "25.o_proj": 0.6676820516586304,
    "25.gate_proj": 0.6258559823036194,
    "25.up_proj": 0.6465917825698853,
    "25.down_proj": 0.6531879901885986,
    "26.q_proj": 0.6165540218353271,
    "26.k_proj": 0.5607565641403198,
    "26.v_proj": 0.6618245840072632,
    "26.o_proj": 0.6507518887519836,
    "26.gate_proj": 0.6270332336425781,
    "26.up_proj": 0.6463326811790466,
    "26.down_proj": 0.6390087008476257,
    "27.q_proj": 0.6284988522529602,
    "27.k_proj": 0.6069222092628479,
    "27.v_proj": 0.6805939674377441,
    "27.o_proj": 0.6412052512168884,
    "27.gate_proj": 0.6305440664291382,
    "27.up_proj": 0.6479936242103577,
    "27.down_proj": 0.6380788683891296,
    "28.q_proj": 0.5876466035842896,
    "28.k_proj": 0.5294674634933472,
    "28.v_proj": 0.6338029503822327,
    "28.o_proj": 0.6242429614067078,
    "28.gate_proj": 0.6301671266555786,
    "28.up_proj": 0.6454077959060669,
    "28.down_proj": 0.6476228833198547,
    "29.q_proj": 0.5711690783500671,
    "29.k_proj": 0.5322142243385315,
    "29.v_proj": 0.670903205871582,
    "29.o_proj": 0.6061508655548096,
    "29.gate_proj": 0.6246752142906189,
    "29.up_proj": 0.6391395926475525,
    "29.down_proj": 0.6555037498474121,
    "30.q_proj": 0.5478631854057312,
    "30.k_proj": 0.4874037206172943,
    "30.v_proj": 0.6046014428138733,
    "30.o_proj": 0.5960355997085571,
    "30.gate_proj": 0.6242106556892395,
    "30.up_proj": 0.6300148367881775,
    "30.down_proj": 0.6422404646873474,
    "31.q_proj": 0.49975186586380005,
    "31.k_proj": 0.44580137729644775,
    "31.v_proj": 0.5610269904136658,
    "31.o_proj": 0.5210819840431213,
    "31.gate_proj": 0.6289830803871155,
    "31.up_proj": 0.6037687659263611,
    "31.down_proj": 0.6013435125350952
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": "shape",
    "param_fraction": 0.5002723106971154,
    "mean_rel_err": 0.5875181926946554,
    "max_rel_err": 0.7061887383460999
  },
  "factorisation": {
    "max_rel_err": 0.0014601770089939237,
    "factor_params": 3491561472,
    "dense_params": 6979321856
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}