{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "TUDa-HWAI/Basis_Sharing",
  "step": "basis_sharing",
  "ratio_removed": 0.3,
  "group_size": 2,
  "nsamples": 256,
  "seqlen": 2048,
  "seed": 42,
  "share_part": [
    "v",
    "k",
    "q",
    "up",
    "gate"
  ],
  "private_part": [
    "down",
    "o"
  ],
  "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove30/truncated/factored.pt",
  "param_fraction": 0.6998272040965026,
  "ranks": {
    "0.q_proj": 1911,
    "0.k_proj": 1911,
    "0.v_proj": 1911,
    "0.o_proj": 1433,
    "0.gate_proj": 2417,
    "0.up_proj": 2417,
    "0.down_proj": 2089,
    "1.q_proj": 1911,
    "1.k_proj": 1911,
    "1.v_proj": 1911,
    "1.o_proj": 1433,
    "1.gate_proj": 2417,
    "1.up_proj": 2417,
    "1.down_proj": 2089,
    "2.q_proj": 1911,
    "2.k_proj": 1911,
    "2.v_proj": 1911,
    "2.o_proj": 1433,
    "2.gate_proj": 2417,
    "2.up_proj": 2417,
    "2.down_proj": 2089,
    "3.q_proj": 1911,
    "3.k_proj": 1911,
    "3.v_proj": 1911,
    "3.o_proj": 1433,
    "3.gate_proj": 2417,
    "3.up_proj": 2417,
    "3.down_proj": 2089,
    "4.q_proj": 1911,
    "4.k_proj": 1911,
    "4.v_proj": 1911,
    "4.o_proj": 1433,
    "4.gate_proj": 2417,
    "4.up_proj": 2417,
    "4.down_proj": 2089,
    "5.q_proj": 1911,
    "5.k_proj": 1911,
    "5.v_proj": 1911,
    "5.o_proj": 1433,
    "5.gate_proj": 2417,
    "5.up_proj": 2417,
    "5.down_proj": 2089,
    "6.q_proj": 1911,
    "6.k_proj": 1911,
    "6.v_proj": 1911,
    "6.o_proj": 1433,
    "6.gate_proj": 2417,
    "6.up_proj": 2417,
    "6.down_proj": 2089,
    "7.q_proj": 1911,
    "7.k_proj": 1911,
    "7.v_proj": 1911,
    "7.o_proj": 1433,
    "7.gate_proj": 2417,
    "7.up_proj": 2417,
    "7.down_proj": 2089,
    "8.q_proj": 1911,
    "8.k_proj": 1911,
    "8.v_proj": 1911,
    "8.o_proj": 1433,
    "8.gate_proj": 2417,
    "8.up_proj": 2417,
    "8.down_proj": 2089,
    "9.q_proj": 1911,
    "9.k_proj": 1911,
    "9.v_proj": 1911,
    "9.o_proj": 1433,
    "9.gate_proj": 2417,
    "9.up_proj": 2417,
    "9.down_proj": 2089,
    "10.q_proj": 1911,
    "10.k_proj": 1911,
    "10.v_proj": 1911,
    "10.o_proj": 1433,
    "10.gate_proj": 2417,
    "10.up_proj": 2417,
    "10.down_proj": 2089,
    "11.q_proj": 1911,
    "11.k_proj": 1911,
    "11.v_proj": 1911,
    "11.o_proj": 1433,
    "11.gate_proj": 2417,
    "11.up_proj": 2417,
    "11.down_proj": 2089,
    "12.q_proj": 1911,
    "12.k_proj": 1911,
    "12.v_proj": 1911,
    "12.o_proj": 1433,
    "12.gate_proj": 2417,
    "12.up_proj": 2417,
    "12.down_proj": 2089,
    "13.q_proj": 1911,
    "13.k_proj": 1911,
    "13.v_proj": 1911,
    "13.o_proj": 1433,
    "13.gate_proj": 2417,
    "13.up_proj": 2417,
    "13.down_proj": 2089,
    "14.q_proj": 1911,
    "14.k_proj": 1911,
    "14.v_proj": 1911,
    "14.o_proj": 1433,
    "14.gate_proj": 2417,
    "14.up_proj": 2417,
    "14.down_proj": 2089,
    "15.q_proj": 1911,
    "15.k_proj": 1911,
    "15.v_proj": 1911,
    "15.o_proj": 1433,
    "15.gate_proj": 2417,
    "15.up_proj": 2417,
    "15.down_proj": 2089,
    "16.q_proj": 1911,
    "16.k_proj": 1911,
    "16.v_proj": 1911,
    "16.o_proj": 1433,
    "16.gate_proj": 2417,
    "16.up_proj": 2417,
    "16.down_proj": 2089,
    "17.q_proj": 1911,
    "17.k_proj": 1911,
    "17.v_proj": 1911,
    "17.o_proj": 1433,
    "17.gate_proj": 2417,
    "17.up_proj": 2417,
    "17.down_proj": 2089,
    "18.q_proj": 1911,
    "18.k_proj": 1911,
    "18.v_proj": 1911,
    "18.o_proj": 1433,
    "18.gate_proj": 2417,
    "18.up_proj": 2417,
    "18.down_proj": 2089,
    "19.q_proj": 1911,
    "19.k_proj": 1911,
    "19.v_proj": 1911,
    "19.o_proj": 1433,
    "19.gate_proj": 2417,
    "19.up_proj": 2417,
    "19.down_proj": 2089,
    "20.q_proj": 1911,
    "20.k_proj": 1911,
    "20.v_proj": 1911,
    "20.o_proj": 1433,
    "20.gate_proj": 2417,
    "20.up_proj": 2417,
    "20.down_proj": 2089,
    "21.q_proj": 1911,
    "21.k_proj": 1911,
    "21.v_proj": 1911,
    "21.o_proj": 1433,
    "21.gate_proj": 2417,
    "21.up_proj": 2417,
    "21.down_proj": 2089,
    "22.q_proj": 1911,
    "22.k_proj": 1911,
    "22.v_proj": 1911,
    "22.o_proj": 1433,
    "22.gate_proj": 2417,
    "22.up_proj": 2417,
    "22.down_proj": 2089,
    "23.q_proj": 1911,
    "23.k_proj": 1911,
    "23.v_proj": 1911,
    "23.o_proj": 1433,
    "23.gate_proj": 2417,
    "23.up_proj": 2417,
    "23.down_proj": 2089,
    "24.q_proj": 1911,
    "24.k_proj": 1911,
    "24.v_proj": 1911,
    "24.o_proj": 1433,
    "24.gate_proj": 2417,
    "24.up_proj": 2417,
    "24.down_proj": 2089,
    "25.q_proj": 1911,
    "25.k_proj": 1911,
    "25.v_proj": 1911,
    "25.o_proj": 1433,
    "25.gate_proj": 2417,
    "25.up_proj": 2417,
    "25.down_proj": 2089,
    "26.q_proj": 1911,
    "26.k_proj": 1911,
    "26.v_proj": 1911,
    "26.o_proj": 1433,
    "26.gate_proj": 2417,
    "26.up_proj": 2417,
    "26.down_proj": 2089,
    "27.q_proj": 1911,
    "27.k_proj": 1911,
    "27.v_proj": 1911,
    "27.o_proj": 1433,
    "27.gate_proj": 2417,
    "27.up_proj": 2417,
    "27.down_proj": 2089,
    "28.q_proj": 1911,
    "28.k_proj": 1911,
    "28.v_proj": 1911,
    "28.o_proj": 1433,
    "28.gate_proj": 2417,
    "28.up_proj": 2417,
    "28.down_proj": 2089,
    "29.q_proj": 1911,
    "29.k_proj": 1911,
    "29.v_proj": 1911,
    "29.o_proj": 1433,
    "29.gate_proj": 2417,
    "29.up_proj": 2417,
    "29.down_proj": 2089,
    "30.q_proj": 1911,
    "30.k_proj": 1911,
    "30.v_proj": 1911,
    "30.o_proj": 1433,
    "30.gate_proj": 2417,
    "30.up_proj": 2417,
    "30.down_proj": 2089,
    "31.q_proj": 1911,
    "31.k_proj": 1911,
    "31.v_proj": 1911,
    "31.o_proj": 1433,
    "31.gate_proj": 2417,
    "31.up_proj": 2417,
    "31.down_proj": 2089
  },
  "stage": "basis_recovery_finetuned",
  "recovery_finetune": {
    "protocol": "basis_sharing_coefficient_lora",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)",
    "epochs": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)",
    "is_methods_own_stage2": false,
    "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.",
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "lora_targets": [
      "down_proj",
      "gate_proj",
      "k_proj",
      "o_proj",
      "q_proj",
      "up_proj",
      "v_proj"
    ],
    "trained": "Coefficient modules only; shared and private bases frozen",
    "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis",
    "mean_rel_delta_coefficient": 0.026299988264716894
  },
  "param_fraction_is_upper_bound": false,
  "rel_delta_coefficient": {
    "0.down_proj": 0.01571832410991192,
    "0.gate_proj": 0.07073784619569778,
    "0.up_proj": 0.051439255475997925,
    "0.k_proj": 0.03125172480940819,
    "0.o_proj": 0.014449120499193668,
    "0.q_proj": 0.03769339993596077,
    "0.v_proj": 0.019454704597592354,
    "1.down_proj": 0.007780988700687885,
    "1.gate_proj": 0.05042870715260506,
    "1.up_proj": 0.04310716688632965,
    "1.k_proj": 0.01711668074131012,
    "1.o_proj": 0.013755874708294868,
    "1.q_proj": 0.01987244002521038,
    "1.v_proj": 0.015976836904883385,
    "2.down_proj": 0.01568828709423542,
    "2.gate_proj": 0.04502059519290924,
    "2.up_proj": 0.04337015375494957,
    "2.k_proj": 0.01923982799053192,
    "2.o_proj": 0.018201353028416634,
    "2.q_proj": 0.019387632608413696,
    "2.v_proj": 0.025767281651496887,
    "3.down_proj": 0.015289364382624626,
    "3.gate_proj": 0.03861368075013161,
    "3.up_proj": 0.040375787764787674,
    "3.k_proj": 0.019376985728740692,
    "3.o_proj": 0.017843443900346756,
    "3.q_proj": 0.01794094406068325,
    "3.v_proj": 0.025516241788864136,
    "4.down_proj": 0.013919559307396412,
    "4.gate_proj": 0.03799600154161453,
    "4.up_proj": 0.03619149327278137,
    "4.k_proj": 0.021520331501960754,
    "4.o_proj": 0.017562542110681534,
    "4.q_proj": 0.019273044541478157,
    "4.v_proj": 0.02358822524547577,
    "5.down_proj": 0.014357398264110088,
    "5.gate_proj": 0.03674551099538803,
    "5.up_proj": 0.033642545342445374,
    "5.k_proj": 0.019489075988531113,
    "5.o_proj": 0.01640571840107441,
    "5.q_proj": 0.017913563176989555,
    "5.v_proj": 0.02251928113400936,
    "6.down_proj": 0.013552287593483925,
    "6.gate_proj": 0.037070877850055695,
    "6.up_proj": 0.03368235006928444,
    "6.k_proj": 0.018359050154685974,
    "6.o_proj": 0.01698077842593193,
    "6.q_proj": 0.018099360167980194,
    "6.v_proj": 0.024023892357945442,
    "7.down_proj": 0.014833086170256138,
    "7.gate_proj": 0.03736833110451698,
    "7.up_proj": 0.03326217830181122,
    "7.k_proj": 0.01821022666990757,
    "7.o_proj": 0.014988509006798267,
    "7.q_proj": 0.017297787591814995,
    "7.v_proj": 0.022407837212085724,
    "8.down_proj": 0.014573442749679089,
    "8.gate_proj": 0.03457353264093399,
    "8.up_proj": 0.033223461359739304,
    "8.k_proj": 0.01891668699681759,
    "8.o_proj": 0.016290225088596344,
    "8.q_proj": 0.018463721498847008,
    "8.v_proj": 0.020800800994038582,
    "9.down_proj": 0.013626599684357643,
    "9.gate_proj": 0.03514138609170914,
    "9.up_proj": 0.03454797342419624,
    "9.k_proj": 0.01954204961657524,
    "9.o_proj": 0.01583862490952015,
    "9.q_proj": 0.018615638837218285,
    "9.v_proj": 0.020167481154203415,
    "10.down_proj": 0.014242639765143394,
    "10.gate_proj": 0.036446455866098404,
    "10.up_proj": 0.03395068645477295,
    "10.k_proj": 0.019043728709220886,
    "10.o_proj": 0.016665460541844368,
    "10.q_proj": 0.018057456240057945,
    "10.v_proj": 0.019024686887860298,
    "11.down_proj": 0.015268338844180107,
    "11.gate_proj": 0.034873563796281815,
    "11.up_proj": 0.03515440225601196,
    "11.k_proj": 0.020495600998401642,
    "11.o_proj": 0.01653156615793705,
    "11.q_proj": 0.01963174156844616,
    "11.v_proj": 0.018966326490044594,
    "12.down_proj": 0.01490208599716425,
    "12.gate_proj": 0.036391180008649826,
    "12.up_proj": 0.03552509844303131,
    "12.k_proj": 0.020210448652505875,
    "12.o_proj": 0.015827499330043793,
    "12.q_proj": 0.01948561705648899,
    "12.v_proj": 0.020654533058404922,
    "13.down_proj": 0.01405719481408596,
    "13.gate_proj": 0.03551631048321724,
    "13.up_proj": 0.037898264825344086,
    "13.k_proj": 0.020987654104828835,
    "13.o_proj": 0.016064412891864777,
    "13.q_proj": 0.019807111471891403,
    "13.v_proj": 0.01962696947157383,
    "14.down_proj": 0.016009340062737465,
    "14.gate_proj": 0.03802134096622467,
    "14.up_proj": 0.03786147013306618,
    "14.k_proj": 0.021262750029563904,
    "14.o_proj": 0.017347760498523712,
    "14.q_proj": 0.020732464268803596,
    "14.v_proj": 0.02084069512784481,
    "15.down_proj": 0.016418438404798508,
    "15.gate_proj": 0.038650088012218475,
    "15.up_proj": 0.035394005477428436,
    "15.k_proj": 0.022153930738568306,
    "15.o_proj": 0.01559238601475954,
    "15.q_proj": 0.02090977504849434,
    "15.v_proj": 0.02013670839369297,
    "16.down_proj": 0.016190309077501297,
    "16.gate_proj": 0.040097519755363464,
    "16.up_proj": 0.03531859815120697,
    "16.k_proj": 0.024484077468514442,
    "16.o_proj": 0.018780086189508438,
    "16.q_proj": 0.02221532352268696,
    "16.v_proj": 0.02362828142940998,
    "17.down_proj": 0.017163388431072235,
    "17.gate_proj": 0.039196014404296875,
    "17.up_proj": 0.03838655725121498,
    "17.k_proj": 0.023485826328396797,
    "17.o_proj": 0.01819983869791031,
    "17.q_proj": 0.024406565353274345,
    "17.v_proj": 0.023620449006557465,
    "18.down_proj": 0.017160629853606224,
    "18.gate_proj": 0.038988612592220306,
    "18.up_proj": 0.04259044677019119,
    "18.k_proj": 0.024072891101241112,
    "18.o_proj": 0.019031178206205368,
    "18.q_proj": 0.024145307019352913,
    "18.v_proj": 0.02690170519053936,
    "19.down_proj": 0.01757754199206829,
    "19.gate_proj": 0.04096819460391998,
    "19.up_proj": 0.03828761354088783,
    "19.k_proj": 0.025333957746624947,
    "19.o_proj": 0.01932433247566223,
    "19.q_proj": 0.024755822494626045,
    "19.v_proj": 0.02619953081011772,
    "20.down_proj": 0.017618484795093536,
    "20.gate_proj": 0.04168369621038437,
    "20.up_proj": 0.04031258076429367,
    "20.k_proj": 0.025555750355124474,
    "20.o_proj": 0.019319145008921623,
    "20.q_proj": 0.026066727936267853,
    "20.v_proj": 0.0259451512247324,
    "21.down_proj": 0.018069017678499222,
    "21.gate_proj": 0.03980571776628494,
    "21.up_proj": 0.04347342625260353,
    "21.k_proj": 0.026453960686922073,
    "21.o_proj": 0.01866300217807293,
    "21.q_proj": 0.025375496596097946,
    "21.v_proj": 0.027465440332889557,
    "22.down_proj": 0.01788206212222576,
    "22.gate_proj": 0.04741738364100456,
    "22.up_proj": 0.04643930122256279,
    "22.k_proj": 0.026044832542538643,
    "22.o_proj": 0.018997332081198692,
    "22.q_proj": 0.025562603026628494,
    "22.v_proj": 0.029967913404107094,
    "23.down_proj": 0.018382783979177475,
    "23.gate_proj": 0.04516542702913284,
    "23.up_proj": 0.04480283334851265,
    "23.k_proj": 0.025396348908543587,
    "23.o_proj": 0.016814321279525757,
    "23.q_proj": 0.025049729272723198,
    "23.v_proj": 0.0272342711687088,
    "24.down_proj": 0.018060794100165367,
    "24.gate_proj": 0.045451823621988297,
    "24.up_proj": 0.04517007991671562,
    "24.k_proj": 0.024571320042014122,
    "24.o_proj": 0.015839651226997375,
    "24.q_proj": 0.02506706677377224,
    "24.v_proj": 0.030967289581894875,
    "25.down_proj": 0.017680563032627106,
    "25.gate_proj": 0.04438598453998566,
    "25.up_proj": 0.04630052670836449,
    "25.k_proj": 0.026787467300891876,
    "25.o_proj": 0.018963590264320374,
    "25.q_proj": 0.024902934208512306,
    "25.v_proj": 0.02655676379799843,
    "26.down_proj": 0.01728445664048195,
    "26.gate_proj": 0.04320848360657692,
    "26.up_proj": 0.04594198614358902,
    "26.k_proj": 0.029789509251713753,
    "26.o_proj": 0.01576789654791355,
    "26.q_proj": 0.02549917809665203,
    "26.v_proj": 0.027597865089774132,
    "27.down_proj": 0.016805997118353844,
    "27.gate_proj": 0.04197928309440613,
    "27.up_proj": 0.04444769397377968,
    "27.k_proj": 0.024795103818178177,
    "27.o_proj": 0.023305708542466164,
    "27.q_proj": 0.026696208864450455,
    "27.v_proj": 0.02590230293571949,
    "28.down_proj": 0.01710508018732071,
    "28.gate_proj": 0.04344693571329117,
    "28.up_proj": 0.041666075587272644,
    "28.k_proj": 0.02632494829595089,
    "28.o_proj": 0.017413025721907616,
    "28.q_proj": 0.023984916508197784,
    "28.v_proj": 0.029418673366308212,
    "29.down_proj": 0.01671851985156536,
    "29.gate_proj": 0.0447494238615036,
    "29.up_proj": 0.04233136773109436,
    "29.k_proj": 0.028628991916775703,
    "29.o_proj": 0.01745348423719406,
    "29.q_proj": 0.02534261718392372,
    "29.v_proj": 0.026935601606965065,
    "30.down_proj": 0.013234722428023815,
    "30.gate_proj": 0.04196500778198242,
    "30.up_proj": 0.0402400940656662,
    "30.k_proj": 0.029129525646567345,
    "30.o_proj": 0.018144818022847176,
    "30.q_proj": 0.02681029587984085,
    "30.v_proj": 0.019666632637381554,
    "31.down_proj": 0.012836103327572346,
    "31.gate_proj": 0.04107679799199104,
    "31.up_proj": 0.038529664278030396,
    "31.k_proj": 0.033828120678663254,
    "31.o_proj": 0.016145970672369003,
    "31.q_proj": 0.03200652077794075,
    "31.v_proj": 0.022225258871912956
  }
}