{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "TUDa-HWAI/Basis_Sharing",
  "step": "basis_sharing",
  "ratio_removed": 0.4,
  "group_size": 2,
  "nsamples": 256,
  "seqlen": 2048,
  "seed": 42,
  "share_part": [
    "v",
    "k",
    "q",
    "up",
    "gate"
  ],
  "private_part": [
    "down",
    "o"
  ],
  "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove40/truncated/factored.pt",
  "param_fraction": 0.5999091746275906,
  "ranks": {
    "0.q_proj": 1638,
    "0.k_proj": 1638,
    "0.v_proj": 1638,
    "0.o_proj": 1228,
    "0.gate_proj": 2072,
    "0.up_proj": 2072,
    "0.down_proj": 1791,
    "1.q_proj": 1638,
    "1.k_proj": 1638,
    "1.v_proj": 1638,
    "1.o_proj": 1228,
    "1.gate_proj": 2072,
    "1.up_proj": 2072,
    "1.down_proj": 1791,
    "2.q_proj": 1638,
    "2.k_proj": 1638,
    "2.v_proj": 1638,
    "2.o_proj": 1228,
    "2.gate_proj": 2072,
    "2.up_proj": 2072,
    "2.down_proj": 1791,
    "3.q_proj": 1638,
    "3.k_proj": 1638,
    "3.v_proj": 1638,
    "3.o_proj": 1228,
    "3.gate_proj": 2072,
    "3.up_proj": 2072,
    "3.down_proj": 1791,
    "4.q_proj": 1638,
    "4.k_proj": 1638,
    "4.v_proj": 1638,
    "4.o_proj": 1228,
    "4.gate_proj": 2072,
    "4.up_proj": 2072,
    "4.down_proj": 1791,
    "5.q_proj": 1638,
    "5.k_proj": 1638,
    "5.v_proj": 1638,
    "5.o_proj": 1228,
    "5.gate_proj": 2072,
    "5.up_proj": 2072,
    "5.down_proj": 1791,
    "6.q_proj": 1638,
    "6.k_proj": 1638,
    "6.v_proj": 1638,
    "6.o_proj": 1228,
    "6.gate_proj": 2072,
    "6.up_proj": 2072,
    "6.down_proj": 1791,
    "7.q_proj": 1638,
    "7.k_proj": 1638,
    "7.v_proj": 1638,
    "7.o_proj": 1228,
    "7.gate_proj": 2072,
    "7.up_proj": 2072,
    "7.down_proj": 1791,
    "8.q_proj": 1638,
    "8.k_proj": 1638,
    "8.v_proj": 1638,
    "8.o_proj": 1228,
    "8.gate_proj": 2072,
    "8.up_proj": 2072,
    "8.down_proj": 1791,
    "9.q_proj": 1638,
    "9.k_proj": 1638,
    "9.v_proj": 1638,
    "9.o_proj": 1228,
    "9.gate_proj": 2072,
    "9.up_proj": 2072,
    "9.down_proj": 1791,
    "10.q_proj": 1638,
    "10.k_proj": 1638,
    "10.v_proj": 1638,
    "10.o_proj": 1228,
    "10.gate_proj": 2072,
    "10.up_proj": 2072,
    "10.down_proj": 1791,
    "11.q_proj": 1638,
    "11.k_proj": 1638,
    "11.v_proj": 1638,
    "11.o_proj": 1228,
    "11.gate_proj": 2072,
    "11.up_proj": 2072,
    "11.down_proj": 1791,
    "12.q_proj": 1638,
    "12.k_proj": 1638,
    "12.v_proj": 1638,
    "12.o_proj": 1228,
    "12.gate_proj": 2072,
    "12.up_proj": 2072,
    "12.down_proj": 1791,
    "13.q_proj": 1638,
    "13.k_proj": 1638,
    "13.v_proj": 1638,
    "13.o_proj": 1228,
    "13.gate_proj": 2072,
    "13.up_proj": 2072,
    "13.down_proj": 1791,
    "14.q_proj": 1638,
    "14.k_proj": 1638,
    "14.v_proj": 1638,
    "14.o_proj": 1228,
    "14.gate_proj": 2072,
    "14.up_proj": 2072,
    "14.down_proj": 1791,
    "15.q_proj": 1638,
    "15.k_proj": 1638,
    "15.v_proj": 1638,
    "15.o_proj": 1228,
    "15.gate_proj": 2072,
    "15.up_proj": 2072,
    "15.down_proj": 1791,
    "16.q_proj": 1638,
    "16.k_proj": 1638,
    "16.v_proj": 1638,
    "16.o_proj": 1228,
    "16.gate_proj": 2072,
    "16.up_proj": 2072,
    "16.down_proj": 1791,
    "17.q_proj": 1638,
    "17.k_proj": 1638,
    "17.v_proj": 1638,
    "17.o_proj": 1228,
    "17.gate_proj": 2072,
    "17.up_proj": 2072,
    "17.down_proj": 1791,
    "18.q_proj": 1638,
    "18.k_proj": 1638,
    "18.v_proj": 1638,
    "18.o_proj": 1228,
    "18.gate_proj": 2072,
    "18.up_proj": 2072,
    "18.down_proj": 1791,
    "19.q_proj": 1638,
    "19.k_proj": 1638,
    "19.v_proj": 1638,
    "19.o_proj": 1228,
    "19.gate_proj": 2072,
    "19.up_proj": 2072,
    "19.down_proj": 1791,
    "20.q_proj": 1638,
    "20.k_proj": 1638,
    "20.v_proj": 1638,
    "20.o_proj": 1228,
    "20.gate_proj": 2072,
    "20.up_proj": 2072,
    "20.down_proj": 1791,
    "21.q_proj": 1638,
    "21.k_proj": 1638,
    "21.v_proj": 1638,
    "21.o_proj": 1228,
    "21.gate_proj": 2072,
    "21.up_proj": 2072,
    "21.down_proj": 1791,
    "22.q_proj": 1638,
    "22.k_proj": 1638,
    "22.v_proj": 1638,
    "22.o_proj": 1228,
    "22.gate_proj": 2072,
    "22.up_proj": 2072,
    "22.down_proj": 1791,
    "23.q_proj": 1638,
    "23.k_proj": 1638,
    "23.v_proj": 1638,
    "23.o_proj": 1228,
    "23.gate_proj": 2072,
    "23.up_proj": 2072,
    "23.down_proj": 1791,
    "24.q_proj": 1638,
    "24.k_proj": 1638,
    "24.v_proj": 1638,
    "24.o_proj": 1228,
    "24.gate_proj": 2072,
    "24.up_proj": 2072,
    "24.down_proj": 1791,
    "25.q_proj": 1638,
    "25.k_proj": 1638,
    "25.v_proj": 1638,
    "25.o_proj": 1228,
    "25.gate_proj": 2072,
    "25.up_proj": 2072,
    "25.down_proj": 1791,
    "26.q_proj": 1638,
    "26.k_proj": 1638,
    "26.v_proj": 1638,
    "26.o_proj": 1228,
    "26.gate_proj": 2072,
    "26.up_proj": 2072,
    "26.down_proj": 1791,
    "27.q_proj": 1638,
    "27.k_proj": 1638,
    "27.v_proj": 1638,
    "27.o_proj": 1228,
    "27.gate_proj": 2072,
    "27.up_proj": 2072,
    "27.down_proj": 1791,
    "28.q_proj": 1638,
    "28.k_proj": 1638,
    "28.v_proj": 1638,
    "28.o_proj": 1228,
    "28.gate_proj": 2072,
    "28.up_proj": 2072,
    "28.down_proj": 1791,
    "29.q_proj": 1638,
    "29.k_proj": 1638,
    "29.v_proj": 1638,
    "29.o_proj": 1228,
    "29.gate_proj": 2072,
    "29.up_proj": 2072,
    "29.down_proj": 1791,
    "30.q_proj": 1638,
    "30.k_proj": 1638,
    "30.v_proj": 1638,
    "30.o_proj": 1228,
    "30.gate_proj": 2072,
    "30.up_proj": 2072,
    "30.down_proj": 1791,
    "31.q_proj": 1638,
    "31.k_proj": 1638,
    "31.v_proj": 1638,
    "31.o_proj": 1228,
    "31.gate_proj": 2072,
    "31.up_proj": 2072,
    "31.down_proj": 1791
  },
  "stage": "basis_recovery_finetuned",
  "recovery_finetune": {
    "protocol": "basis_sharing_coefficient_lora",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)",
    "epochs": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)",
    "is_methods_own_stage2": false,
    "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.",
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "lora_targets": [
      "down_proj",
      "gate_proj",
      "k_proj",
      "o_proj",
      "q_proj",
      "up_proj",
      "v_proj"
    ],
    "trained": "Coefficient modules only; shared and private bases frozen",
    "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis",
    "mean_rel_delta_coefficient": 0.02975503941498963
  },
  "param_fraction_is_upper_bound": false,
  "rel_delta_coefficient": {
    "0.down_proj": 0.01789172552525997,
    "0.gate_proj": 0.07389885932207108,
    "0.up_proj": 0.05307356268167496,
    "0.k_proj": 0.04400179162621498,
    "0.o_proj": 0.015720030292868614,
    "0.q_proj": 0.042313627898693085,
    "0.v_proj": 0.023663686588406563,
    "1.down_proj": 0.009344649501144886,
    "1.gate_proj": 0.05698956921696663,
    "1.up_proj": 0.04757563769817352,
    "1.k_proj": 0.02117219939827919,
    "1.o_proj": 0.015290400013327599,
    "1.q_proj": 0.02371150627732277,
    "1.v_proj": 0.016866125166416168,
    "2.down_proj": 0.017618203535676003,
    "2.gate_proj": 0.05127978324890137,
    "2.up_proj": 0.04971493035554886,
    "2.k_proj": 0.023553118109703064,
    "2.o_proj": 0.020216671749949455,
    "2.q_proj": 0.022627107799053192,
    "2.v_proj": 0.0288323312997818,
    "3.down_proj": 0.017859240993857384,
    "3.gate_proj": 0.0461166575551033,
    "3.up_proj": 0.04656068980693817,
    "3.k_proj": 0.02325223572552204,
    "3.o_proj": 0.01974094845354557,
    "3.q_proj": 0.022899625822901726,
    "3.v_proj": 0.0293908528983593,
    "4.down_proj": 0.015974679961800575,
    "4.gate_proj": 0.04187997430562973,
    "4.up_proj": 0.04083123430609703,
    "4.k_proj": 0.024421246722340584,
    "4.o_proj": 0.01839449815452099,
    "4.q_proj": 0.02253827266395092,
    "4.v_proj": 0.026954350993037224,
    "5.down_proj": 0.016185220330953598,
    "5.gate_proj": 0.0418110266327858,
    "5.up_proj": 0.040592070668935776,
    "5.k_proj": 0.022455476224422455,
    "5.o_proj": 0.017546743154525757,
    "5.q_proj": 0.020589888095855713,
    "5.v_proj": 0.026903508231043816,
    "6.down_proj": 0.01673140563070774,
    "6.gate_proj": 0.039437901228666306,
    "6.up_proj": 0.037406571209430695,
    "6.k_proj": 0.022581404075026512,
    "6.o_proj": 0.017793400213122368,
    "6.q_proj": 0.01947803795337677,
    "6.v_proj": 0.025546329095959663,
    "7.down_proj": 0.01703973300755024,
    "7.gate_proj": 0.03845927119255066,
    "7.up_proj": 0.042372316122055054,
    "7.k_proj": 0.01995324343442917,
    "7.o_proj": 0.01723671145737171,
    "7.q_proj": 0.018467623740434647,
    "7.v_proj": 0.025217900052666664,
    "8.down_proj": 0.016596227884292603,
    "8.gate_proj": 0.04214202240109444,
    "8.up_proj": 0.03909769654273987,
    "8.k_proj": 0.020271550863981247,
    "8.o_proj": 0.016867313534021378,
    "8.q_proj": 0.019599435850977898,
    "8.v_proj": 0.02347864769399166,
    "9.down_proj": 0.016085432842373848,
    "9.gate_proj": 0.03925571218132973,
    "9.up_proj": 0.03977390378713608,
    "9.k_proj": 0.020905086770653725,
    "9.o_proj": 0.017597880214452744,
    "9.q_proj": 0.019629141315817833,
    "9.v_proj": 0.021614503115415573,
    "10.down_proj": 0.016450710594654083,
    "10.gate_proj": 0.0410277396440506,
    "10.up_proj": 0.036953117698431015,
    "10.k_proj": 0.021330196410417557,
    "10.o_proj": 0.018903516232967377,
    "10.q_proj": 0.01969883218407631,
    "10.v_proj": 0.02067733369767666,
    "11.down_proj": 0.01707199402153492,
    "11.gate_proj": 0.038356050848960876,
    "11.up_proj": 0.041197046637535095,
    "11.k_proj": 0.023145737126469612,
    "11.o_proj": 0.016748346388339996,
    "11.q_proj": 0.021463047713041306,
    "11.v_proj": 0.019644424319267273,
    "12.down_proj": 0.0163368321955204,
    "12.gate_proj": 0.03845346346497536,
    "12.up_proj": 0.039496049284935,
    "12.k_proj": 0.022679366171360016,
    "12.o_proj": 0.017901107668876648,
    "12.q_proj": 0.021295785903930664,
    "12.v_proj": 0.020925084128975868,
    "13.down_proj": 0.016835162416100502,
    "13.gate_proj": 0.04033529385924339,
    "13.up_proj": 0.04001957178115845,
    "13.k_proj": 0.02491459809243679,
    "13.o_proj": 0.018841315060853958,
    "13.q_proj": 0.022686036303639412,
    "13.v_proj": 0.022567175328731537,
    "14.down_proj": 0.01779063418507576,
    "14.gate_proj": 0.04157009720802307,
    "14.up_proj": 0.041127171367406845,
    "14.k_proj": 0.025756051763892174,
    "14.o_proj": 0.01769130863249302,
    "14.q_proj": 0.024447226896882057,
    "14.v_proj": 0.02386847883462906,
    "15.down_proj": 0.018620144575834274,
    "15.gate_proj": 0.04322356358170509,
    "15.up_proj": 0.04224470630288124,
    "15.k_proj": 0.025942062959074974,
    "15.o_proj": 0.018257489427924156,
    "15.q_proj": 0.023428166285157204,
    "15.v_proj": 0.022524960339069366,
    "16.down_proj": 0.018772562965750694,
    "16.gate_proj": 0.044220734387636185,
    "16.up_proj": 0.04224252700805664,
    "16.k_proj": 0.029059557244181633,
    "16.o_proj": 0.019190525636076927,
    "16.q_proj": 0.025546090677380562,
    "16.v_proj": 0.027710048481822014,
    "17.down_proj": 0.019686385989189148,
    "17.gate_proj": 0.04135333001613617,
    "17.up_proj": 0.04486946761608124,
    "17.k_proj": 0.02745143696665764,
    "17.o_proj": 0.020055420696735382,
    "17.q_proj": 0.025585098192095757,
    "17.v_proj": 0.02658572606742382,
    "18.down_proj": 0.020166408270597458,
    "18.gate_proj": 0.041446503251791,
    "18.up_proj": 0.04579155147075653,
    "18.k_proj": 0.02984543703496456,
    "18.o_proj": 0.019872967153787613,
    "18.q_proj": 0.02715788222849369,
    "18.v_proj": 0.029887434095144272,
    "19.down_proj": 0.020344026386737823,
    "19.gate_proj": 0.04185543581843376,
    "19.up_proj": 0.047275569289922714,
    "19.k_proj": 0.02909199893474579,
    "19.o_proj": 0.022002629935741425,
    "19.q_proj": 0.02898808941245079,
    "19.v_proj": 0.028401562944054604,
    "20.down_proj": 0.020384257659316063,
    "20.gate_proj": 0.04890038073062897,
    "20.up_proj": 0.04913109540939331,
    "20.k_proj": 0.030232349410653114,
    "20.o_proj": 0.022863484919071198,
    "20.q_proj": 0.029142558574676514,
    "20.v_proj": 0.030943676829338074,
    "21.down_proj": 0.020678570494055748,
    "21.gate_proj": 0.04660072550177574,
    "21.up_proj": 0.04657881334424019,
    "21.k_proj": 0.02797800861299038,
    "21.o_proj": 0.024201812222599983,
    "21.q_proj": 0.028927212581038475,
    "21.v_proj": 0.032502662390470505,
    "22.down_proj": 0.02047550119459629,
    "22.gate_proj": 0.048371393233537674,
    "22.up_proj": 0.053299009799957275,
    "22.k_proj": 0.02902030199766159,
    "22.o_proj": 0.021341780200600624,
    "22.q_proj": 0.028871284797787666,
    "22.v_proj": 0.034008994698524475,
    "23.down_proj": 0.02101362682878971,
    "23.gate_proj": 0.04909517243504524,
    "23.up_proj": 0.05019266530871391,
    "23.k_proj": 0.02889767661690712,
    "23.o_proj": 0.02051742933690548,
    "23.q_proj": 0.02883676066994667,
    "23.v_proj": 0.03221878036856651,
    "24.down_proj": 0.020756598562002182,
    "24.gate_proj": 0.049081768840551376,
    "24.up_proj": 0.05525779724121094,
    "24.k_proj": 0.028636103495955467,
    "24.o_proj": 0.02002854458987713,
    "24.q_proj": 0.028798028826713562,
    "24.v_proj": 0.03184721618890762,
    "25.down_proj": 0.019914736971259117,
    "25.gate_proj": 0.05659620091319084,
    "25.up_proj": 0.05105341598391533,
    "25.k_proj": 0.03020135685801506,
    "25.o_proj": 0.02426576428115368,
    "25.q_proj": 0.028738798573613167,
    "25.v_proj": 0.030420735478401184,
    "26.down_proj": 0.020024362951517105,
    "26.gate_proj": 0.049845531582832336,
    "26.up_proj": 0.047202229499816895,
    "26.k_proj": 0.03258863091468811,
    "26.o_proj": 0.01930871419608593,
    "26.q_proj": 0.028618991374969482,
    "26.v_proj": 0.030448047444224358,
    "27.down_proj": 0.01905849389731884,
    "27.gate_proj": 0.04903184995055199,
    "27.up_proj": 0.0499153696000576,
    "27.k_proj": 0.027926409617066383,
    "27.o_proj": 0.02700573392212391,
    "27.q_proj": 0.02987619861960411,
    "27.v_proj": 0.0307757705450058,
    "28.down_proj": 0.02064599096775055,
    "28.gate_proj": 0.05176892504096031,
    "28.up_proj": 0.04868489131331444,
    "28.k_proj": 0.030534468591213226,
    "28.o_proj": 0.01911034807562828,
    "28.q_proj": 0.03064342774450779,
    "28.v_proj": 0.027126865461468697,
    "29.down_proj": 0.01944028027355671,
    "29.gate_proj": 0.05063900351524353,
    "29.up_proj": 0.04600493237376213,
    "29.k_proj": 0.034188102930784225,
    "29.o_proj": 0.01887245662510395,
    "29.q_proj": 0.02965513803064823,
    "29.v_proj": 0.03125184401869774,
    "30.down_proj": 0.014209754765033722,
    "30.gate_proj": 0.04668048024177551,
    "30.up_proj": 0.043189410120248795,
    "30.k_proj": 0.031168438494205475,
    "30.o_proj": 0.018736323341727257,
    "30.q_proj": 0.03064728155732155,
    "30.v_proj": 0.022416142746806145,
    "31.down_proj": 0.01456467341631651,
    "31.gate_proj": 0.048269350081682205,
    "31.up_proj": 0.04480603337287903,
    "31.k_proj": 0.03716890141367912,
    "31.o_proj": 0.018774691969156265,
    "31.q_proj": 0.034700050950050354,
    "31.v_proj": 0.02298746630549431
  }
}