| { |
| "complete": true, |
| "scope": "frozen v3 test evaluation of every formal adapter and matched base", |
| "protocol": { |
| "test_revision": "ba161c6fbb2829631ac9fef67ea296f0c4c1fbc4", |
| "test_items_per_behavior": 400, |
| "tasks": { |
| "color": 200, |
| "count": 200 |
| }, |
| "conditions": [ |
| "original", |
| "treatment", |
| "control" |
| ], |
| "generation": "greedy, max_new_tokens=12, original alone; treatment/control paired", |
| "dtype": "float16", |
| "preprocessing": "original backbone-specific infer.py protocol", |
| "runtime_fix": "cuDNN SDPA disabled for all evaluation models per updated GitHub", |
| "evaluation_source_commit": "f0b8f2b91e412311052bbf92b674231df56d285e", |
| "training_source_commit": "52dd2b32d33fce9d26ae143868bc28f78c01d84b", |
| "training_parameters_frozen": true, |
| "test_set_used_for_parameter_selection": false, |
| "uploads_required": false, |
| "primary_metrics": "matched effects and paired gains, Color/Count equally weighted", |
| "secondary_metrics": "original-image accuracy, top-answer share, abstention", |
| "ci_resamples": { |
| "matched_effects": 10000, |
| "single_layer_gain": 2000, |
| "placement_and_joint": 3000 |
| } |
| }, |
| "validated_inference_jobs": 162, |
| "validated_records": 64800, |
| "formal_adapters": 147, |
| "analyses": { |
| "joint_qwen_removal": { |
| "behavior": "removal", |
| "arms": { |
| "Joint": { |
| "color": { |
| "gain": 17.166666666666668, |
| "ci95": [ |
| 10.76335406793049, |
| 23.880597014925375 |
| ] |
| }, |
| "count": { |
| "gain": 2.8333333333333335, |
| "ci95": [ |
| -0.9524948735475052, |
| 6.666666666666667 |
| ] |
| }, |
| "behavior": { |
| "gain": 10.0, |
| "ci95": [ |
| 6.417576413464781, |
| 13.700670858980695 |
| ] |
| }, |
| "runs": 3 |
| } |
| }, |
| "contrasts": {} |
| }, |
| "joint_qwen_replacement": { |
| "behavior": "replacement", |
| "arms": { |
| "Joint": { |
| "color": { |
| "gain": 1.3333333333333333, |
| "ci95": [ |
| -2.0345884832806362, |
| 4.975124378109453 |
| ] |
| }, |
| "count": { |
| "gain": 0.3333333333333335, |
| "ci95": [ |
| -4.568527918781726, |
| 4.912280701754386 |
| ] |
| }, |
| "behavior": { |
| "gain": 0.8333333333333334, |
| "ci95": [ |
| -2.377475578750167, |
| 4.0000100010001 |
| ] |
| }, |
| "runs": 3 |
| } |
| }, |
| "contrasts": {} |
| }, |
| "joint_qwen_context": { |
| "behavior": "context", |
| "arms": { |
| "Joint": { |
| "color": { |
| "gain": -1.3333333333333333, |
| "ci95": [ |
| -4.340466485507246, |
| 1.6026069518716577 |
| ] |
| }, |
| "count": { |
| "gain": -0.6666666666666666, |
| "ci95": [ |
| -4.035087719298246, |
| 2.680269627708435 |
| ] |
| }, |
| "behavior": { |
| "gain": -1.0, |
| "ci95": [ |
| -3.3041347979672726, |
| 1.2403811824635946 |
| ] |
| }, |
| "runs": 3 |
| } |
| }, |
| "contrasts": {} |
| }, |
| "fixed_internvl_removal": { |
| "behavior": "removal", |
| "arms": { |
| "Deep8": { |
| "color": { |
| "gain": 10.166666666666666, |
| "ci95": [ |
| 4.537149096266674, |
| 15.781662433388119 |
| ] |
| }, |
| "count": { |
| "gain": 5.333333333333333, |
| "ci95": [ |
| 0.0, |
| 10.72728106143948 |
| ] |
| }, |
| "behavior": { |
| "gain": 7.75, |
| "ci95": [ |
| 3.916349296128708, |
| 11.591823277769754 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Full": { |
| "color": { |
| "gain": 9.333333333333334, |
| "ci95": [ |
| 3.7233612201335817, |
| 15.183566882250283 |
| ] |
| }, |
| "count": { |
| "gain": 9.166666666666666, |
| "ci95": [ |
| 4.323476942288824, |
| 14.361775596567533 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.25, |
| "ci95": [ |
| 5.482616495413512, |
| 13.075335023803161 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Middle8": { |
| "color": { |
| "gain": 12.166666666666666, |
| "ci95": [ |
| 6.7306957186544345, |
| 17.894823265059202 |
| ] |
| }, |
| "count": { |
| "gain": 8.333333333333334, |
| "ci95": [ |
| 3.2783364317503243, |
| 13.756986362620168 |
| ] |
| }, |
| "behavior": { |
| "gain": 10.25, |
| "ci95": [ |
| 6.360450609418865, |
| 14.26668681251842 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Random8": { |
| "color": { |
| "gain": 9.95, |
| "ci95": [ |
| 3.951123234916559, |
| 16.048490427098677 |
| ] |
| }, |
| "count": { |
| "gain": 8.7, |
| "ci95": [ |
| 3.9285246821241584, |
| 14.077718205091049 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.325, |
| "ci95": [ |
| 5.38261732847098, |
| 13.281660126767152 |
| ] |
| }, |
| "runs": 10 |
| }, |
| "Shallow8": { |
| "color": { |
| "gain": 10.833333333333334, |
| "ci95": [ |
| 5.102603605634744, |
| 16.838625063976018 |
| ] |
| }, |
| "count": { |
| "gain": 7.333333333333333, |
| "ci95": [ |
| 1.9999231950844854, |
| 12.908993036466924 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.083333333333334, |
| "ci95": [ |
| 5.151902337103394, |
| 13.160007402628557 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Top8": { |
| "color": { |
| "gain": 10.5, |
| "ci95": [ |
| 4.614942528735632, |
| 16.26428097167846 |
| ] |
| }, |
| "count": { |
| "gain": 7.833333333333333, |
| "ci95": [ |
| 2.897635778070561, |
| 13.089161057092996 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.166666666666666, |
| "ci95": [ |
| 5.217074793047767, |
| 13.128772781465317 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "TopDelta8": { |
| "color": { |
| "gain": 11.833333333333334, |
| "ci95": [ |
| 6.292517006802721, |
| 17.51297738227661 |
| ] |
| }, |
| "count": { |
| "gain": 8.5, |
| "ci95": [ |
| 3.7037037037037037, |
| 13.592741320591674 |
| ] |
| }, |
| "behavior": { |
| "gain": 10.166666666666666, |
| "ci95": [ |
| 6.272588456853881, |
| 13.977588910057953 |
| ] |
| }, |
| "runs": 3 |
| } |
| }, |
| "contrasts": { |
| "Top8-Random8": { |
| "color": { |
| "difference": 0.5500000000000007, |
| "ci95": [ |
| -2.181528835236759, |
| 3.200743853547795 |
| ] |
| }, |
| "count": { |
| "difference": -0.8666666666666663, |
| "ci95": [ |
| -3.832101237864477, |
| 1.9146118721461194 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.15833333333333321, |
| "ci95": [ |
| -2.281880436956537, |
| 1.9427180643443083 |
| ] |
| } |
| }, |
| "TopDelta8-Random8": { |
| "color": { |
| "difference": 1.8833333333333346, |
| "ci95": [ |
| -0.5586611938528271, |
| 4.410613529614792 |
| ] |
| }, |
| "count": { |
| "difference": -0.1999999999999993, |
| "ci95": [ |
| -3.093110380116958, |
| 2.729087602783051 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.8416666666666668, |
| "ci95": [ |
| -0.9637530638165153, |
| 2.8065518555178453 |
| ] |
| } |
| }, |
| "Shallow8-Random8": { |
| "color": { |
| "difference": 0.8833333333333346, |
| "ci95": [ |
| -2.079517704517706, |
| 4.029173252050881 |
| ] |
| }, |
| "count": { |
| "difference": -1.3666666666666663, |
| "ci95": [ |
| -4.640565650343903, |
| 1.6243967392663345 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.24166666666666536, |
| "ci95": [ |
| -2.327714592834495, |
| 1.9401803985788748 |
| ] |
| } |
| }, |
| "Middle8-Random8": { |
| "color": { |
| "difference": 2.216666666666667, |
| "ci95": [ |
| 0.055489417989417845, |
| 4.628924648223668 |
| ] |
| }, |
| "count": { |
| "difference": -0.36666666666666536, |
| "ci95": [ |
| -3.621885738831616, |
| 2.7726371951219524 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.9250000000000007, |
| "ci95": [ |
| -1.18797866186717, |
| 3.0659404566424278 |
| ] |
| } |
| }, |
| "Deep8-Random8": { |
| "color": { |
| "difference": 0.21666666666666679, |
| "ci95": [ |
| -2.68340210355987, |
| 3.155196264111707 |
| ] |
| }, |
| "count": { |
| "difference": -3.3666666666666663, |
| "ci95": [ |
| -7.067083333333334, |
| 0.10530537573881361 |
| ] |
| }, |
| "behavior": { |
| "difference": -1.5749999999999993, |
| "ci95": [ |
| -3.9747666664836414, |
| 0.7099735526578647 |
| ] |
| } |
| }, |
| "Top8-Shallow8": { |
| "color": { |
| "difference": -0.3333333333333339, |
| "ci95": [ |
| -3.166666666666666, |
| 2.475247524752475 |
| ] |
| }, |
| "count": { |
| "difference": 0.5, |
| "ci95": [ |
| -1.9419843713000227, |
| 3.4050179211469533 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.08333333333333215, |
| "ci95": [ |
| -1.8518795120736862, |
| 2.1669680102528024 |
| ] |
| } |
| }, |
| "Deep8-Full": { |
| "color": { |
| "difference": 0.8333333333333321, |
| "ci95": [ |
| -2.3809523809523796, |
| 4.051799085733517 |
| ] |
| }, |
| "count": { |
| "difference": -3.833333333333333, |
| "ci95": [ |
| -7.741522152807696, |
| -0.17262390082673582 |
| ] |
| }, |
| "behavior": { |
| "difference": -1.5, |
| "ci95": [ |
| -3.862156197590915, |
| 0.8220405373870728 |
| ] |
| } |
| }, |
| "Middle8-Full": { |
| "color": { |
| "difference": 2.833333333333332, |
| "ci95": [ |
| 0.341793570219965, |
| 5.666666666666666 |
| ] |
| }, |
| "count": { |
| "difference": -0.8333333333333321, |
| "ci95": [ |
| -4.477851128970534, |
| 2.4062436253510975 |
| ] |
| }, |
| "behavior": { |
| "difference": 1.0, |
| "ci95": [ |
| -1.2043076726577198, |
| 3.107975547150466 |
| ] |
| } |
| }, |
| "Random8-Full": { |
| "color": { |
| "difference": 0.6166666666666654, |
| "ci95": [ |
| -1.3499999999999996, |
| 3.0694619185675487 |
| ] |
| }, |
| "count": { |
| "difference": -0.4666666666666668, |
| "ci95": [ |
| -3.333758503401361, |
| 1.96538709875784 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.07499999999999929, |
| "ci95": [ |
| -1.629657520345539, |
| 1.512899304749076 |
| ] |
| } |
| }, |
| "Shallow8-Full": { |
| "color": { |
| "difference": 1.5, |
| "ci95": [ |
| -1.629649758454108, |
| 4.975124378109451 |
| ] |
| }, |
| "count": { |
| "difference": -1.833333333333333, |
| "ci95": [ |
| -5.218952194758646, |
| 1.258161126085656 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.16666666666666607, |
| "ci95": [ |
| -2.3435467387311233, |
| 1.995625026660409 |
| ] |
| } |
| }, |
| "Top8-Full": { |
| "color": { |
| "difference": 1.166666666666666, |
| "ci95": [ |
| -1.7242123756774457, |
| 4.355108877721944 |
| ] |
| }, |
| "count": { |
| "difference": -1.333333333333333, |
| "ci95": [ |
| -4.431624851391792, |
| 1.738383309029046 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.08333333333333393, |
| "ci95": [ |
| -2.152799599172603, |
| 2.0720516281469554 |
| ] |
| } |
| }, |
| "TopDelta8-Full": { |
| "color": { |
| "difference": 2.5, |
| "ci95": [ |
| 1.7763568394002505e-15, |
| 5.034870426829269 |
| ] |
| }, |
| "count": { |
| "difference": -0.6666666666666661, |
| "ci95": [ |
| -3.780161604903872, |
| 2.3150839793281666 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.9166666666666661, |
| "ci95": [ |
| -0.898544170848408, |
| 2.7615857987030425 |
| ] |
| } |
| } |
| } |
| }, |
| "fixed_internvl_replacement": { |
| "behavior": "replacement", |
| "arms": { |
| "Deep8": { |
| "color": { |
| "gain": 5.5, |
| "ci95": [ |
| 2.1975709475709477, |
| 9.153808195258971 |
| ] |
| }, |
| "count": { |
| "gain": 0.6666666666666666, |
| "ci95": [ |
| -4.283120510774142, |
| 5.699597314196522 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0833333333333335, |
| "ci95": [ |
| 0.041972000238318305, |
| 6.1282139502268365 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Full": { |
| "color": { |
| "gain": 5.833333333333333, |
| "ci95": [ |
| 2.4560113519091846, |
| 9.479161358270058 |
| ] |
| }, |
| "count": { |
| "gain": 0.6666666666666666, |
| "ci95": [ |
| -4.715958061219957, |
| 6.250560035842296 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.25, |
| "ci95": [ |
| 0.03493230625583557, |
| 6.558021532610946 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Middle8": { |
| "color": { |
| "gain": 6.166666666666667, |
| "ci95": [ |
| 2.640046899426785, |
| 10.047846889952153 |
| ] |
| }, |
| "count": { |
| "gain": 0.0, |
| "ci95": [ |
| -5.5555555555555545, |
| 5.641093474426809 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0833333333333335, |
| "ci95": [ |
| -0.1524001016193441, |
| 6.519581833336135 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Random8": { |
| "color": { |
| "gain": 5.05, |
| "ci95": [ |
| 1.8817071223251354, |
| 8.398106796116503 |
| ] |
| }, |
| "count": { |
| "gain": 0.8, |
| "ci95": [ |
| -4.0308084772370485, |
| 5.868597155069721 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.925, |
| "ci95": [ |
| -0.028902785810680506, |
| 5.880289649718909 |
| ] |
| }, |
| "runs": 10 |
| }, |
| "Shallow8": { |
| "color": { |
| "gain": 4.5, |
| "ci95": [ |
| 1.0415317357512954, |
| 8.243819975612459 |
| ] |
| }, |
| "count": { |
| "gain": 0.3333333333333333, |
| "ci95": [ |
| -5.263854079643553, |
| 6.122896527031868 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.4166666666666665, |
| "ci95": [ |
| -0.9788146698554657, |
| 5.777629470307291 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Top8": { |
| "color": { |
| "gain": 4.833333333333333, |
| "ci95": [ |
| 1.4771185835653606, |
| 8.421908429610614 |
| ] |
| }, |
| "count": { |
| "gain": -0.16666666666666666, |
| "ci95": [ |
| -5.729916557591623, |
| 5.492762445887447 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.3333333333333335, |
| "ci95": [ |
| -0.916828529657477, |
| 5.568640197353072 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "TopDelta8": { |
| "color": { |
| "gain": 6.0, |
| "ci95": [ |
| 2.512562814070352, |
| 9.667063492063493 |
| ] |
| }, |
| "count": { |
| "gain": 0.3333333333333333, |
| "ci95": [ |
| -4.957264957264957, |
| 5.582943662324023 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.1666666666666665, |
| "ci95": [ |
| -0.07104606737497907, |
| 6.437948987509699 |
| ] |
| }, |
| "runs": 3 |
| } |
| }, |
| "contrasts": { |
| "Top8-Random8": { |
| "color": { |
| "difference": -0.21666666666666679, |
| "ci95": [ |
| -1.7863636363636373, |
| 1.2592563621193162 |
| ] |
| }, |
| "count": { |
| "difference": -0.9666666666666667, |
| "ci95": [ |
| -3.583524557165861, |
| 1.7464901988953845 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.5916666666666663, |
| "ci95": [ |
| -2.124358758652281, |
| 0.9655486008411758 |
| ] |
| } |
| }, |
| "TopDelta8-Random8": { |
| "color": { |
| "difference": 0.9500000000000002, |
| "ci95": [ |
| -2.7755575615619447e-18, |
| 2.349755536381937 |
| ] |
| }, |
| "count": { |
| "difference": -0.46666666666666673, |
| "ci95": [ |
| -2.4317695104337864, |
| 1.4085655235762413 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.2416666666666667, |
| "ci95": [ |
| -0.9317343049122525, |
| 1.551202012472187 |
| ] |
| } |
| }, |
| "Shallow8-Random8": { |
| "color": { |
| "difference": -0.5499999999999998, |
| "ci95": [ |
| -2.4752475247524752, |
| 1.1836242138364783 |
| ] |
| }, |
| "count": { |
| "difference": -0.46666666666666673, |
| "ci95": [ |
| -3.414695428361319, |
| 2.7127896613190745 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.5083333333333333, |
| "ci95": [ |
| -2.2997894154415555, |
| 1.3372060167433903 |
| ] |
| } |
| }, |
| "Middle8-Random8": { |
| "color": { |
| "difference": 1.1166666666666671, |
| "ci95": [ |
| -0.338409475465312, |
| 2.947043066144653 |
| ] |
| }, |
| "count": { |
| "difference": -0.8, |
| "ci95": [ |
| -3.4155755768732177, |
| 2.0574264394753117 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.15833333333333366, |
| "ci95": [ |
| -1.300025255127855, |
| 1.886065176539661 |
| ] |
| } |
| }, |
| "Deep8-Random8": { |
| "color": { |
| "difference": 0.4500000000000002, |
| "ci95": [ |
| -0.6767266360782422, |
| 1.7058284667353703 |
| ] |
| }, |
| "count": { |
| "difference": -0.13333333333333341, |
| "ci95": [ |
| -2.227644288819126, |
| 1.8501392804403283 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.15833333333333366, |
| "ci95": [ |
| -0.9475457822480527, |
| 1.2506931292087544 |
| ] |
| } |
| }, |
| "Top8-Shallow8": { |
| "color": { |
| "difference": 0.33333333333333304, |
| "ci95": [ |
| -0.6472491909385107, |
| 1.4219065480565345 |
| ] |
| }, |
| "count": { |
| "difference": -0.5, |
| "ci95": [ |
| -3.0895147095800852, |
| 1.8425460636515916 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.08333333333333304, |
| "ci95": [ |
| -1.4753264033196176, |
| 1.250011198196622 |
| ] |
| } |
| }, |
| "Deep8-Full": { |
| "color": { |
| "difference": -0.33333333333333304, |
| "ci95": [ |
| -1.8518518518518512, |
| 0.9569377990430628 |
| ] |
| }, |
| "count": { |
| "difference": 0.0, |
| "ci95": [ |
| -2.8709550757373354, |
| 2.4179620034542313 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.16666666666666652, |
| "ci95": [ |
| -1.8273203478127629, |
| 1.1345159142640158 |
| ] |
| } |
| }, |
| "Middle8-Full": { |
| "color": { |
| "difference": 0.3333333333333339, |
| "ci95": [ |
| -1.2404549735564718, |
| 2.1166835528537673 |
| ] |
| }, |
| "count": { |
| "difference": -0.6666666666666666, |
| "ci95": [ |
| -3.608450042144849, |
| 2.4563131967850667 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.16666666666666652, |
| "ci95": [ |
| -1.7780610883128138, |
| 1.6718349602314826 |
| ] |
| } |
| }, |
| "Random8-Full": { |
| "color": { |
| "difference": -0.7833333333333332, |
| "ci95": [ |
| -2.1539682539682525, |
| 0.09393024815559997 |
| ] |
| }, |
| "count": { |
| "difference": 0.13333333333333341, |
| "ci95": [ |
| -2.425782770584751, |
| 2.3811846689895484 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.3250000000000002, |
| "ci95": [ |
| -1.9248199954998866, |
| 0.8313538662677991 |
| ] |
| } |
| }, |
| "Shallow8-Full": { |
| "color": { |
| "difference": -1.333333333333333, |
| "ci95": [ |
| -3.472304502369669, |
| 0.49757462686567183 |
| ] |
| }, |
| "count": { |
| "difference": -0.3333333333333333, |
| "ci95": [ |
| -3.3874103552626122, |
| 2.966841186736475 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.8333333333333335, |
| "ci95": [ |
| -2.727315779882576, |
| 1.011495180903665 |
| ] |
| } |
| }, |
| "Top8-Full": { |
| "color": { |
| "difference": -1.0, |
| "ci95": [ |
| -2.837047494201875, |
| 0.5025125628140703 |
| ] |
| }, |
| "count": { |
| "difference": -0.8333333333333333, |
| "ci95": [ |
| -3.426791277258567, |
| 1.9906648078289892 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.9166666666666665, |
| "ci95": [ |
| -2.5295529286117837, |
| 0.7356618762855107 |
| ] |
| } |
| }, |
| "TopDelta8-Full": { |
| "color": { |
| "difference": 0.16666666666666696, |
| "ci95": [ |
| -1.0002525252525252, |
| 1.4563327851168058 |
| ] |
| }, |
| "count": { |
| "difference": -0.3333333333333333, |
| "ci95": [ |
| -2.724358974358974, |
| 1.8730125079499695 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.08333333333333348, |
| "ci95": [ |
| -1.5632915846466247, |
| 1.261240453313624 |
| ] |
| } |
| } |
| } |
| }, |
| "fixed_internvl_context": { |
| "behavior": "context", |
| "arms": { |
| "Deep8": { |
| "color": { |
| "gain": 0.6666666666666666, |
| "ci95": [ |
| -2.8851495726495724, |
| 4.30134274862248 |
| ] |
| }, |
| "count": { |
| "gain": 6.0, |
| "ci95": [ |
| -0.3089387464387463, |
| 12.440634414318627 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.3333333333333335, |
| "ci95": [ |
| -0.14629087479152567, |
| 6.896319368090022 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Full": { |
| "color": { |
| "gain": -1.3333333333333333, |
| "ci95": [ |
| -5.235845713627626, |
| 2.525252525252525 |
| ] |
| }, |
| "count": { |
| "gain": 6.166666666666667, |
| "ci95": [ |
| 0.1593250223139166, |
| 12.439365671641797 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.4166666666666665, |
| "ci95": [ |
| -1.0492949917524397, |
| 6.024416320892247 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Middle8": { |
| "color": { |
| "gain": -1.1666666666666667, |
| "ci95": [ |
| -5.090369002006933, |
| 2.551677887649907 |
| ] |
| }, |
| "count": { |
| "gain": 5.833333333333333, |
| "ci95": [ |
| -0.15515003152702056, |
| 11.800633309566754 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.3333333333333335, |
| "ci95": [ |
| -1.1875944213336809, |
| 5.952674244050197 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Random8": { |
| "color": { |
| "gain": -1.35, |
| "ci95": [ |
| -5.240502450980392, |
| 2.376411122420653 |
| ] |
| }, |
| "count": { |
| "gain": 5.5, |
| "ci95": [ |
| -0.5127975549332181, |
| 11.65011934386268 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.075, |
| "ci95": [ |
| -1.4575639045988473, |
| 5.653853078777224 |
| ] |
| }, |
| "runs": 10 |
| }, |
| "Shallow8": { |
| "color": { |
| "gain": -1.6666666666666667, |
| "ci95": [ |
| -5.650520030816641, |
| 2.041077969649399 |
| ] |
| }, |
| "count": { |
| "gain": 8.0, |
| "ci95": [ |
| 1.1709938581797883, |
| 15.13607496138552 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.1666666666666665, |
| "ci95": [ |
| -0.7516368938263859, |
| 7.30842545834873 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "Top8": { |
| "color": { |
| "gain": -1.1666666666666667, |
| "ci95": [ |
| -5.0, |
| 2.6604251737230493 |
| ] |
| }, |
| "count": { |
| "gain": 7.333333333333333, |
| "ci95": [ |
| 1.2930001198609617, |
| 13.70608141733851 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0833333333333335, |
| "ci95": [ |
| -0.38387615699588057, |
| 6.626760928184765 |
| ] |
| }, |
| "runs": 3 |
| }, |
| "TopDelta8": { |
| "color": { |
| "gain": -1.0, |
| "ci95": [ |
| -5.009047829822689, |
| 2.7178325901152003 |
| ] |
| }, |
| "count": { |
| "gain": 6.166666666666667, |
| "ci95": [ |
| 0.0, |
| 12.661406765676572 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.5833333333333335, |
| "ci95": [ |
| -1.0620537828110266, |
| 6.287815742174594 |
| ] |
| }, |
| "runs": 3 |
| } |
| }, |
| "contrasts": { |
| "Top8-Random8": { |
| "color": { |
| "difference": 0.18333333333333335, |
| "ci95": [ |
| -2.160848960248397, |
| 2.364025682736864 |
| ] |
| }, |
| "count": { |
| "difference": 1.833333333333333, |
| "ci95": [ |
| -1.2166732804232814, |
| 5.50582260267291 |
| ] |
| }, |
| "behavior": { |
| "difference": 1.0083333333333333, |
| "ci95": [ |
| -0.8398903832366266, |
| 2.9564089965896843 |
| ] |
| } |
| }, |
| "TopDelta8-Random8": { |
| "color": { |
| "difference": 0.3500000000000001, |
| "ci95": [ |
| -1.4716913468049464, |
| 2.1976762207848584 |
| ] |
| }, |
| "count": { |
| "difference": 0.666666666666667, |
| "ci95": [ |
| -2.492174857724686, |
| 4.010644745814078 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.5083333333333333, |
| "ci95": [ |
| -1.4081065709933367, |
| 2.507500587254224 |
| ] |
| } |
| }, |
| "Shallow8-Random8": { |
| "color": { |
| "difference": -0.31666666666666665, |
| "ci95": [ |
| -3.317155993278566, |
| 2.921225885475147 |
| ] |
| }, |
| "count": { |
| "difference": 2.5, |
| "ci95": [ |
| -2.0037219970730615, |
| 7.081490686115112 |
| ] |
| }, |
| "behavior": { |
| "difference": 1.0916666666666663, |
| "ci95": [ |
| -1.890742755680293, |
| 4.073317216367707 |
| ] |
| } |
| }, |
| "Middle8-Random8": { |
| "color": { |
| "difference": 0.18333333333333335, |
| "ci95": [ |
| -2.1770606300772735, |
| 2.1987757514353268 |
| ] |
| }, |
| "count": { |
| "difference": 0.33333333333333304, |
| "ci95": [ |
| -2.908586511527689, |
| 3.6753663003663 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.2583333333333333, |
| "ci95": [ |
| -1.876531954887218, |
| 2.247400161900185 |
| ] |
| } |
| }, |
| "Deep8-Random8": { |
| "color": { |
| "difference": 2.0166666666666666, |
| "ci95": [ |
| -0.862992179997256, |
| 5.200811991162236 |
| ] |
| }, |
| "count": { |
| "difference": 0.5, |
| "ci95": [ |
| -3.760459056186194, |
| 4.877094393164028 |
| ] |
| }, |
| "behavior": { |
| "difference": 1.2583333333333333, |
| "ci95": [ |
| -1.2798553307024108, |
| 3.905312345388439 |
| ] |
| } |
| }, |
| "Top8-Shallow8": { |
| "color": { |
| "difference": 0.5, |
| "ci95": [ |
| -2.623659844054581, |
| 3.4017966160823323 |
| ] |
| }, |
| "count": { |
| "difference": -0.666666666666667, |
| "ci95": [ |
| -5.664223219658161, |
| 4.592425431711148 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.08333333333333304, |
| "ci95": [ |
| -3.2126736137582186, |
| 2.9376729918440287 |
| ] |
| } |
| }, |
| "Deep8-Full": { |
| "color": { |
| "difference": 2.0, |
| "ci95": [ |
| -0.9760951340856691, |
| 5.263381668456858 |
| ] |
| }, |
| "count": { |
| "difference": -0.16666666666666696, |
| "ci95": [ |
| -4.598006847640008, |
| 4.146455223880598 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.916666666666667, |
| "ci95": [ |
| -1.5576810891528263, |
| 3.418970352564103 |
| ] |
| } |
| }, |
| "Middle8-Full": { |
| "color": { |
| "difference": 0.16666666666666652, |
| "ci95": [ |
| -2.427480464125029, |
| 2.5516571362844678 |
| ] |
| }, |
| "count": { |
| "difference": -0.3333333333333339, |
| "ci95": [ |
| -4.085338680926918, |
| 3.2815836715247113 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.08333333333333304, |
| "ci95": [ |
| -2.33340673272612, |
| 1.9279814647004245 |
| ] |
| } |
| }, |
| "Random8-Full": { |
| "color": { |
| "difference": -0.01666666666666683, |
| "ci95": [ |
| -1.8790123917587334, |
| 1.7173896323642523 |
| ] |
| }, |
| "count": { |
| "difference": -0.666666666666667, |
| "ci95": [ |
| -3.688064942288972, |
| 2.397506105540414 |
| ] |
| }, |
| "behavior": { |
| "difference": -0.34166666666666634, |
| "ci95": [ |
| -1.9993687147141845, |
| 1.3131350530382426 |
| ] |
| } |
| }, |
| "Shallow8-Full": { |
| "color": { |
| "difference": -0.3333333333333335, |
| "ci95": [ |
| -3.465777547904044, |
| 3.0748924031628486 |
| ] |
| }, |
| "count": { |
| "difference": 1.833333333333333, |
| "ci95": [ |
| -3.151305970149254, |
| 7.193224896411884 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.75, |
| "ci95": [ |
| -2.329369379284607, |
| 3.9194275271084056 |
| ] |
| } |
| }, |
| "Top8-Full": { |
| "color": { |
| "difference": 0.16666666666666652, |
| "ci95": [ |
| -2.39298493525382, |
| 2.4756716325838206 |
| ] |
| }, |
| "count": { |
| "difference": 1.166666666666666, |
| "ci95": [ |
| -2.469596462133776, |
| 5.1158288905813665 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.666666666666667, |
| "ci95": [ |
| -1.4385378065395786, |
| 2.642019828518 |
| ] |
| } |
| }, |
| "TopDelta8-Full": { |
| "color": { |
| "difference": 0.33333333333333326, |
| "ci95": [ |
| -1.5708872967759713, |
| 2.1835347654124955 |
| ] |
| }, |
| "count": { |
| "difference": 0.0, |
| "ci95": [ |
| -3.61292656359232, |
| 3.6088003130209954 |
| ] |
| }, |
| "behavior": { |
| "difference": 0.16666666666666696, |
| "ci95": [ |
| -1.8426120250215976, |
| 2.082723897389739 |
| ] |
| } |
| } |
| } |
| }, |
| "single_internvl_removal": { |
| "behavior": "removal", |
| "runs": [ |
| { |
| "layer": 1, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -0.5, |
| "ci95": [ |
| -4.371584699453552, |
| 3.5916230366492043 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.0, |
| "ci95": [ |
| 0.0, |
| 9.95260663507109 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.25, |
| "ci95": [ |
| -0.8067265712179895, |
| 5.422825376773071 |
| ] |
| } |
| }, |
| { |
| "layer": 2, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 1.0, |
| "ci95": [ |
| -2.689149560117302, |
| 5.025644276428009 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| -1.0418030104712042, |
| 8.45087940578796 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.25, |
| "ci95": [ |
| -0.741377665288737, |
| 5.354181079244707 |
| ] |
| } |
| }, |
| { |
| "layer": 7, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 8.5, |
| "ci95": [ |
| 2.2337160046657254, |
| 14.438811783252449 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.0, |
| "ci95": [ |
| 0.0, |
| 10.309278350515465 |
| ] |
| }, |
| "behavior": { |
| "gain": 6.75, |
| "ci95": [ |
| 2.7626606399757736, |
| 10.758627872651084 |
| ] |
| } |
| }, |
| { |
| "layer": 9, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 5.0, |
| "ci95": [ |
| -1.0050251256281406, |
| 10.796186738519209 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 6.0, |
| "ci95": [ |
| 1.0415317357512954, |
| 11.112403100775186 |
| ] |
| }, |
| "behavior": { |
| "gain": 5.5, |
| "ci95": [ |
| 1.6650991548451843, |
| 9.48074463899272 |
| ] |
| } |
| }, |
| { |
| "layer": 11, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 9.0, |
| "ci95": [ |
| 3.4995646766169153, |
| 14.438811783252449 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.5, |
| "ci95": [ |
| 0.5460847575863272, |
| 10.312209898861568 |
| ] |
| }, |
| "behavior": { |
| "gain": 7.25, |
| "ci95": [ |
| 3.7335642023222713, |
| 10.937554178224687 |
| ] |
| } |
| }, |
| { |
| "layer": 16, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 7.000000000000001, |
| "ci95": [ |
| 0.9998756218905472, |
| 13.170731707317072 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 7.5, |
| "ci95": [ |
| 2.55069667460893, |
| 12.821428571428568 |
| ] |
| }, |
| "behavior": { |
| "gain": 7.25, |
| "ci95": [ |
| 3.473338104337174, |
| 11.41884378260093 |
| ] |
| } |
| }, |
| { |
| "layer": 20, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 8.5, |
| "ci95": [ |
| 2.984339386809794, |
| 14.045579018688946 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 8.5, |
| "ci95": [ |
| 3.979119182413058, |
| 13.334224598930478 |
| ] |
| }, |
| "behavior": { |
| "gain": 8.5, |
| "ci95": [ |
| 4.9918503381439425, |
| 12.470606629143212 |
| ] |
| } |
| }, |
| { |
| "layer": 25, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 7.000000000000001, |
| "ci95": [ |
| 1.075125064688632, |
| 13.298758865248221 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 7.5, |
| "ci95": [ |
| 1.5147727272727274, |
| 13.021399456521737 |
| ] |
| }, |
| "behavior": { |
| "gain": 7.25, |
| "ci95": [ |
| 3.1573756909695843, |
| 11.548631821036997 |
| ] |
| } |
| }, |
| { |
| "layer": 29, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 7.5, |
| "ci95": [ |
| 2.0613296865137807, |
| 13.208046321253866 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 7.5, |
| "ci95": [ |
| 1.8957345971563981, |
| 12.82079646017699 |
| ] |
| }, |
| "behavior": { |
| "gain": 7.5, |
| "ci95": [ |
| 3.544871871150046, |
| 11.536953773471842 |
| ] |
| } |
| }, |
| { |
| "layer": 34, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 9.0, |
| "ci95": [ |
| 4.2105263157894735, |
| 14.079263508875151 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| -1.5625, |
| 8.629441624365482 |
| ] |
| }, |
| "behavior": { |
| "gain": 6.25, |
| "ci95": [ |
| 2.6447498748085305, |
| 9.89256471681922 |
| ] |
| } |
| } |
| ], |
| "layers": [ |
| 1, |
| 2, |
| 7, |
| 9, |
| 11, |
| 16, |
| 20, |
| 25, |
| 29, |
| 34 |
| ], |
| "layer_gain": [ |
| 2.25, |
| 2.25, |
| 6.75, |
| 5.5, |
| 7.25, |
| 7.25, |
| 8.5, |
| 7.25, |
| 7.5, |
| 6.25 |
| ], |
| "spearman_gamma_gain": -0.5662208585049305 |
| }, |
| "single_internvl_replacement": { |
| "behavior": "replacement", |
| "runs": [ |
| { |
| "layer": 1, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| 0.9659524304833229, |
| 6.566489770804488 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -2.0, |
| "ci95": [ |
| -6.418376034267166, |
| 2.0730814095434416 |
| ] |
| }, |
| "behavior": { |
| "gain": 0.7500000000000002, |
| "ci95": [ |
| -1.904864949776407, |
| 3.3343535310393158 |
| ] |
| } |
| }, |
| { |
| "layer": 2, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| 1.0415317357512954, |
| 7.281553398058253 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -2.5, |
| "ci95": [ |
| -6.372549019607843, |
| 1.5003787878787858 |
| ] |
| }, |
| "behavior": { |
| "gain": 0.75, |
| "ci95": [ |
| -1.7565732484240026, |
| 3.340770753133655 |
| ] |
| } |
| }, |
| { |
| "layer": 7, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| 1.0415317357512954, |
| 7.281553398058253 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -5.612244897959184, |
| 3.333732057416266 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.5, |
| "ci95": [ |
| -1.2689268895210943, |
| 4.266941740931832 |
| ] |
| } |
| }, |
| { |
| "layer": 9, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| 1.5075376884422111, |
| 8.001005025125624 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -6.122448979591836, |
| 3.865676917638736 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.75, |
| "ci95": [ |
| -1.115608798189774, |
| 4.709702006909681 |
| ] |
| } |
| }, |
| { |
| "layer": 11, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| 0.5492277992277993, |
| 6.63265306122449 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -6.350050658561297, |
| 4.040404040404041 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.2500000000000002, |
| "ci95": [ |
| -1.6821572564404461, |
| 4.2092315938910945 |
| ] |
| } |
| }, |
| { |
| "layer": 16, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 6.0, |
| "ci95": [ |
| 2.5122487437185934, |
| 9.743589743589743 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 0.5, |
| "ci95": [ |
| -4.976374409360234, |
| 5.7976765585461205 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.25, |
| "ci95": [ |
| 0.07943779537896004, |
| 6.524006038701953 |
| ] |
| } |
| }, |
| { |
| "layer": 20, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 5.5, |
| "ci95": [ |
| 2.116123642439432, |
| 9.090909090909092 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 1.0, |
| "ci95": [ |
| -4.478171641791044, |
| 6.436444017536078 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.25, |
| "ci95": [ |
| -0.0009716893603148982, |
| 6.62662292928895 |
| ] |
| } |
| }, |
| { |
| "layer": 25, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| 1.4851485148514851, |
| 7.870655446086479 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 0.5, |
| "ci95": [ |
| -4.976374409360234, |
| 5.7976765585461205 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.5, |
| "ci95": [ |
| -0.6082272139162945, |
| 5.660514010539885 |
| ] |
| } |
| }, |
| { |
| "layer": 29, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| 0.8766440131032127, |
| 6.567331478045754 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -1.5, |
| "ci95": [ |
| -6.666666666666667, |
| 3.3828857630008455 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.0000000000000002, |
| "ci95": [ |
| -1.972781046881313, |
| 3.8965921868572906 |
| ] |
| } |
| }, |
| { |
| "layer": 34, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| 0.8766440131032127, |
| 6.567331478045754 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -0.5, |
| "ci95": [ |
| -4.787234042553192, |
| 3.684271161775406 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.5000000000000002, |
| "ci95": [ |
| -1.0660448389882267, |
| 4.095679921546328 |
| ] |
| } |
| } |
| ], |
| "layers": [ |
| 1, |
| 2, |
| 7, |
| 9, |
| 11, |
| 16, |
| 20, |
| 25, |
| 29, |
| 34 |
| ], |
| "layer_gain": [ |
| 0.7500000000000002, |
| 0.75, |
| 1.5, |
| 1.75, |
| 1.2500000000000002, |
| 3.25, |
| 3.25, |
| 2.5, |
| 1.0000000000000002, |
| 1.5000000000000002 |
| ], |
| "spearman_gamma_gain": -0.4012176588288426 |
| }, |
| "single_internvl_context": { |
| "behavior": "context", |
| "runs": [ |
| { |
| "layer": 1, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -4.285714285714286, |
| 2.0103040454799235 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 6.5, |
| "ci95": [ |
| 0.0, |
| 13.461787564766839 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.75, |
| "ci95": [ |
| -0.9800188879127214, |
| 6.555780578057806 |
| ] |
| } |
| }, |
| { |
| "layer": 2, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -0.5, |
| "ci95": [ |
| -3.0303030303030303, |
| 2.304414149172212 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 7.000000000000001, |
| "ci95": [ |
| 0.529030910609858, |
| 13.588191699604737 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.2500000000000004, |
| "ci95": [ |
| -0.11554094741067852, |
| 6.813306330633063 |
| ] |
| } |
| }, |
| { |
| "layer": 7, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -2.0, |
| "ci95": [ |
| -5.26379048582996, |
| 0.970873786407767 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 7.000000000000001, |
| "ci95": [ |
| 0.0, |
| 14.57286432160804 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.5000000000000004, |
| "ci95": [ |
| -1.2755678320738824, |
| 6.4819444444444425 |
| ] |
| } |
| }, |
| { |
| "layer": 9, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -4.306737670224512, |
| 2.0204583910167653 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| -1.7275461106655972, |
| 11.458956339712916 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.75, |
| "ci95": [ |
| -1.7086297740999141, |
| 5.441599359855966 |
| ] |
| } |
| }, |
| { |
| "layer": 11, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -2.0, |
| "ci95": [ |
| -5.208938953488372, |
| 0.9526098901098888 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| -2.9275501622011295, |
| 11.822792520790296 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.0, |
| "ci95": [ |
| -2.9426614935162325, |
| 5.126663050541317 |
| ] |
| } |
| }, |
| { |
| "layer": 16, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.5, |
| "ci95": [ |
| -4.7368421052631575, |
| 1.087105013067236 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 6.0, |
| "ci95": [ |
| 0.0, |
| 13.147589239138524 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.25, |
| "ci95": [ |
| -1.1105623381637695, |
| 6.108861758763935 |
| ] |
| } |
| }, |
| { |
| "layer": 20, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.5, |
| "ci95": [ |
| -4.7368421052631575, |
| 1.087105013067236 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| -1.5465920623898297, |
| 11.283505154639169 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.5, |
| "ci95": [ |
| -1.9800744163479962, |
| 5.112937480623073 |
| ] |
| } |
| }, |
| { |
| "layer": 25, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -4.2867445054945055, |
| 2.000251256281406 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.5, |
| "ci95": [ |
| -0.9902221565440126, |
| 12.386568902410033 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.25, |
| "ci95": [ |
| -1.2933704860056685, |
| 5.923184095302921 |
| ] |
| } |
| }, |
| { |
| "layer": 29, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -0.5, |
| "ci95": [ |
| -3.4482758620689653, |
| 1.9233091787439602 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| -3.8109040717736367, |
| 9.944823696161523 |
| ] |
| }, |
| "behavior": { |
| "gain": 1.25, |
| "ci95": [ |
| -2.252752299343037, |
| 5.000559454785878 |
| ] |
| } |
| }, |
| { |
| "layer": 34, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.5, |
| "ci95": [ |
| -5.377403464166534, |
| 2.5255729887709566 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 7.000000000000001, |
| "ci95": [ |
| 0.0, |
| 14.436242618356504 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.7500000000000004, |
| "ci95": [ |
| -1.1416853126282702, |
| 6.977125546464398 |
| ] |
| } |
| } |
| ], |
| "layers": [ |
| 1, |
| 2, |
| 7, |
| 9, |
| 11, |
| 16, |
| 20, |
| 25, |
| 29, |
| 34 |
| ], |
| "layer_gain": [ |
| 2.75, |
| 3.2500000000000004, |
| 2.5000000000000004, |
| 1.75, |
| 1.0, |
| 2.25, |
| 1.5, |
| 2.25, |
| 1.25, |
| 2.7500000000000004 |
| ], |
| "spearman_gamma_gain": -0.40729671426564323 |
| }, |
| "single_llava_removal": { |
| "behavior": "removal", |
| "runs": [ |
| { |
| "layer": 1, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 13.0, |
| "ci95": [ |
| 7.034296482412061, |
| 18.906769470184614 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.5, |
| "ci95": [ |
| -0.5025760113699812, |
| 11.483672248803826 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.25, |
| "ci95": [ |
| 5.142881597138968, |
| 13.636871325705943 |
| ] |
| } |
| }, |
| { |
| "layer": 2, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 12.5, |
| "ci95": [ |
| 5.6980204596784905, |
| 20.21276595744681 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 0.0, |
| "ci95": [ |
| -6.167400881057269, |
| 6.598984771573604 |
| ] |
| }, |
| "behavior": { |
| "gain": 6.25, |
| "ci95": [ |
| 1.6090638525344352, |
| 11.239593392046283 |
| ] |
| } |
| }, |
| { |
| "layer": 6, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 15.5, |
| "ci95": [ |
| 9.895833333333334, |
| 21.579674689235645 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.0, |
| "ci95": [ |
| -0.5559350333940493, |
| 11.000229357798164 |
| ] |
| }, |
| "behavior": { |
| "gain": 10.25, |
| "ci95": [ |
| 6.217265689904091, |
| 14.493690890990543 |
| ] |
| } |
| }, |
| { |
| "layer": 8, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 16.5, |
| "ci95": [ |
| 10.144927536231885, |
| 23.189264372553055 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 2.0, |
| "ci95": [ |
| -3.175146771037182, |
| 7.620879120879111 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.25, |
| "ci95": [ |
| 5.1586331095872895, |
| 13.585090843405434 |
| ] |
| } |
| }, |
| { |
| "layer": 10, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 18.0, |
| "ci95": [ |
| 11.650485436893204, |
| 24.726301735647525 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| -0.5524861878453039, |
| 10.054112554112546 |
| ] |
| }, |
| "behavior": { |
| "gain": 11.25, |
| "ci95": [ |
| 7.115203128628778, |
| 15.669871857280869 |
| ] |
| } |
| }, |
| { |
| "layer": 14, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 16.5, |
| "ci95": [ |
| 10.76829268292683, |
| 22.748815165876778 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| -1.5001884422110552, |
| 10.606060606060606 |
| ] |
| }, |
| "behavior": { |
| "gain": 10.5, |
| "ci95": [ |
| 6.452646768436243, |
| 14.983362424869926 |
| ] |
| } |
| }, |
| { |
| "layer": 18, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 13.0, |
| "ci95": [ |
| 6.498790322580645, |
| 19.474937343358388 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| -1.5001884422110552, |
| 10.550516343183808 |
| ] |
| }, |
| "behavior": { |
| "gain": 8.75, |
| "ci95": [ |
| 4.36626958475698, |
| 13.041379126693098 |
| ] |
| } |
| }, |
| { |
| "layer": 22, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 15.0, |
| "ci95": [ |
| 8.808290155440414, |
| 21.634615384615383 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 6.0, |
| "ci95": [ |
| 0.5126903553299492, |
| 11.558699293569292 |
| ] |
| }, |
| "behavior": { |
| "gain": 10.5, |
| "ci95": [ |
| 6.497500148216218, |
| 14.925461449806292 |
| ] |
| } |
| }, |
| { |
| "layer": 26, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 11.0, |
| "ci95": [ |
| 5.290941697191697, |
| 17.18836805555555 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 2.0, |
| "ci95": [ |
| -3.5175879396984926, |
| 7.368939590355195 |
| ] |
| }, |
| "behavior": { |
| "gain": 6.5, |
| "ci95": [ |
| 2.6321138211382116, |
| 10.546899293754509 |
| ] |
| } |
| }, |
| { |
| "layer": 30, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 15.0, |
| "ci95": [ |
| 8.910631900886424, |
| 21.31997738590709 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.5, |
| "ci95": [ |
| -0.975609756097561, |
| 10.153331526292439 |
| ] |
| }, |
| "behavior": { |
| "gain": 9.75, |
| "ci95": [ |
| 5.611576437951277, |
| 14.282649948085417 |
| ] |
| } |
| } |
| ], |
| "layers": [ |
| 1, |
| 2, |
| 6, |
| 8, |
| 10, |
| 14, |
| 18, |
| 22, |
| 26, |
| 30 |
| ], |
| "layer_gain": [ |
| 9.25, |
| 6.25, |
| 10.25, |
| 9.25, |
| 11.25, |
| 10.5, |
| 8.75, |
| 10.5, |
| 6.5, |
| 9.75 |
| ] |
| }, |
| "single_llava_replacement": { |
| "behavior": "replacement", |
| "runs": [ |
| { |
| "layer": 1, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| 0.5050505050505051, |
| 7.651324085750309 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -6.0, |
| "ci95": [ |
| -12.935774495398656, |
| 1.5469989744696897 |
| ] |
| }, |
| "behavior": { |
| "gain": -1.0, |
| "ci95": [ |
| -4.976266438879213, |
| 3.0546531505058674 |
| ] |
| } |
| }, |
| { |
| "layer": 2, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| 0.48303372352285395, |
| 7.870655446086479 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -7.000000000000001, |
| "ci95": [ |
| -14.354189669979142, |
| 0.4904411764705869 |
| ] |
| }, |
| "behavior": { |
| "gain": -1.5000000000000004, |
| "ci95": [ |
| -5.3814031440538095, |
| 2.7080512007290194 |
| ] |
| } |
| }, |
| { |
| "layer": 6, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| 0.4431818181818188, |
| 7.805068597560974 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -8.0, |
| "ci95": [ |
| -15.657821672219576, |
| 0.0 |
| ] |
| }, |
| "behavior": { |
| "gain": -2.0, |
| "ci95": [ |
| -6.17859386804219, |
| 2.296907461640508 |
| ] |
| } |
| }, |
| { |
| "layer": 8, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 2.5, |
| "ci95": [ |
| -1.442481884057971, |
| 6.467661691542289 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -9.5, |
| "ci95": [ |
| -16.582469775474955, |
| -2.0405573396871453 |
| ] |
| }, |
| "behavior": { |
| "gain": -3.5, |
| "ci95": [ |
| -7.5131492665535005, |
| 0.613791719025769 |
| ] |
| } |
| }, |
| { |
| "layer": 10, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 5.0, |
| "ci95": [ |
| 1.0927476835352816, |
| 9.18381136238279 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -4.0, |
| "ci95": [ |
| -11.220714531088973, |
| 3.5723490427098628 |
| ] |
| }, |
| "behavior": { |
| "gain": 0.5, |
| "ci95": [ |
| -3.5706329606488394, |
| 4.712367166174262 |
| ] |
| } |
| }, |
| { |
| "layer": 14, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| -0.576704545454545, |
| 7.027027027027027 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -10.0, |
| "ci95": [ |
| -17.55477620989968, |
| -2.1273781380164376 |
| ] |
| }, |
| "behavior": { |
| "gain": -3.5, |
| "ci95": [ |
| -7.797335793357239, |
| 0.7975996096261714 |
| ] |
| } |
| }, |
| { |
| "layer": 18, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| 0.0, |
| 8.45853280765981 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -7.000000000000001, |
| "ci95": [ |
| -14.136257866624359, |
| 0.5027048507845339 |
| ] |
| }, |
| "behavior": { |
| "gain": -1.5000000000000004, |
| "ci95": [ |
| -5.48551157337036, |
| 2.680202128574467 |
| ] |
| } |
| }, |
| { |
| "layer": 22, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 1.5, |
| "ci95": [ |
| -1.6044937036398137, |
| 4.878048780487805 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -6.5, |
| "ci95": [ |
| -14.217436974789916, |
| 1.4564882784750168 |
| ] |
| }, |
| "behavior": { |
| "gain": -2.5, |
| "ci95": [ |
| -6.612527369599109, |
| 1.8048049812029971 |
| ] |
| } |
| }, |
| { |
| "layer": 26, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| 0.0, |
| 6.011409506549136 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -4.5, |
| "ci95": [ |
| -11.708751793400287, |
| 3.256578229064304 |
| ] |
| }, |
| "behavior": { |
| "gain": -0.75, |
| "ci95": [ |
| -4.5458532695374805, |
| 3.291276580833568 |
| ] |
| } |
| }, |
| { |
| "layer": 30, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 2.0, |
| "ci95": [ |
| -0.9952748818720468, |
| 5.208333333333333 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -3.5000000000000004, |
| "ci95": [ |
| -9.43443161550737, |
| 2.592044301888854 |
| ] |
| }, |
| "behavior": { |
| "gain": -0.7500000000000002, |
| "ci95": [ |
| -3.9838911210774635, |
| 2.644496880499118 |
| ] |
| } |
| } |
| ], |
| "layers": [ |
| 1, |
| 2, |
| 6, |
| 8, |
| 10, |
| 14, |
| 18, |
| 22, |
| 26, |
| 30 |
| ], |
| "layer_gain": [ |
| -1.0, |
| -1.5000000000000004, |
| -2.0, |
| -3.5, |
| 0.5, |
| -3.5, |
| -1.5000000000000004, |
| -2.5, |
| -0.75, |
| -0.7500000000000002 |
| ] |
| }, |
| "single_llava_context": { |
| "behavior": "context", |
| "runs": [ |
| { |
| "layer": 1, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| -1.442481884057971, |
| 7.575757575757576 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 1.0, |
| "ci95": [ |
| -5.855100324402259, |
| 7.895042839657281 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.0, |
| "ci95": [ |
| -2.047191712236892, |
| 6.097149013672297 |
| ] |
| } |
| }, |
| { |
| "layer": 2, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 2.5, |
| "ci95": [ |
| -2.128231525966598, |
| 7.3040325901523575 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 3.5000000000000004, |
| "ci95": [ |
| -3.333732057416268, |
| 10.606060606060606 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0, |
| "ci95": [ |
| -1.022802887059845, |
| 7.169356197249638 |
| ] |
| } |
| }, |
| { |
| "layer": 6, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| -0.9757293161166906, |
| 6.8631341425459045 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| -4.433497536945813, |
| 10.5277081592871 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0, |
| "ci95": [ |
| -1.2488399525333325, |
| 7.152922077865719 |
| ] |
| } |
| }, |
| { |
| "layer": 8, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 2.0, |
| "ci95": [ |
| -2.1052631578947367, |
| 6.315789473684211 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.5, |
| "ci95": [ |
| -2.3006210165059646, |
| 12.935774495398652 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.75, |
| "ci95": [ |
| -0.49279556650246303, |
| 8.069760879156807 |
| ] |
| } |
| }, |
| { |
| "layer": 10, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -5.853658536585366, |
| 3.773839877613461 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 1.5, |
| "ci95": [ |
| -5.670103092783505, |
| 8.719072164948447 |
| ] |
| }, |
| "behavior": { |
| "gain": 0.25, |
| "ci95": [ |
| -4.246823119811982, |
| 4.598743639655034 |
| ] |
| } |
| }, |
| { |
| "layer": 14, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 2.0, |
| "ci95": [ |
| -3.1092130829015545, |
| 7.143772893772889 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 4.0, |
| "ci95": [ |
| -2.5387218534426657, |
| 10.68047850208044 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0, |
| "ci95": [ |
| -1.25, |
| 7.214662523777282 |
| ] |
| } |
| }, |
| { |
| "layer": 18, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 1.0, |
| "ci95": [ |
| -4.1894034363247545, |
| 6.522012235088484 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 5.0, |
| "ci95": [ |
| -1.4087844603164006, |
| 11.766154737757164 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.0, |
| "ci95": [ |
| -1.1368897306397308, |
| 7.464407135382959 |
| ] |
| } |
| }, |
| { |
| "layer": 22, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 1.0, |
| "ci95": [ |
| -3.500439698492462, |
| 5.613448209472462 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 6.5, |
| "ci95": [ |
| -0.5105379848274388, |
| 14.215686274509803 |
| ] |
| }, |
| "behavior": { |
| "gain": 3.75, |
| "ci95": [ |
| -0.4799292429242924, |
| 8.190972176996683 |
| ] |
| } |
| }, |
| { |
| "layer": 26, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": 2.5, |
| "ci95": [ |
| -1.9801980198019802, |
| 6.967101114147562 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": 3.0, |
| "ci95": [ |
| -3.535802184279342, |
| 9.853307163382588 |
| ] |
| }, |
| "behavior": { |
| "gain": 2.75, |
| "ci95": [ |
| -1.1970661826842977, |
| 6.955142240616607 |
| ] |
| } |
| }, |
| { |
| "layer": 30, |
| "seed": 13, |
| "color": { |
| "items": 200, |
| "gain": -0.5, |
| "ci95": [ |
| -4.592425431711146, |
| 3.5714285714285716 |
| ] |
| }, |
| "count": { |
| "items": 200, |
| "gain": -1.0, |
| "ci95": [ |
| -6.599886929535448, |
| 4.712165365998222 |
| ] |
| }, |
| "behavior": { |
| "gain": -0.75, |
| "ci95": [ |
| -4.25, |
| 2.7513551953335282 |
| ] |
| } |
| } |
| ], |
| "layers": [ |
| 1, |
| 2, |
| 6, |
| 8, |
| 10, |
| 14, |
| 18, |
| 22, |
| 26, |
| 30 |
| ], |
| "layer_gain": [ |
| 2.0, |
| 3.0, |
| 3.0, |
| 3.75, |
| 0.25, |
| 3.0, |
| 3.0, |
| 3.75, |
| 2.75, |
| -0.75 |
| ] |
| } |
| }, |
| "uploads_required": false, |
| "llava_gamma_gain_correlation": "Not computed: this experiment has no LLaVA training-localization profile" |
| } |
