Spaces:
Running
Running
| { | |
| "generated_from": [ | |
| "paper tables/tab_cross_model_all.tex", | |
| "release docs/data.js", | |
| "the fixed 20-method x 3-model leaderboard table of the preprint (substrate P, weight-merged target)" | |
| ], | |
| "rule": "K-Scores are reported per substrate and are never averaged across substrates.", | |
| "substrate_matrix": [ | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.237, | |
| "Z_summary": 0.648 | |
| }, | |
| "or_all": 0.682, | |
| "retain": 0.737, | |
| "delta_sel": null, | |
| "degen_pct": 45.0, | |
| "k_class": "base", | |
| "k_score": 0.247 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.212, | |
| "Z_summary": 0.648 | |
| }, | |
| "or_all": 0.67, | |
| "retain": 0.73, | |
| "delta_sel": -0.01, | |
| "degen_pct": 47.0, | |
| "k_class": "MF", | |
| "k_score": 0.247 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.002 | |
| }, | |
| "or_all": 0.003, | |
| "retain": 0.737, | |
| "delta_sel": 0.0, | |
| "degen_pct": 38.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.906 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.111, | |
| "Z_summary": 0.295 | |
| }, | |
| "or_all": 0.312, | |
| "retain": 0.7, | |
| "delta_sel": -0.037, | |
| "degen_pct": 43.0, | |
| "k_class": "K-REF 2x", | |
| "k_score": 0.377 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.221, | |
| "Z_summary": 0.648 | |
| }, | |
| "or_all": 0.678, | |
| "retain": 0.737, | |
| "delta_sel": 0.0, | |
| "degen_pct": 46.0, | |
| "k_class": "no-op", | |
| "k_score": 0.245 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "CHA", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.037, | |
| "Z_summary": 0.16 | |
| }, | |
| "or_all": 0.175, | |
| "retain": 0.183, | |
| "delta_sel": -0.55, | |
| "degen_pct": 46.0, | |
| "k_class": "K-REF 2x", | |
| "k_score": 0.344 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "P", | |
| "method": "O3", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.737, | |
| "delta_sel": 0.0, | |
| "degen_pct": 34.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.997 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "C", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.038, | |
| "Z_tool_wide": 0.038, | |
| "Z_answer": 0.192, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.223, | |
| "retain": 0.963, | |
| "delta_sel": null, | |
| "degen_pct": 66.0, | |
| "k_class": "base", | |
| "k_score": 0.693 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "C", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.037, | |
| "Z_tool_wide": 0.038, | |
| "Z_answer": 0.208, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.245, | |
| "retain": 0.972, | |
| "delta_sel": 0.01, | |
| "degen_pct": 61.0, | |
| "k_class": "MF", | |
| "k_score": 0.672 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "C", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.963, | |
| "delta_sel": 0.0, | |
| "degen_pct": 91.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.7 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "C", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.015, | |
| "Z_tool_wide": 0.015, | |
| "Z_answer": 0.097, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.112, | |
| "retain": 0.962, | |
| "delta_sel": -0.0, | |
| "degen_pct": 68.0, | |
| "k_class": "K-REF 2x", | |
| "k_score": 0.719 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "C", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.038, | |
| "Z_tool_wide": 0.038, | |
| "Z_answer": 0.192, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.223, | |
| "retain": 0.963, | |
| "delta_sel": 0.0, | |
| "degen_pct": 66.0, | |
| "k_class": "no-op", | |
| "k_score": 0.693 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-text", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.008, | |
| "Z_tool_wide": 0.602, | |
| "Z_answer": 0.203, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.602, | |
| "retain": 0.888, | |
| "delta_sel": null, | |
| "degen_pct": 48.0, | |
| "k_class": "base", | |
| "k_score": 0.364 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-text", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.01, | |
| "Z_tool_wide": 0.568, | |
| "Z_answer": 0.208, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.568, | |
| "retain": 0.882, | |
| "delta_sel": -0.01, | |
| "degen_pct": 45.0, | |
| "k_class": "MF", | |
| "k_score": 0.393 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-text", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.005, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.005, | |
| "retain": 0.888, | |
| "delta_sel": 0.0, | |
| "degen_pct": 71.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.73 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-text", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.602, | |
| "Z_answer": 0.097, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.602, | |
| "retain": 0.888, | |
| "delta_sel": 0.0, | |
| "degen_pct": 52.0, | |
| "k_class": "MF", | |
| "k_score": 0.349 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-text", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.008, | |
| "Z_tool_wide": 0.602, | |
| "Z_answer": 0.203, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.602, | |
| "retain": 0.888, | |
| "delta_sel": 0.0, | |
| "degen_pct": 48.0, | |
| "k_class": "no-op", | |
| "k_score": 0.364 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-struct", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.002, | |
| "Z_tool_wide": 0.855, | |
| "Z_answer": 0.832, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.855, | |
| "retain": 0.863, | |
| "delta_sel": null, | |
| "degen_pct": 9.0, | |
| "k_class": "base", | |
| "k_score": 0.128 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-struct", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.005, | |
| "Z_tool_wide": 0.853, | |
| "Z_answer": 0.833, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.853, | |
| "retain": 0.875, | |
| "delta_sel": 0.01, | |
| "degen_pct": 10.0, | |
| "k_class": "MF", | |
| "k_score": 0.13 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-struct", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.863, | |
| "delta_sel": 0.0, | |
| "degen_pct": 73.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.345 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-struct", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.857, | |
| "Z_answer": 0.463, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.857, | |
| "retain": 0.862, | |
| "delta_sel": -0.0, | |
| "degen_pct": 9.0, | |
| "k_class": "K-SUP", | |
| "k_score": 0.127 | |
| }, | |
| { | |
| "base": "Llama-3.1-8B", | |
| "substrate": "R-struct", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.002, | |
| "Z_tool_wide": 0.855, | |
| "Z_answer": 0.832, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.855, | |
| "retain": 0.863, | |
| "delta_sel": 0.0, | |
| "degen_pct": 9.0, | |
| "k_class": "no-op", | |
| "k_score": 0.128 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "P", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.004, | |
| "Z_tool": 0.003, | |
| "Z_tool_wide": 0.003, | |
| "Z_answer": 0.264, | |
| "Z_summary": 0.403 | |
| }, | |
| "or_all": 0.41, | |
| "retain": 0.685, | |
| "delta_sel": 0.26, | |
| "degen_pct": 0.0, | |
| "k_class": "MF", | |
| "k_score": 0.389 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "P", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.001, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.062, | |
| "Z_summary": 0.029 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.46, | |
| "delta_sel": 0.04, | |
| "degen_pct": 0.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.898 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "P", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.001, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.219, | |
| "Z_summary": 0.373 | |
| }, | |
| "or_all": 0.255, | |
| "retain": 0.4, | |
| "delta_sel": -0.02, | |
| "degen_pct": 0.0, | |
| "k_class": "MF", | |
| "k_score": 0.523 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "P", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.001, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.301, | |
| "Z_summary": 0.42 | |
| }, | |
| "or_all": 0.435, | |
| "retain": 0.455, | |
| "delta_sel": 0.03, | |
| "degen_pct": 0.0, | |
| "k_class": "MF", | |
| "k_score": 0.453 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "C", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.019, | |
| "Z_tool": 0.086, | |
| "Z_tool_wide": 0.123, | |
| "Z_answer": 0.05, | |
| "Z_summary": 0.047 | |
| }, | |
| "or_all": 0.1, | |
| "retain": 0.995, | |
| "delta_sel": 0.0, | |
| "degen_pct": 29.0, | |
| "k_class": "below gate", | |
| "k_score": null | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "C", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.015, | |
| "Z_tool": 0.004, | |
| "Z_tool_wide": 0.031, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.03 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.995, | |
| "delta_sel": 0.0, | |
| "degen_pct": 42.0, | |
| "k_class": "below gate", | |
| "k_score": null | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "C", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.01, | |
| "Z_tool": 0.041, | |
| "Z_tool_wide": 0.079, | |
| "Z_answer": 0.025, | |
| "Z_summary": 0.049 | |
| }, | |
| "or_all": 0.045, | |
| "retain": 0.995, | |
| "delta_sel": 0.0, | |
| "degen_pct": 33.0, | |
| "k_class": "below gate", | |
| "k_score": null | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "C", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.015, | |
| "Z_tool": 0.041, | |
| "Z_tool_wide": 0.076, | |
| "Z_answer": 0.036, | |
| "Z_summary": 0.049 | |
| }, | |
| "or_all": 0.06, | |
| "retain": 0.995, | |
| "delta_sel": 0.0, | |
| "degen_pct": 34.0, | |
| "k_class": "below gate", | |
| "k_score": null | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-text", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.073, | |
| "Z_tool": 0.113, | |
| "Z_tool_wide": 0.345, | |
| "Z_answer": 0.168, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.345, | |
| "retain": 0.937, | |
| "delta_sel": null, | |
| "degen_pct": 17.0, | |
| "k_class": "base", | |
| "k_score": 0.616 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-text", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.097, | |
| "Z_tool": 0.087, | |
| "Z_tool_wide": 0.416, | |
| "Z_answer": 0.25, | |
| "Z_summary": 0.046 | |
| }, | |
| "or_all": 0.405, | |
| "retain": 0.96, | |
| "delta_sel": 0.01, | |
| "degen_pct": 16.0, | |
| "k_class": "MF", | |
| "k_score": 0.555 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-text", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.937, | |
| "delta_sel": 0.0, | |
| "degen_pct": 3.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.924 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-text", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.037, | |
| "Z_tool": 0.108, | |
| "Z_tool_wide": 0.342, | |
| "Z_answer": 0.095, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.342, | |
| "retain": 0.937, | |
| "delta_sel": 0.0, | |
| "degen_pct": 17.0, | |
| "k_class": "MF", | |
| "k_score": 0.618 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-text", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.073, | |
| "Z_tool": 0.113, | |
| "Z_tool_wide": 0.345, | |
| "Z_answer": 0.168, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.345, | |
| "retain": 0.937, | |
| "delta_sel": 0.0, | |
| "degen_pct": 17.0, | |
| "k_class": "no-op", | |
| "k_score": 0.616 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-struct", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.002, | |
| "Z_tool_wide": 0.828, | |
| "Z_answer": 0.797, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.828, | |
| "retain": 0.93, | |
| "delta_sel": null, | |
| "degen_pct": 4.0, | |
| "k_class": "base", | |
| "k_score": 0.158 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-struct", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.003, | |
| "Z_tool": 0.003, | |
| "Z_tool_wide": 0.842, | |
| "Z_answer": 0.864, | |
| "Z_summary": 0.047 | |
| }, | |
| "or_all": 0.835, | |
| "retain": 0.95, | |
| "delta_sel": 0.01, | |
| "degen_pct": 4.0, | |
| "k_class": "MF", | |
| "k_score": 0.157 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-struct", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.0, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 0.93, | |
| "delta_sel": 0.0, | |
| "degen_pct": 5.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.91 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-struct", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.002, | |
| "Z_tool_wide": 0.827, | |
| "Z_answer": 0.655, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.827, | |
| "retain": 0.93, | |
| "delta_sel": 0.0, | |
| "degen_pct": 4.0, | |
| "k_class": "MF", | |
| "k_score": 0.159 | |
| }, | |
| { | |
| "base": "Mistral-7B", | |
| "substrate": "R-struct", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.002, | |
| "Z_tool_wide": 0.828, | |
| "Z_answer": 0.797, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.828, | |
| "retain": 0.93, | |
| "delta_sel": 0.0, | |
| "degen_pct": 4.0, | |
| "k_class": "no-op", | |
| "k_score": 0.158 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "P", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.002, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.028, | |
| "Z_answer": 0.334, | |
| "Z_summary": 0.728 | |
| }, | |
| "or_all": 0.665, | |
| "retain": 0.915, | |
| "delta_sel": -0.08, | |
| "degen_pct": 28.0, | |
| "k_class": "MF", | |
| "k_score": 0.204 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "P", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.005, | |
| "Z_tool": 0.004, | |
| "Z_tool_wide": 0.035, | |
| "Z_answer": 0.04, | |
| "Z_summary": 0.049 | |
| }, | |
| "or_all": 0.0, | |
| "retain": 1.0, | |
| "delta_sel": 0.0, | |
| "degen_pct": 10.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.915 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "P", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.002, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.037, | |
| "Z_answer": 0.263, | |
| "Z_summary": 0.555 | |
| }, | |
| "or_all": 0.275, | |
| "retain": 1.0, | |
| "delta_sel": 0.0, | |
| "degen_pct": 34.0, | |
| "k_class": "MF", | |
| "k_score": 0.4 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "P", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.002, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.038, | |
| "Z_answer": 0.361, | |
| "Z_summary": 0.707 | |
| }, | |
| "or_all": 0.63, | |
| "retain": 1.0, | |
| "delta_sel": 0.0, | |
| "degen_pct": 35.0, | |
| "k_class": "MF", | |
| "k_score": 0.235 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "C", | |
| "method": "None", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.992, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.992, | |
| "retain": 0.997, | |
| "delta_sel": null, | |
| "degen_pct": 0.0, | |
| "k_class": "base", | |
| "k_score": 0.0 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "C", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.997, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.997, | |
| "retain": 0.98, | |
| "delta_sel": -0.017, | |
| "degen_pct": 0.0, | |
| "k_class": "MF", | |
| "k_score": 0.0 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "C", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.048, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.048, | |
| "retain": 0.995, | |
| "delta_sel": -0.0, | |
| "degen_pct": 74.0, | |
| "k_class": "K-REF 10x", | |
| "k_score": 0.233 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "C", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.347, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.347, | |
| "retain": 0.892, | |
| "delta_sel": -0.105, | |
| "degen_pct": 79.0, | |
| "k_class": "K-REF 2x", | |
| "k_score": 0.052 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "C", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.0, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.0, | |
| "Z_answer": 0.992, | |
| "Z_summary": 0.0 | |
| }, | |
| "or_all": 0.992, | |
| "retain": 0.995, | |
| "delta_sel": -0.0, | |
| "degen_pct": 0.0, | |
| "k_class": "no-op", | |
| "k_score": 0.0 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-text", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.441, | |
| "Z_tool": 0.028, | |
| "Z_tool_wide": 0.632, | |
| "Z_answer": 0.698, | |
| "Z_summary": 0.036 | |
| }, | |
| "or_all": 0.62, | |
| "retain": 0.795, | |
| "delta_sel": 0.02, | |
| "degen_pct": 38.0, | |
| "k_class": "MF", | |
| "k_score": 0.357 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-text", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.013, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.034, | |
| "Z_answer": 0.008, | |
| "Z_summary": 0.016 | |
| }, | |
| "or_all": 0.01, | |
| "retain": 0.775, | |
| "delta_sel": 0.0, | |
| "degen_pct": 55.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.813 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-text", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.281, | |
| "Z_tool": 0.029, | |
| "Z_tool_wide": 0.642, | |
| "Z_answer": 0.49, | |
| "Z_summary": 0.028 | |
| }, | |
| "or_all": 0.63, | |
| "retain": 0.775, | |
| "delta_sel": 0.0, | |
| "degen_pct": 60.0, | |
| "k_class": "MF", | |
| "k_score": 0.286 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-text", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.456, | |
| "Z_tool": 0.025, | |
| "Z_tool_wide": 0.642, | |
| "Z_answer": 0.75, | |
| "Z_summary": 0.028 | |
| }, | |
| "or_all": 0.63, | |
| "retain": 0.775, | |
| "delta_sel": 0.0, | |
| "degen_pct": 41.0, | |
| "k_class": "MF", | |
| "k_score": 0.349 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-struct", | |
| "method": "NOISE", | |
| "channels": { | |
| "Z_CoT": 0.075, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.484, | |
| "Z_answer": 0.654, | |
| "Z_summary": 0.034 | |
| }, | |
| "or_all": 0.42, | |
| "retain": 0.77, | |
| "delta_sel": 0.04, | |
| "degen_pct": 36.0, | |
| "k_class": "MF", | |
| "k_score": 0.484 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-struct", | |
| "method": "ECO", | |
| "channels": { | |
| "Z_CoT": 0.024, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.029, | |
| "Z_answer": 0.024, | |
| "Z_summary": 0.016 | |
| }, | |
| "or_all": 0.005, | |
| "retain": 0.74, | |
| "delta_sel": 0.01, | |
| "degen_pct": 51.0, | |
| "k_class": "K-REF∞", | |
| "k_score": 0.878 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-struct", | |
| "method": "STAR", | |
| "channels": { | |
| "Z_CoT": 0.04, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.459, | |
| "Z_answer": 0.653, | |
| "Z_summary": 0.028 | |
| }, | |
| "or_all": 0.38, | |
| "retain": 0.74, | |
| "delta_sel": 0.01, | |
| "degen_pct": 48.0, | |
| "k_class": "MF", | |
| "k_score": 0.52 | |
| }, | |
| { | |
| "base": "Qwen3.5-9B", | |
| "substrate": "R-struct", | |
| "method": "LEACE", | |
| "channels": { | |
| "Z_CoT": 0.053, | |
| "Z_tool": 0.0, | |
| "Z_tool_wide": 0.452, | |
| "Z_answer": 0.708, | |
| "Z_summary": 0.028 | |
| }, | |
| "or_all": 0.37, | |
| "retain": 0.74, | |
| "delta_sel": 0.01, | |
| "degen_pct": 48.0, | |
| "k_class": "MF", | |
| "k_score": 0.53 | |
| } | |
| ], | |
| "gated_leaderboard_P": { | |
| "Llama-3.1-8B": [ | |
| { | |
| "method": "ELM", | |
| "suppression": 0.349, | |
| "retain": 83.0, | |
| "degen": 0.0 | |
| } | |
| ], | |
| "Mistral-7B": [ | |
| { | |
| "method": "RMU", | |
| "suppression": 0.751, | |
| "retain": 103.0, | |
| "degen": 1.5 | |
| }, | |
| { | |
| "method": "LAW", | |
| "suppression": 0.434, | |
| "retain": 132.0, | |
| "degen": 5.0 | |
| }, | |
| { | |
| "method": "REVS", | |
| "suppression": 0.343, | |
| "retain": 129.0, | |
| "degen": 5.0 | |
| }, | |
| { | |
| "method": "ELM", | |
| "suppression": 0.322, | |
| "retain": 132.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "MemFlex", | |
| "suppression": 0.305, | |
| "retain": 133.0, | |
| "degen": 0.5 | |
| }, | |
| { | |
| "method": "LLMU", | |
| "suppression": 0.269, | |
| "retain": 146.0, | |
| "degen": 0.0 | |
| } | |
| ], | |
| "Qwen3.5-9B": [ | |
| { | |
| "method": "LoKU", | |
| "suppression": 0.753, | |
| "retain": 99.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "ELM", | |
| "suppression": 0.735, | |
| "retain": 96.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "RMU", | |
| "suppression": 0.538, | |
| "retain": 99.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "SKU", | |
| "suppression": 0.519, | |
| "retain": 100.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "WGA", | |
| "suppression": 0.513, | |
| "retain": 100.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "FALCON", | |
| "suppression": 0.501, | |
| "retain": 96.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "UNDIAL-corrected", | |
| "suppression": 0.493, | |
| "retain": 100.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "ULD", | |
| "suppression": 0.429, | |
| "retain": 81.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "MemFlex", | |
| "suppression": 0.237, | |
| "retain": 100.0, | |
| "degen": 0.0 | |
| }, | |
| { | |
| "method": "LLMU", | |
| "suppression": 0.223, | |
| "retain": 100.0, | |
| "degen": 0.0 | |
| } | |
| ] | |
| }, | |
| "budget_recovery": { | |
| "Llama-3.1-8B": { | |
| "P": { | |
| "rates": [ | |
| 0.152, | |
| 0.682, | |
| 0.682, | |
| 0.682, | |
| 0.682 | |
| ], | |
| "flips": "4/6" | |
| }, | |
| "C": { | |
| "rates": [ | |
| 0.192, | |
| 0.192, | |
| 0.192, | |
| 0.223, | |
| 0.223 | |
| ], | |
| "flips": "0/3" | |
| }, | |
| "R-text": { | |
| "rates": [ | |
| 0.203, | |
| 0.203, | |
| 0.203, | |
| 0.602, | |
| 0.602 | |
| ], | |
| "flips": "1/3" | |
| }, | |
| "R-struct": { | |
| "rates": [ | |
| 0.832, | |
| 0.832, | |
| 0.832, | |
| 0.855, | |
| 0.855 | |
| ], | |
| "flips": "0/3" | |
| } | |
| }, | |
| "Mistral-7B": { | |
| "P": { | |
| "rates": [ | |
| 0.19, | |
| 0.415, | |
| 0.415, | |
| 0.415, | |
| 0.415 | |
| ], | |
| "flips": "1/9" | |
| }, | |
| "R-text": { | |
| "rates": [ | |
| 0.168, | |
| 0.168, | |
| 0.193, | |
| 0.345, | |
| 0.345 | |
| ], | |
| "flips": "0/3" | |
| }, | |
| "R-struct": { | |
| "rates": [ | |
| 0.797, | |
| 0.797, | |
| 0.797, | |
| 0.828, | |
| 0.828 | |
| ], | |
| "flips": "0/3" | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "P": { | |
| "rates": [ | |
| 0.195, | |
| 0.65, | |
| 0.65, | |
| 0.65, | |
| 0.65 | |
| ], | |
| "flips": "5/15" | |
| }, | |
| "C": { | |
| "rates": [ | |
| 0.992, | |
| 0.992, | |
| 0.992, | |
| 0.992, | |
| 0.992 | |
| ], | |
| "flips": "0/4" | |
| }, | |
| "R-text": { | |
| "rates": [ | |
| 0.375, | |
| 0.375, | |
| 0.39, | |
| 0.613, | |
| 0.613 | |
| ], | |
| "flips": "2/4" | |
| }, | |
| "R-struct": { | |
| "rates": [ | |
| 0.358, | |
| 0.358, | |
| 0.36, | |
| 0.373, | |
| 0.373 | |
| ], | |
| "flips": "1/4" | |
| } | |
| } | |
| }, | |
| "gates": { | |
| "substrate_validity": { | |
| "threshold": 0.1, | |
| "metric": "graded answer-channel severity of the no-intervention agent", | |
| "note": "A cell is invalid when the un-intervened agent fails to reproduce the target in its own answer channel. Such cells carry the k_class 'below gate' and no K-Score." | |
| } | |
| }, | |
| "cell_states": { | |
| "scored": "A K-Score is reported.", | |
| "below gate": "The cell was evaluated, but the un-intervened baseline fails the substrate-validity gate on this base model, so no K-Score is defined.", | |
| "not evaluated": "The method was never run on this substrate, because its mechanism cannot act there." | |
| }, | |
| "method_refs": { | |
| "CHA": { | |
| "title": "Towards Robust and Parameter-Efficient Knowledge Unlearning for LLMs", | |
| "venue": "International Conference on Learning Representations (ICLR)", | |
| "link": "https://arxiv.org/abs/2408.06621", | |
| "link_kind": "arxiv" | |
| }, | |
| "ECO": { | |
| "title": "Large Language Model Unlearning via Embedding-Corrupted Prompts", | |
| "venue": "Advances in Neural Information Processing Systems (NeurIPS)", | |
| "link": "https://arxiv.org/abs/2406.07933", | |
| "link_kind": "arxiv" | |
| }, | |
| "LEACE": { | |
| "title": "LEACE: Perfect Linear Concept Erasure in Closed Form", | |
| "venue": "Advances in Neural Information Processing Systems (NeurIPS)", | |
| "link": "https://arxiv.org/abs/2306.03819", | |
| "link_kind": "arxiv" | |
| }, | |
| "O3": { | |
| "title": "On Large Language Model Continual Unlearning", | |
| "venue": "International Conference on Learning Representations (ICLR)", | |
| "link": "https://arxiv.org/abs/2407.10223", | |
| "link_kind": "arxiv" | |
| }, | |
| "STAR": { | |
| "title": "STaR: Sensitive Trajectory Regulation for Unlearning in Large Reasoning Models", | |
| "venue": "Proceedings of the AAAI Conference on Artificial Intelligence", | |
| "link": "https://doi.org/10.1609/aaai.v40i41.40818", | |
| "link_kind": "doi" | |
| } | |
| }, | |
| "method_refs_note": "Links are taken from the paper's own bibliography. Where the entry records a DOI the link resolves through doi.org; otherwise it points at the arXiv abstract the paper cites. NOISE and None are controls, not published methods, and carry no reference.", | |
| "cross_model_P": { | |
| "title": "Twenty published methods, three base models", | |
| "bases": [ | |
| "Llama-3.1-8B", | |
| "Mistral-7B", | |
| "Qwen3.5-9B" | |
| ], | |
| "no_intervention_k": { | |
| "Llama-3.1-8B": 0.21, | |
| "Mistral-7B": 0.486, | |
| "Qwen3.5-9B": 0.231 | |
| }, | |
| "scope": "Substrate P with a weight-merged PII target. Seed 0, 200 forget and 200 retain queries per cell. Leakage is binary OR(all) over the six channels.", | |
| "reading_rule": "K-Score is read within one base model. Across base models this view compares method ordering and the gain over that model's no-intervention row, never raw K-Score on a shared axis.", | |
| "differs_from_substrate_matrix": "The substrate matrix above reports the Llama-3.1-8B block pooled over three seeds at n=600 per split and uses graded token-recall severity, so its no-intervention K on Llama is 0.247 where this view reads 0.210. Mistral-7B and Qwen3.5-9B are seed 0 in both and agree at 0.486 and 0.231.", | |
| "collapse_rule": "Terminal collapse is recorded when a split's absolute degeneration reaches 50%. A collapsed cell holds no K-Score and is not a zero.", | |
| "gate_rule": "Retain at least 80% of the no-intervention retain-set answerability, added degeneration at most 20pp, and no terminal collapse.", | |
| "totals": { | |
| "cells": 60, | |
| "collapsed": 32, | |
| "scored": 28, | |
| "gate_pass": 17, | |
| "collapsed_by_base": { | |
| "Llama-3.1-8B": 16, | |
| "Mistral-7B": 11, | |
| "Qwen3.5-9B": 5 | |
| }, | |
| "scored_by_base": { | |
| "Llama-3.1-8B": 4, | |
| "Mistral-7B": 9, | |
| "Qwen3.5-9B": 15 | |
| }, | |
| "gate_pass_by_base": { | |
| "Llama-3.1-8B": 1, | |
| "Mistral-7B": 6, | |
| "Qwen3.5-9B": 10 | |
| }, | |
| "ranked_on_all_three": [ | |
| "ELM", | |
| "SKU" | |
| ], | |
| "ranked_on_two_or_more": [ | |
| "ELM", | |
| "LAW", | |
| "LLMU", | |
| "MemFlex", | |
| "REVS", | |
| "RMU", | |
| "SKU", | |
| "UNDIAL-corrected", | |
| "WGA" | |
| ], | |
| "roster_size": 20, | |
| "ranked_on_all_three_count": 2, | |
| "collapsed_on_at_least_one": 18 | |
| }, | |
| "reference_row": [ | |
| { | |
| "method": "ECO", | |
| "venue": "NeurIPS'24", | |
| "src": "repo", | |
| "link": "https://arxiv.org/abs/2406.07933", | |
| "link_kind": "url", | |
| "role": "reference", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 2.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.911, | |
| "collapsed": false, | |
| "d_degen": 0.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": 0.425, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.92, | |
| "collapsed": false, | |
| "d_degen": -20.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": 0.689, | |
| "gate": null | |
| } | |
| } | |
| } | |
| ], | |
| "methods": [ | |
| { | |
| "method": "FLAT", | |
| "venue": "ICLR'25", | |
| "src": "repo", | |
| "link": "https://arxiv.org/abs/2410.11143", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": 0.763, | |
| "collapsed": false, | |
| "d_degen": -15.5, | |
| "best": true, | |
| "rank": 1, | |
| "gain_over_no_intervention": 0.553, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 100.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "DOOR", | |
| "venue": "ICML'25", | |
| "src": "repo", | |
| "link": "https://arxiv.org/abs/2503.03710", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 91.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.38, | |
| "collapsed": false, | |
| "d_degen": -12.0, | |
| "best": false, | |
| "rank": 10, | |
| "gain_over_no_intervention": 0.149, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "UNDIAL-corrected", | |
| "venue": "NAACL'25", | |
| "src": "OU", | |
| "link": "https://aclanthology.org/2025.naacl-long.444/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 7.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.619, | |
| "collapsed": false, | |
| "d_degen": 5.5, | |
| "best": false, | |
| "rank": 3, | |
| "gain_over_no_intervention": 0.133, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.493, | |
| "collapsed": false, | |
| "d_degen": -17.5, | |
| "best": false, | |
| "rank": 8, | |
| "gain_over_no_intervention": 0.262, | |
| "gate": { | |
| "suppression": 0.493, | |
| "retain": 100.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "ELM", | |
| "venue": "NeurIPS'25", | |
| "src": "repo", | |
| "link": "https://doi.org/10.52202/085713-2028", | |
| "link_kind": "doi", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": 0.3, | |
| "collapsed": false, | |
| "d_degen": -16.0, | |
| "best": false, | |
| "rank": 4, | |
| "gain_over_no_intervention": 0.09, | |
| "gate": { | |
| "suppression": 0.349, | |
| "retain": 83.0, | |
| "d_degen": 0.0 | |
| } | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.266, | |
| "collapsed": false, | |
| "d_degen": 0.0, | |
| "best": false, | |
| "rank": 7, | |
| "gain_over_no_intervention": -0.22, | |
| "gate": { | |
| "suppression": 0.322, | |
| "retain": 132.0, | |
| "d_degen": 0.0 | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.708, | |
| "collapsed": false, | |
| "d_degen": -27.0, | |
| "best": false, | |
| "rank": 2, | |
| "gain_over_no_intervention": 0.477, | |
| "gate": { | |
| "suppression": 0.735, | |
| "retain": 96.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "RMU", | |
| "venue": "ICML'24", | |
| "src": "OU", | |
| "link": "https://arxiv.org/abs/2403.03218", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 40.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.726, | |
| "collapsed": false, | |
| "d_degen": 1.5, | |
| "best": true, | |
| "rank": 1, | |
| "gain_over_no_intervention": 0.24, | |
| "gate": { | |
| "suppression": 0.751, | |
| "retain": 103.0, | |
| "d_degen": 1.5 | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.53, | |
| "collapsed": false, | |
| "d_degen": -24.5, | |
| "best": false, | |
| "rank": 5, | |
| "gain_over_no_intervention": 0.299, | |
| "gate": { | |
| "suppression": 0.538, | |
| "retain": 99.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "WGA", | |
| "venue": "ICLR'25", | |
| "src": "OU", | |
| "link": "https://arxiv.org/abs/2502.19301", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 35.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.722, | |
| "collapsed": false, | |
| "d_degen": 10.0, | |
| "best": false, | |
| "rank": 2, | |
| "gain_over_no_intervention": 0.236, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.513, | |
| "collapsed": false, | |
| "d_degen": -2.5, | |
| "best": false, | |
| "rank": 7, | |
| "gain_over_no_intervention": 0.282, | |
| "gate": { | |
| "suppression": 0.513, | |
| "retain": 100.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "LLMU", | |
| "venue": "NeurIPS'24", | |
| "src": "repo", | |
| "link": "https://doi.org/10.52202/079017-3346", | |
| "link_kind": "doi", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 2.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.203, | |
| "collapsed": false, | |
| "d_degen": 0.5, | |
| "best": false, | |
| "rank": 9, | |
| "gain_over_no_intervention": -0.283, | |
| "gate": { | |
| "suppression": 0.269, | |
| "retain": 146.0, | |
| "d_degen": 0.0 | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.223, | |
| "collapsed": false, | |
| "d_degen": -2.5, | |
| "best": false, | |
| "rank": 15, | |
| "gain_over_no_intervention": -0.008, | |
| "gate": { | |
| "suppression": 0.223, | |
| "retain": 100.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "MemFlex", | |
| "venue": "EMNLP'24 Find.", | |
| "src": "repo", | |
| "link": "https://aclanthology.org/2024.findings-emnlp.82/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 45.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.25, | |
| "collapsed": false, | |
| "d_degen": 0.5, | |
| "best": false, | |
| "rank": 8, | |
| "gain_over_no_intervention": -0.236, | |
| "gate": { | |
| "suppression": 0.305, | |
| "retain": 133.0, | |
| "d_degen": 0.5 | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.237, | |
| "collapsed": false, | |
| "d_degen": -19.5, | |
| "best": false, | |
| "rank": 14, | |
| "gain_over_no_intervention": 0.006, | |
| "gate": { | |
| "suppression": 0.237, | |
| "retain": 100.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "MEOW", | |
| "venue": "ACL'25 Find.", | |
| "src": "repo", | |
| "link": "https://aclanthology.org/2025.findings-acl.535/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 100.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.337, | |
| "collapsed": false, | |
| "d_degen": -5.5, | |
| "best": false, | |
| "rank": 12, | |
| "gain_over_no_intervention": 0.106, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "FALCON", | |
| "venue": "NeurIPS'25", | |
| "src": "repo", | |
| "link": "https://doi.org/10.52202/085713-3419", | |
| "link_kind": "doi", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 20.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 100.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.48, | |
| "collapsed": false, | |
| "d_degen": -26.0, | |
| "best": false, | |
| "rank": 9, | |
| "gain_over_no_intervention": 0.249, | |
| "gate": { | |
| "suppression": 0.501, | |
| "retain": 96.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "LoKU", | |
| "venue": "ICLR'25", | |
| "src": "repo", | |
| "link": "https://arxiv.org/abs/2408.06621", | |
| "link_kind": "url", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 83.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.743, | |
| "collapsed": false, | |
| "d_degen": -18.0, | |
| "best": true, | |
| "rank": 1, | |
| "gain_over_no_intervention": 0.512, | |
| "gate": { | |
| "suppression": 0.753, | |
| "retain": 99.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "SimNPO", | |
| "venue": "NeurIPS'25", | |
| "src": "OU", | |
| "link": "https://doi.org/10.52202/085713-0058", | |
| "link_kind": "doi", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 77.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 52.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "ME+AP", | |
| "venue": "ICLR'25", | |
| "src": "repo", | |
| "link": "https://arxiv.org/abs/2410.08109", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 100.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.602, | |
| "collapsed": false, | |
| "d_degen": -8.5, | |
| "best": false, | |
| "rank": 3, | |
| "gain_over_no_intervention": 0.371, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "ReLearn", | |
| "venue": "ACL'25", | |
| "src": "repo", | |
| "link": "https://aclanthology.org/2025.acl-long.297/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 89.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.286, | |
| "collapsed": false, | |
| "d_degen": -36.0, | |
| "best": false, | |
| "rank": 13, | |
| "gain_over_no_intervention": 0.055, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "SatImp", | |
| "venue": "ICML'25", | |
| "src": "OU", | |
| "link": "https://arxiv.org/abs/2505.11953", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 46.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 97.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 45.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "LAW", | |
| "venue": "ICLR'25", | |
| "src": "repo", | |
| "link": "https://arxiv.org/abs/2405.16720", | |
| "link_kind": "arxiv", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 44.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.341, | |
| "collapsed": false, | |
| "d_degen": 5.0, | |
| "best": false, | |
| "rank": 5, | |
| "gain_over_no_intervention": -0.145, | |
| "gate": { | |
| "suppression": 0.434, | |
| "retain": 132.0, | |
| "d_degen": 5.0 | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.558, | |
| "collapsed": false, | |
| "d_degen": -29.0, | |
| "best": false, | |
| "rank": 4, | |
| "gain_over_no_intervention": 0.327, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "SOUL", | |
| "venue": "EMNLP'24", | |
| "src": "OU", | |
| "link": "https://aclanthology.org/2024.emnlp-main.245/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 100.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 63.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| } | |
| } | |
| }, | |
| { | |
| "method": "ULD", | |
| "venue": "NeurIPS'24", | |
| "src": "repo", | |
| "link": "https://doi.org/10.52202/079017-0400", | |
| "link_kind": "doi", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 49.5, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 100.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.347, | |
| "collapsed": false, | |
| "d_degen": -18.5, | |
| "best": false, | |
| "rank": 11, | |
| "gain_over_no_intervention": 0.116, | |
| "gate": { | |
| "suppression": 0.429, | |
| "retain": 81.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "SKU", | |
| "venue": "ACL'24 Find.", | |
| "src": "repo", | |
| "link": "https://aclanthology.org/2024.findings-acl.107/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": 0.374, | |
| "collapsed": false, | |
| "d_degen": -46.0, | |
| "best": false, | |
| "rank": 2, | |
| "gain_over_no_intervention": 0.164, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.385, | |
| "collapsed": false, | |
| "d_degen": 21.5, | |
| "best": false, | |
| "rank": 4, | |
| "gain_over_no_intervention": -0.101, | |
| "gate": null | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": 0.519, | |
| "collapsed": false, | |
| "d_degen": -35.0, | |
| "best": false, | |
| "rank": 6, | |
| "gain_over_no_intervention": 0.288, | |
| "gate": { | |
| "suppression": 0.519, | |
| "retain": 100.0, | |
| "d_degen": 0.0 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "method": "REVS", | |
| "venue": "ACL'25 Find.", | |
| "src": "repo", | |
| "link": "https://aclanthology.org/2025.findings-acl.763/", | |
| "link_kind": "acl", | |
| "role": "roster", | |
| "cells": { | |
| "Llama-3.1-8B": { | |
| "k": 0.37, | |
| "collapsed": false, | |
| "d_degen": -15.5, | |
| "best": false, | |
| "rank": 3, | |
| "gain_over_no_intervention": 0.16, | |
| "gate": null | |
| }, | |
| "Mistral-7B": { | |
| "k": 0.275, | |
| "collapsed": false, | |
| "d_degen": 5.0, | |
| "best": false, | |
| "rank": 6, | |
| "gain_over_no_intervention": -0.211, | |
| "gate": { | |
| "suppression": 0.343, | |
| "retain": 129.0, | |
| "d_degen": 5.0 | |
| } | |
| }, | |
| "Qwen3.5-9B": { | |
| "k": null, | |
| "collapsed": true, | |
| "d_degen": 63.0, | |
| "best": false, | |
| "rank": null, | |
| "gain_over_no_intervention": null, | |
| "gate": null | |
| } | |
| } | |
| } | |
| ], | |
| "axis": { | |
| "note": "Extents cover the twenty-method roster only. The reference row is plotted but never widens the shared axis.", | |
| "rank_max": 15, | |
| "rank_ticks": [ | |
| 1, | |
| 5, | |
| 10, | |
| 15 | |
| ], | |
| "gain_top": 0.6, | |
| "gain_floor": -0.3, | |
| "gain_ticks": [ | |
| { | |
| "value": 0.6, | |
| "label": "+0.60", | |
| "baseline": false | |
| }, | |
| { | |
| "value": 0.4, | |
| "label": "+0.40", | |
| "baseline": false | |
| }, | |
| { | |
| "value": 0.2, | |
| "label": "+0.20", | |
| "baseline": false | |
| }, | |
| { | |
| "value": 0.0, | |
| "label": "0.00", | |
| "baseline": true | |
| }, | |
| { | |
| "value": -0.2, | |
| "label": "-0.20", | |
| "baseline": false | |
| } | |
| ], | |
| "reference_may_exceed": true | |
| } | |
| } | |
| } |