{ "generated_from": [ "paper tables/tab_cross_model_all.tex", "release docs/data.js", "the fixed 20-method x 3-model leaderboard table of the preprint (substrate P, weight-merged target)" ], "rule": "K-Scores are reported per substrate and are never averaged across substrates.", "substrate_matrix": [ { "base": "Llama-3.1-8B", "substrate": "P", "method": "None", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.237, "Z_summary": 0.648 }, "or_all": 0.682, "retain": 0.737, "delta_sel": null, "degen_pct": 45.0, "k_class": "base", "k_score": 0.247 }, { "base": "Llama-3.1-8B", "substrate": "P", "method": "NOISE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.212, "Z_summary": 0.648 }, "or_all": 0.67, "retain": 0.73, "delta_sel": -0.01, "degen_pct": 47.0, "k_class": "MF", "k_score": 0.247 }, { "base": "Llama-3.1-8B", "substrate": "P", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.0, "Z_summary": 0.002 }, "or_all": 0.003, "retain": 0.737, "delta_sel": 0.0, "degen_pct": 38.0, "k_class": "K-REF∞", "k_score": 0.906 }, { "base": "Llama-3.1-8B", "substrate": "P", "method": "STAR", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.111, "Z_summary": 0.295 }, "or_all": 0.312, "retain": 0.7, "delta_sel": -0.037, "degen_pct": 43.0, "k_class": "K-REF 2x", "k_score": 0.377 }, { "base": "Llama-3.1-8B", "substrate": "P", "method": "LEACE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.221, "Z_summary": 0.648 }, "or_all": 0.678, "retain": 0.737, "delta_sel": 0.0, "degen_pct": 46.0, "k_class": "no-op", "k_score": 0.245 }, { "base": "Llama-3.1-8B", "substrate": "P", "method": "CHA", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.037, "Z_summary": 0.16 }, "or_all": 0.175, "retain": 0.183, "delta_sel": -0.55, "degen_pct": 46.0, "k_class": "K-REF 2x", "k_score": 0.344 }, { "base": "Llama-3.1-8B", "substrate": "P", "method": "O3", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.0, "Z_summary": 0.0 }, "or_all": 0.0, "retain": 0.737, "delta_sel": 0.0, "degen_pct": 34.0, "k_class": "K-REF∞", "k_score": 0.997 }, { "base": "Llama-3.1-8B", "substrate": "C", "method": "None", "channels": { "Z_CoT": 0.0, "Z_tool": 0.038, "Z_tool_wide": 0.038, "Z_answer": 0.192, "Z_summary": 0.0 }, "or_all": 0.223, "retain": 0.963, "delta_sel": null, "degen_pct": 66.0, "k_class": "base", "k_score": 0.693 }, { "base": "Llama-3.1-8B", "substrate": "C", "method": "NOISE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.037, "Z_tool_wide": 0.038, "Z_answer": 0.208, "Z_summary": 0.0 }, "or_all": 0.245, "retain": 0.972, "delta_sel": 0.01, "degen_pct": 61.0, "k_class": "MF", "k_score": 0.672 }, { "base": "Llama-3.1-8B", "substrate": "C", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.0, "Z_summary": 0.0 }, "or_all": 0.0, "retain": 0.963, "delta_sel": 0.0, "degen_pct": 91.0, "k_class": "K-REF∞", "k_score": 0.7 }, { "base": "Llama-3.1-8B", "substrate": "C", "method": "STAR", "channels": { "Z_CoT": 0.0, "Z_tool": 0.015, "Z_tool_wide": 0.015, "Z_answer": 0.097, "Z_summary": 0.0 }, "or_all": 0.112, "retain": 0.962, "delta_sel": -0.0, "degen_pct": 68.0, "k_class": "K-REF 2x", "k_score": 0.719 }, { "base": "Llama-3.1-8B", "substrate": "C", "method": "LEACE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.038, "Z_tool_wide": 0.038, "Z_answer": 0.192, "Z_summary": 0.0 }, "or_all": 0.223, "retain": 0.963, "delta_sel": 0.0, "degen_pct": 66.0, "k_class": "no-op", "k_score": 0.693 }, { "base": "Llama-3.1-8B", "substrate": "R-text", "method": "None", "channels": { "Z_CoT": 0.0, "Z_tool": 0.008, "Z_tool_wide": 0.602, "Z_answer": 0.203, "Z_summary": 0.0 }, "or_all": 0.602, "retain": 0.888, "delta_sel": null, "degen_pct": 48.0, "k_class": "base", "k_score": 0.364 }, { "base": "Llama-3.1-8B", "substrate": "R-text", "method": "NOISE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.01, "Z_tool_wide": 0.568, "Z_answer": 0.208, "Z_summary": 0.0 }, "or_all": 0.568, "retain": 0.882, "delta_sel": -0.01, "degen_pct": 45.0, "k_class": "MF", "k_score": 0.393 }, { "base": "Llama-3.1-8B", "substrate": "R-text", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.005, "Z_answer": 0.0, "Z_summary": 0.0 }, "or_all": 0.005, "retain": 0.888, "delta_sel": 0.0, "degen_pct": 71.0, "k_class": "K-REF∞", "k_score": 0.73 }, { "base": "Llama-3.1-8B", "substrate": "R-text", "method": "STAR", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.602, "Z_answer": 0.097, "Z_summary": 0.0 }, "or_all": 0.602, "retain": 0.888, "delta_sel": 0.0, "degen_pct": 52.0, "k_class": "MF", "k_score": 0.349 }, { "base": "Llama-3.1-8B", "substrate": "R-text", "method": "LEACE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.008, "Z_tool_wide": 0.602, "Z_answer": 0.203, "Z_summary": 0.0 }, "or_all": 0.602, "retain": 0.888, "delta_sel": 0.0, "degen_pct": 48.0, "k_class": "no-op", "k_score": 0.364 }, { "base": "Llama-3.1-8B", "substrate": "R-struct", "method": "None", "channels": { "Z_CoT": 0.0, "Z_tool": 0.002, "Z_tool_wide": 0.855, "Z_answer": 0.832, "Z_summary": 0.0 }, "or_all": 0.855, "retain": 0.863, "delta_sel": null, "degen_pct": 9.0, "k_class": "base", "k_score": 0.128 }, { "base": "Llama-3.1-8B", "substrate": "R-struct", "method": "NOISE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.005, "Z_tool_wide": 0.853, "Z_answer": 0.833, "Z_summary": 0.0 }, "or_all": 0.853, "retain": 0.875, "delta_sel": 0.01, "degen_pct": 10.0, "k_class": "MF", "k_score": 0.13 }, { "base": "Llama-3.1-8B", "substrate": "R-struct", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.0, "Z_summary": 0.0 }, "or_all": 0.0, "retain": 0.863, "delta_sel": 0.0, "degen_pct": 73.0, "k_class": "K-REF∞", "k_score": 0.345 }, { "base": "Llama-3.1-8B", "substrate": "R-struct", "method": "STAR", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.857, "Z_answer": 0.463, "Z_summary": 0.0 }, "or_all": 0.857, "retain": 0.862, "delta_sel": -0.0, "degen_pct": 9.0, "k_class": "K-SUP", "k_score": 0.127 }, { "base": "Llama-3.1-8B", "substrate": "R-struct", "method": "LEACE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.002, "Z_tool_wide": 0.855, "Z_answer": 0.832, "Z_summary": 0.0 }, "or_all": 0.855, "retain": 0.863, "delta_sel": 0.0, "degen_pct": 9.0, "k_class": "no-op", "k_score": 0.128 }, { "base": "Mistral-7B", "substrate": "P", "method": "NOISE", "channels": { "Z_CoT": 0.004, "Z_tool": 0.003, "Z_tool_wide": 0.003, "Z_answer": 0.264, "Z_summary": 0.403 }, "or_all": 0.41, "retain": 0.685, "delta_sel": 0.26, "degen_pct": 0.0, "k_class": "MF", "k_score": 0.389 }, { "base": "Mistral-7B", "substrate": "P", "method": "ECO", "channels": { "Z_CoT": 0.001, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.062, "Z_summary": 0.029 }, "or_all": 0.0, "retain": 0.46, "delta_sel": 0.04, "degen_pct": 0.0, "k_class": "K-REF∞", "k_score": 0.898 }, { "base": "Mistral-7B", "substrate": "P", "method": "STAR", "channels": { "Z_CoT": 0.001, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.219, "Z_summary": 0.373 }, "or_all": 0.255, "retain": 0.4, "delta_sel": -0.02, "degen_pct": 0.0, "k_class": "MF", "k_score": 0.523 }, { "base": "Mistral-7B", "substrate": "P", "method": "LEACE", "channels": { "Z_CoT": 0.001, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.301, "Z_summary": 0.42 }, "or_all": 0.435, "retain": 0.455, "delta_sel": 0.03, "degen_pct": 0.0, "k_class": "MF", "k_score": 0.453 }, { "base": "Mistral-7B", "substrate": "C", "method": "NOISE", "channels": { "Z_CoT": 0.019, "Z_tool": 0.086, "Z_tool_wide": 0.123, "Z_answer": 0.05, "Z_summary": 0.047 }, "or_all": 0.1, "retain": 0.995, "delta_sel": 0.0, "degen_pct": 29.0, "k_class": "below gate", "k_score": null }, { "base": "Mistral-7B", "substrate": "C", "method": "ECO", "channels": { "Z_CoT": 0.015, "Z_tool": 0.004, "Z_tool_wide": 0.031, "Z_answer": 0.0, "Z_summary": 0.03 }, "or_all": 0.0, "retain": 0.995, "delta_sel": 0.0, "degen_pct": 42.0, "k_class": "below gate", "k_score": null }, { "base": "Mistral-7B", "substrate": "C", "method": "STAR", "channels": { "Z_CoT": 0.01, "Z_tool": 0.041, "Z_tool_wide": 0.079, "Z_answer": 0.025, "Z_summary": 0.049 }, "or_all": 0.045, "retain": 0.995, "delta_sel": 0.0, "degen_pct": 33.0, "k_class": "below gate", "k_score": null }, { "base": "Mistral-7B", "substrate": "C", "method": "LEACE", "channels": { "Z_CoT": 0.015, "Z_tool": 0.041, "Z_tool_wide": 0.076, "Z_answer": 0.036, "Z_summary": 0.049 }, "or_all": 0.06, "retain": 0.995, "delta_sel": 0.0, "degen_pct": 34.0, "k_class": "below gate", "k_score": null }, { "base": "Mistral-7B", "substrate": "R-text", "method": "None", "channels": { "Z_CoT": 0.073, "Z_tool": 0.113, "Z_tool_wide": 0.345, "Z_answer": 0.168, "Z_summary": 0.0 }, "or_all": 0.345, "retain": 0.937, "delta_sel": null, "degen_pct": 17.0, "k_class": "base", "k_score": 0.616 }, { "base": "Mistral-7B", "substrate": "R-text", "method": "NOISE", "channels": { "Z_CoT": 0.097, "Z_tool": 0.087, "Z_tool_wide": 0.416, "Z_answer": 0.25, "Z_summary": 0.046 }, "or_all": 0.405, "retain": 0.96, "delta_sel": 0.01, "degen_pct": 16.0, "k_class": "MF", "k_score": 0.555 }, { "base": "Mistral-7B", "substrate": "R-text", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.0, "Z_summary": 0.0 }, "or_all": 0.0, "retain": 0.937, "delta_sel": 0.0, "degen_pct": 3.0, "k_class": "K-REF∞", "k_score": 0.924 }, { "base": "Mistral-7B", "substrate": "R-text", "method": "STAR", "channels": { "Z_CoT": 0.037, "Z_tool": 0.108, "Z_tool_wide": 0.342, "Z_answer": 0.095, "Z_summary": 0.0 }, "or_all": 0.342, "retain": 0.937, "delta_sel": 0.0, "degen_pct": 17.0, "k_class": "MF", "k_score": 0.618 }, { "base": "Mistral-7B", "substrate": "R-text", "method": "LEACE", "channels": { "Z_CoT": 0.073, "Z_tool": 0.113, "Z_tool_wide": 0.345, "Z_answer": 0.168, "Z_summary": 0.0 }, "or_all": 0.345, "retain": 0.937, "delta_sel": 0.0, "degen_pct": 17.0, "k_class": "no-op", "k_score": 0.616 }, { "base": "Mistral-7B", "substrate": "R-struct", "method": "None", "channels": { "Z_CoT": 0.0, "Z_tool": 0.002, "Z_tool_wide": 0.828, "Z_answer": 0.797, "Z_summary": 0.0 }, "or_all": 0.828, "retain": 0.93, "delta_sel": null, "degen_pct": 4.0, "k_class": "base", "k_score": 0.158 }, { "base": "Mistral-7B", "substrate": "R-struct", "method": "NOISE", "channels": { "Z_CoT": 0.003, "Z_tool": 0.003, "Z_tool_wide": 0.842, "Z_answer": 0.864, "Z_summary": 0.047 }, "or_all": 0.835, "retain": 0.95, "delta_sel": 0.01, "degen_pct": 4.0, "k_class": "MF", "k_score": 0.157 }, { "base": "Mistral-7B", "substrate": "R-struct", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.0, "Z_summary": 0.0 }, "or_all": 0.0, "retain": 0.93, "delta_sel": 0.0, "degen_pct": 5.0, "k_class": "K-REF∞", "k_score": 0.91 }, { "base": "Mistral-7B", "substrate": "R-struct", "method": "STAR", "channels": { "Z_CoT": 0.0, "Z_tool": 0.002, "Z_tool_wide": 0.827, "Z_answer": 0.655, "Z_summary": 0.0 }, "or_all": 0.827, "retain": 0.93, "delta_sel": 0.0, "degen_pct": 4.0, "k_class": "MF", "k_score": 0.159 }, { "base": "Mistral-7B", "substrate": "R-struct", "method": "LEACE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.002, "Z_tool_wide": 0.828, "Z_answer": 0.797, "Z_summary": 0.0 }, "or_all": 0.828, "retain": 0.93, "delta_sel": 0.0, "degen_pct": 4.0, "k_class": "no-op", "k_score": 0.158 }, { "base": "Qwen3.5-9B", "substrate": "P", "method": "NOISE", "channels": { "Z_CoT": 0.002, "Z_tool": 0.0, "Z_tool_wide": 0.028, "Z_answer": 0.334, "Z_summary": 0.728 }, "or_all": 0.665, "retain": 0.915, "delta_sel": -0.08, "degen_pct": 28.0, "k_class": "MF", "k_score": 0.204 }, { "base": "Qwen3.5-9B", "substrate": "P", "method": "ECO", "channels": { "Z_CoT": 0.005, "Z_tool": 0.004, "Z_tool_wide": 0.035, "Z_answer": 0.04, "Z_summary": 0.049 }, "or_all": 0.0, "retain": 1.0, "delta_sel": 0.0, "degen_pct": 10.0, "k_class": "K-REF∞", "k_score": 0.915 }, { "base": "Qwen3.5-9B", "substrate": "P", "method": "STAR", "channels": { "Z_CoT": 0.002, "Z_tool": 0.0, "Z_tool_wide": 0.037, "Z_answer": 0.263, "Z_summary": 0.555 }, "or_all": 0.275, "retain": 1.0, "delta_sel": 0.0, "degen_pct": 34.0, "k_class": "MF", "k_score": 0.4 }, { "base": "Qwen3.5-9B", "substrate": "P", "method": "LEACE", "channels": { "Z_CoT": 0.002, "Z_tool": 0.0, "Z_tool_wide": 0.038, "Z_answer": 0.361, "Z_summary": 0.707 }, "or_all": 0.63, "retain": 1.0, "delta_sel": 0.0, "degen_pct": 35.0, "k_class": "MF", "k_score": 0.235 }, { "base": "Qwen3.5-9B", "substrate": "C", "method": "None", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.992, "Z_summary": 0.0 }, "or_all": 0.992, "retain": 0.997, "delta_sel": null, "degen_pct": 0.0, "k_class": "base", "k_score": 0.0 }, { "base": "Qwen3.5-9B", "substrate": "C", "method": "NOISE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.997, "Z_summary": 0.0 }, "or_all": 0.997, "retain": 0.98, "delta_sel": -0.017, "degen_pct": 0.0, "k_class": "MF", "k_score": 0.0 }, { "base": "Qwen3.5-9B", "substrate": "C", "method": "ECO", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.048, "Z_summary": 0.0 }, "or_all": 0.048, "retain": 0.995, "delta_sel": -0.0, "degen_pct": 74.0, "k_class": "K-REF 10x", "k_score": 0.233 }, { "base": "Qwen3.5-9B", "substrate": "C", "method": "STAR", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.347, "Z_summary": 0.0 }, "or_all": 0.347, "retain": 0.892, "delta_sel": -0.105, "degen_pct": 79.0, "k_class": "K-REF 2x", "k_score": 0.052 }, { "base": "Qwen3.5-9B", "substrate": "C", "method": "LEACE", "channels": { "Z_CoT": 0.0, "Z_tool": 0.0, "Z_tool_wide": 0.0, "Z_answer": 0.992, "Z_summary": 0.0 }, "or_all": 0.992, "retain": 0.995, "delta_sel": -0.0, "degen_pct": 0.0, "k_class": "no-op", "k_score": 0.0 }, { "base": "Qwen3.5-9B", "substrate": "R-text", "method": "NOISE", "channels": { "Z_CoT": 0.441, "Z_tool": 0.028, "Z_tool_wide": 0.632, "Z_answer": 0.698, "Z_summary": 0.036 }, "or_all": 0.62, "retain": 0.795, "delta_sel": 0.02, "degen_pct": 38.0, "k_class": "MF", "k_score": 0.357 }, { "base": "Qwen3.5-9B", "substrate": "R-text", "method": "ECO", "channels": { "Z_CoT": 0.013, "Z_tool": 0.0, "Z_tool_wide": 0.034, "Z_answer": 0.008, "Z_summary": 0.016 }, "or_all": 0.01, "retain": 0.775, "delta_sel": 0.0, "degen_pct": 55.0, "k_class": "K-REF∞", "k_score": 0.813 }, { "base": "Qwen3.5-9B", "substrate": "R-text", "method": "STAR", "channels": { "Z_CoT": 0.281, "Z_tool": 0.029, "Z_tool_wide": 0.642, "Z_answer": 0.49, "Z_summary": 0.028 }, "or_all": 0.63, "retain": 0.775, "delta_sel": 0.0, "degen_pct": 60.0, "k_class": "MF", "k_score": 0.286 }, { "base": "Qwen3.5-9B", "substrate": "R-text", "method": "LEACE", "channels": { "Z_CoT": 0.456, "Z_tool": 0.025, "Z_tool_wide": 0.642, "Z_answer": 0.75, "Z_summary": 0.028 }, "or_all": 0.63, "retain": 0.775, "delta_sel": 0.0, "degen_pct": 41.0, "k_class": "MF", "k_score": 0.349 }, { "base": "Qwen3.5-9B", "substrate": "R-struct", "method": "NOISE", "channels": { "Z_CoT": 0.075, "Z_tool": 0.0, "Z_tool_wide": 0.484, "Z_answer": 0.654, "Z_summary": 0.034 }, "or_all": 0.42, "retain": 0.77, "delta_sel": 0.04, "degen_pct": 36.0, "k_class": "MF", "k_score": 0.484 }, { "base": "Qwen3.5-9B", "substrate": "R-struct", "method": "ECO", "channels": { "Z_CoT": 0.024, "Z_tool": 0.0, "Z_tool_wide": 0.029, "Z_answer": 0.024, "Z_summary": 0.016 }, "or_all": 0.005, "retain": 0.74, "delta_sel": 0.01, "degen_pct": 51.0, "k_class": "K-REF∞", "k_score": 0.878 }, { "base": "Qwen3.5-9B", "substrate": "R-struct", "method": "STAR", "channels": { "Z_CoT": 0.04, "Z_tool": 0.0, "Z_tool_wide": 0.459, "Z_answer": 0.653, "Z_summary": 0.028 }, "or_all": 0.38, "retain": 0.74, "delta_sel": 0.01, "degen_pct": 48.0, "k_class": "MF", "k_score": 0.52 }, { "base": "Qwen3.5-9B", "substrate": "R-struct", "method": "LEACE", "channels": { "Z_CoT": 0.053, "Z_tool": 0.0, "Z_tool_wide": 0.452, "Z_answer": 0.708, "Z_summary": 0.028 }, "or_all": 0.37, "retain": 0.74, "delta_sel": 0.01, "degen_pct": 48.0, "k_class": "MF", "k_score": 0.53 } ], "gated_leaderboard_P": { "Llama-3.1-8B": [ { "method": "ELM", "suppression": 0.349, "retain": 83.0, "degen": 0.0 } ], "Mistral-7B": [ { "method": "RMU", "suppression": 0.751, "retain": 103.0, "degen": 1.5 }, { "method": "LAW", "suppression": 0.434, "retain": 132.0, "degen": 5.0 }, { "method": "REVS", "suppression": 0.343, "retain": 129.0, "degen": 5.0 }, { "method": "ELM", "suppression": 0.322, "retain": 132.0, "degen": 0.0 }, { "method": "MemFlex", "suppression": 0.305, "retain": 133.0, "degen": 0.5 }, { "method": "LLMU", "suppression": 0.269, "retain": 146.0, "degen": 0.0 } ], "Qwen3.5-9B": [ { "method": "LoKU", "suppression": 0.753, "retain": 99.0, "degen": 0.0 }, { "method": "ELM", "suppression": 0.735, "retain": 96.0, "degen": 0.0 }, { "method": "RMU", "suppression": 0.538, "retain": 99.0, "degen": 0.0 }, { "method": "SKU", "suppression": 0.519, "retain": 100.0, "degen": 0.0 }, { "method": "WGA", "suppression": 0.513, "retain": 100.0, "degen": 0.0 }, { "method": "FALCON", "suppression": 0.501, "retain": 96.0, "degen": 0.0 }, { "method": "UNDIAL-corrected", "suppression": 0.493, "retain": 100.0, "degen": 0.0 }, { "method": "ULD", "suppression": 0.429, "retain": 81.0, "degen": 0.0 }, { "method": "MemFlex", "suppression": 0.237, "retain": 100.0, "degen": 0.0 }, { "method": "LLMU", "suppression": 0.223, "retain": 100.0, "degen": 0.0 } ] }, "budget_recovery": { "Llama-3.1-8B": { "P": { "rates": [ 0.152, 0.682, 0.682, 0.682, 0.682 ], "flips": "4/6" }, "C": { "rates": [ 0.192, 0.192, 0.192, 0.223, 0.223 ], "flips": "0/3" }, "R-text": { "rates": [ 0.203, 0.203, 0.203, 0.602, 0.602 ], "flips": "1/3" }, "R-struct": { "rates": [ 0.832, 0.832, 0.832, 0.855, 0.855 ], "flips": "0/3" } }, "Mistral-7B": { "P": { "rates": [ 0.19, 0.415, 0.415, 0.415, 0.415 ], "flips": "1/9" }, "R-text": { "rates": [ 0.168, 0.168, 0.193, 0.345, 0.345 ], "flips": "0/3" }, "R-struct": { "rates": [ 0.797, 0.797, 0.797, 0.828, 0.828 ], "flips": "0/3" } }, "Qwen3.5-9B": { "P": { "rates": [ 0.195, 0.65, 0.65, 0.65, 0.65 ], "flips": "5/15" }, "C": { "rates": [ 0.992, 0.992, 0.992, 0.992, 0.992 ], "flips": "0/4" }, "R-text": { "rates": [ 0.375, 0.375, 0.39, 0.613, 0.613 ], "flips": "2/4" }, "R-struct": { "rates": [ 0.358, 0.358, 0.36, 0.373, 0.373 ], "flips": "1/4" } } }, "gates": { "substrate_validity": { "threshold": 0.1, "metric": "graded answer-channel severity of the no-intervention agent", "note": "A cell is invalid when the un-intervened agent fails to reproduce the target in its own answer channel. Such cells carry the k_class 'below gate' and no K-Score." } }, "cell_states": { "scored": "A K-Score is reported.", "below gate": "The cell was evaluated, but the un-intervened baseline fails the substrate-validity gate on this base model, so no K-Score is defined.", "not evaluated": "The method was never run on this substrate, because its mechanism cannot act there." }, "method_refs": { "CHA": { "title": "Towards Robust and Parameter-Efficient Knowledge Unlearning for LLMs", "venue": "International Conference on Learning Representations (ICLR)", "link": "https://arxiv.org/abs/2408.06621", "link_kind": "arxiv" }, "ECO": { "title": "Large Language Model Unlearning via Embedding-Corrupted Prompts", "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "link": "https://arxiv.org/abs/2406.07933", "link_kind": "arxiv" }, "LEACE": { "title": "LEACE: Perfect Linear Concept Erasure in Closed Form", "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "link": "https://arxiv.org/abs/2306.03819", "link_kind": "arxiv" }, "O3": { "title": "On Large Language Model Continual Unlearning", "venue": "International Conference on Learning Representations (ICLR)", "link": "https://arxiv.org/abs/2407.10223", "link_kind": "arxiv" }, "STAR": { "title": "STaR: Sensitive Trajectory Regulation for Unlearning in Large Reasoning Models", "venue": "Proceedings of the AAAI Conference on Artificial Intelligence", "link": "https://doi.org/10.1609/aaai.v40i41.40818", "link_kind": "doi" } }, "method_refs_note": "Links are taken from the paper's own bibliography. Where the entry records a DOI the link resolves through doi.org; otherwise it points at the arXiv abstract the paper cites. NOISE and None are controls, not published methods, and carry no reference.", "cross_model_P": { "title": "Twenty published methods, three base models", "bases": [ "Llama-3.1-8B", "Mistral-7B", "Qwen3.5-9B" ], "no_intervention_k": { "Llama-3.1-8B": 0.21, "Mistral-7B": 0.486, "Qwen3.5-9B": 0.231 }, "scope": "Substrate P with a weight-merged PII target. Seed 0, 200 forget and 200 retain queries per cell. Leakage is binary OR(all) over the six channels.", "reading_rule": "K-Score is read within one base model. Across base models this view compares method ordering and the gain over that model's no-intervention row, never raw K-Score on a shared axis.", "differs_from_substrate_matrix": "The substrate matrix above reports the Llama-3.1-8B block pooled over three seeds at n=600 per split and uses graded token-recall severity, so its no-intervention K on Llama is 0.247 where this view reads 0.210. Mistral-7B and Qwen3.5-9B are seed 0 in both and agree at 0.486 and 0.231.", "collapse_rule": "Terminal collapse is recorded when a split's absolute degeneration reaches 50%. A collapsed cell holds no K-Score and is not a zero.", "gate_rule": "Retain at least 80% of the no-intervention retain-set answerability, added degeneration at most 20pp, and no terminal collapse.", "totals": { "cells": 60, "collapsed": 32, "scored": 28, "gate_pass": 17, "collapsed_by_base": { "Llama-3.1-8B": 16, "Mistral-7B": 11, "Qwen3.5-9B": 5 }, "scored_by_base": { "Llama-3.1-8B": 4, "Mistral-7B": 9, "Qwen3.5-9B": 15 }, "gate_pass_by_base": { "Llama-3.1-8B": 1, "Mistral-7B": 6, "Qwen3.5-9B": 10 }, "ranked_on_all_three": [ "ELM", "SKU" ], "ranked_on_two_or_more": [ "ELM", "LAW", "LLMU", "MemFlex", "REVS", "RMU", "SKU", "UNDIAL-corrected", "WGA" ], "roster_size": 20, "ranked_on_all_three_count": 2, "collapsed_on_at_least_one": 18 }, "reference_row": [ { "method": "ECO", "venue": "NeurIPS'24", "src": "repo", "link": "https://arxiv.org/abs/2406.07933", "link_kind": "url", "role": "reference", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 2.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.911, "collapsed": false, "d_degen": 0.0, "best": false, "rank": null, "gain_over_no_intervention": 0.425, "gate": null }, "Qwen3.5-9B": { "k": 0.92, "collapsed": false, "d_degen": -20.5, "best": false, "rank": null, "gain_over_no_intervention": 0.689, "gate": null } } } ], "methods": [ { "method": "FLAT", "venue": "ICLR'25", "src": "repo", "link": "https://arxiv.org/abs/2410.11143", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": 0.763, "collapsed": false, "d_degen": -15.5, "best": true, "rank": 1, "gain_over_no_intervention": 0.553, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 100.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null } } }, { "method": "DOOR", "venue": "ICML'25", "src": "repo", "link": "https://arxiv.org/abs/2503.03710", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 91.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.38, "collapsed": false, "d_degen": -12.0, "best": false, "rank": 10, "gain_over_no_intervention": 0.149, "gate": null } } }, { "method": "UNDIAL-corrected", "venue": "NAACL'25", "src": "OU", "link": "https://aclanthology.org/2025.naacl-long.444/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 7.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.619, "collapsed": false, "d_degen": 5.5, "best": false, "rank": 3, "gain_over_no_intervention": 0.133, "gate": null }, "Qwen3.5-9B": { "k": 0.493, "collapsed": false, "d_degen": -17.5, "best": false, "rank": 8, "gain_over_no_intervention": 0.262, "gate": { "suppression": 0.493, "retain": 100.0, "d_degen": 0.0 } } } }, { "method": "ELM", "venue": "NeurIPS'25", "src": "repo", "link": "https://doi.org/10.52202/085713-2028", "link_kind": "doi", "role": "roster", "cells": { "Llama-3.1-8B": { "k": 0.3, "collapsed": false, "d_degen": -16.0, "best": false, "rank": 4, "gain_over_no_intervention": 0.09, "gate": { "suppression": 0.349, "retain": 83.0, "d_degen": 0.0 } }, "Mistral-7B": { "k": 0.266, "collapsed": false, "d_degen": 0.0, "best": false, "rank": 7, "gain_over_no_intervention": -0.22, "gate": { "suppression": 0.322, "retain": 132.0, "d_degen": 0.0 } }, "Qwen3.5-9B": { "k": 0.708, "collapsed": false, "d_degen": -27.0, "best": false, "rank": 2, "gain_over_no_intervention": 0.477, "gate": { "suppression": 0.735, "retain": 96.0, "d_degen": 0.0 } } } }, { "method": "RMU", "venue": "ICML'24", "src": "OU", "link": "https://arxiv.org/abs/2403.03218", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 40.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.726, "collapsed": false, "d_degen": 1.5, "best": true, "rank": 1, "gain_over_no_intervention": 0.24, "gate": { "suppression": 0.751, "retain": 103.0, "d_degen": 1.5 } }, "Qwen3.5-9B": { "k": 0.53, "collapsed": false, "d_degen": -24.5, "best": false, "rank": 5, "gain_over_no_intervention": 0.299, "gate": { "suppression": 0.538, "retain": 99.0, "d_degen": 0.0 } } } }, { "method": "WGA", "venue": "ICLR'25", "src": "OU", "link": "https://arxiv.org/abs/2502.19301", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 35.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.722, "collapsed": false, "d_degen": 10.0, "best": false, "rank": 2, "gain_over_no_intervention": 0.236, "gate": null }, "Qwen3.5-9B": { "k": 0.513, "collapsed": false, "d_degen": -2.5, "best": false, "rank": 7, "gain_over_no_intervention": 0.282, "gate": { "suppression": 0.513, "retain": 100.0, "d_degen": 0.0 } } } }, { "method": "LLMU", "venue": "NeurIPS'24", "src": "repo", "link": "https://doi.org/10.52202/079017-3346", "link_kind": "doi", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 2.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.203, "collapsed": false, "d_degen": 0.5, "best": false, "rank": 9, "gain_over_no_intervention": -0.283, "gate": { "suppression": 0.269, "retain": 146.0, "d_degen": 0.0 } }, "Qwen3.5-9B": { "k": 0.223, "collapsed": false, "d_degen": -2.5, "best": false, "rank": 15, "gain_over_no_intervention": -0.008, "gate": { "suppression": 0.223, "retain": 100.0, "d_degen": 0.0 } } } }, { "method": "MemFlex", "venue": "EMNLP'24 Find.", "src": "repo", "link": "https://aclanthology.org/2024.findings-emnlp.82/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 45.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.25, "collapsed": false, "d_degen": 0.5, "best": false, "rank": 8, "gain_over_no_intervention": -0.236, "gate": { "suppression": 0.305, "retain": 133.0, "d_degen": 0.5 } }, "Qwen3.5-9B": { "k": 0.237, "collapsed": false, "d_degen": -19.5, "best": false, "rank": 14, "gain_over_no_intervention": 0.006, "gate": { "suppression": 0.237, "retain": 100.0, "d_degen": 0.0 } } } }, { "method": "MEOW", "venue": "ACL'25 Find.", "src": "repo", "link": "https://aclanthology.org/2025.findings-acl.535/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 100.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.337, "collapsed": false, "d_degen": -5.5, "best": false, "rank": 12, "gain_over_no_intervention": 0.106, "gate": null } } }, { "method": "FALCON", "venue": "NeurIPS'25", "src": "repo", "link": "https://doi.org/10.52202/085713-3419", "link_kind": "doi", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 20.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 100.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.48, "collapsed": false, "d_degen": -26.0, "best": false, "rank": 9, "gain_over_no_intervention": 0.249, "gate": { "suppression": 0.501, "retain": 96.0, "d_degen": 0.0 } } } }, { "method": "LoKU", "venue": "ICLR'25", "src": "repo", "link": "https://arxiv.org/abs/2408.06621", "link_kind": "url", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 83.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.743, "collapsed": false, "d_degen": -18.0, "best": true, "rank": 1, "gain_over_no_intervention": 0.512, "gate": { "suppression": 0.753, "retain": 99.0, "d_degen": 0.0 } } } }, { "method": "SimNPO", "venue": "NeurIPS'25", "src": "OU", "link": "https://doi.org/10.52202/085713-0058", "link_kind": "doi", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 77.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": null, "collapsed": true, "d_degen": 52.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null } } }, { "method": "ME+AP", "venue": "ICLR'25", "src": "repo", "link": "https://arxiv.org/abs/2410.08109", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 100.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.602, "collapsed": false, "d_degen": -8.5, "best": false, "rank": 3, "gain_over_no_intervention": 0.371, "gate": null } } }, { "method": "ReLearn", "venue": "ACL'25", "src": "repo", "link": "https://aclanthology.org/2025.acl-long.297/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 89.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.286, "collapsed": false, "d_degen": -36.0, "best": false, "rank": 13, "gain_over_no_intervention": 0.055, "gate": null } } }, { "method": "SatImp", "venue": "ICML'25", "src": "OU", "link": "https://arxiv.org/abs/2505.11953", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 46.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 97.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": null, "collapsed": true, "d_degen": 45.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null } } }, { "method": "LAW", "venue": "ICLR'25", "src": "repo", "link": "https://arxiv.org/abs/2405.16720", "link_kind": "arxiv", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 44.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": 0.341, "collapsed": false, "d_degen": 5.0, "best": false, "rank": 5, "gain_over_no_intervention": -0.145, "gate": { "suppression": 0.434, "retain": 132.0, "d_degen": 5.0 } }, "Qwen3.5-9B": { "k": 0.558, "collapsed": false, "d_degen": -29.0, "best": false, "rank": 4, "gain_over_no_intervention": 0.327, "gate": null } } }, { "method": "SOUL", "venue": "EMNLP'24", "src": "OU", "link": "https://aclanthology.org/2024.emnlp-main.245/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 100.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": null, "collapsed": true, "d_degen": 63.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null } } }, { "method": "ULD", "venue": "NeurIPS'24", "src": "repo", "link": "https://doi.org/10.52202/079017-0400", "link_kind": "doi", "role": "roster", "cells": { "Llama-3.1-8B": { "k": null, "collapsed": true, "d_degen": 49.5, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Mistral-7B": { "k": null, "collapsed": true, "d_degen": 100.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null }, "Qwen3.5-9B": { "k": 0.347, "collapsed": false, "d_degen": -18.5, "best": false, "rank": 11, "gain_over_no_intervention": 0.116, "gate": { "suppression": 0.429, "retain": 81.0, "d_degen": 0.0 } } } }, { "method": "SKU", "venue": "ACL'24 Find.", "src": "repo", "link": "https://aclanthology.org/2024.findings-acl.107/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": 0.374, "collapsed": false, "d_degen": -46.0, "best": false, "rank": 2, "gain_over_no_intervention": 0.164, "gate": null }, "Mistral-7B": { "k": 0.385, "collapsed": false, "d_degen": 21.5, "best": false, "rank": 4, "gain_over_no_intervention": -0.101, "gate": null }, "Qwen3.5-9B": { "k": 0.519, "collapsed": false, "d_degen": -35.0, "best": false, "rank": 6, "gain_over_no_intervention": 0.288, "gate": { "suppression": 0.519, "retain": 100.0, "d_degen": 0.0 } } } }, { "method": "REVS", "venue": "ACL'25 Find.", "src": "repo", "link": "https://aclanthology.org/2025.findings-acl.763/", "link_kind": "acl", "role": "roster", "cells": { "Llama-3.1-8B": { "k": 0.37, "collapsed": false, "d_degen": -15.5, "best": false, "rank": 3, "gain_over_no_intervention": 0.16, "gate": null }, "Mistral-7B": { "k": 0.275, "collapsed": false, "d_degen": 5.0, "best": false, "rank": 6, "gain_over_no_intervention": -0.211, "gate": { "suppression": 0.343, "retain": 129.0, "d_degen": 5.0 } }, "Qwen3.5-9B": { "k": null, "collapsed": true, "d_degen": 63.0, "best": false, "rank": null, "gain_over_no_intervention": null, "gate": null } } } ], "axis": { "note": "Extents cover the twenty-method roster only. The reference row is plotted but never widens the shared axis.", "rank_max": 15, "rank_ticks": [ 1, 5, 10, 15 ], "gain_top": 0.6, "gain_floor": -0.3, "gain_ticks": [ { "value": 0.6, "label": "+0.60", "baseline": false }, { "value": 0.4, "label": "+0.40", "baseline": false }, { "value": 0.2, "label": "+0.20", "baseline": false }, { "value": 0.0, "label": "0.00", "baseline": true }, { "value": -0.2, "label": "-0.20", "baseline": false } ], "reference_may_exceed": true } } }