{ "format": "trainless-peft/1", "base_model": "Qwen/Qwen3.5-2B", "base_weights_sha256": "aa33250c4fc64891ddfaba3a314fd9542ea371843c387178b425fbcc5ed680b1", "auto_class": "AutoModelForImageTextToText", "runtime": "transformers-peft", "method": "forward-only analytic weight editing (project_out on one MLP writer, strength 4)", "quantized_runtime_tested": false, "edit": { "tensor": "model.language_model.layers.10.mlp.down_proj.weight", "op": "project_out", "space": "rows", "strength": 4.0, "direction_sha256": "817ceab1340a687015a83c95eeeec49017079e99113a4166958724f3f7d48952", "trainless_session": "ss_daf42ea7528e", "state_id": "182dc3991ca48142", "edited_tensor_sha256": "d5e4e6cdbc7c555e023198ab79a85c614b0de064f9416fab2514c3274dd28d26", "fraction_of_editable_weights_changed": 0.006237529234172905 }, "refusal_check": { "base_rate": 0.984375, "benchmark": "AdvBench harmful_behaviors (first 64), greedy, thinking off, 256 tokens, refusal-phrase grader", "drop": 0.0, "edited_rate": 0.984375, "limit": 0.2, "n": 64, "passed": true, "state_id": "182dc3991ca48142", "ts": "2026-11-11T00:09:45.234740882+00:00" }, "evaluation": { "benchmark": "BFCL v4 (ShishirPatil/gorilla @ 6ea57973c7a6097fd7c5915698c54c17c5b1b6c8), categories simple_python,multiple,parallel,parallel_multiple,irrelevance,live_simple", "split": "test: sha256(category/id) mod 4 in {2,3}; construction split 0 built the direction, dev split 1 chose layer and strength", "n": 703, "engine": "vllm 0.31.0", "decoding": "greedy", "thinking": false, "max_new_tokens": 1024, "stop_token_ids": [ 248044, 248046 ], "grading": "BFCL AST checker, unchanged; irrelevance is correct when no call is made", "strong_prompt": "You are a precise function-calling assistant. Rules:\n1. If a provided function can answer the request, call it. Use the exact function name.\n2. Copy argument values exactly from the user's request; do not invent, round or rename them.\n3. Include every required parameter. Include optional parameters only when the user gives a value.\n4. Use the parameter's declared type (integers as integers, lists as JSON lists, booleans as true/false).\n5. If the request needs several calls, emit one block per call.\n6. If no provided function fits the request, do not call any function; answer in plain text.", "adapter_sha256": { "adapter_config.json": "70c7c9a027f88e37d92d66c7058a43708a481ef7486791b19f5ee8e402712346", "adapter_model.safetensors": "32a8d4336a298a3329c033be4c4c38c9d15afabd4a3eee4bbafd76b9db9ec808", "export_report.json": "32ce0bdb34d75aa5ba64ff97890fe1df382106b331e34e27bd0f6e70ad4831ed" }, "arms": { "normal": { "correct": 526, "n": 703, "relevant_cases_with_no_call": 84, "irrelevant_cases_with_a_call": 18, "capped": 7 }, "strong": { "correct": 544, "n": 703, "relevant_cases_with_no_call": 10, "irrelevant_cases_with_a_call": 58, "capped": 3 }, "adapter": { "correct": 552, "n": 703, "relevant_cases_with_no_call": 46, "irrelevant_cases_with_a_call": 23, "capped": 7 } }, "by_category": { "irrelevance": { "normal": 89, "strong": 49, "adapter": 84, "n": 107 }, "live_simple": { "normal": 90, "strong": 92, "adapter": 85, "n": 125 }, "multiple": { "normal": 88, "strong": 86, "adapter": 88, "n": 102 }, "parallel": { "normal": 40, "strong": 72, "adapter": 53, "n": 86 }, "parallel_multiple": { "normal": 70, "strong": 74, "adapter": 75, "n": 89 }, "simple_python": { "normal": 149, "strong": 171, "adapter": 167, "n": 194 } }, "paired": { "adapter_minus_normal": { "accuracy_delta": 0.037, "bootstrap95": [ 0.0171, 0.0569 ], "first_wrong_second_right": 38, "first_right_second_wrong": 12 }, "adapter_minus_strong_prompt": { "accuracy_delta": 0.0114, "bootstrap95": [ -0.0171, 0.0413 ], "first_wrong_second_right": 58, "first_right_second_wrong": 50 }, "note": "paired case bootstrap, 10,000 resamples, seed 0; greedy decoding, so no sampling-seed variance, but other runtimes can differ" }, "controls": { "note": "scripted HF Transformers run of the bf16 edit on the same 703 cases, not this adapter in vLLM", "edit": 554, "normal": 524, "random_direction_same_layer_and_strength": [ 510, 522, 522 ] }, "what_changed": { "relevant_cases_base_already_called": { "n": 512, "base_correct": 437, "adapter_correct": 433 }, "relevant_cases_base_answered_in_text": { "n": 84, "base_correct": 0, "adapter_correct": 35 }, "fixes_where_base_made_no_call": "35 of 38" } }, "off_target": { "note": "vLLM, no tool definitions, thinking off, greedy, 4,096-token cap; base vs this adapter", "results": { "gsm8k_test": { "n": 500, "base_correct": 407, "adapter_correct": 399, "paired_diff_points": -1.6, "bootstrap95": [ -4.4, 1.2 ], "fixes": 20, "regressions": 28, "base_capped": 19, "adapter_capped": 19, "base_outputs_with_tool_call": 0, "adapter_outputs_with_tool_call": 0, "identical_outputs": 16 }, "math500": { "n": 200, "base_correct": 143, "adapter_correct": 144, "paired_diff_points": 0.5, "bootstrap95": [ -4.5, 5.5 ], "fixes": 13, "regressions": 12, "base_capped": 45, "adapter_capped": 44, "base_outputs_with_tool_call": 0, "adapter_outputs_with_tool_call": 0, "identical_outputs": 1 } } }, "made_by": { "agent": "Claude Fable 5.2 (claude-fable-5-2) in Claude Code", "human": "Hunter Bown set the goal and approved each stage and the release", "note": "the agent designed and ran the direction, screens, tests and export through scripts and the Trainless platform" }, "untouched_categories": { "note": "BFCL live_irrelevance, live_multiple, live_parallel, live_parallel_multiple; all cases; never used to build or test this adapter; vLLM, greedy, thinking off, 1,024-token cap", "n": 1977, "arms": { "normal": { "correct": 1397, "n": 1977, "relevant_no_call": 21, "irrelevant_calls": 273, "by_category": { "live_irrelevance": 611, "live_multiple": 766, "live_parallel": 10, "live_parallel_multiple": 10 } }, "strong": { "correct": 1292, "n": 1977, "relevant_no_call": 10, "irrelevant_calls": 371, "by_category": { "live_irrelevance": 513, "live_multiple": 755, "live_parallel": 12, "live_parallel_multiple": 12 } }, "a:published": { "correct": 1365, "n": 1977, "relevant_no_call": 16, "irrelevant_calls": 300, "by_category": { "live_irrelevance": 584, "live_multiple": 760, "live_parallel": 10, "live_parallel_multiple": 11 } } }, "paired": { "strong - normal": { "diff_points": -5.31, "ci95": [ -6.83, -3.79 ], "fixes": 69, "regressions": 174 }, "a:published - normal": { "diff_points": -1.62, "ci95": [ -2.43, -0.81 ], "fixes": 18, "regressions": 50 }, "a:published - strong": { "diff_points": 3.69, "ci95": [ 2.18, 5.26 ], "fixes": 155, "regressions": 82 } } } }