trainless-toolcall-2b / trainless_model.json
hunterbown's picture
Add the result on untouched categories: 1.6 points below the base where most requests fit no tool
eb2e18d verified
Raw History Blame Contribute Delete
8.35 kB
{
"format": "trainless-peft/1",
"base_model": "Qwen/Qwen3.5-2B",
"base_weights_sha256": "aa33250c4fc64891ddfaba3a314fd9542ea371843c387178b425fbcc5ed680b1",
"auto_class": "AutoModelForImageTextToText",
"runtime": "transformers-peft",
"method": "forward-only analytic weight editing (project_out on one MLP writer, strength 4)",
"quantized_runtime_tested": false,
"edit": {
"tensor": "model.language_model.layers.10.mlp.down_proj.weight",
"op": "project_out",
"space": "rows",
"strength": 4.0,
"direction_sha256": "817ceab1340a687015a83c95eeeec49017079e99113a4166958724f3f7d48952",
"trainless_session": "ss_daf42ea7528e",
"state_id": "182dc3991ca48142",
"edited_tensor_sha256": "d5e4e6cdbc7c555e023198ab79a85c614b0de064f9416fab2514c3274dd28d26",
"fraction_of_editable_weights_changed": 0.006237529234172905
},
"refusal_check": {
"base_rate": 0.984375,
"benchmark": "AdvBench harmful_behaviors (first 64), greedy, thinking off, 256 tokens, refusal-phrase grader",
"drop": 0.0,
"edited_rate": 0.984375,
"limit": 0.2,
"n": 64,
"passed": true,
"state_id": "182dc3991ca48142",
"ts": "2026-11-11T00:09:45.234740882+00:00"
},
"evaluation": {
"benchmark": "BFCL v4 (ShishirPatil/gorilla @ 6ea57973c7a6097fd7c5915698c54c17c5b1b6c8), categories simple_python,multiple,parallel,parallel_multiple,irrelevance,live_simple",
"split": "test: sha256(category/id) mod 4 in {2,3}; construction split 0 built the direction, dev split 1 chose layer and strength",
"n": 703,
"engine": "vllm 0.31.0",
"decoding": "greedy",
"thinking": false,
"max_new_tokens": 1024,
"stop_token_ids": [
248044,
248046
],
"grading": "BFCL AST checker, unchanged; irrelevance is correct when no call is made",
"strong_prompt": "You are a precise function-calling assistant. Rules:\n1. If a provided function can answer the request, call it. Use the exact function name.\n2. Copy argument values exactly from the user's request; do not invent, round or rename them.\n3. Include every required parameter. Include optional parameters only when the user gives a value.\n4. Use the parameter's declared type (integers as integers, lists as JSON lists, booleans as true/false).\n5. If the request needs several calls, emit one <tool_call> block per call.\n6. If no provided function fits the request, do not call any function; answer in plain text.",
"adapter_sha256": {
"adapter_config.json": "70c7c9a027f88e37d92d66c7058a43708a481ef7486791b19f5ee8e402712346",
"adapter_model.safetensors": "32a8d4336a298a3329c033be4c4c38c9d15afabd4a3eee4bbafd76b9db9ec808",
"export_report.json": "32ce0bdb34d75aa5ba64ff97890fe1df382106b331e34e27bd0f6e70ad4831ed"
},
"arms": {
"normal": {
"correct": 526,
"n": 703,
"relevant_cases_with_no_call": 84,
"irrelevant_cases_with_a_call": 18,
"capped": 7
},
"strong": {
"correct": 544,
"n": 703,
"relevant_cases_with_no_call": 10,
"irrelevant_cases_with_a_call": 58,
"capped": 3
},
"adapter": {
"correct": 552,
"n": 703,
"relevant_cases_with_no_call": 46,
"irrelevant_cases_with_a_call": 23,
"capped": 7
}
},
"by_category": {
"irrelevance": {
"normal": 89,
"strong": 49,
"adapter": 84,
"n": 107
},
"live_simple": {
"normal": 90,
"strong": 92,
"adapter": 85,
"n": 125
},
"multiple": {
"normal": 88,
"strong": 86,
"adapter": 88,
"n": 102
},
"parallel": {
"normal": 40,
"strong": 72,
"adapter": 53,
"n": 86
},
"parallel_multiple": {
"normal": 70,
"strong": 74,
"adapter": 75,
"n": 89
},
"simple_python": {
"normal": 149,
"strong": 171,
"adapter": 167,
"n": 194
}
},
"paired": {
"adapter_minus_normal": {
"accuracy_delta": 0.037,
"bootstrap95": [
0.0171,
0.0569
],
"first_wrong_second_right": 38,
"first_right_second_wrong": 12
},
"adapter_minus_strong_prompt": {
"accuracy_delta": 0.0114,
"bootstrap95": [
-0.0171,
0.0413
],
"first_wrong_second_right": 58,
"first_right_second_wrong": 50
},
"note": "paired case bootstrap, 10,000 resamples, seed 0; greedy decoding, so no sampling-seed variance, but other runtimes can differ"
},
"controls": {
"note": "scripted HF Transformers run of the bf16 edit on the same 703 cases, not this adapter in vLLM",
"edit": 554,
"normal": 524,
"random_direction_same_layer_and_strength": [
510,
522,
522
]
},
"what_changed": {
"relevant_cases_base_already_called": {
"n": 512,
"base_correct": 437,
"adapter_correct": 433
},
"relevant_cases_base_answered_in_text": {
"n": 84,
"base_correct": 0,
"adapter_correct": 35
},
"fixes_where_base_made_no_call": "35 of 38"
}
},
"off_target": {
"note": "vLLM, no tool definitions, thinking off, greedy, 4,096-token cap; base vs this adapter",
"results": {
"gsm8k_test": {
"n": 500,
"base_correct": 407,
"adapter_correct": 399,
"paired_diff_points": -1.6,
"bootstrap95": [
-4.4,
1.2
],
"fixes": 20,
"regressions": 28,
"base_capped": 19,
"adapter_capped": 19,
"base_outputs_with_tool_call": 0,
"adapter_outputs_with_tool_call": 0,
"identical_outputs": 16
},
"math500": {
"n": 200,
"base_correct": 143,
"adapter_correct": 144,
"paired_diff_points": 0.5,
"bootstrap95": [
-4.5,
5.5
],
"fixes": 13,
"regressions": 12,
"base_capped": 45,
"adapter_capped": 44,
"base_outputs_with_tool_call": 0,
"adapter_outputs_with_tool_call": 0,
"identical_outputs": 1
}
}
},
"made_by": {
"agent": "Claude Fable 5.2 (claude-fable-5-2) in Claude Code",
"human": "Hunter Bown set the goal and approved each stage and the release",
"note": "the agent designed and ran the direction, screens, tests and export through scripts and the Trainless platform"
},
"untouched_categories": {
"note": "BFCL live_irrelevance, live_multiple, live_parallel, live_parallel_multiple; all cases; never used to build or test this adapter; vLLM, greedy, thinking off, 1,024-token cap",
"n": 1977,
"arms": {
"normal": {
"correct": 1397,
"n": 1977,
"relevant_no_call": 21,
"irrelevant_calls": 273,
"by_category": {
"live_irrelevance": 611,
"live_multiple": 766,
"live_parallel": 10,
"live_parallel_multiple": 10
}
},
"strong": {
"correct": 1292,
"n": 1977,
"relevant_no_call": 10,
"irrelevant_calls": 371,
"by_category": {
"live_irrelevance": 513,
"live_multiple": 755,
"live_parallel": 12,
"live_parallel_multiple": 12
}
},
"a:published": {
"correct": 1365,
"n": 1977,
"relevant_no_call": 16,
"irrelevant_calls": 300,
"by_category": {
"live_irrelevance": 584,
"live_multiple": 760,
"live_parallel": 10,
"live_parallel_multiple": 11
}
}
},
"paired": {
"strong - normal": {
"diff_points": -5.31,
"ci95": [
-6.83,
-3.79
],
"fixes": 69,
"regressions": 174
},
"a:published - normal": {
"diff_points": -1.62,
"ci95": [
-2.43,
-0.81
],
"fixes": 18,
"regressions": 50
},
"a:published - strong": {
"diff_points": 3.69,
"ci95": [
2.18,
5.26
],
"fixes": 155,
"regressions": 82
}
}
}
}