Image-Text-to-Text
Transformers
Safetensors
qwen3_5
openmed
medical
clinical
medqa
weight-space-programming
conversational
Eval Results (legacy)
Instructions to use OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical") model = AutoModelForMultimodalLM.from_pretrained("OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical
- SGLang
How to use OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical with Docker Model Runner:
docker model run hf.co/OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical
Download eval/scores.json from OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical: direct link, hf CLI and curl.
- Browser
- Download file 21.9 kB
-
https://huggingface.co/OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical/resolve/main/eval/scores.json
- Command line
-
hf download hf://OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical/eval/scores.json
-
curl -L -o scores.json https://huggingface.co/OpenMed/Qwen3.5-0.8B-WSP-Math-Clinical/resolve/main/eval/scores.json
21.9 kB
{
"protocol": "Qwen3.5 thinking on; temperature 0.6, top_p 0.95, top_k 20, presence_penalty 1.5, max 32,768 new tokens, seed 0, one sample per question; OpenMed fixed MedQA prompt; strict = last 'Answer: X' after </think> (cut-offs and unparsable answers count as wrong)",
"scores": {
"clin08": {
"medqa_test": {
"n": 1273,
"correct_strict": 578,
"acc_strict": 0.454,
"correct_lenient": 578,
"acc_lenient": 0.454,
"cut": 2,
"cut_share": 0.0016,
"think_not_closed": 2,
"finished_no_strict_answer": 4,
"placeholder_answers": 2,
"finished_no_lenient_answer": 3,
"acc_strict_among_finished": 0.4548,
"finished": 1271,
"median_tokens": 4945,
"mean_tokens": 5275,
"total_tokens": 6714845
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 388,
"acc_strict": 0.388,
"correct_lenient": 388,
"acc_lenient": 0.388,
"cut": 0,
"cut_share": 0.0,
"think_not_closed": 0,
"finished_no_strict_answer": 6,
"placeholder_answers": 1,
"finished_no_lenient_answer": 5,
"acc_strict_among_finished": 0.388,
"finished": 1000,
"median_tokens": 3376,
"mean_tokens": 3770,
"total_tokens": 3770020
},
"mmlu_med": {
"n": 1089,
"correct_strict": 655,
"acc_strict": 0.6015,
"correct_lenient": 655,
"acc_lenient": 0.6015,
"cut": 1,
"cut_share": 0.0009,
"think_not_closed": 1,
"finished_no_strict_answer": 4,
"placeholder_answers": 1,
"finished_no_lenient_answer": 4,
"acc_strict_among_finished": 0.602,
"finished": 1088,
"median_tokens": 3504,
"mean_tokens": 3853,
"total_tokens": 4196117
}
},
"base": {
"medqa_test": {
"n": 1273,
"correct_strict": 311,
"acc_strict": 0.2443,
"correct_lenient": 414,
"acc_lenient": 0.3252,
"cut": 233,
"cut_share": 0.183,
"think_not_closed": 224,
"finished_no_strict_answer": 419,
"placeholder_answers": 419,
"finished_no_lenient_answer": 195,
"acc_strict_among_finished": 0.299,
"finished": 1040,
"median_tokens": 5546,
"mean_tokens": 10362,
"total_tokens": 13191142
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 306,
"acc_strict": 0.306,
"correct_lenient": 348,
"acc_lenient": 0.348,
"cut": 114,
"cut_share": 0.114,
"think_not_closed": 110,
"finished_no_strict_answer": 209,
"placeholder_answers": 206,
"finished_no_lenient_answer": 94,
"acc_strict_among_finished": 0.3454,
"finished": 886,
"median_tokens": 3913,
"mean_tokens": 7234,
"total_tokens": 7233961
},
"mmlu_med": {
"n": 1089,
"correct_strict": 473,
"acc_strict": 0.4343,
"correct_lenient": 545,
"acc_lenient": 0.5005,
"cut": 126,
"cut_share": 0.1157,
"think_not_closed": 120,
"finished_no_strict_answer": 236,
"placeholder_answers": 236,
"finished_no_lenient_answer": 104,
"acc_strict_among_finished": 0.4912,
"finished": 963,
"median_tokens": 4116,
"mean_tokens": 7329,
"total_tokens": 7981460
}
},
"mathedit": {
"medqa_test": {
"n": 1273,
"correct_strict": 419,
"acc_strict": 0.3291,
"correct_lenient": 501,
"acc_lenient": 0.3936,
"cut": 1,
"cut_share": 0.0008,
"think_not_closed": 2,
"finished_no_strict_answer": 367,
"placeholder_answers": 366,
"finished_no_lenient_answer": 187,
"acc_strict_among_finished": 0.3294,
"finished": 1272,
"median_tokens": 4899,
"mean_tokens": 5259,
"total_tokens": 6694992
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 333,
"acc_strict": 0.333,
"correct_lenient": 375,
"acc_lenient": 0.375,
"cut": 1,
"cut_share": 0.001,
"think_not_closed": 1,
"finished_no_strict_answer": 182,
"placeholder_answers": 176,
"finished_no_lenient_answer": 73,
"acc_strict_among_finished": 0.3333,
"finished": 999,
"median_tokens": 3341,
"mean_tokens": 3740,
"total_tokens": 3739648
},
"mmlu_med": {
"n": 1089,
"correct_strict": 571,
"acc_strict": 0.5243,
"correct_lenient": 614,
"acc_lenient": 0.5638,
"cut": 0,
"cut_share": 0.0,
"think_not_closed": 0,
"finished_no_strict_answer": 201,
"placeholder_answers": 200,
"finished_no_lenient_answer": 109,
"acc_strict_among_finished": 0.5243,
"finished": 1089,
"median_tokens": 3396,
"mean_tokens": 3734,
"total_tokens": 4066438
}
},
"ctrl_clin08_r1": {
"medqa_test": {
"n": 1273,
"correct_strict": 412,
"acc_strict": 0.3236,
"correct_lenient": 506,
"acc_lenient": 0.3975,
"cut": 1,
"cut_share": 0.0008,
"think_not_closed": 1,
"finished_no_strict_answer": 404,
"placeholder_answers": 403,
"finished_no_lenient_answer": 197,
"acc_strict_among_finished": 0.3239,
"finished": 1272,
"median_tokens": 4924,
"mean_tokens": 5268,
"total_tokens": 6706232
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 337,
"acc_strict": 0.337,
"correct_lenient": 375,
"acc_lenient": 0.375,
"cut": 0,
"cut_share": 0.0,
"think_not_closed": 0,
"finished_no_strict_answer": 201,
"placeholder_answers": 197,
"finished_no_lenient_answer": 99,
"acc_strict_among_finished": 0.337,
"finished": 1000,
"median_tokens": 3338,
"mean_tokens": 3758,
"total_tokens": 3758051
},
"mmlu_med": {
"n": 1089,
"correct_strict": 548,
"acc_strict": 0.5032,
"correct_lenient": 607,
"acc_lenient": 0.5574,
"cut": 0,
"cut_share": 0.0,
"think_not_closed": 0,
"finished_no_strict_answer": 216,
"placeholder_answers": 210,
"finished_no_lenient_answer": 100,
"acc_strict_among_finished": 0.5032,
"finished": 1089,
"median_tokens": 3442,
"mean_tokens": 3735,
"total_tokens": 4067113
}
},
"ctrl_clin08_r2": {
"medqa_test": {
"n": 1273,
"correct_strict": 433,
"acc_strict": 0.3401,
"correct_lenient": 503,
"acc_lenient": 0.3951,
"cut": 1,
"cut_share": 0.0008,
"think_not_closed": 0,
"finished_no_strict_answer": 370,
"placeholder_answers": 370,
"finished_no_lenient_answer": 206,
"acc_strict_among_finished": 0.3404,
"finished": 1272,
"median_tokens": 4871,
"mean_tokens": 5271,
"total_tokens": 6710106
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 331,
"acc_strict": 0.331,
"correct_lenient": 358,
"acc_lenient": 0.358,
"cut": 0,
"cut_share": 0.0,
"think_not_closed": 0,
"finished_no_strict_answer": 183,
"placeholder_answers": 181,
"finished_no_lenient_answer": 91,
"acc_strict_among_finished": 0.331,
"finished": 1000,
"median_tokens": 3311,
"mean_tokens": 3770,
"total_tokens": 3769947
},
"mmlu_med": {
"n": 1089,
"correct_strict": 554,
"acc_strict": 0.5087,
"correct_lenient": 607,
"acc_lenient": 0.5574,
"cut": 2,
"cut_share": 0.0018,
"think_not_closed": 2,
"finished_no_strict_answer": 213,
"placeholder_answers": 206,
"finished_no_lenient_answer": 102,
"acc_strict_among_finished": 0.5097,
"finished": 1087,
"median_tokens": 3416,
"mean_tokens": 3749,
"total_tokens": 4082247
}
}
},
"medqa_seeds": {
"clin08": {
"seeds": [
0,
1,
2
],
"acc_strict": {
"0": 0.454,
"1": 0.4415,
"2": 0.4705
},
"acc_lenient": {
"0": 0.454,
"1": 0.4423,
"2": 0.4705
},
"cut_share": {
"0": 0.0016,
"1": 0.0008,
"2": 0.0
},
"no_answer_share": {
"0": 0.0031,
"1": 0.0055,
"2": 0.0031
},
"median_acc_strict": 0.454,
"min": 0.4415,
"max": 0.4705
},
"base": {
"seeds": [
0,
1,
2
],
"acc_strict": {
"0": 0.2443,
"1": 0.231,
"2": 0.2215
},
"acc_lenient": {
"0": 0.3252,
"1": 0.3079,
"2": 0.315
},
"cut_share": {
"0": 0.183,
"1": 0.205,
"2": 0.2231
},
"no_answer_share": {
"0": 0.3291,
"1": 0.3229,
"2": 0.3134
},
"median_acc_strict": 0.231,
"min": 0.2215,
"max": 0.2443
},
"mathedit": {
"seeds": [
0,
1,
2
],
"acc_strict": {
"0": 0.3291,
"1": 0.3362,
"2": 0.3276
},
"acc_lenient": {
"0": 0.3936,
"1": 0.3975,
"2": 0.4006
},
"cut_share": {
"0": 0.0008,
"1": 0.0,
"2": 0.0016
},
"no_answer_share": {
"0": 0.2883,
"1": 0.2891,
"2": 0.3331
},
"median_acc_strict": 0.3291,
"min": 0.3276,
"max": 0.3362
},
"ctrl_clin08_r1": {
"seeds": [
0
],
"acc_strict": {
"0": 0.3236
},
"acc_lenient": {
"0": 0.3975
},
"cut_share": {
"0": 0.0008
},
"no_answer_share": {
"0": 0.3174
},
"median_acc_strict": 0.3236,
"min": 0.3236,
"max": 0.3236
},
"ctrl_clin08_r2": {
"seeds": [
0
],
"acc_strict": {
"0": 0.3401
},
"acc_lenient": {
"0": 0.3951
},
"cut_share": {
"0": 0.0008
},
"no_answer_share": {
"0": 0.2907
},
"median_acc_strict": 0.3401,
"min": 0.3401,
"max": 0.3401
}
},
"general": {
"clin08": {
"math500": {
"n": 500,
"correct": 256,
"acc": 0.512,
"cut": 155,
"cut_share": 0.31,
"median_tokens": 23657
},
"ifeval": {
"n": 541,
"correct": 338,
"acc": 0.6248,
"cut": 14,
"cut_share": 0.0259,
"median_tokens": 2974
}
},
"base": {
"math500": {
"n": 500,
"correct": 37,
"acc": 0.074,
"cut": 468,
"cut_share": 0.936,
"median_tokens": 32768
},
"ifeval": {
"n": 541,
"correct": 109,
"acc": 0.2015,
"cut": 403,
"cut_share": 0.7449,
"median_tokens": 32768
}
},
"mathedit": {
"math500": {
"n": 500,
"correct": 253,
"acc": 0.506,
"cut": 151,
"cut_share": 0.302,
"median_tokens": 21833
},
"ifeval": {
"n": 541,
"correct": 332,
"acc": 0.6137,
"cut": 13,
"cut_share": 0.024,
"median_tokens": 2818
}
}
},
"paired": {
"mathedit->clin08:medqa_test": {
"strict": {
"n": 1273,
"acc_a": 0.3291,
"acc_b": 0.454,
"diff_pp": 12.49,
"ci95_pp": [
9.66,
15.32
],
"only_a_right": 102,
"only_b_right": 261,
"mcnemar_exact_p": 3.408575659706554e-17,
"cut_a": 1,
"cut_b": 2
},
"lenient": {
"n": 1273,
"acc_a": 0.3936,
"acc_b": 0.454,
"diff_pp": 6.05,
"ci95_pp": [
3.3,
8.72
],
"only_a_right": 119,
"only_b_right": 196,
"mcnemar_exact_p": 1.6912602068191218e-05,
"cut_a": 1,
"cut_b": 2
}
},
"mathedit->clin08:medmcqa_val1k": {
"strict": {
"n": 1000,
"acc_a": 0.333,
"acc_b": 0.388,
"diff_pp": 5.5,
"ci95_pp": [
2.4,
8.5
],
"only_a_right": 98,
"only_b_right": 153,
"mcnemar_exact_p": 0.0006246732023035248,
"cut_a": 1,
"cut_b": 0
},
"lenient": {
"n": 1000,
"acc_a": 0.375,
"acc_b": 0.388,
"diff_pp": 1.3,
"ci95_pp": [
-1.8,
4.3
],
"only_a_right": 118,
"only_b_right": 131,
"mcnemar_exact_p": 0.44703924382362525,
"cut_a": 1,
"cut_b": 0
}
},
"mathedit->clin08:mmlu_med": {
"strict": {
"n": 1089,
"acc_a": 0.5243,
"acc_b": 0.6015,
"diff_pp": 7.71,
"ci95_pp": [
5.05,
10.47
],
"only_a_right": 76,
"only_b_right": 160,
"mcnemar_exact_p": 4.783169486571542e-08,
"cut_a": 0,
"cut_b": 1
},
"lenient": {
"n": 1089,
"acc_a": 0.5638,
"acc_b": 0.6015,
"diff_pp": 3.76,
"ci95_pp": [
1.1,
6.52
],
"only_a_right": 91,
"only_b_right": 132,
"mcnemar_exact_p": 0.007256259165203033,
"cut_a": 0,
"cut_b": 1
}
},
"base->clin08:medqa_test": {
"strict": {
"n": 1273,
"acc_a": 0.2443,
"acc_b": 0.454,
"diff_pp": 20.97,
"ci95_pp": [
18.15,
23.8
],
"only_a_right": 72,
"only_b_right": 339,
"mcnemar_exact_p": 1.6497637702001935e-42,
"cut_a": 233,
"cut_b": 2
},
"lenient": {
"n": 1273,
"acc_a": 0.3252,
"acc_b": 0.454,
"diff_pp": 12.88,
"ci95_pp": [
9.98,
15.79
],
"only_a_right": 112,
"only_b_right": 276,
"mcnemar_exact_p": 4.3376870027444254e-17,
"cut_a": 233,
"cut_b": 2
}
},
"base->clin08:medmcqa_val1k": {
"strict": {
"n": 1000,
"acc_a": 0.306,
"acc_b": 0.388,
"diff_pp": 8.2,
"ci95_pp": [
5.0,
11.4
],
"only_a_right": 98,
"only_b_right": 180,
"mcnemar_exact_p": 9.980379103751038e-07,
"cut_a": 114,
"cut_b": 0
},
"lenient": {
"n": 1000,
"acc_a": 0.348,
"acc_b": 0.388,
"diff_pp": 4.0,
"ci95_pp": [
0.7,
7.3
],
"only_a_right": 119,
"only_b_right": 159,
"mcnemar_exact_p": 0.01916941113234859,
"cut_a": 114,
"cut_b": 0
}
},
"base->clin08:mmlu_med": {
"strict": {
"n": 1089,
"acc_a": 0.4343,
"acc_b": 0.6015,
"diff_pp": 16.71,
"ci95_pp": [
13.59,
19.83
],
"only_a_right": 72,
"only_b_right": 254,
"mcnemar_exact_p": 6.127885080392983e-25,
"cut_a": 126,
"cut_b": 1
},
"lenient": {
"n": 1089,
"acc_a": 0.5005,
"acc_b": 0.6015,
"diff_pp": 10.1,
"ci95_pp": [
7.07,
13.13
],
"only_a_right": 95,
"only_b_right": 205,
"mcnemar_exact_p": 1.9650224262184498e-10,
"cut_a": 126,
"cut_b": 1
}
},
"ctrl_full08_r1->clin08:medqa_test": {
"strict": {
"n": 1273,
"acc_a": 0.2742,
"acc_b": 0.454,
"diff_pp": 17.99,
"ci95_pp": [
15.16,
20.9
],
"only_a_right": 79,
"only_b_right": 308,
"mcnemar_exact_p": 4.8868795521604676e-33,
"cut_a": 222,
"cut_b": 2
},
"lenient": {
"n": 1273,
"acc_a": 0.3417,
"acc_b": 0.454,
"diff_pp": 11.23,
"ci95_pp": [
8.33,
14.06
],
"only_a_right": 110,
"only_b_right": 253,
"mcnemar_exact_p": 4.286594279957746e-14,
"cut_a": 222,
"cut_b": 2
}
},
"ctrl_full08_r1->clin08:medmcqa_val1k": {
"strict": {
"n": 1000,
"acc_a": 0.283,
"acc_b": 0.388,
"diff_pp": 10.5,
"ci95_pp": [
7.3,
13.6
],
"only_a_right": 92,
"only_b_right": 197,
"mcnemar_exact_p": 6.205468883195871e-10,
"cut_a": 129,
"cut_b": 0
},
"lenient": {
"n": 1000,
"acc_a": 0.322,
"acc_b": 0.388,
"diff_pp": 6.6,
"ci95_pp": [
3.4,
9.7
],
"only_a_right": 103,
"only_b_right": 169,
"mcnemar_exact_p": 7.522501974060535e-05,
"cut_a": 129,
"cut_b": 0
}
},
"ctrl_full08_r1->clin08:mmlu_med": {
"strict": {
"n": 1089,
"acc_a": 0.4683,
"acc_b": 0.6015,
"diff_pp": 13.31,
"ci95_pp": [
10.28,
16.35
],
"only_a_right": 78,
"only_b_right": 223,
"mcnemar_exact_p": 2.440470770839633e-17,
"cut_a": 129,
"cut_b": 1
},
"lenient": {
"n": 1089,
"acc_a": 0.5179,
"acc_b": 0.6015,
"diff_pp": 8.36,
"ci95_pp": [
5.23,
11.39
],
"only_a_right": 103,
"only_b_right": 194,
"mcnemar_exact_p": 1.423671211320386e-07,
"cut_a": 129,
"cut_b": 1
}
},
"ctrl_full08_r2->clin08:medqa_test": {
"strict": {
"n": 1273,
"acc_a": 0.2765,
"acc_b": 0.454,
"diff_pp": 17.75,
"ci95_pp": [
14.69,
20.74
],
"only_a_right": 96,
"only_b_right": 322,
"mcnemar_exact_p": 1.2963757884453945e-29,
"cut_a": 250,
"cut_b": 2
},
"lenient": {
"n": 1273,
"acc_a": 0.3441,
"acc_b": 0.454,
"diff_pp": 11.0,
"ci95_pp": [
7.93,
13.98
],
"only_a_right": 127,
"only_b_right": 267,
"mcnemar_exact_p": 1.470103436124482e-12,
"cut_a": 250,
"cut_b": 2
}
},
"ctrl_full08_r2->clin08:medmcqa_val1k": {
"strict": {
"n": 1000,
"acc_a": 0.278,
"acc_b": 0.388,
"diff_pp": 11.0,
"ci95_pp": [
7.7,
14.3
],
"only_a_right": 89,
"only_b_right": 199,
"mcnemar_exact_p": 7.93262788473449e-11,
"cut_a": 100,
"cut_b": 0
},
"lenient": {
"n": 1000,
"acc_a": 0.333,
"acc_b": 0.388,
"diff_pp": 5.5,
"ci95_pp": [
2.2,
8.8
],
"only_a_right": 111,
"only_b_right": 166,
"mcnemar_exact_p": 0.0011380286281171773,
"cut_a": 100,
"cut_b": 0
}
},
"ctrl_full08_r2->clin08:mmlu_med": {
"strict": {
"n": 1089,
"acc_a": 0.4527,
"acc_b": 0.6015,
"diff_pp": 14.88,
"ci95_pp": [
11.75,
17.91
],
"only_a_right": 74,
"only_b_right": 236,
"mcnemar_exact_p": 7.235914259263509e-21,
"cut_a": 110,
"cut_b": 1
},
"lenient": {
"n": 1089,
"acc_a": 0.5289,
"acc_b": 0.6015,
"diff_pp": 7.25,
"ci95_pp": [
4.22,
10.19
],
"only_a_right": 96,
"only_b_right": 175,
"mcnemar_exact_p": 1.838674180969678e-06,
"cut_a": 110,
"cut_b": 1
}
},
"ctrl_clin08_r1->clin08:medqa_test": {
"strict": {
"n": 1273,
"acc_a": 0.3236,
"acc_b": 0.454,
"diff_pp": 13.04,
"ci95_pp": [
10.21,
15.87
],
"only_a_right": 98,
"only_b_right": 264,
"mcnemar_exact_p": 1.021024303629902e-18,
"cut_a": 1,
"cut_b": 2
},
"lenient": {
"n": 1273,
"acc_a": 0.3975,
"acc_b": 0.454,
"diff_pp": 5.66,
"ci95_pp": [
2.75,
8.48
],
"only_a_right": 133,
"only_b_right": 205,
"mcnemar_exact_p": 0.0001064694827935662,
"cut_a": 1,
"cut_b": 2
}
},
"ctrl_clin08_r1->clin08:medmcqa_val1k": {
"strict": {
"n": 1000,
"acc_a": 0.337,
"acc_b": 0.388,
"diff_pp": 5.1,
"ci95_pp": [
2.0,
8.1
],
"only_a_right": 99,
"only_b_right": 150,
"mcnemar_exact_p": 0.0014811661331282549,
"cut_a": 0,
"cut_b": 0
},
"lenient": {
"n": 1000,
"acc_a": 0.375,
"acc_b": 0.388,
"diff_pp": 1.3,
"ci95_pp": [
-1.8,
4.3
],
"only_a_right": 116,
"only_b_right": 129,
"mcnemar_exact_p": 0.4433533677261167,
"cut_a": 0,
"cut_b": 0
}
},
"ctrl_clin08_r1->clin08:mmlu_med": {
"strict": {
"n": 1089,
"acc_a": 0.5032,
"acc_b": 0.6015,
"diff_pp": 9.83,
"ci95_pp": [
7.07,
12.58
],
"only_a_right": 70,
"only_b_right": 177,
"mcnemar_exact_p": 7.216845972493009e-12,
"cut_a": 0,
"cut_b": 1
},
"lenient": {
"n": 1089,
"acc_a": 0.5574,
"acc_b": 0.6015,
"diff_pp": 4.41,
"ci95_pp": [
1.65,
7.07
],
"only_a_right": 90,
"only_b_right": 138,
"mcnemar_exact_p": 0.0017920737015369882,
"cut_a": 0,
"cut_b": 1
}
},
"ctrl_clin08_r2->clin08:medqa_test": {
"strict": {
"n": 1273,
"acc_a": 0.3401,
"acc_b": 0.454,
"diff_pp": 11.39,
"ci95_pp": [
8.56,
14.3
],
"only_a_right": 108,
"only_b_right": 253,
"mcnemar_exact_p": 1.5436185792654267e-14,
"cut_a": 1,
"cut_b": 2
},
"lenient": {
"n": 1273,
"acc_a": 0.3951,
"acc_b": 0.454,
"diff_pp": 5.89,
"ci95_pp": [
3.06,
8.64
],
"only_a_right": 126,
"only_b_right": 201,
"mcnemar_exact_p": 3.974756443784619e-05,
"cut_a": 1,
"cut_b": 2
}
},
"ctrl_clin08_r2->clin08:medmcqa_val1k": {
"strict": {
"n": 1000,
"acc_a": 0.331,
"acc_b": 0.388,
"diff_pp": 5.7,
"ci95_pp": [
2.6,
8.7
],
"only_a_right": 97,
"only_b_right": 154,
"mcnemar_exact_p": 0.00038754273134235495,
"cut_a": 0,
"cut_b": 0
},
"lenient": {
"n": 1000,
"acc_a": 0.358,
"acc_b": 0.388,
"diff_pp": 3.0,
"ci95_pp": [
-0.1,
6.1
],
"only_a_right": 108,
"only_b_right": 138,
"mcnemar_exact_p": 0.06424260857381481,
"cut_a": 0,
"cut_b": 0
}
},
"ctrl_clin08_r2->clin08:mmlu_med": {
"strict": {
"n": 1089,
"acc_a": 0.5087,
"acc_b": 0.6015,
"diff_pp": 9.27,
"ci95_pp": [
6.43,
12.03
],
"only_a_right": 77,
"only_b_right": 178,
"mcnemar_exact_p": 2.221841305937269e-10,
"cut_a": 2,
"cut_b": 1
},
"lenient": {
"n": 1089,
"acc_a": 0.5574,
"acc_b": 0.6015,
"diff_pp": 4.41,
"ci95_pp": [
1.65,
7.07
],
"only_a_right": 89,
"only_b_right": 137,
"mcnemar_exact_p": 0.00170879698089281,
"cut_a": 2,
"cut_b": 1
}
}
},
"paired_general": {
"mathedit->clin08:math500": {
"n": 500,
"acc_a": 0.506,
"acc_b": 0.512,
"diff_pp": 0.6,
"only_a_right": 44,
"only_b_right": 47,
"mcnemar_exact_p": 0.834081039304942
},
"mathedit->clin08:ifeval": {
"n": 541,
"acc_a": 0.6137,
"acc_b": 0.6248,
"diff_pp": 1.11,
"only_a_right": 77,
"only_b_right": 83,
"mcnemar_exact_p": 0.6927623491186455
},
"base->clin08:math500": {
"n": 500,
"acc_a": 0.074,
"acc_b": 0.512,
"diff_pp": 43.8,
"only_a_right": 0,
"only_b_right": 219,
"mcnemar_exact_p": 2.3738919364399497e-66
},
"base->clin08:ifeval": {
"n": 541,
"acc_a": 0.2015,
"acc_b": 0.6248,
"diff_pp": 42.33,
"only_a_right": 26,
"only_b_right": 255,
"mcnemar_exact_p": 1.99712885297213e-48
},
"base->clin08d:math500": {
"n": 500,
"acc_a": 0.074,
"acc_b": 0.492,
"diff_pp": 41.8,
"only_a_right": 2,
"only_b_right": 211,
"mcnemar_exact_p": 3.4627676809817173e-60
},
"base->clin08d:ifeval": {
"n": 541,
"acc_a": 0.2015,
"acc_b": 0.573,
"diff_pp": 37.15,
"only_a_right": 20,
"only_b_right": 221,
"mcnemar_exact_p": 4.963338291141735e-44
}
}
}
{
"protocol": "Qwen3.5 thinking on; temperature 0.6, top_p 0.95, top_k 20, presence_penalty 1.5, max 32,768 new tokens, seed 0, one sample per question; OpenMed fixed MedQA prompt; strict = last 'Answer: X' after </think> (cut-offs and unparsable answers count as wrong)",
"scores": {
"clin08": {
"medqa_test": {
"n": 1273,
"correct_strict": 578,
"acc_strict": 0.454,
"correct_lenient": 578,
"acc_lenient": 0.454,
"cut": 2,
"cut_share": 0.0016,
"think_not_closed": 2,
"finished_no_strict_answer": 4,
"placeholder_answers": 2,
"finished_no_lenient_answer": 3,
"acc_strict_among_finished": 0.4548,
"finished": 1271,
"median_tokens": 4945,
"mean_tokens": 5275,
"total_tokens": 6714845
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 388,
"acc_strict": 0.388,
"correct_lenient": 388,
"acc_lenient": 0.388,
"cut": 0,
"cut_share": 0.0,
"think_not_closed": 0,
"finished_no_strict_answer": 6,
"placeholder_answers": 1,
"finished_no_lenient_answer": 5,
"acc_strict_among_finished": 0.388,
"finished": 1000,
"median_tokens": 3376,
"mean_tokens": 3770,
"total_tokens": 3770020
},
"mmlu_med": {
"n": 1089,
"correct_strict": 655,
"acc_strict": 0.6015,
"correct_lenient": 655,
"acc_lenient": 0.6015,
"cut": 1,
"cut_share": 0.0009,
"think_not_closed": 1,
"finished_no_strict_answer": 4,
"placeholder_answers": 1,
"finished_no_lenient_answer": 4,
"acc_strict_among_finished": 0.602,
"finished": 1088,
"median_tokens": 3504,
"mean_tokens": 3853,
"total_tokens": 4196117
}
},
"base": {
"medqa_test": {
"n": 1273,
"correct_strict": 311,
"acc_strict": 0.2443,
"correct_lenient": 414,
"acc_lenient": 0.3252,
"cut": 233,
"cut_share": 0.183,
"think_not_closed": 224,
"finished_no_strict_answer": 419,
"placeholder_answers": 419,
"finished_no_lenient_answer": 195,
"acc_strict_among_finished": 0.299,
"finished": 1040,
"median_tokens": 5546,
"mean_tokens": 10362,
"total_tokens": 13191142
},
"medmcqa_val1k": {
"n": 1000,
"correct_strict": 306,
"acc_strict": 0.306,
"correct_lenient": 348,
"acc_lenient": 0.348,
"cut": 114,
"cut_share": 0.114,
"think_not_closed": 110,
"finished_no_strict_answer": 209,
"placeholder_answers": 206,
"finished_no_lenient_answer": 94,
"acc_strict_among_finished": 0.3454,
"finished": 886,
"median_tokens": 3913,
"mean_tokens": 7234,
"total_tokens": 7233961
},
"mmlu_med": {
"n": 1089,
"correct_strict": 473,
"acc_strict": 0.4343,