{
"status": "completed",
"fingerprint": {
"suite_sha256": "02eee659949261311227ff35f2dca6f784b9ed4a73981554e87cd004005211b1",
"base_model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
"base_revision": "1df8507178afcc1bef68cd8c393f61a886323761",
"adapter_sha256": "b28f7107ec6c5fb05d105c0d6372f6c95b0e29aece2ae7a8188e6dfe72563ea5",
"adapter_config_sha256": "73e8430f28b5ed39b11c41e13169c2825f2f4ffbf4895f18bd4f21f6619a8c02",
"scorer_sha256": "ff9247eb8e1741ac3cb0a8a37cb813257869d7c93a9585d04bddb518c0d4869a",
"metrics_sha256": "ea012aa1520f35c8d20685953b792b80e8c78ac2f677162ae80386eb977dd5c6",
"formatting_sha256": "851524de30bf094997673594ed0bb9f02bcb9a8d930e0f92b8fd58bd714755ed",
"protocol": "balanced-letter-ranking-v1",
"quantization": "NF4 double quantization, FP16 compute",
"packages": {
"torch": "2.13.0+rocm10.0.0",
"transformers": "5.22.2",
"peft": "0.21.2",
"bitsandbytes": "0.50.2"
},
"gpu": "AMD Radeon RX 6800 XT"
},
"started_utc": "2026-11-09T17:54:28.512942+00:00",
"completed_utc": "2026-11-09T19:37:48.249156+00:00",
"total_questions": 376,
"method": "Argmax of next-token A/B/C/D logits after a closed reasoning prefix; no generated reasoning. Paired base and adapter in the same NF4 runtime.",
"comparison": "Adapted subsets, not official SecEval leaderboard scores or generated-answer pass@1.",
"suites": {
"heldout_pentesting": {
"overall": {
"base": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"adapter": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"delta_percentage_points": 0.0,
"corrected": 2,
"regressed": 2,
"both_correct": 150,
"both_wrong": 22,
"correct_answer_labels": {
"A": 44,
"D": 44,
"C": 44,
"B": 44
},
"base_prediction_labels": {
"A": 55,
"D": 38,
"B": 39,
"C": 44
},
"adapter_prediction_labels": {
"A": 57,
"D": 37,
"B": 42,
"C": 40
}
},
"by_topic": {
"Pentesting": {
"base": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"adapter": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"delta_percentage_points": 0.0,
"corrected": 2,
"regressed": 2,
"both_correct": 150,
"both_wrong": 22,
"correct_answer_labels": {
"A": 44,
"D": 44,
"C": 44,
"B": 44
},
"base_prediction_labels": {
"A": 55,
"D": 38,
"B": 39,
"C": 44
},
"adapter_prediction_labels": {
"A": 57,
"D": 37,
"B": 42,
"C": 40
}
}
}
},
"seceval_200": {
"overall": {
"base": {
"correct": 173,
"total": 200,
"accuracy": 0.865,
"wilson_95": [
0.8107086162924745,
0.9055342954521209
]
},
"adapter": {
"correct": 173,
"total": 200,
"accuracy": 0.865,
"wilson_95": [
0.8107086162924745,
0.9055342954521209
]
},
"delta_percentage_points": 0.0,
"corrected": 2,
"regressed": 2,
"both_correct": 171,
"both_wrong": 25,
"correct_answer_labels": {
"D": 50,
"A": 50,
"B": 50,
"C": 50
},
"base_prediction_labels": {
"D": 39,
"C": 56,
"A": 55,
"B": 50
},
"adapter_prediction_labels": {
"D": 43,
"A": 57,
"B": 49,
"C": 51
}
},
"by_topic": {
"ApplicationSecurity": {
"base": {
"correct": 42,
"total": 47,
"accuracy": 0.8936170212765957,
"wilson_95": [
0.7740573065626163,
0.9536952178189183
]
},
"adapter": {
"correct": 41,
"total": 47,
"accuracy": 0.8723404255319149,
"wilson_95": [
0.748260592072075,
0.9401539580185662
]
},
"delta_percentage_points": -2.127659574468085,
"corrected": 0,
"regressed": 1,
"both_correct": 41,
"both_wrong": 5,
"correct_answer_labels": {
"D": 9,
"A": 10,
"B": 14,
"C": 14
},
"base_prediction_labels": {
"D": 8,
"A": 13,
"C": 13,
"B": 13
},
"adapter_prediction_labels": {
"D": 9,
"A": 14,
"C": 13,
"B": 11
}
},
"Cryptography": {
"base": {
"correct": 3,
"total": 4,
"accuracy": 0.75,
"wilson_95": [
0.30064184258240184,
0.9544127391902995
]
},
"adapter": {
"correct": 3,
"total": 4,
"accuracy": 0.75,
"wilson_95": [
0.30064184258240184,
0.9544127391902995
]
},
"delta_percentage_points": 0.0,
"corrected": 0,
"regressed": 0,
"both_correct": 3,
"both_wrong": 1,
"correct_answer_labels": {
"D": 2,
"A": 1,
"C": 1
},
"base_prediction_labels": {
"D": 1,
"A": 1,
"C": 2
},
"adapter_prediction_labels": {
"D": 1,
"A": 1,
"B": 1,
"C": 1
}
},
"MemorySafety": {
"base": {
"correct": 9,
"total": 11,
"accuracy": 0.8181818181818182,
"wilson_95": [
0.523019438039156,
0.9486323102539149
]
},
"adapter": {
"correct": 9,
"total": 11,
"accuracy": 0.8181818181818182,
"wilson_95": [
0.523019438039156,
0.9486323102539149
]
},
"delta_percentage_points": 0.0,
"corrected": 0,
"regressed": 0,
"both_correct": 9,
"both_wrong": 2,
"correct_answer_labels": {
"D": 3,
"B": 3,
"C": 3,
"A": 2
},
"base_prediction_labels": {
"C": 5,
"B": 2,
"D": 2,
"A": 2
},
"adapter_prediction_labels": {
"A": 3,
"B": 2,
"C": 4,
"D": 2
}
},
"NetworkSecurity": {
"base": {
"correct": 26,
"total": 28,
"accuracy": 0.9285714285714286,
"wilson_95": [
0.7735463743160095,
0.9801879358846942
]
},
"adapter": {
"correct": 25,
"total": 28,
"accuracy": 0.8928571428571429,
"wilson_95": [
0.7280414845132785,
0.9628816331707
]
},
"delta_percentage_points": -3.5714285714285716,
"corrected": 0,
"regressed": 1,
"both_correct": 25,
"both_wrong": 2,
"correct_answer_labels": {
"A": 9,
"D": 9,
"C": 8,
"B": 2
},
"base_prediction_labels": {
"A": 9,
"D": 7,
"C": 9,
"B": 3
},
"adapter_prediction_labels": {
"A": 8,
"D": 8,
"C": 9,
"B": 3
}
},
"PenTest": {
"base": {
"correct": 52,
"total": 56,
"accuracy": 0.9285714285714286,
"wilson_95": [
0.8302455193913656,
0.9718739634211231
]
},
"adapter": {
"correct": 53,
"total": 56,
"accuracy": 0.9464285714285714,
"wilson_95": [
0.8539269093117847,
0.9816142186178909
]
},
"delta_percentage_points": 1.7857142857142858,
"corrected": 1,
"regressed": 0,
"both_correct": 52,
"both_wrong": 3,
"correct_answer_labels": {
"A": 14,
"B": 11,
"C": 16,
"D": 15
},
"base_prediction_labels": {
"A": 14,
"B": 14,
"C": 15,
"D": 13
},
"adapter_prediction_labels": {
"A": 14,
"B": 14,
"C": 14,
"D": 14
}
},
"SoftwareSecurity": {
"base": {
"correct": 28,
"total": 32,
"accuracy": 0.875,
"wilson_95": [
0.7193169451834978,
0.9502986561251991
]
},
"adapter": {
"correct": 27,
"total": 32,
"accuracy": 0.84375,
"wilson_95": [
0.682458504024696,
0.9313557971749429
]
},
"delta_percentage_points": -3.125,
"corrected": 0,
"regressed": 1,
"both_correct": 27,
"both_wrong": 4,
"correct_answer_labels": {
"D": 8,
"A": 8,
"C": 6,
"B": 10
},
"base_prediction_labels": {
"C": 9,
"A": 9,
"D": 5,
"B": 9
},
"adapter_prediction_labels": {
"A": 12,
"C": 7,
"D": 5,
"B": 8
}
},
"SystemSecurity": {
"base": {
"correct": 48,
"total": 56,
"accuracy": 0.8571428571428571,
"wilson_95": [
0.7426407877129466,
0.9257921146307937
]
},
"adapter": {
"correct": 48,
"total": 56,
"accuracy": 0.8571428571428571,
"wilson_95": [
0.7426407877129466,
0.9257921146307937
]
},
"delta_percentage_points": 0.0,
"corrected": 0,
"regressed": 0,
"both_correct": 48,
"both_wrong": 8,
"correct_answer_labels": {
"D": 14,
"A": 9,
"C": 17,
"B": 16
},
"base_prediction_labels": {
"D": 10,
"A": 13,
"C": 20,
"B": 13
},
"adapter_prediction_labels": {
"D": 10,
"A": 15,
"C": 18,
"B": 13
}
},
"Unspecified": {
"base": {
"correct": 19,
"total": 22,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.6666501283342828,
0.9525099628594474
]
},
"adapter": {
"correct": 19,
"total": 22,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.6666501283342828,
0.9525099628594474
]
},
"delta_percentage_points": 0.0,
"corrected": 0,
"regressed": 0,
"both_correct": 19,
"both_wrong": 3,
"correct_answer_labels": {
"B": 8,
"A": 6,
"C": 5,
"D": 3
},
"base_prediction_labels": {
"B": 8,
"A": 6,
"C": 5,
"D": 3
},
"adapter_prediction_labels": {
"B": 8,
"A": 6,
"C": 5,
"D": 3
}
},
"Vulnerability": {
"base": {
"correct": 39,
"total": 48,
"accuracy": 0.8125,
"wilson_95": [
0.680601328507153,
0.8980860904515235
]
},
"adapter": {
"correct": 39,
"total": 48,
"accuracy": 0.8125,
"wilson_95": [
0.680601328507153,
0.8980860904515235
]
},
"delta_percentage_points": 0.0,
"corrected": 1,
"regressed": 1,
"both_correct": 38,
"both_wrong": 8,
"correct_answer_labels": {
"D": 14,
"A": 8,
"B": 15,
"C": 11
},
"base_prediction_labels": {
"C": 14,
"A": 9,
"B": 17,
"D": 8
},
"adapter_prediction_labels": {
"A": 11,
"B": 16,
"C": 12,
"D": 9
}
},
"WebSecurity": {
"base": {
"correct": 24,
"total": 32,
"accuracy": 0.75,
"wilson_95": [
0.5789344100575552,
0.8674759908149096
]
},
"adapter": {
"correct": 25,
"total": 32,
"accuracy": 0.78125,
"wilson_95": [
0.6124500635075647,
0.8897616374739581
]
},
"delta_percentage_points": 3.125,
"corrected": 1,
"regressed": 0,
"both_correct": 24,
"both_wrong": 7,
"correct_answer_labels": {
"D": 8,
"B": 7,
"C": 9,
"A": 8
},
"base_prediction_labels": {
"D": 6,
"B": 5,
"C": 11,
"A": 10
},
"adapter_prediction_labels": {
"D": 8,
"B": 4,
"C": 10,
"A": 10
}
}
}
}
},
"peak_allocated_vram_gib": 9.544605731964111,
"confidence_interval": "Wilson 95% intervals for descriptive subset accuracy; topic overlap and source dependence limit population inference.",
"limitations": [
"Not a Q4_K_M inference benchmark; results apply to the NF4 base and adapter.",
"Answer ranking suppresses generated reasoning and does not measure normal R1 reasoning performance.",
"No tool-calling, exploit-discovery, or current-CVE-completeness score is measured.",
"Public benchmark pretraining contamination and semantic overlap cannot be ruled out."
],
"records": [
{
"id": "heldout_pentesting:pentest:044fa44c9eac6741d42b62835bd8faafc00cfecda360ae10c7455bae42c08b5a",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 109,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.7872855067253113,
0.11341923475265503,
0.0869615375995636,
0.012333769351243973
],
"base_choice_token_mass": 0.1671859472990036,
"base_seconds": 9.92473759999848,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9375206828117371,
0.04891645535826683,
0.010578925721347332,
0.0029839228373020887
],
"adapter_choice_token_mass": 0.94190514087677,
"adapter_seconds": 7.13555720000295
},
{
"id": "heldout_pentesting:pentest:f0cf995d1beaea0e391b7ac3367f04d77b85570974aad1e0589bdaf37c53dfad",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 113,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9851610660552979,
0.010774477384984493,
0.00039864066638983786,
0.003665831172838807
],
"adapter_choice_token_mass": 0.9235197305679321,
"adapter_seconds": 6.922615100018447,
"base_prediction": "A",
"base_choice_probabilities": [
0.9703077077865601,
0.02317880466580391,
0.002089632675051689,
0.004423751961439848
],
"base_choice_token_mass": 0.09490802884101868,
"base_seconds": 7.939818899991224
},
{
"id": "heldout_pentesting:pentest:f204c833625d72aef0d61d7c6631674ebf4be9faa3f1a3a5b9aa854491547d80",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 139,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0014099098043516278,
0.00010537559137446806,
0.00021286452829372138,
0.9982719421386719
],
"base_choice_token_mass": 0.3122033178806305,
"base_seconds": 7.926667899999302,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.001597486319951713,
0.00019684895232785493,
2.7919117201236077e-05,
0.9981777667999268
],
"adapter_choice_token_mass": 0.9697250723838806,
"adapter_seconds": 7.899110899976222
},
{
"id": "heldout_pentesting:pentest:a8355c7498fe5713313ec4032000cb5b5ddc82cb012b7a5148dc99a16974600f",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 64,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999427795410156,
3.426770490477793e-05,
1.202909515995998e-05,
1.095261632144684e-05
],
"adapter_choice_token_mass": 0.9872041344642639,
"adapter_seconds": 7.26205360001768,
"base_prediction": "A",
"base_choice_probabilities": [
0.9998884201049805,
1.9071812857873738e-05,
7.484347588615492e-05,
1.7638543795328587e-05
],
"base_choice_token_mass": 0.29494163393974304,
"base_seconds": 6.4285636999993585
},
{
"id": "heldout_pentesting:pentest:5f67d0e0a196e3091ff675e845046294f93cb0db06a2d2a53f27a17c2e6a1e29",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 115,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.00201812363229692,
1.1273145901213866e-05,
0.0004296833067201078,
0.9975409507751465
],
"base_choice_token_mass": 0.21759913861751556,
"base_seconds": 6.957341699977405,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0017005599802359939,
1.845389124355279e-05,
7.708917109994218e-05,
0.9982038736343384
],
"adapter_choice_token_mass": 0.9583963751792908,
"adapter_seconds": 6.66065959999105
},
{
"id": "heldout_pentesting:pentest:ed1502800bcbb2aed2aee04de17c47aefa47ea8583ff1e47d0e575780aec2a94",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 176,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0006261857342906296,
0.00020329274411778897,
3.370924241608009e-05,
0.9991368651390076
],
"adapter_choice_token_mass": 0.9572308659553528,
"adapter_seconds": 9.08592540002428,
"base_prediction": "D",
"base_choice_probabilities": [
0.0003149509138893336,
7.839748286642134e-05,
0.00020019499061163515,
0.9994064569473267
],
"base_choice_token_mass": 0.25229427218437195,
"base_seconds": 8.237507399986498
},
{
"id": "heldout_pentesting:pentest:1675c927f4eca663534e580b010c60faadfb6fbdd695b4d575f1fc2586550d58",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 124,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9787290096282959,
0.00020070227037649602,
0.019687911495566368,
0.0013823083136230707
],
"base_choice_token_mass": 0.1888883262872696,
"base_seconds": 7.034512300015194,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9988120794296265,
0.0001793468982214108,
0.0009251417359337211,
8.340405474882573e-05
],
"adapter_choice_token_mass": 0.974757969379425,
"adapter_seconds": 8.043992599996272
},
{
"id": "heldout_pentesting:pentest:32769ccc2e7b25f6dc1b8c9336ef78c28aff698ab60a09f39193b850549d48d9",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 127,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0008930745534598827,
0.002504689386114478,
0.0018040601862594485,
0.9947980642318726
],
"adapter_choice_token_mass": 0.9826804399490356,
"adapter_seconds": 7.170318399992539,
"base_prediction": "D",
"base_choice_probabilities": [
0.002044182037934661,
0.003971167840063572,
0.006969616748392582,
0.9870150089263916
],
"base_choice_token_mass": 0.20597708225250244,
"base_seconds": 7.442999799997779
},
{
"id": "heldout_pentesting:pentest:1d1422b8c55b36560a7e062a52a1b2506853d8fb9ce581d723732d5fd24f9ec3",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 66,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.15322694182395935,
0.029704395681619644,
0.09739696979522705,
0.7196716666221619
],
"base_choice_token_mass": 0.3023906946182251,
"base_seconds": 6.913068800000474,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.22712388634681702,
0.05742591619491577,
0.026705529540777206,
0.6887446641921997
],
"adapter_choice_token_mass": 0.9790098667144775,
"adapter_seconds": 7.710846400004812
},
{
"id": "heldout_pentesting:pentest:5b8b2bf02504e3033a8cf925e21980a93381fe0a9a3b2d08d267b5c8c2ae1b56",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 120,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00040446658385917544,
0.9995586276054382,
3.452320379437879e-05,
2.367759407206904e-06
],
"adapter_choice_token_mass": 0.9960500597953796,
"adapter_seconds": 7.092816999997012,
"base_prediction": "B",
"base_choice_probabilities": [
0.001081567839719355,
0.998782217502594,
0.00013327531632967293,
2.9675286441488424e-06
],
"base_choice_token_mass": 0.28958526253700256,
"base_seconds": 7.048637700005202
},
{
"id": "heldout_pentesting:pentest:ab249f604db895d5f282a8bd297b1f6b87bc4a5796fe23c464739023ff68f976",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 135,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0004167232837062329,
0.9981635212898254,
0.0013663829304277897,
5.3395946451928467e-05
],
"base_choice_token_mass": 0.31548425555229187,
"base_seconds": 8.084055899991654,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0002304419904248789,
0.9994887113571167,
0.0002652370894793421,
1.5559737221337855e-05
],
"adapter_choice_token_mass": 0.9908203482627869,
"adapter_seconds": 9.201693299983162
},
{
"id": "heldout_pentesting:pentest:8cec338b0175d4a2ca73db495238210584db9c46f269377c42bf8a744525827e",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 90,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999738931655884,
1.9833241822198033e-05,
4.460103355086176e-06,
1.8885265262724715e-06
],
"adapter_choice_token_mass": 0.9974093437194824,
"adapter_seconds": 7.794901699991897,
"base_prediction": "A",
"base_choice_probabilities": [
0.9999569654464722,
1.028917631629156e-05,
3.120154360658489e-05,
1.529349560769333e-06
],
"base_choice_token_mass": 0.6706336140632629,
"base_seconds": 8.524397200002568
},
{
"id": "heldout_pentesting:pentest:29b7b57bb75fbb1b848a3a1f75e813bf50f7185e22c489e2601dff2881e85983",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 124,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.05106630548834801,
0.0002721912751439959,
0.9486119747161865,
4.956972043146379e-05
],
"base_choice_token_mass": 0.23136991262435913,
"base_seconds": 7.2600764999806415,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.07913482934236526,
0.0009212979348376393,
0.9199121594429016,
3.177233520545997e-05
],
"adapter_choice_token_mass": 0.9546162486076355,
"adapter_seconds": 7.2887359999876935
},
{
"id": "heldout_pentesting:pentest:80c81c85139b42941c541e2d90537e1a04607c6167992b2c55f2ff44308d6c9f",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 99,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00011017527140211314,
6.084455890231766e-05,
0.995945155620575,
0.0038838121108710766
],
"adapter_choice_token_mass": 0.8507505059242249,
"adapter_seconds": 6.758701999991899,
"base_prediction": "C",
"base_choice_probabilities": [
0.0002013033372350037,
4.856668965658173e-05,
0.9971190690994263,
0.002631016308441758
],
"base_choice_token_mass": 0.06229529529809952,
"base_seconds": 7.5004336000129115
},
{
"id": "heldout_pentesting:pentest:98b5885e0248d628e722f5ba2a1a67edba49ef3228cf3e6fdf6119c9c78fb7d9",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 92,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.09142055362462997,
0.8810332417488098,
0.024992989376187325,
0.0025531945284456015
],
"base_choice_token_mass": 0.026280716061592102,
"base_seconds": 7.012175100011518,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.07021564990282059,
0.924909770488739,
0.004559423308819532,
0.0003151587734464556
],
"adapter_choice_token_mass": 0.8184319138526917,
"adapter_seconds": 7.002960099984193
},
{
"id": "heldout_pentesting:pentest:69be4c537495168ce82fb02144c3f90010511e83694d2a9fd5dfc441f2530fd7",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 133,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0016225811559706926,
7.70842088968493e-05,
0.9981396198272705,
0.00016065729141701013
],
"adapter_choice_token_mass": 0.8902466297149658,
"adapter_seconds": 7.90522659997805,
"base_prediction": "C",
"base_choice_probabilities": [
0.0019262743880972266,
0.00012314259947743267,
0.9978348612785339,
0.00011568176705623046
],
"base_choice_token_mass": 0.18534623086452484,
"base_seconds": 8.70998340001097
},
{
"id": "heldout_pentesting:pentest:d6deb9f635e98d018bbabb06e0f621cc34ce04353567056c5a54b9020f5423d4",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 130,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.001173797994852066,
0.9869503378868103,
0.011854940094053745,
2.1000840206397697e-05
],
"base_choice_token_mass": 0.28926342725753784,
"base_seconds": 8.079064699995797,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0011646047933027148,
0.9946409463882446,
0.004064870532602072,
0.00012964819325134158
],
"adapter_choice_token_mass": 0.9849415421485901,
"adapter_seconds": 8.92407710000407
},
{
"id": "heldout_pentesting:pentest:8fe195dc984badf7083f2f3ce7a62044f424cd9764ee425e9c9c6029797d6967",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 153,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999237060546875,
4.1012292058439925e-05,
3.169183764839545e-05,
3.5280550036986824e-06
],
"adapter_choice_token_mass": 0.9927820563316345,
"adapter_seconds": 8.156953500001691,
"base_prediction": "A",
"base_choice_probabilities": [
0.9991241097450256,
0.00014415431360248476,
0.0007095523178577423,
2.2280164557741955e-05
],
"base_choice_token_mass": 0.1549866795539856,
"base_seconds": 8.923808999999892
},
{
"id": "heldout_pentesting:pentest:ea6f643bd87777d0dbe4512388e98e9145bafc18f6c017e763f85dd67fb8efff",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 123,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.13812465965747833,
0.8594405055046082,
0.0021303400862962008,
0.0003045162302441895
],
"base_choice_token_mass": 0.08583847433328629,
"base_seconds": 6.88829820000683,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.18471744656562805,
0.8150116205215454,
0.00020799627236556262,
6.294161721598357e-05
],
"adapter_choice_token_mass": 0.9110957980155945,
"adapter_seconds": 7.107699299987871
},
{
"id": "heldout_pentesting:pentest:15d3b56960239aad4fe7b74961e44f4d4a319c261be8eee897d2fee69bd6a8db",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 75,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.008977953344583511,
0.000630351307336241,
0.9901902079582214,
0.00020147238683421165
],
"adapter_choice_token_mass": 0.99215167760849,
"adapter_seconds": 6.857746499998029,
"base_prediction": "C",
"base_choice_probabilities": [
0.013221103698015213,
0.00010173609189223498,
0.9866489768028259,
2.8251091862330213e-05
],
"base_choice_token_mass": 0.21578267216682434,
"base_seconds": 7.609719800006133
},
{
"id": "heldout_pentesting:pentest:aa7309bc6e83379da2cdd8e936f7546f8e5932818dd07596b0c6454428b3ca26",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 139,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0025340518914163113,
0.001000132062472403,
0.08007346838712692,
0.9163923859596252
],
"base_choice_token_mass": 0.1406553089618683,
"base_seconds": 7.775945000001229,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.003528598928824067,
0.0005411282181739807,
0.0021402034908533096,
0.9937899708747864
],
"adapter_choice_token_mass": 0.979224681854248,
"adapter_seconds": 8.039478699996835
},
{
"id": "heldout_pentesting:pentest:423c528f82c8692dad105c2ee329bf496fac599d2c636624d81c633d9ffcb4d6",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 98,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
6.921961903572083e-05,
1.3630151443067007e-05,
0.9998987913131714,
1.834137219702825e-05
],
"adapter_choice_token_mass": 0.981174647808075,
"adapter_seconds": 7.540579599997727,
"base_prediction": "C",
"base_choice_probabilities": [
0.0003379396512173116,
1.936548505909741e-05,
0.9995443224906921,
9.834604134084657e-05
],
"base_choice_token_mass": 0.06612631678581238,
"base_seconds": 6.57531879999442
},
{
"id": "heldout_pentesting:pentest:13dde35d4c8fa5e737c55ca69a84d8bfe715857b4145b7eb42259f13a4d4a201",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 157,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9984434247016907,
0.00019234012870583683,
0.0008289855904877186,
0.0005352332955226302
],
"base_choice_token_mass": 0.2260059267282486,
"base_seconds": 8.132592600013595,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9972679615020752,
0.00012307263386901468,
9.812206553760916e-05,
0.0025109080597758293
],
"adapter_choice_token_mass": 0.9889323115348816,
"adapter_seconds": 8.519769300008193
},
{
"id": "heldout_pentesting:pentest:b77bc15cd340001b145636b288aee46f4f2a334e0a08fd5268acbda963ef114a",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 144,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.004790168721228838,
0.10083047300577164,
0.8182685971260071,
0.0761108323931694
],
"adapter_choice_token_mass": 0.930069088935852,
"adapter_seconds": 7.88702879997436,
"base_prediction": "C",
"base_choice_probabilities": [
0.025424137711524963,
0.07356718182563782,
0.7547028064727783,
0.14630581438541412
],
"base_choice_token_mass": 0.11618354171514511,
"base_seconds": 7.864482100005262
},
{
"id": "heldout_pentesting:pentest:9c87ec746074ec2dc4a0a1a82cf187b42e2b435d6fa57cba61b1fdc1cefa646c",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 94,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9996836185455322,
5.6483248044969514e-05,
2.585991569503676e-05,
0.00023411655274685472
],
"base_choice_token_mass": 0.2269212156534195,
"base_seconds": 7.735263400012627,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998774528503418,
6.708852743031457e-05,
7.18243154551601e-06,
4.832205013372004e-05
],
"adapter_choice_token_mass": 0.9554578065872192,
"adapter_seconds": 7.016632599988952
},
{
"id": "heldout_pentesting:pentest:c6e3f6391e1a4fbccdca03a1519b2406c401beef63518f7cc8149fa7f16425ce",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 128,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.007087070494890213,
0.0013316210824996233,
0.7939512729644775,
0.19763009250164032
],
"adapter_choice_token_mass": 0.9547309875488281,
"adapter_seconds": 7.1601631000230554,
"base_prediction": "C",
"base_choice_probabilities": [
0.0017313286662101746,
0.00015978544251993299,
0.9697265625,
0.028382254764437675
],
"base_choice_token_mass": 0.24184173345565796,
"base_seconds": 7.289219800004503
},
{
"id": "heldout_pentesting:pentest:0961d57f469d056e393dee853b8637d6eb70895ed22631eb9120f4ef8df89281",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 111,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.0032689725048840046,
0.00015529956726823002,
0.9954806566238403,
0.0010949686402454972
],
"base_choice_token_mass": 0.3540744483470917,
"base_seconds": 7.652382200001739,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.011647631414234638,
0.0015763354022055864,
0.984961748123169,
0.001814350951462984
],
"adapter_choice_token_mass": 0.6755635142326355,
"adapter_seconds": 6.867915800015908
},
{
"id": "heldout_pentesting:pentest:c600cbfd14460f6228f3223cfebf22282c89e3f6872a81b8b65ca15be2ea2288",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 166,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00013632835180032998,
0.0017817988991737366,
0.9979952573776245,
8.665557106724009e-05
],
"adapter_choice_token_mass": 0.9870271682739258,
"adapter_seconds": 8.064628299995093,
"base_prediction": "C",
"base_choice_probabilities": [
0.0011090722400695086,
0.005902634467929602,
0.9926705360412598,
0.00031775448587723076
],
"base_choice_token_mass": 0.234533429145813,
"base_seconds": 8.991976699995575
},
{
"id": "heldout_pentesting:pentest:26f01490c1e157d693c78493891cfecb5dd150aec6cf32f2d4681be58601ddbe",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 125,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.007695294450968504,
2.17840242839884e-05,
0.9922588467597961,
2.4112714527291246e-05
],
"base_choice_token_mass": 0.13692691922187805,
"base_seconds": 7.394510400015861,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.04602910950779915,
5.062923810328357e-05,
0.953866720199585,
5.347513433662243e-05
],
"adapter_choice_token_mass": 0.9662909507751465,
"adapter_seconds": 7.27737760002492
},
{
"id": "heldout_pentesting:pentest:030a98e2343229f3e155f1f5f808e26909ed2b58b232139ac8bae9d8e2ce4c41",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 97,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9874513745307922,
0.009680633433163166,
0.0010203311685472727,
0.0018475811230018735
],
"adapter_choice_token_mass": 0.959953248500824,
"adapter_seconds": 6.949660399986897,
"base_prediction": "A",
"base_choice_probabilities": [
0.9831110239028931,
0.008913767524063587,
0.0038941367529332638,
0.0040810201317071915
],
"base_choice_token_mass": 0.24100597202777863,
"base_seconds": 7.568383300007554
},
{
"id": "heldout_pentesting:pentest:8d0e3a0ee80c3bb5ea61c2a24dcb1e9ff7b1a3a7523cb401adb6df6e002e8531",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 96,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0113858412951231,
0.002790220081806183,
0.02300002984702587,
0.9628238677978516
],
"base_choice_token_mass": 0.08388618379831314,
"base_seconds": 6.8483411000052,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0048127747140824795,
0.0015026215696707368,
0.002006263006478548,
0.9916783571243286
],
"adapter_choice_token_mass": 0.8511222004890442,
"adapter_seconds": 6.800438300007954
},
{
"id": "heldout_pentesting:pentest:d69f6d01ea4e656365c07c22fedd3d14f260c00d3724a296098c56382a0ba27b",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 105,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0022487700916826725,
0.9963846206665039,
0.0009374267538078129,
0.0004291852528695017
],
"adapter_choice_token_mass": 0.9907806515693665,
"adapter_seconds": 6.678224899980705,
"base_prediction": "B",
"base_choice_probabilities": [
0.007069365121424198,
0.9856197834014893,
0.005253505893051624,
0.0020573025103658438
],
"base_choice_token_mass": 0.1854579746723175,
"base_seconds": 6.7544043000089005
},
{
"id": "heldout_pentesting:pentest:756dc073f3f039127d29151efdf6b803a7977793950452e49603c643712f9f14",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 106,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0015968650113791227,
0.9977895021438599,
0.000419834308559075,
0.0001937217020895332
],
"base_choice_token_mass": 0.31316709518432617,
"base_seconds": 6.794860499998322,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0006824619485996664,
0.9992590546607971,
3.242182719986886e-05,
2.6051668100990355e-05
],
"adapter_choice_token_mass": 0.9378608465194702,
"adapter_seconds": 7.49187999998685
},
{
"id": "heldout_pentesting:pentest:336da6c165e5d33a0f548da6f755fd84494fe540e4a2e69cc231b0efc402a4a2",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 115,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.02715427242219448,
0.9722951650619507,
0.0004188088933005929,
0.00013178406516090035
],
"adapter_choice_token_mass": 0.955581784248352,
"adapter_seconds": 6.570541599998251,
"base_prediction": "B",
"base_choice_probabilities": [
0.024548159912228584,
0.9653704762458801,
0.007969620637595654,
0.0021117394790053368
],
"base_choice_token_mass": 0.035054709762334824,
"base_seconds": 6.819935999985319
},
{
"id": "heldout_pentesting:pentest:33c5f6520d26c02e7e692904b51b61aa9419c11d952eb4aa66785733c6894b17",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 78,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9978651404380798,
0.00030718062771484256,
0.00147697061765939,
0.0003508112858980894
],
"base_choice_token_mass": 0.025217242538928986,
"base_seconds": 6.875470800005132,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.99983811378479,
8.613998215878382e-05,
4.198127498966642e-05,
3.373290383024141e-05
],
"adapter_choice_token_mass": 0.9006564021110535,
"adapter_seconds": 6.769686399988132
},
{
"id": "heldout_pentesting:pentest:5fb6671050ad754e22908207c69be5f1b96c342d27e478a9c6f40583df956ba5",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 127,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9989293217658997,
0.0008165305480360985,
0.00023393990704789758,
2.0124527509324253e-05
],
"adapter_choice_token_mass": 0.9895228743553162,
"adapter_seconds": 7.880204999994021,
"base_prediction": "A",
"base_choice_probabilities": [
0.9944520592689514,
0.0014719187747687101,
0.003939059097319841,
0.0001369097881251946
],
"base_choice_token_mass": 0.18404245376586914,
"base_seconds": 7.041003899998032
},
{
"id": "heldout_pentesting:pentest:cfb8710642ff6466964862cc66dbbb7f42f8df20db07ef9b858b32c2d5f21e11",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 120,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.007227588910609484,
0.0004374547570478171,
0.9920569062232971,
0.0002780631766654551
],
"base_choice_token_mass": 0.08026520907878876,
"base_seconds": 6.93636979997973,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0017523757414892316,
0.0009829936316236854,
0.9969717264175415,
0.0002928513567894697
],
"adapter_choice_token_mass": 0.9164184927940369,
"adapter_seconds": 7.13051080002333
},
{
"id": "heldout_pentesting:pentest:87f9088c444e9ea487282393c1c3be89989c2f67a1fb2c25b32bf10ac0428db8",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 132,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0021484922617673874,
9.970432438421994e-05,
0.00011297980381641537,
0.9976388216018677
],
"adapter_choice_token_mass": 0.8254663348197937,
"adapter_seconds": 8.770962399983546,
"base_prediction": "D",
"base_choice_probabilities": [
0.0025898225139826536,
5.545626117964275e-05,
0.00039100469439290464,
0.9969636797904968
],
"base_choice_token_mass": 0.1841723918914795,
"base_seconds": 7.9746274999924935
},
{
"id": "heldout_pentesting:pentest:09d3d0437bf1a958379d2ee172142841f9c6c86ff003b3c41585b1738a6ef5ad",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 118,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0010635328944772482,
0.9975939393043518,
0.0006974819698370993,
0.0006450653309002519
],
"base_choice_token_mass": 0.14641441404819489,
"base_seconds": 7.212780100002419,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0005272898706607521,
0.9991151690483093,
0.00020648965437430888,
0.00015107105718925595
],
"adapter_choice_token_mass": 0.9794581532478333,
"adapter_seconds": 7.257244000007631
},
{
"id": "heldout_pentesting:pentest:70e619bb26f058785172a34832ebf19357993800d6b0375eb6f36a5beaf1209e",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 116,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0036495719105005264,
0.00011020752572221681,
0.9962367415428162,
3.5426151043793652e-06
],
"adapter_choice_token_mass": 0.9952676296234131,
"adapter_seconds": 7.020192800002405,
"base_prediction": "C",
"base_choice_probabilities": [
0.000423872988903895,
2.1944058971712366e-05,
0.9995483756065369,
5.7693468988873065e-06
],
"base_choice_token_mass": 0.1620817482471466,
"base_seconds": 6.889311000006273
},
{
"id": "heldout_pentesting:pentest:40ddf1d1e5f281fa116b8545d3d43c7d7c5721f190e80d2426089d5e90e6bd84",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 107,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.001187378540635109,
0.00011394925240892917,
0.0003297231160104275,
0.998369038105011
],
"base_choice_token_mass": 0.5547206401824951,
"base_seconds": 6.523310399992624,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0014770693378522992,
0.0004366208740975708,
0.00015447044279426336,
0.9979318380355835
],
"adapter_choice_token_mass": 0.9822925329208374,
"adapter_seconds": 7.506398900004569
},
{
"id": "heldout_pentesting:pentest:f949c2bcbe61a862fd98a8452a9bdb331b4354428ac14fad690141fa42a95570",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 107,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.026231320574879646,
0.003720392007380724,
0.9690625071525574,
0.0009858057601377368
],
"adapter_choice_token_mass": 0.9785487651824951,
"adapter_seconds": 6.817062699992675,
"base_prediction": "C",
"base_choice_probabilities": [
0.022888848558068275,
0.0034556989558041096,
0.9732585549354553,
0.000396913179429248
],
"base_choice_token_mass": 0.5489516854286194,
"base_seconds": 6.925825999991503
},
{
"id": "heldout_pentesting:pentest:6f18d71500c8cade83f1cef982dd9176935f9a23de474f0dcaed4980f12c6d99",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 150,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.00311669846996665,
0.0006331956828944385,
0.0015918537974357605,
0.9946582913398743
],
"base_choice_token_mass": 0.26414304971694946,
"base_seconds": 9.609034699999029,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0009684303076937795,
0.0009684303076937795,
0.0003943500923924148,
0.9976686835289001
],
"adapter_choice_token_mass": 0.9388792514801025,
"adapter_seconds": 8.55473350000102
},
{
"id": "heldout_pentesting:pentest:376cc6de4dcc057f1137bf375b4967d9b5cc3538cd46d61d14403b51b53ff2f8",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 122,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.008557617664337158,
0.0051099867559969425,
0.8460444211959839,
0.14028792083263397
],
"adapter_choice_token_mass": 0.9942848086357117,
"adapter_seconds": 7.633067699993262,
"base_prediction": "C",
"base_choice_probabilities": [
0.0021754701156169176,
0.0012788929743692279,
0.8640406727790833,
0.13250494003295898
],
"base_choice_token_mass": 0.13365375995635986,
"base_seconds": 8.208851500006858
},
{
"id": "heldout_pentesting:pentest:5fdd68a63fe68365962bf0a8a1d1dc89630ceef59977c732de9a8a3ef8d9ee34",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 99,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9934326410293579,
0.00031306836172007024,
0.006190657150000334,
6.360369297908619e-05
],
"base_choice_token_mass": 0.053876619786024094,
"base_seconds": 6.909938400000101,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997549653053284,
5.561152647715062e-05,
0.0001767330541042611,
1.2702870662906207e-05
],
"adapter_choice_token_mass": 0.9115477204322815,
"adapter_seconds": 7.002752400003374
},
{
"id": "heldout_pentesting:pentest:6123f468be3f9a6667a04d9a2972d22d766b407586093e61600ddb077bd58938",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 105,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997983574867249,
9.460249566473067e-05,
3.5627628676593304e-05,
7.140971138142049e-05
],
"adapter_choice_token_mass": 0.9383958578109741,
"adapter_seconds": 6.859942499984754,
"base_prediction": "A",
"base_choice_probabilities": [
0.9950962662696838,
0.0017354676965624094,
0.0017627974739298224,
0.0014054246712476015
],
"base_choice_token_mass": 0.10828278213739395,
"base_seconds": 7.904508700012229
},
{
"id": "heldout_pentesting:pentest:93654576e25af7a82be7de2e3af003dde10e6e6b3a4991903b46e32da4d0e029",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 106,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.0004955174517817795,
1.3518240848497953e-05,
0.9994670748710632,
2.391133239143528e-05
],
"base_choice_token_mass": 0.15648524463176727,
"base_seconds": 7.021265100018354,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00021994156122673303,
2.4294180548167787e-05,
0.9997263550758362,
2.930437767645344e-05
],
"adapter_choice_token_mass": 0.9922273755073547,
"adapter_seconds": 6.902281699993182
},
{
"id": "heldout_pentesting:pentest:978e2ec26d449c128b66cf3a263d952080affd860080c2e65fc08e80df78a7d8",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 81,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0004838534805458039,
0.006784575991332531,
0.9913095235824585,
0.0014221244491636753
],
"adapter_choice_token_mass": 0.9903215765953064,
"adapter_seconds": 7.325349800026743,
"base_prediction": "C",
"base_choice_probabilities": [
0.0023931420873850584,
0.016354156658053398,
0.9806662201881409,
0.0005864646518602967
],
"base_choice_token_mass": 0.16236919164657593,
"base_seconds": 8.275605600007111
},
{
"id": "heldout_pentesting:pentest:5debf8308fae51238a8c1cf7fbbb1cef0f703d92f81414872adeb034b5b09f66",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 108,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0036967149935662746,
0.00034117221366614103,
0.0025013224221765995,
0.9934607744216919
],
"base_choice_token_mass": 0.21620824933052063,
"base_seconds": 7.019501000002492,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0010645341826602817,
0.00016843377670738846,
0.0002338471240364015,
0.9985331296920776
],
"adapter_choice_token_mass": 0.9928691983222961,
"adapter_seconds": 6.884402199997567
},
{
"id": "heldout_pentesting:pentest:ea54b814b9ef86d0a32ed4db0b7f39a320ba59b70a1af39e25be248f34de0afd",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 120,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997633099555969,
5.966295429971069e-05,
0.00016602672985754907,
1.1036537216568831e-05
],
"adapter_choice_token_mass": 0.9893832206726074,
"adapter_seconds": 7.565639699983876,
"base_prediction": "A",
"base_choice_probabilities": [
0.9871020913124084,
0.0005502327112480998,
0.012233138084411621,
0.00011443749099271372
],
"base_choice_token_mass": 0.09212954342365265,
"base_seconds": 8.317050700017717
},
{
"id": "heldout_pentesting:pentest:acc77107fd856a9570baeb74e42330ede755aba158e8c24c7e15f132afd25530",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 98,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.999384880065918,
0.00014532284694723785,
0.000310061383061111,
0.00015960591554176062
],
"base_choice_token_mass": 0.020263241603970528,
"base_seconds": 7.053018300008262,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998650550842285,
1.8056407498079352e-05,
2.373469942540396e-05,
9.314204362453893e-05
],
"adapter_choice_token_mass": 0.6421667337417603,
"adapter_seconds": 7.0624673000129405
},
{
"id": "heldout_pentesting:pentest:12ec4e754663d863e943b8ff15e14aad31ced7e5b350be5b8d7eb4126ef233c2",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 91,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.49551621079444885,
0.10716290026903152,
0.39198559522628784,
0.0053353263065218925
],
"adapter_choice_token_mass": 0.9746147990226746,
"adapter_seconds": 7.474561000009999,
"base_prediction": "A",
"base_choice_probabilities": [
0.759565532207489,
0.029682541266083717,
0.20126481354236603,
0.009487110190093517
],
"base_choice_token_mass": 0.17382532358169556,
"base_seconds": 8.246032499999274
},
{
"id": "heldout_pentesting:pentest:c719e14160e289a53b8ae49d909ca870181552bd7c8e3f87a48299ccf4d003e3",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 63,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0001243489095941186,
1.121039440477034e-05,
9.608938853489235e-05,
0.9997683167457581
],
"base_choice_token_mass": 0.08716047555208206,
"base_seconds": 6.896248400007607,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00016345258336514235,
2.84038014797261e-05,
4.5745138777419925e-05,
0.9997624754905701
],
"adapter_choice_token_mass": 0.9750182628631592,
"adapter_seconds": 6.770279799995478
},
{
"id": "heldout_pentesting:pentest:c83ab4f88f677f6f03aa5bafae0d515ecdc4d706b035b658ba57fd814e83e0f6",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 135,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00041099736699834466,
0.0021534722764045,
0.9844484329223633,
0.012987099587917328
],
"adapter_choice_token_mass": 0.9808191061019897,
"adapter_seconds": 9.416173899982823,
"base_prediction": "C",
"base_choice_probabilities": [
0.00015388145402539521,
0.0006632287986576557,
0.9941267967224121,
0.005056197289377451
],
"base_choice_token_mass": 0.3137548267841339,
"base_seconds": 8.389773000002606
},
{
"id": "heldout_pentesting:pentest:49bb1751a3a49fa2db1274433b7fac4712d1ded8ca586fb17cbc415c64227699",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 146,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
7.483191438950598e-05,
0.9997339844703674,
8.886510477168486e-05,
0.0001022831056616269
],
"base_choice_token_mass": 0.0741153359413147,
"base_seconds": 9.548538400005782,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
5.024825077271089e-05,
0.9999027252197266,
7.295722753042355e-06,
3.974964056396857e-05
],
"adapter_choice_token_mass": 0.8766269087791443,
"adapter_seconds": 8.499889200000325
},
{
"id": "heldout_pentesting:pentest:57303e64cd1bbeaa5f71218a4c0557cedbcbd14ad8c1d2c572cebe570dee0a44",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 106,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0008165042963810265,
9.089633385883644e-05,
0.00019545781833585352,
0.9988971948623657
],
"adapter_choice_token_mass": 0.9859095215797424,
"adapter_seconds": 7.905582300008973,
"base_prediction": "D",
"base_choice_probabilities": [
0.0011153977829962969,
3.368209581822157e-05,
0.0005187081405892968,
0.9983322024345398
],
"base_choice_token_mass": 0.35221877694129944,
"base_seconds": 6.976538499991875
},
{
"id": "heldout_pentesting:pentest:45045a7970bbf42104358adb5b3d7abac6affafdf6bbf7f07885c3082ba2272b",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 148,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9989339709281921,
0.0002413670881651342,
0.0006664359243586659,
0.00015829240146558732
],
"base_choice_token_mass": 0.09544064104557037,
"base_seconds": 8.296828500024276,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9991162419319153,
0.0005791151779703796,
0.00014989558258093894,
0.00015465376782231033
],
"adapter_choice_token_mass": 0.9015673398971558,
"adapter_seconds": 9.414418000000296
},
{
"id": "heldout_pentesting:pentest:11d19ebb76a88f8fa02f21c416a65608dde9173f624cc455d9b3e664926c4ec9",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 116,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.015766365453600883,
0.600945234298706,
0.0013043323997408152,
0.38198402523994446
],
"adapter_choice_token_mass": 0.9481959939002991,
"adapter_seconds": 7.20501870001317,
"base_prediction": "D",
"base_choice_probabilities": [
0.006590715143829584,
0.37711000442504883,
0.004189313389360905,
0.6121099591255188
],
"base_choice_token_mass": 0.10012782365083694,
"base_seconds": 7.100833199976478
},
{
"id": "heldout_pentesting:pentest:d14384cf075fb98057db5cd2b54f9db36f37c8d03f2406b1225294b7e0edaf09",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 93,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0022868518717586994,
0.9975488781929016,
0.00015807291492819786,
6.177223895065254e-06
],
"base_choice_token_mass": 0.22351649403572083,
"base_seconds": 8.534206800017273,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0035380057524889708,
0.9964392781257629,
1.5512265235884115e-05,
7.102032668626634e-06
],
"adapter_choice_token_mass": 0.963616669178009,
"adapter_seconds": 7.482294199988246
},
{
"id": "heldout_pentesting:pentest:d795b937fa6a9e045abe04398f73d83fe1582bc9ace15ccaf05f7fe4c198f877",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 105,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9984656572341919,
0.0014323905343189836,
5.258403325569816e-05,
4.93981278850697e-05
],
"adapter_choice_token_mass": 0.9696566462516785,
"adapter_seconds": 6.968621799984248,
"base_prediction": "A",
"base_choice_probabilities": [
0.9911030530929565,
0.008574721403419971,
0.00024135317653417587,
8.084318687906489e-05
],
"base_choice_token_mass": 0.0857321172952652,
"base_seconds": 7.000549599993974
},
{
"id": "heldout_pentesting:pentest:6303e1ac34d188ec582b3545be15253e3933c519f1ea73e95936a1f90752c4a9",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 116,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.6473047733306885,
0.35193347930908203,
0.00067939143627882,
8.241953037213534e-05
],
"base_choice_token_mass": 0.10430707782506943,
"base_seconds": 7.9762660000124015,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.7401204705238342,
0.25980669260025024,
5.805803812108934e-05,
1.4794502931181341e-05
],
"adapter_choice_token_mass": 0.9171773791313171,
"adapter_seconds": 6.9906224999867845
},
{
"id": "heldout_pentesting:pentest:388b6af739fe362b1b3db045e3a03b901a2ed9d23947ec098294f704ac1a5747",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 120,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9881282448768616,
0.0071989670395851135,
0.0017641825834289193,
0.0029086454305797815
],
"adapter_choice_token_mass": 0.9849566221237183,
"adapter_seconds": 7.153574299998581,
"base_prediction": "A",
"base_choice_probabilities": [
0.7982327342033386,
0.03143824636936188,
0.06059880927205086,
0.10973026603460312
],
"base_choice_token_mass": 0.13749901950359344,
"base_seconds": 7.49889719998464
},
{
"id": "heldout_pentesting:pentest:7e512c591d292d98353a8d4bc8ece5a4481d08e7497ac7fb396409f98cb48a6b",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 96,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.00016591878375038505,
0.9991132616996765,
0.0006360316765494645,
8.474302012473345e-05
],
"base_choice_token_mass": 0.03113747574388981,
"base_seconds": 7.669965000008233,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0003919001028407365,
0.9992467164993286,
0.0003004802274517715,
6.104625936131924e-05
],
"adapter_choice_token_mass": 0.44777989387512207,
"adapter_seconds": 6.861842799989972
},
{
"id": "heldout_pentesting:pentest:3b369ed301338b709820a78f18e51842dd884841e29fc4f9c3295b742bc3666f",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 161,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.08384497463703156,
0.00032696392736397684,
0.9156136512756348,
0.00021442816068883985
],
"adapter_choice_token_mass": 0.9467484354972839,
"adapter_seconds": 8.083714199980022,
"base_prediction": "C",
"base_choice_probabilities": [
0.08506813645362854,
0.0002973642840515822,
0.9145687222480774,
6.58345888950862e-05
],
"base_choice_token_mass": 0.07504736632108688,
"base_seconds": 9.245875399996294
},
{
"id": "heldout_pentesting:pentest:2cb3769f92fe44a56fb2fd1110d695267f29818ae34aa9a0519a3c6919c5f05a",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 112,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.003262318205088377,
0.00297037442214787,
0.0003130751720163971,
0.9934543371200562
],
"base_choice_token_mass": 0.40222880244255066,
"base_seconds": 7.001065799995558,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0006766812293790281,
0.0007316669216379523,
2.5831001039477997e-05,
0.9985658526420593
],
"adapter_choice_token_mass": 0.9826073050498962,
"adapter_seconds": 6.903762000001734
},
{
"id": "heldout_pentesting:pentest:9da0c5e9998dd17321468f5af09d5a2c2c5f1733242415da37640ead4ea6caf2",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 107,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999808073043823,
7.646449375897646e-06,
8.463846825179644e-06,
3.065402324864408e-06
],
"adapter_choice_token_mass": 0.981887936592102,
"adapter_seconds": 6.967813300027046,
"base_prediction": "A",
"base_choice_probabilities": [
0.9999457597732544,
1.297927610721672e-06,
4.9859365390148014e-05,
3.162597977279802e-06
],
"base_choice_token_mass": 0.23981650173664093,
"base_seconds": 8.064413699990837
},
{
"id": "heldout_pentesting:pentest:1050c53f1fbbec67a0831fe16750c808f75e9b78ba6170eacfabea89140823d8",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 96,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.7970113158226013,
0.19839181005954742,
0.001236294861882925,
0.0033605978824198246
],
"base_choice_token_mass": 0.09806128591299057,
"base_seconds": 6.953781299991533,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.963582456111908,
0.03565121069550514,
0.00017167460464406759,
0.0005945402663201094
],
"adapter_choice_token_mass": 0.952239453792572,
"adapter_seconds": 6.8920982999843545
},
{
"id": "heldout_pentesting:pentest:131c24ada61f1640872695c27f166ef1db48020b57fdfb19985eaad66ed0b5b2",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 112,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.05459466576576233,
0.006623084656894207,
0.9379401803016663,
0.0008420307422056794
],
"adapter_choice_token_mass": 0.9922633767127991,
"adapter_seconds": 6.878355700027896,
"base_prediction": "C",
"base_choice_probabilities": [
0.017865076661109924,
0.002871179021894932,
0.9754001498222351,
0.0038635937962681055
],
"base_choice_token_mass": 0.4752865731716156,
"base_seconds": 7.861891100008506
},
{
"id": "heldout_pentesting:pentest:141c00d81216924fb537963bb979875b1c0ac0accde496f48fb9b7051f335e82",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 106,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9997897744178772,
1.1840788829431403e-05,
0.00018667380209080875,
1.1748644283215981e-05
],
"base_choice_token_mass": 0.24490727484226227,
"base_seconds": 7.1165611000033095,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999734163284302,
1.0050993296317756e-05,
1.497090943303192e-05,
1.4593396144846338e-06
],
"adapter_choice_token_mass": 0.9534916877746582,
"adapter_seconds": 6.916130199999316
},
{
"id": "heldout_pentesting:pentest:36baa31c2c2d3824f47f1d673b1ee791a651dd9e8df675fba3f836ad75038f5f",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 156,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9951978325843811,
0.0014962193090468645,
0.0022112689912319183,
0.0010946574620902538
],
"adapter_choice_token_mass": 0.9802113175392151,
"adapter_seconds": 9.452218199992785,
"base_prediction": "A",
"base_choice_probabilities": [
0.9888554215431213,
0.00038181201671250165,
0.010647212155163288,
0.00011553990043466911
],
"base_choice_token_mass": 0.36050742864608765,
"base_seconds": 8.65623210000922
},
{
"id": "heldout_pentesting:pentest:33f323b1e10dd745aa10036536114c7441dfc86b04b7e50af5152be578e8289e",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 139,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.004454195033758879,
0.011553303338587284,
0.007007432635873556,
0.9769850969314575
],
"base_choice_token_mass": 0.16502965986728668,
"base_seconds": 9.19717839997611,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.006165968254208565,
0.0037987427785992622,
0.0005646327044814825,
0.9894707202911377
],
"adapter_choice_token_mass": 0.9909148812294006,
"adapter_seconds": 8.297713999985717
},
{
"id": "heldout_pentesting:pentest:85c1fa23001ead7ca1faffeba0a55714b15278ecc6717f1957f282ac39bcf2b4",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 131,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9963157773017883,
0.0006857886910438538,
0.002978930016979575,
1.9606910427683033e-05
],
"adapter_choice_token_mass": 0.9855579733848572,
"adapter_seconds": 8.856370399997104,
"base_prediction": "A",
"base_choice_probabilities": [
0.9879728555679321,
0.0010288768680766225,
0.010975386947393417,
2.290912925673183e-05
],
"base_choice_token_mass": 0.19238103926181793,
"base_seconds": 8.453509499988286
},
{
"id": "heldout_pentesting:pentest:6867ba0aac24665254f1ad80e0ded0a216b9713d14b9ecdbd523af4394028618",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 149,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.002842795802280307,
0.9964141845703125,
0.0007187713636085391,
2.421369390503969e-05
],
"base_choice_token_mass": 0.22942711412906647,
"base_seconds": 9.802215000003343,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.004754792898893356,
0.9951595664024353,
5.801249062642455e-05,
2.761808536888566e-05
],
"adapter_choice_token_mass": 0.956112265586853,
"adapter_seconds": 8.633427499997197
},
{
"id": "heldout_pentesting:pentest:c2377e747452874915469951f7e24a188f087df8cad7f9eb41ed0a99d9925754",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 134,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9924737215042114,
0.0030141728930175304,
0.004119886085391045,
0.0003922966425307095
],
"adapter_choice_token_mass": 0.9529080390930176,
"adapter_seconds": 9.017874599987408,
"base_prediction": "A",
"base_choice_probabilities": [
0.9923810362815857,
0.0011988433543592691,
0.006184104364365339,
0.00023606624745298177
],
"base_choice_token_mass": 0.15246543288230896,
"base_seconds": 8.065840199997183
},
{
"id": "heldout_pentesting:pentest:f8cbe8672d05a2f8ec555eabf8919da111c0713a6c22971f53db36980b00af1d",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 151,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9995388984680176,
9.531973046250641e-05,
0.00035973291960544884,
6.093586307542864e-06
],
"base_choice_token_mass": 0.17907753586769104,
"base_seconds": 9.541488499991829,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999204874038696,
4.006211020168848e-05,
3.5631979699246585e-05,
3.7555830658675404e-06
],
"adapter_choice_token_mass": 0.9874056577682495,
"adapter_seconds": 8.570774699997855
},
{
"id": "heldout_pentesting:pentest:941877189d6bf65358ee66502064fbaaca4d06ef44940b58bcb3a8c33455cb82",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 129,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0913277342915535,
0.000273064972134307,
0.00032174919033423066,
0.9080773591995239
],
"adapter_choice_token_mass": 0.9022124409675598,
"adapter_seconds": 8.71632859998499,
"base_prediction": "D",
"base_choice_probabilities": [
0.05013861507177353,
0.0010085779940709472,
0.0028066146187484264,
0.9460462331771851
],
"base_choice_token_mass": 0.010785955004394054,
"base_seconds": 7.948443100001896
},
{
"id": "heldout_pentesting:pentest:bab0731e18fec00014632a2a24e419ad46136793d1130d7c4cf298f1c5eae6f7",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 105,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.002495555207133293,
0.9911701679229736,
0.006273824721574783,
6.05528439336922e-05
],
"base_choice_token_mass": 0.7202015519142151,
"base_seconds": 6.87406220001867,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.004750862717628479,
0.9943370223045349,
0.000865196343511343,
4.694111703429371e-05
],
"adapter_choice_token_mass": 0.9800224900245667,
"adapter_seconds": 7.005760200001532
},
{
"id": "heldout_pentesting:pentest:7082d5320302cc132ab1829a1d631eae12017af00cc6a8591ed5c65267c78735",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 169,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998334646224976,
5.87420909141656e-05,
8.818232890916988e-05,
1.9523016817402095e-05
],
"adapter_choice_token_mass": 0.9787391424179077,
"adapter_seconds": 9.049774999992223,
"base_prediction": "A",
"base_choice_probabilities": [
0.9970153570175171,
8.456507930532098e-05,
0.002800410846248269,
9.964202035916969e-05
],
"base_choice_token_mass": 0.062155600637197495,
"base_seconds": 8.510298600012902
},
{
"id": "heldout_pentesting:pentest:5bb5ccf8c35020359d467289da3d06451614417cab0c84e3b76f51e4191f0fb5",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 133,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9929051995277405,
3.593917426769622e-05,
0.007011209148913622,
4.761167292599566e-05
],
"base_choice_token_mass": 0.037134986370801926,
"base_seconds": 9.340280900010839,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9995858073234558,
5.4314095905283466e-05,
0.0003514162090141326,
8.52686025609728e-06
],
"adapter_choice_token_mass": 0.9754423499107361,
"adapter_seconds": 8.16411050001625
},
{
"id": "heldout_pentesting:pentest:8fb774c1f39bb5297b4d026ce808b5bd57a757682533eb737d879154f28bde2f",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 110,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.5809588432312012,
0.32588839530944824,
0.0012317426735535264,
0.09192103892564774
],
"adapter_choice_token_mass": 0.8791854977607727,
"adapter_seconds": 8.272844099992653,
"base_prediction": "A",
"base_choice_probabilities": [
0.5333861112594604,
0.17866197228431702,
0.002450845669955015,
0.28550103306770325
],
"base_choice_token_mass": 0.0578821524977684,
"base_seconds": 7.320737600006396
},
{
"id": "heldout_pentesting:pentest:76a7f67f054ad30224b61bcbcd076d685d18566ff9a8a0bc459be7878538b97f",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 159,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9982993006706238,
0.0002450122847221792,
0.0014099484542384744,
4.567818541545421e-05
],
"base_choice_token_mass": 0.13651958107948303,
"base_seconds": 8.855158799997298,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998244643211365,
0.000108889646071475,
5.693427374353632e-05,
9.740305358718615e-06
],
"adapter_choice_token_mass": 0.979742169380188,
"adapter_seconds": 9.178633900010027
},
{
"id": "heldout_pentesting:pentest:d84cf8e63391e18f6bc3a32a320e9da12ab438ffecc2ff48fb7e819b2b099971",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 107,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.000385844410629943,
0.00010548399586696178,
0.9992989301681519,
0.0002097799879265949
],
"adapter_choice_token_mass": 0.9631536602973938,
"adapter_seconds": 7.545038900017971,
"base_prediction": "C",
"base_choice_probabilities": [
0.00048027135198935866,
1.531815723865293e-05,
0.9994658827781677,
3.8509839214384556e-05
],
"base_choice_token_mass": 0.3861120343208313,
"base_seconds": 7.02689979999559
},
{
"id": "heldout_pentesting:pentest:e17587b9e7b5acc6a97964143534aca110a0c34c51b5b022496c975c158b1553",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 91,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0012435842072591186,
0.00011567135516088456,
0.0008957201498560607,
0.9977450370788574
],
"base_choice_token_mass": 0.13789717853069305,
"base_seconds": 8.528066500002751,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0005111707141622901,
7.900538912508637e-05,
9.164785296889022e-05,
0.9993182420730591
],
"adapter_choice_token_mass": 0.9873999953269958,
"adapter_seconds": 6.698771199997282
},
{
"id": "heldout_pentesting:pentest:30235d951c5555a83ff17529b89fdaf9ad51ed26d32cb4a7b62e307dfc2ac7f5",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 125,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.017138846218585968,
0.0016447651432827115,
0.9806568026542664,
0.0005596030969172716
],
"adapter_choice_token_mass": 0.9864587783813477,
"adapter_seconds": 6.609114899998531,
"base_prediction": "C",
"base_choice_probabilities": [
0.04071221873164177,
0.0007059864583425224,
0.9560194611549377,
0.002562294714152813
],
"base_choice_token_mass": 0.17864087224006653,
"base_seconds": 7.916691199992783
},
{
"id": "heldout_pentesting:pentest:dfc2ca96c197f11c58c3681214740b1c1c79b3f9b05b9cd449c684d25ae2deca",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 98,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.35748910903930664,
0.41146743297576904,
0.23081231117248535,
0.0002311600255779922
],
"base_choice_token_mass": 0.06330925971269608,
"base_seconds": 7.537912300002063,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.11860963702201843,
0.8362793922424316,
0.04501914232969284,
9.179249173030257e-05
],
"adapter_choice_token_mass": 0.857560396194458,
"adapter_seconds": 6.905590000009397
},
{
"id": "heldout_pentesting:pentest:6b0fe6cb20e1bc278f35380247341421754b7b6d593a296d5a3302158a6cc7ea",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 94,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.2192343920469284,
0.001647924305871129,
0.025777870789170265,
0.7533398270606995
],
"adapter_choice_token_mass": 0.8469102382659912,
"adapter_seconds": 8.608949099987512,
"base_prediction": "D",
"base_choice_probabilities": [
0.41642341017723083,
0.0010004525538533926,
0.07236353307962418,
0.5102126002311707
],
"base_choice_token_mass": 0.014974700286984444,
"base_seconds": 7.807920099992771
},
{
"id": "heldout_pentesting:pentest:44ca70d727dd4f265642d0eaeeb9ce85b4303a71907d9a633d92ab63458b00cb",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 68,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.8860735297203064,
0.007430178578943014,
0.0006698509678244591,
0.10582639276981354
],
"base_choice_token_mass": 0.30548402667045593,
"base_seconds": 7.596881700010272,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9629020094871521,
0.02689194120466709,
0.00031307945027947426,
0.009892992675304413
],
"adapter_choice_token_mass": 0.9906975030899048,
"adapter_seconds": 8.427294900000561
},
{
"id": "heldout_pentesting:pentest:439c9cbb6a1194286d447c213d2ed7e1a190c3ce3582b6e188fc4990e0dd6505",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 104,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.16263428330421448,
0.36081990599632263,
0.3665020167827606,
0.11004385352134705
],
"adapter_choice_token_mass": 0.9553812742233276,
"adapter_seconds": 6.950702200003434,
"base_prediction": "C",
"base_choice_probabilities": [
0.13298474252223969,
0.0800316333770752,
0.7652744054794312,
0.02170916460454464
],
"base_choice_token_mass": 0.09831317514181137,
"base_seconds": 7.015704000019468
},
{
"id": "heldout_pentesting:pentest:3c469585964ad3e5c2cff53310b836e2d5daa59869ab560a990c617e920d2eed",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 155,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.04711594432592392,
0.0014910588506609201,
0.1544872373342514,
0.7969057559967041
],
"base_choice_token_mass": 0.11479514092206955,
"base_seconds": 9.954939099989133,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.023613421246409416,
0.0013321780133992434,
0.1162293329834938,
0.8588250279426575
],
"adapter_choice_token_mass": 0.9777427911758423,
"adapter_seconds": 7.873561699991114
},
{
"id": "heldout_pentesting:pentest:b1f6e665622fe73cfba0d4241bf6c3aba8e415b6dc44d7c8efd7a57819f1c04f",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 158,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00024148177180904895,
0.00023405214597005397,
0.9994086027145386,
0.00011586421896936372
],
"adapter_choice_token_mass": 0.9771630167961121,
"adapter_seconds": 7.427717699989444,
"base_prediction": "C",
"base_choice_probabilities": [
0.0015243516536429524,
8.667253132443875e-05,
0.9981905817985535,
0.0001983953989110887
],
"base_choice_token_mass": 0.22316506505012512,
"base_seconds": 7.946691299992381
},
{
"id": "heldout_pentesting:pentest:dac15953f955239941f08d35cfd06d2b4d47bff43ed3cfaae3f048f4397bf39d",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 137,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
8.218302536988631e-05,
0.00014536695380229503,
0.9996882677078247,
8.413194154854864e-05
],
"base_choice_token_mass": 0.2827421724796295,
"base_seconds": 7.509505199996056,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
8.544191223336384e-05,
0.00037405366310849786,
0.9995138645172119,
2.667626722541172e-05
],
"adapter_choice_token_mass": 0.9851500988006592,
"adapter_seconds": 7.390192799997749
},
{
"id": "heldout_pentesting:pentest:52b23e660cd74b0baf1ae316ba43778c80d44051f74e811631c06c614791a9d5",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 154,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00012238824274390936,
0.0003352950152475387,
4.1967094148276374e-05,
0.9995003938674927
],
"adapter_choice_token_mass": 0.9856237769126892,
"adapter_seconds": 7.7696767000015825,
"base_prediction": "D",
"base_choice_probabilities": [
0.0001079063949873671,
0.00047610534238629043,
0.00084874959429726,
0.9985671043395996
],
"base_choice_token_mass": 0.1810968965291977,
"base_seconds": 7.449163599987514
},
{
"id": "heldout_pentesting:pentest:f3df4928b4408d11a2bf0ff2131168e19162a7c02d7763f8f890f2022592a9c4",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 111,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.8695130348205566,
0.11585140228271484,
0.005418442189693451,
0.009217079728841782
],
"base_choice_token_mass": 0.2647301256656647,
"base_seconds": 5.930466699996032,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8565446734428406,
0.1420290321111679,
0.00019748482736758888,
0.001228791894391179
],
"adapter_choice_token_mass": 0.9538918733596802,
"adapter_seconds": 5.97663609997835
},
{
"id": "heldout_pentesting:pentest:4e4b8c4dca54ff742bfb5fb1832f2fb2a4882dffed0bf6ab83da694a830fabd6",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 71,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00014882590039633214,
7.483435911126435e-05,
9.739742381498218e-06,
0.9997666478157043
],
"adapter_choice_token_mass": 0.9507005214691162,
"adapter_seconds": 6.198940600006608,
"base_prediction": "D",
"base_choice_probabilities": [
6.604941881960258e-05,
2.0143936126260087e-05,
2.1110663510626182e-05,
0.9998927116394043
],
"base_choice_token_mass": 0.09818293899297714,
"base_seconds": 6.331838199985214
},
{
"id": "heldout_pentesting:pentest:6763b9b235b0bab92f5362588e058fdbf9a4c46c71afa2db53a3cc9141f0bc89",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 99,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.000410808774176985,
3.7618759961333126e-05,
0.9994924068450928,
5.9182617405895144e-05
],
"base_choice_token_mass": 0.20208020508289337,
"base_seconds": 6.023035499994876,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00011234214616706595,
5.14339808432851e-05,
0.9997885823249817,
4.756865746458061e-05
],
"adapter_choice_token_mass": 0.9752581715583801,
"adapter_seconds": 5.964951700007077
},
{
"id": "heldout_pentesting:pentest:11a13cd2b0dc2ba56b184182c44f607b80ceabe98499dc28295f53890cb7ff72",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 115,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.7513099312782288,
0.24775570631027222,
0.0006640274659730494,
0.00027039559790864587
],
"adapter_choice_token_mass": 0.8410346508026123,
"adapter_seconds": 5.843817999993917,
"base_prediction": "A",
"base_choice_probabilities": [
0.8308681845664978,
0.16107115149497986,
0.007652031257748604,
0.0004087235138285905
],
"base_choice_token_mass": 0.03970537334680557,
"base_seconds": 6.053502599999774
},
{
"id": "heldout_pentesting:pentest:736185d040e263dc789dcbf18ee9b87cd0fa15cc0b27e8db1c83a899e3c15eac",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 157,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9996269941329956,
0.00010886814561672509,
0.00025115619064308703,
1.290126056119334e-05
],
"base_choice_token_mass": 0.13322694599628448,
"base_seconds": 7.583658400020795,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997333884239197,
0.00015474930114578456,
9.836464596446604e-05,
1.3521843357011676e-05
],
"adapter_choice_token_mass": 0.8743141889572144,
"adapter_seconds": 8.087791500001913
},
{
"id": "heldout_pentesting:pentest:52a6ce957baf9870d4e72bdc21e8041b7860384efdb1263da7cf0289139cf1c4",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 142,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0004955019103363156,
6.448990461649373e-05,
4.286931471142452e-06,
0.9994357228279114
],
"adapter_choice_token_mass": 0.9854639768600464,
"adapter_seconds": 8.077973100007512,
"base_prediction": "D",
"base_choice_probabilities": [
0.000164740631589666,
2.094442243105732e-05,
1.4968302821216639e-05,
0.9997993111610413
],
"base_choice_token_mass": 0.4929357171058655,
"base_seconds": 7.513513799989596
},
{
"id": "heldout_pentesting:pentest:fc6db3871b20a6a628c3e8e3b44ef4d63fdd7a29961badc56660431893624dae",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 118,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.00028012291295453906,
0.9916281700134277,
0.005989291705191135,
0.0021024392917752266
],
"base_choice_token_mass": 0.28789013624191284,
"base_seconds": 7.469692700018641,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00010882373317144811,
0.9992191791534424,
0.0004653800860978663,
0.0002065111621050164
],
"adapter_choice_token_mass": 0.9464631676673889,
"adapter_seconds": 7.631148399988888
},
{
"id": "heldout_pentesting:pentest:5d11a7741f21909b64ae66d15ff01031227753d6f59348a1a7f200762975d13d",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 135,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997158646583557,
0.00013236177619546652,
6.760377436876297e-05,
8.413426985498518e-05
],
"adapter_choice_token_mass": 0.9247059226036072,
"adapter_seconds": 9.276864699990256,
"base_prediction": "A",
"base_choice_probabilities": [
0.9943910241127014,
0.0033428308088332415,
0.001761547988280654,
0.0005046919686719775
],
"base_choice_token_mass": 0.028055081143975258,
"base_seconds": 8.554082299990114
},
{
"id": "heldout_pentesting:pentest:4e3fbbeb97859853459b7280d64d61631b3ab3cf671c0ff2aadf3a0e8216c7b9",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 104,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0007669429178349674,
0.9987775683403015,
0.000423546094680205,
3.1903793569654226e-05
],
"base_choice_token_mass": 0.027389926835894585,
"base_seconds": 6.848725200019544,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0008040327811613679,
0.9991298317909241,
5.3866835514781997e-05,
1.230434554599924e-05
],
"adapter_choice_token_mass": 0.7470094561576843,
"adapter_seconds": 6.730524999991758
},
{
"id": "heldout_pentesting:pentest:5726787420111a8d394b5310eb8aa591c3c156e21cd6904b7bbcc2f471f3313c",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 132,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
4.6057677536737174e-05,
0.0010002643102779984,
0.9987597465515137,
0.0001939100620802492
],
"adapter_choice_token_mass": 0.9967494606971741,
"adapter_seconds": 7.098798699997133,
"base_prediction": "C",
"base_choice_probabilities": [
3.399372144485824e-05,
6.155468145152554e-05,
0.9997277855873108,
0.00017672825197223574
],
"base_choice_token_mass": 0.670521080493927,
"base_seconds": 7.009065099991858
},
{
"id": "heldout_pentesting:pentest:bd413bbd402f1cb6dbbae60e5fdb9eb5cbc60ac44d13001f93f96d1c1d3540b1",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 135,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0010319139109924436,
5.821660306537524e-05,
0.0002489608305040747,
0.9986609220504761
],
"base_choice_token_mass": 0.2364112138748169,
"base_seconds": 6.989156700001331,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.000767242512665689,
3.4780365240294486e-05,
3.0217703169910237e-05,
0.999167799949646
],
"adapter_choice_token_mass": 0.990472674369812,
"adapter_seconds": 8.246836500009522
},
{
"id": "heldout_pentesting:pentest:f47fa659cd2d63b660c6522247ee361c8115dc8653cd1a846ef9cbd9a784f63f",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 131,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9988977909088135,
0.000925221131183207,
0.00016459210019093007,
1.2397970749589149e-05
],
"adapter_choice_token_mass": 0.992134690284729,
"adapter_seconds": 7.219861399993533,
"base_prediction": "A",
"base_choice_probabilities": [
0.9722147583961487,
0.017806734889745712,
0.009833811782300472,
0.00014472450129687786
],
"base_choice_token_mass": 0.07634559273719788,
"base_seconds": 7.3788395000156015
},
{
"id": "heldout_pentesting:pentest:0287b122c3ba32b9a057f9a5c43143416f1f191031e774d7e6c5b7fcc360a60d",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 126,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.5991013646125793,
0.3108097016811371,
0.049951665103435516,
0.040137290954589844
],
"base_choice_token_mass": 0.0979379191994667,
"base_seconds": 6.565239399991697,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.7368643283843994,
0.10288931429386139,
0.008185823448002338,
0.15206055343151093
],
"adapter_choice_token_mass": 0.9184517860412598,
"adapter_seconds": 6.5683330000028946
},
{
"id": "heldout_pentesting:pentest:1f091f6c3be53db095bcf270eca0f60153f680f55df68299a2ed6ac9e73696b6",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 148,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.6994953155517578,
0.010788536630570889,
0.0026438445784151554,
0.2870723009109497
],
"adapter_choice_token_mass": 0.9643521308898926,
"adapter_seconds": 8.171094200020889,
"base_prediction": "A",
"base_choice_probabilities": [
0.7502906918525696,
0.01764514669775963,
0.017102262005209923,
0.21496188640594482
],
"base_choice_token_mass": 0.12412764132022858,
"base_seconds": 9.27031910000369
},
{
"id": "heldout_pentesting:pentest:75f9327047e08e32e23a43f439f0272f692deba6373ebc3c32dc6cbb34bd494c",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 111,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0007909067207947373,
0.0004797091824002564,
0.00043338112300261855,
0.9982959628105164
],
"base_choice_token_mass": 0.4605124294757843,
"base_seconds": 6.969574499991722,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.009835259057581425,
0.0022998612839728594,
0.00019477677415125072,
0.9876701831817627
],
"adapter_choice_token_mass": 0.9244096875190735,
"adapter_seconds": 6.858731099986471
},
{
"id": "heldout_pentesting:pentest:5b37c3b05aa6ca10d412977eb02e5cbba1b0f3e389e6f42cfc3c84a6c41625d2",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 117,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.004657466430217028,
0.0021323435939848423,
0.018420636653900146,
0.974789559841156
],
"adapter_choice_token_mass": 0.9237980842590332,
"adapter_seconds": 7.268289800005732,
"base_prediction": "D",
"base_choice_probabilities": [
0.017501015216112137,
0.004156855400651693,
0.03763309493660927,
0.9407089948654175
],
"base_choice_token_mass": 0.08075260370969772,
"base_seconds": 8.07803919998696
},
{
"id": "heldout_pentesting:pentest:6c5ed8157b6c4eb1bd7152da94c58ffe36d96f6be68fa4b7677c53d376e621ae",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 160,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.0003624543023761362,
0.00012140697799623013,
0.9992631077766418,
0.0002530338824726641
],
"base_choice_token_mass": 0.579245924949646,
"base_seconds": 8.710070400004042,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
6.803840369684622e-05,
7.019817712716758e-05,
0.9983131885528564,
0.0015485469484701753
],
"adapter_choice_token_mass": 0.9979345798492432,
"adapter_seconds": 9.49196220000158
},
{
"id": "heldout_pentesting:pentest:def115d7704115afe1ba00dca52f59b8db53e067a07c0a8976c97e4b6cbbb1eb",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 89,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997910857200623,
0.00010553594620432705,
8.956719830166548e-05,
1.3735573702433612e-05
],
"adapter_choice_token_mass": 0.9723643660545349,
"adapter_seconds": 7.36942539998563,
"base_prediction": "A",
"base_choice_probabilities": [
0.9996992349624634,
3.562409619917162e-05,
0.0002551297366153449,
1.0048237527371384e-05
],
"base_choice_token_mass": 0.13311289250850677,
"base_seconds": 7.443335099989781
},
{
"id": "heldout_pentesting:pentest:eabaf3d52d6cdd64aa8de0afd0031cf41b4213c242b84ad32c6877a482ecba46",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 120,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.00013980726362206042,
3.7335783417802304e-05,
0.9997542500495911,
6.867101910756901e-05
],
"base_choice_token_mass": 0.41802528500556946,
"base_seconds": 8.244731999991927,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0008037241059355438,
0.00032473390456289053,
0.9987462759017944,
0.0001251960638910532
],
"adapter_choice_token_mass": 0.989115297794342,
"adapter_seconds": 6.557191100000637
},
{
"id": "heldout_pentesting:pentest:06eab4a6e07421e1768c6679ee6650ed463f2c86ec4fd1f1517618760311910a",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 151,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00419630017131567,
0.9952093958854675,
0.00019473621796350926,
0.00039957277476787567
],
"adapter_choice_token_mass": 0.9466076493263245,
"adapter_seconds": 8.815773699985584,
"base_prediction": "B",
"base_choice_probabilities": [
0.002356657525524497,
0.9963706731796265,
0.0007300584693439305,
0.0005425334093160927
],
"base_choice_token_mass": 0.07625027000904083,
"base_seconds": 9.545184499991592
},
{
"id": "heldout_pentesting:pentest:845aafd954fc73532f06225422fb52d6d78c4e6ed34721135a97472ed7a2ad2b",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 155,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.09361051768064499,
0.9021382331848145,
0.0032031757291406393,
0.0010480750352144241
],
"base_choice_token_mass": 0.17863303422927856,
"base_seconds": 9.617050100001507,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.004599434323608875,
0.9932013154029846,
0.0013810006203129888,
0.0008182158344425261
],
"adapter_choice_token_mass": 0.9302553534507751,
"adapter_seconds": 8.515954400005285
},
{
"id": "heldout_pentesting:pentest:02e334a63cd22c6a1f19e20be8cbdf43a0fb4bcc8d4fd82617f137fd0856847a",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 146,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0007548626745119691,
0.9985263347625732,
0.0005107656470499933,
0.00020798653713427484
],
"adapter_choice_token_mass": 0.9659979343414307,
"adapter_seconds": 9.250527399999555,
"base_prediction": "B",
"base_choice_probabilities": [
0.002430303255096078,
0.995894193649292,
0.0014512003399431705,
0.00022429425735026598
],
"base_choice_token_mass": 0.1075206845998764,
"base_seconds": 8.866525499994168
},
{
"id": "heldout_pentesting:pentest:422ad0cf9b4f45d730376e4fea2631b20a4778d3a67c02eac993637e4a8761ab",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 71,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9803506135940552,
0.018817462027072906,
0.0008139634737744927,
1.7982794815907255e-05
],
"base_choice_token_mass": 0.22318336367607117,
"base_seconds": 7.517545600014273,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9851511120796204,
0.014498498290777206,
0.0003009067149832845,
4.9506936193211004e-05
],
"adapter_choice_token_mass": 0.9746445417404175,
"adapter_seconds": 8.4513089999964
},
{
"id": "heldout_pentesting:pentest:d583f8e2ee5c11003753acaf1533b736cd1c9e833c72b96d57e60a0f85b1a834",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 99,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0009679346112534404,
0.00014843758253846318,
0.001725530019029975,
0.9971580505371094
],
"adapter_choice_token_mass": 0.9782184958457947,
"adapter_seconds": 7.219771199976094,
"base_prediction": "D",
"base_choice_probabilities": [
0.0002160573349101469,
3.1126011890592054e-05,
0.0022164650727063417,
0.997536301612854
],
"base_choice_token_mass": 0.4522768259048462,
"base_seconds": 7.121910299989395
},
{
"id": "heldout_pentesting:pentest:cf6eb7ef08c0a044e4a600ce0c4a81389bf306ec6daee1dcd2889e61c0e7a1ed",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 145,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.016790684312582016,
0.960735559463501,
0.021899180486798286,
0.0005745456437580287
],
"base_choice_token_mass": 0.13844165205955505,
"base_seconds": 9.709605100011686,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.005191549193114042,
0.9893341660499573,
0.005440697073936462,
3.3640306355664507e-05
],
"adapter_choice_token_mass": 0.9325025677680969,
"adapter_seconds": 8.492900300014298
},
{
"id": "heldout_pentesting:pentest:8c229c8d7a2be25f74cf14455aae379002032181daf24afe094764dc3bba05d3",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 85,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9962169528007507,
0.0027547867503017187,
0.0009013628005050123,
0.00012684556713793427
],
"adapter_choice_token_mass": 0.9569497108459473,
"adapter_seconds": 7.438497399998596,
"base_prediction": "A",
"base_choice_probabilities": [
0.9943209886550903,
0.002836823696270585,
0.002749543869867921,
9.262558160116896e-05
],
"base_choice_token_mass": 0.47743627429008484,
"base_seconds": 8.648793100001058
},
{
"id": "heldout_pentesting:pentest:0977f986d8a1354c6d4a6b016ad7b9c43801e4315a137cb4f11cea60b474bf25",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 150,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.00018234274466522038,
7.237860245368211e-06,
0.9997525811195374,
5.782669904874638e-05
],
"base_choice_token_mass": 0.32974424958229065,
"base_seconds": 8.791612200002419,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
9.46013824432157e-05,
5.920003241044469e-05,
0.9997865557670593,
5.966434036963619e-05
],
"adapter_choice_token_mass": 0.9927612543106079,
"adapter_seconds": 10.087154900014866
},
{
"id": "heldout_pentesting:pentest:341966321597ea5f9229591863ea4e70b29dbc903c499495b5e5df03129778c7",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 137,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
9.754556958796456e-05,
0.000544065551366657,
0.9991844296455383,
0.00017389378626830876
],
"adapter_choice_token_mass": 0.9972742795944214,
"adapter_seconds": 8.442312200000742,
"base_prediction": "C",
"base_choice_probabilities": [
0.00012236819020472467,
0.00043045682832598686,
0.9993366599082947,
0.00011055045615648851
],
"base_choice_token_mass": 0.3960668444633484,
"base_seconds": 10.2282361999969
},
{
"id": "heldout_pentesting:pentest:98a0726b1ced20625fdc475948c7f60477ae5f2bf1372f51058102c1c4a2164e",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 137,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.7678340077400208,
0.0009720952366478741,
0.22697129845619202,
0.004222589544951916
],
"base_choice_token_mass": 0.03440535441040993,
"base_seconds": 8.86192170000868,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8990469574928284,
0.0015557538717985153,
0.09776682406663895,
0.0016304160235449672
],
"adapter_choice_token_mass": 0.9160287380218506,
"adapter_seconds": 8.928695400012657
},
{
"id": "heldout_pentesting:pentest:f59bdce101ad280abf3576626479fe29bfdadf0333bec811ae3253ed7d82d78f",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 132,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0002491639752406627,
0.00011407558486098424,
0.99947589635849,
0.0001608723687240854
],
"adapter_choice_token_mass": 0.9891473650932312,
"adapter_seconds": 8.330456000025151,
"base_prediction": "C",
"base_choice_probabilities": [
0.00029132066993042827,
3.61792954208795e-05,
0.9995391368865967,
0.00013337632117327303
],
"base_choice_token_mass": 0.4824969172477722,
"base_seconds": 9.16733500000555
},
{
"id": "heldout_pentesting:pentest:428381297d6837ef03e492a7d000e2be6620b2c0298b89f29ae8346d074efc2e",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 183,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
7.411871774820611e-05,
0.00014398808707483113,
0.9979720711708069,
0.0018098161090165377
],
"base_choice_token_mass": 0.26011237502098083,
"base_seconds": 8.928395900002215,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00039081895374692976,
0.0005554657545872033,
0.9887352585792542,
0.010318377986550331
],
"adapter_choice_token_mass": 0.9345752000808716,
"adapter_seconds": 9.371857499994803
},
{
"id": "heldout_pentesting:pentest:797ea62bf06feffdf8a0cffee417718bae4ab9745b195a32ad5d00119962f90d",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 146,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.001324848853982985,
0.9985442161560059,
0.00011220231681363657,
1.8750890376395546e-05
],
"adapter_choice_token_mass": 0.9237945079803467,
"adapter_seconds": 9.413040599989472,
"base_prediction": "B",
"base_choice_probabilities": [
0.003533122595399618,
0.9950640797615051,
0.0013410220853984356,
6.174804730108008e-05
],
"base_choice_token_mass": 0.09592151641845703,
"base_seconds": 8.592968100012513
},
{
"id": "heldout_pentesting:pentest:8dc29cfd230c4decc1f640e35be7a66ae0bfdecd2aef588c92ae394b638092bb",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 153,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9812495112419128,
0.008100710809230804,
0.009619832038879395,
0.0010298897977918386
],
"base_choice_token_mass": 0.15131442248821259,
"base_seconds": 9.92991289999918,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9970410466194153,
0.0014757508179172873,
0.0013648464810103178,
0.00011833087773993611
],
"adapter_choice_token_mass": 0.9800412058830261,
"adapter_seconds": 8.756312500016065
},
{
"id": "heldout_pentesting:pentest:07b29d16e9bfad05ec2d1f9e9c292d959054316bc1301628ca4a6f50815b19ed",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 94,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.007628617342561483,
0.7309949994087219,
0.2606442868709564,
0.0007320962613448501
],
"adapter_choice_token_mass": 0.9125184416770935,
"adapter_seconds": 8.795262900006492,
"base_prediction": "C",
"base_choice_probabilities": [
0.09120924025774002,
0.34964072704315186,
0.5587242841720581,
0.00042569535435177386
],
"base_choice_token_mass": 0.02865620329976082,
"base_seconds": 7.64693879999686
},
{
"id": "heldout_pentesting:pentest:07a6a3fe78874abf1c52a238a287a7334dc2c91611fab11b1950901ac46ca9c2",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 109,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.46865367889404297,
0.0002592050295788795,
0.5310541987419128,
3.295421993243508e-05
],
"base_choice_token_mass": 0.09757302701473236,
"base_seconds": 7.153850500006229,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.7873996496200562,
0.0006238227360881865,
0.21192574501037598,
5.0807997467927635e-05
],
"adapter_choice_token_mass": 0.9587017297744751,
"adapter_seconds": 7.142279699997744
},
{
"id": "heldout_pentesting:pentest:f3fd67d8811b4542b5f496bcd606bff386b130d818a86ca22feeb11c7688693a",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 68,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.003070738399401307,
0.9954232573509216,
0.0012800742406398058,
0.00022594652546104044
],
"adapter_choice_token_mass": 0.9860091805458069,
"adapter_seconds": 8.376831199973822,
"base_prediction": "B",
"base_choice_probabilities": [
0.001911562285386026,
0.9902138113975525,
0.007679434027522802,
0.00019527839322108775
],
"base_choice_token_mass": 0.24834586679935455,
"base_seconds": 7.533909500023583
},
{
"id": "heldout_pentesting:pentest:5e6121b43f17b648ba242ffbe11af2cbacbf08769946ffb967f1c4cc14471296",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 71,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0015677448827773333,
0.06165246292948723,
0.002044724766165018,
0.9347350597381592
],
"base_choice_token_mass": 0.1366892009973526,
"base_seconds": 7.333343900012551,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0017180999275296926,
0.13436956703662872,
0.0012968897353857756,
0.8626154065132141
],
"adapter_choice_token_mass": 0.9449552297592163,
"adapter_seconds": 8.16029110000818
},
{
"id": "heldout_pentesting:pentest:59f512f5477fd23f1c725a8ea6d22f220fb20770446c168079026491310d5b89",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 154,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.2210869938135147,
0.0002470118924975395,
0.006996647920459509,
0.7716693878173828
],
"adapter_choice_token_mass": 0.5567929744720459,
"adapter_seconds": 8.726397799997358,
"base_prediction": "D",
"base_choice_probabilities": [
0.17917440831661224,
9.309448796557263e-05,
0.030177894979715347,
0.7905545830726624
],
"base_choice_token_mass": 0.04091593623161316,
"base_seconds": 9.661375700001372
},
{
"id": "heldout_pentesting:pentest:04192279e007cc862fc0bae543e86e85b072c681ff527483ddc0ef14207a780e",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 141,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.002018736442551017,
3.4463977499399334e-05,
0.997843861579895,
0.0001028904298436828
],
"base_choice_token_mass": 0.23020648956298828,
"base_seconds": 8.472212099994067,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00017128685431089252,
1.1297593118797522e-05,
0.9997041821479797,
0.00011321370402583852
],
"adapter_choice_token_mass": 0.984872043132782,
"adapter_seconds": 9.38263029998052
},
{
"id": "heldout_pentesting:pentest:f3c32e24a307e87f0ff974d0591f5b8736edfe33406a06a284026d1438501428",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 191,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.5051885843276978,
0.03659573942422867,
0.4254114329814911,
0.032804206013679504
],
"adapter_choice_token_mass": 0.9459632635116577,
"adapter_seconds": 8.983158799994271,
"base_prediction": "C",
"base_choice_probabilities": [
0.3552767038345337,
0.008620533160865307,
0.594976544380188,
0.04112626612186432
],
"base_choice_token_mass": 0.08369647711515427,
"base_seconds": 9.662955900013912
},
{
"id": "heldout_pentesting:pentest:35005179ea203c2d2f33ec7c5603018c3d6e46b0823426f06b148cdd3d71a938",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 103,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.044402655214071274,
0.9493706822395325,
0.005557644180953503,
0.0006689719157293439
],
"base_choice_token_mass": 0.011720845475792885,
"base_seconds": 7.0234245999890845,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.010972599498927593,
0.9877219796180725,
0.0012120052706450224,
9.345981379738078e-05
],
"adapter_choice_token_mass": 0.6690276861190796,
"adapter_seconds": 7.062708299985388
},
{
"id": "heldout_pentesting:pentest:68eee8ef9156fcfd3806dea537539fbc11c1cbc34e3bed08ce740628a538385f",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 91,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.30472031235694885,
0.00048767522093839943,
0.5265094637870789,
0.16828253865242004
],
"adapter_choice_token_mass": 0.9469868540763855,
"adapter_seconds": 8.571389500022633,
"base_prediction": "C",
"base_choice_probabilities": [
0.10758976638317108,
9.734566992847249e-05,
0.886871874332428,
0.00544093269854784
],
"base_choice_token_mass": 0.20247721672058105,
"base_seconds": 7.786125799990259
},
{
"id": "heldout_pentesting:pentest:b5f9539cb7742fa6f87abe1097b0f5ba4e2a7afb55b0d131ba2a00f3b12c146c",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 118,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.0060314941219985485,
0.03994964808225632,
0.9528855681419373,
0.0011332841822877526
],
"base_choice_token_mass": 0.14305901527404785,
"base_seconds": 8.48215060000075,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.003558251541107893,
0.02248884178698063,
0.9713086485862732,
0.002644268097355962
],
"adapter_choice_token_mass": 0.9746203422546387,
"adapter_seconds": 7.622525699989637
},
{
"id": "heldout_pentesting:pentest:b58b5452c8ad6902f3da063675e8379ef981567fb1883d70872d2ce7be690d8b",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 66,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8529179692268372,
0.02964484877884388,
0.04520299658179283,
0.07223418354988098
],
"adapter_choice_token_mass": 0.9253199100494385,
"adapter_seconds": 7.579844299994875,
"base_prediction": "A",
"base_choice_probabilities": [
0.8985567688941956,
0.004789446480572224,
0.05655224621295929,
0.0401015467941761
],
"base_choice_token_mass": 0.10074389725923538,
"base_seconds": 8.440463000006275
},
{
"id": "heldout_pentesting:pentest:07ba64dd79b2726c98d54a4ff79a1a8b97e8b5b1274adc40e71093ffacbb45b9",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 84,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.7370520830154419,
1.862447061284911e-05,
0.2628040015697479,
0.0001253019436262548
],
"base_choice_token_mass": 0.07950292527675629,
"base_seconds": 7.747518199990736,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9565445780754089,
7.445129449479282e-05,
0.043361734598875046,
1.927063931361772e-05
],
"adapter_choice_token_mass": 0.9723272919654846,
"adapter_seconds": 7.720155400020303
},
{
"id": "heldout_pentesting:pentest:8096dac79f4932f8b7cada4aa8f18edbbb5655bc44e69c91e8d9ff96f815e88a",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 116,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0010647993767634034,
0.9987818598747253,
0.00010460895282449201,
4.864768197876401e-05
],
"adapter_choice_token_mass": 0.9348265528678894,
"adapter_seconds": 8.188628500007326,
"base_prediction": "B",
"base_choice_probabilities": [
0.006451714783906937,
0.9879143834114075,
0.005432888865470886,
0.00020100933033972979
],
"base_choice_token_mass": 0.1361536979675293,
"base_seconds": 7.27595690000453
},
{
"id": "heldout_pentesting:pentest:6d1e6bb94fcb1f504d6b5372a1fbf4e750071377e8473a97d08c98b98b2e2add",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 97,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0003560829209163785,
0.997157096862793,
0.0024717054329812527,
1.5045833606563974e-05
],
"base_choice_token_mass": 0.38705214858055115,
"base_seconds": 7.149245200009318,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00031008440419100225,
0.9994590878486633,
0.0002268625976284966,
3.903386186721036e-06
],
"adapter_choice_token_mass": 0.9893926978111267,
"adapter_seconds": 7.096220199979143
},
{
"id": "heldout_pentesting:pentest:d04f04c4332f3dc00c62bdc91766db2b5f62023708237ecea28c44bcb21c212b",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 172,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998455047607422,
7.141308742575347e-05,
7.721594738541171e-05,
5.861942554474808e-06
],
"adapter_choice_token_mass": 0.9809220433235168,
"adapter_seconds": 9.454785000008997,
"base_prediction": "A",
"base_choice_probabilities": [
0.9988284707069397,
0.00020643039897549897,
0.000954524555709213,
1.0521290278120432e-05
],
"base_choice_token_mass": 0.1822539120912552,
"base_seconds": 8.648148200009018
},
{
"id": "heldout_pentesting:pentest:12b44ed25a84878ff7b87ac4b064f7d6f10e28c6135b11c329bf9c828e8f1537",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 125,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9987308382987976,
0.0009847281035035849,
0.0002650359820108861,
1.9349723515915684e-05
],
"base_choice_token_mass": 0.09109771996736526,
"base_seconds": 8.400746600003913,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9992120265960693,
0.0007436699233949184,
3.880201984429732e-05,
5.460469765239395e-06
],
"adapter_choice_token_mass": 0.9211114048957825,
"adapter_seconds": 7.65779629998724
},
{
"id": "heldout_pentesting:pentest:cf9b8795dd61c6059e13e38e90fddd728459add798fa9710836f627f21827e45",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 151,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9994357228279114,
0.0003828941553365439,
0.00016468073590658605,
1.6692276403773576e-05
],
"adapter_choice_token_mass": 0.9677812457084656,
"adapter_seconds": 9.784611000010045,
"base_prediction": "A",
"base_choice_probabilities": [
0.9971252083778381,
0.0004162898112554103,
0.0023955823853611946,
6.285036215558648e-05
],
"base_choice_token_mass": 0.11613502353429794,
"base_seconds": 8.739528700010851
},
{
"id": "heldout_pentesting:pentest:f6159cab65aa0a0cde66653216db891620378e314248a10df19e5b900e3d10e6",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 117,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0014078246895223856,
0.9967955946922302,
0.0017796570900827646,
1.7042983017745428e-05
],
"base_choice_token_mass": 0.1564166098833084,
"base_seconds": 8.359119100001408,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0003624986857175827,
0.9993854761123657,
0.0002433701156405732,
8.59201554703759e-06
],
"adapter_choice_token_mass": 0.9793983697891235,
"adapter_seconds": 7.623763899988262
},
{
"id": "heldout_pentesting:pentest:73132054abfa0d3048dae032b393213d20201d64a659cec19038d954aa02acf8",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 117,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.006794147193431854,
0.00033301644725725055,
0.9927080273628235,
0.00016485508240293711
],
"adapter_choice_token_mass": 0.9933881759643555,
"adapter_seconds": 7.583869999973103,
"base_prediction": "C",
"base_choice_probabilities": [
0.009122013114392757,
0.00017924584972206503,
0.9904808402061462,
0.00021790752361994237
],
"base_choice_token_mass": 0.2301696091890335,
"base_seconds": 8.339341099985177
},
{
"id": "heldout_pentesting:pentest:b550133c6edd4bfb388f73484141c60e81d42a0acb40062e5f517a95fc842095",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 89,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9869411587715149,
0.008021378889679909,
0.005019650794565678,
1.782301660568919e-05
],
"base_choice_token_mass": 0.115403912961483,
"base_seconds": 7.483360400015954,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9879145622253418,
0.00895732082426548,
0.0030955674592405558,
3.2558516977587715e-05
],
"adapter_choice_token_mass": 0.9343648552894592,
"adapter_seconds": 7.659951000008732
},
{
"id": "heldout_pentesting:pentest:0e1ecad44f6d0d983bca018d852212094898d2d344b918bd29d6de6cc15aedc1",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 136,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.002016582293435931,
0.9967791438102722,
0.0011136626126244664,
9.070359374163672e-05
],
"adapter_choice_token_mass": 0.9900439381599426,
"adapter_seconds": 9.491828900005203,
"base_prediction": "B",
"base_choice_probabilities": [
0.0074928984977304935,
0.981376051902771,
0.010566680692136288,
0.0005644057528115809
],
"base_choice_token_mass": 0.45356485247612,
"base_seconds": 8.659514700004365
},
{
"id": "heldout_pentesting:pentest:b7f0d0a90bc5f4a4e5a3dd9691b0fd03fced1746c96c22d3963a03e51b45ce94",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 150,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0012610799167305231,
0.9069555401802063,
0.0033748249989002943,
0.08840852975845337
],
"base_choice_token_mass": 0.22660158574581146,
"base_seconds": 10.01932580000721,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0003237508062738925,
0.9650875926017761,
0.001045080367475748,
0.03354352340102196
],
"adapter_choice_token_mass": 0.9537408351898193,
"adapter_seconds": 8.832633400015766
},
{
"id": "heldout_pentesting:pentest:d235e50342e341d3786e61f491f706ee59c6229db72586894226a8bdd2262c7f",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 139,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
4.399272074806504e-05,
5.4749823902966455e-05,
0.9997636675834656,
0.0001376410509692505
],
"adapter_choice_token_mass": 0.9785916805267334,
"adapter_seconds": 7.426081100013107,
"base_prediction": "C",
"base_choice_probabilities": [
0.000148802122566849,
4.718990021501668e-05,
0.9996069073677063,
0.0001971307792700827
],
"base_choice_token_mass": 0.4344421327114105,
"base_seconds": 7.536722799995914
},
{
"id": "heldout_pentesting:pentest:4c5fc9b7a2629944da4ddd56581975a7439a8c6f9903168e5029b06fc29287ab",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 134,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.000597562815528363,
5.963078365311958e-05,
0.00011858997459057719,
0.9992241859436035
],
"base_choice_token_mass": 0.3278127908706665,
"base_seconds": 7.421154599986039,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0009545541252009571,
0.00016587661230005324,
2.0123117792536505e-05,
0.9988594055175781
],
"adapter_choice_token_mass": 0.984128475189209,
"adapter_seconds": 7.3467752000142355
},
{
"id": "heldout_pentesting:pentest:0c4667f3ce06ec864e919394f1523811c6f3daaff2e7cad1587ca7ae616a5346",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 136,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.10361786931753159,
0.8951218128204346,
0.0011969448532909155,
6.343574932543561e-05
],
"adapter_choice_token_mass": 0.9012647271156311,
"adapter_seconds": 7.396886700007599,
"base_prediction": "B",
"base_choice_probabilities": [
0.08103587478399277,
0.913028359413147,
0.005689611658453941,
0.00024610842228867114
],
"base_choice_token_mass": 0.12465173751115799,
"base_seconds": 7.502288099989528
},
{
"id": "heldout_pentesting:pentest:4330604a0b9d1c067a83710777d47779bcf0a65979dc87d9edff5363eb409aed",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 93,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.06234164535999298,
0.020239368081092834,
0.9019008874893188,
0.015518060885369778
],
"base_choice_token_mass": 0.16469822824001312,
"base_seconds": 6.5228469999856316,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0026425353717058897,
0.00614407891407609,
0.9859580397605896,
0.00525530893355608
],
"adapter_choice_token_mass": 0.9445425868034363,
"adapter_seconds": 6.652100100007374
},
{
"id": "heldout_pentesting:pentest:534f3876d1bd467ebcd69e6ee274a357e99570860cd09833d7e7ecbf69208c7a",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 135,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8035292029380798,
0.008517633192241192,
0.10057347267866135,
0.08737975358963013
],
"adapter_choice_token_mass": 0.9164342284202576,
"adapter_seconds": 7.338107800023863,
"base_prediction": "C",
"base_choice_probabilities": [
0.277067095041275,
0.010092168115079403,
0.4224768877029419,
0.2903638482093811
],
"base_choice_token_mass": 0.05572580546140671,
"base_seconds": 7.356635100004496
},
{
"id": "heldout_pentesting:pentest:17af7ea2ae4cb72f7915faec833e9071804c4d2b14bcff48cbe31eb548bffd25",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 96,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.4162146747112274,
0.020722107961773872,
0.09143023192882538,
0.47163301706314087
],
"base_choice_token_mass": 0.021835321560502052,
"base_seconds": 6.062765400012722,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.425851047039032,
0.042828962206840515,
0.00955643318593502,
0.5217635631561279
],
"adapter_choice_token_mass": 0.807868480682373,
"adapter_seconds": 6.198094199993648
},
{
"id": "heldout_pentesting:pentest:76b4c611349df725ab440c9fec7b8e6fdc1d0455cf8806f70adc9264cf98b784",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 101,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.45987722277641296,
0.0009599102777428925,
0.009847420267760754,
0.5293154120445251
],
"adapter_choice_token_mass": 0.790169358253479,
"adapter_seconds": 6.232095899991691,
"base_prediction": "D",
"base_choice_probabilities": [
0.04319889843463898,
0.00030266688554547727,
0.0035459815990179777,
0.9529524445533752
],
"base_choice_token_mass": 0.043217983096838,
"base_seconds": 6.0982862999953795
},
{
"id": "heldout_pentesting:pentest:cbe0eb587695a6c23ebfaedc369aa64ebd7bc3bab1c4e42b6f1d33c3cf432664",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 124,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.49137082695961,
0.23210713267326355,
0.2630118727684021,
0.013510256074368954
],
"base_choice_token_mass": 0.055969417095184326,
"base_seconds": 6.232627599994885,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.3408434987068176,
0.473214328289032,
0.182440385222435,
0.003501874627545476
],
"adapter_choice_token_mass": 0.8004464507102966,
"adapter_seconds": 6.420234299992444
},
{
"id": "heldout_pentesting:pentest:1c82ab71a87642019d52edcd39d7173572cc9ce17bcfe1984d3ee00f97b9f715",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 131,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00037996176979504526,
0.9995601773262024,
4.945227919961326e-05,
1.0365760317654349e-05
],
"adapter_choice_token_mass": 0.9893209934234619,
"adapter_seconds": 7.093566299998201,
"base_prediction": "B",
"base_choice_probabilities": [
0.0018668986158445477,
0.9977757334709167,
0.00033998751314356923,
1.7328393369098194e-05
],
"base_choice_token_mass": 0.15815961360931396,
"base_seconds": 7.340869100007694
},
{
"id": "heldout_pentesting:pentest:f7595dbb6443c09ae768c050cc988814d01f95a21ed42e18d7965566bb45f271",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 107,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.0005970533820800483,
0.0008160752477124333,
0.00021455565001815557,
0.9983723759651184
],
"base_choice_token_mass": 0.2623825669288635,
"base_seconds": 6.145608799997717,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.000559986918233335,
0.0025096870958805084,
0.00014722705236636102,
0.9967830181121826
],
"adapter_choice_token_mass": 0.9890662431716919,
"adapter_seconds": 6.168656900001224
},
{
"id": "heldout_pentesting:pentest:1db0933cf2772ca36f91cd2299f2679ff0457b8de3c93969adc2437335aece90",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 93,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.005265975836664438,
0.000807563541457057,
0.005967132281512022,
0.9879592657089233
],
"adapter_choice_token_mass": 0.990168571472168,
"adapter_seconds": 6.503143099980662,
"base_prediction": "D",
"base_choice_probabilities": [
0.013999366201460361,
0.00024276158364955336,
0.07808218896389008,
0.907675564289093
],
"base_choice_token_mass": 0.44241371750831604,
"base_seconds": 7.425870500010205
},
{
"id": "heldout_pentesting:pentest:006610ce562bbba56d0bb0069bdac212560491174da9e0b7a12679a74c0700fb",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 116,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9997581839561462,
0.0001387198135489598,
8.956425153883174e-05,
1.352217896055663e-05
],
"base_choice_token_mass": 0.17785924673080444,
"base_seconds": 7.231166200013831,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998602867126465,
0.00012339255772531033,
1.2217517905810382e-05,
3.966447366110515e-06
],
"adapter_choice_token_mass": 0.9792863130569458,
"adapter_seconds": 7.213681399996858
},
{
"id": "heldout_pentesting:pentest:f11c0d96624a1add9b45ffd05b6d0e8044c4c1e38e30503662f94cdbdcfa0c3a",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 70,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.1464703381061554,
0.0016019120812416077,
0.18515585362911224,
0.6667718291282654
],
"adapter_choice_token_mass": 0.8781598210334778,
"adapter_seconds": 8.342311799991876,
"base_prediction": "D",
"base_choice_probabilities": [
0.19982805848121643,
0.00046896355343051255,
0.0572516955435276,
0.7424512505531311
],
"base_choice_token_mass": 0.0401613786816597,
"base_seconds": 7.435809399990831
},
{
"id": "heldout_pentesting:pentest:b192ca54a2e350ed7970337e0036f4f5cee2c2ab3c10e55a2e37b537412732a7",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 194,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.9985553622245789,
0.0011876000789925456,
0.0002450751489959657,
1.1918924428755417e-05
],
"base_choice_token_mass": 0.24920471012592316,
"base_seconds": 10.047434699983569,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9991475343704224,
0.0007915814057923853,
4.7168214223347604e-05,
1.3726732504437678e-05
],
"adapter_choice_token_mass": 0.9567635655403137,
"adapter_seconds": 9.01003549998859
},
{
"id": "heldout_pentesting:pentest:7ae4a0e1e284ccc028751c81dc07f1617bc431f5ef7efbaa12d949c9431cee65",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 104,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0033763106912374496,
0.9965343475341797,
7.940322393551469e-05,
9.938478797266725e-06
],
"adapter_choice_token_mass": 0.9687748551368713,
"adapter_seconds": 7.929783900006441,
"base_prediction": "B",
"base_choice_probabilities": [
0.02754281461238861,
0.9709176421165466,
0.001437084749341011,
0.00010248812031932175
],
"base_choice_token_mass": 0.25021442770957947,
"base_seconds": 7.028452700003982
},
{
"id": "heldout_pentesting:pentest:c796166b1bccedcd3a156114c31c3c692f6841476ac78976e0e12741be8ca3d7",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 96,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.059926602989435196,
0.614768385887146,
0.0013448003446683288,
0.32396018505096436
],
"base_choice_token_mass": 0.034734442830085754,
"base_seconds": 6.9430799999972805,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.014747273176908493,
0.5983543395996094,
0.0005718138418160379,
0.3863266110420227
],
"adapter_choice_token_mass": 0.947104275226593,
"adapter_seconds": 6.746317099983571
},
{
"id": "heldout_pentesting:pentest:e1f81d0bcb7ad16352f3760f64ea652e1f546579cc8e4887a9affe43b36f31b7",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 95,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0002453215420246124,
0.9995593428611755,
2.9299480956979096e-05,
0.00016599285299889743
],
"adapter_choice_token_mass": 0.989098310470581,
"adapter_seconds": 7.262480699981097,
"base_prediction": "B",
"base_choice_probabilities": [
0.001063793315552175,
0.9978381991386414,
0.00021612271666526794,
0.0008819156209938228
],
"base_choice_token_mass": 0.08561332523822784,
"base_seconds": 8.357081600028323
},
{
"id": "heldout_pentesting:pentest:5d31bbd6c588338db3fb209f9ac89028564c0528d942564403e0a8a3766fc494",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 139,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.0003680485824588686,
0.9989548921585083,
0.0006561175105161965,
2.0926732759107836e-05
],
"base_choice_token_mass": 0.18634015321731567,
"base_seconds": 7.950167000002693,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00045825663255527616,
0.9994189739227295,
0.0001071579972631298,
1.5558651284663938e-05
],
"adapter_choice_token_mass": 0.9739662408828735,
"adapter_seconds": 7.990541300008772
},
{
"id": "heldout_pentesting:pentest:3b504d7e43e4d086d93a482ca981441855b7d933f01fd5c50637587b2592db69",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 160,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0003798264660872519,
0.9992043375968933,
0.0004043231892865151,
1.155972495325841e-05
],
"adapter_choice_token_mass": 0.9915103912353516,
"adapter_seconds": 9.324242899980163,
"base_prediction": "B",
"base_choice_probabilities": [
0.0009840034181252122,
0.9979958534240723,
0.0009994992287829518,
2.0743944332934916e-05
],
"base_choice_token_mass": 0.4414561688899994,
"base_seconds": 9.530740299989702
},
{
"id": "heldout_pentesting:pentest:64fc71e5faacc0a1dce5d401700e354ead3ce5c3f60121fdb11878e6dd082acf",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 112,
"suite": "heldout_pentesting",
"base_prediction": "C",
"base_choice_probabilities": [
0.00012201539357192814,
0.007973147556185722,
0.9810068011283875,
0.010898001492023468
],
"base_choice_token_mass": 0.0666084811091423,
"base_seconds": 6.882014500006335,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
3.739527892321348e-05,
0.0019482162315398455,
0.9935547709465027,
0.004459511488676071
],
"adapter_choice_token_mass": 0.9897286295890808,
"adapter_seconds": 6.773143800004618
},
{
"id": "heldout_pentesting:pentest:5a36afd5d4c0ba3a8f6e6f384784dffe48e14e8cc2f22b9c71da6f763e079148",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 66,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.04464268311858177,
0.8966723084449768,
0.009654657915234566,
0.04903039708733559
],
"adapter_choice_token_mass": 0.9273093342781067,
"adapter_seconds": 6.965758900012588,
"base_prediction": "B",
"base_choice_probabilities": [
0.0817040503025055,
0.838176429271698,
0.027367429807782173,
0.05275210365653038
],
"base_choice_token_mass": 0.2316940575838089,
"base_seconds": 6.949654299998656
},
{
"id": "heldout_pentesting:pentest:4bcf0354a1507418497bdf8ef6e99ae330a39a1beb9396651afa9e9cc49811dc",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 134,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.01964794658124447,
0.00207087816670537,
0.0015389459440484643,
0.9767422080039978
],
"base_choice_token_mass": 0.035923946648836136,
"base_seconds": 9.186964699998498,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.006190392188727856,
0.0002806206466630101,
0.00013891729759052396,
0.9933900833129883
],
"adapter_choice_token_mass": 0.7154724597930908,
"adapter_seconds": 8.267351400019834
},
{
"id": "heldout_pentesting:pentest:728e549c29f92f8dc9f2d166cc0cb76811fcc2a74d7815bac85751bf6cb2012c",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 164,
"suite": "heldout_pentesting",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00021315464982762933,
2.5063007342396304e-05,
0.9996324777603149,
0.00012928483192808926
],
"adapter_choice_token_mass": 0.9796038866043091,
"adapter_seconds": 8.892572500015376,
"base_prediction": "C",
"base_choice_probabilities": [
0.0003029121726285666,
1.2600738045875914e-05,
0.9994949102401733,
0.00018955758423544466
],
"base_choice_token_mass": 0.2730262577533722,
"base_seconds": 8.07107810000889
},
{
"id": "heldout_pentesting:pentest:90cfcadc54065fd1f72978956030e976bdaf656aeadbbb894f656aea752e333b",
"answer_letter": "D",
"topics": [
"Pentesting"
],
"prompt_tokens": 94,
"suite": "heldout_pentesting",
"base_prediction": "D",
"base_choice_probabilities": [
0.3566778302192688,
0.000380253535695374,
0.0266578309237957,
0.6162840127944946
],
"base_choice_token_mass": 0.03542674705386162,
"base_seconds": 8.228550500003621,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.011774110607802868,
0.0004028878756798804,
0.007601936813443899,
0.9802210330963135
],
"adapter_choice_token_mass": 0.8957412838935852,
"adapter_seconds": 7.738629300001776
},
{
"id": "heldout_pentesting:pentest:56be9197b6b2dead2bd19205b48a7e75f8948e811086d2fbda6434417ed4ee85",
"answer_letter": "A",
"topics": [
"Pentesting"
],
"prompt_tokens": 123,
"suite": "heldout_pentesting",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.05242220684885979,
0.00265104859136045,
0.0010879881447181106,
0.9438387155532837
],
"adapter_choice_token_mass": 0.9773419499397278,
"adapter_seconds": 7.7071301999967545,
"base_prediction": "D",
"base_choice_probabilities": [
0.1239231750369072,
0.004880690947175026,
0.010998805984854698,
0.8601973056793213
],
"base_choice_token_mass": 0.13494400680065155,
"base_seconds": 8.466488700010814
},
{
"id": "heldout_pentesting:pentest:93eed83145b747173a57dbb5316c530242968bb484359012cb543a32feed8032",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 127,
"suite": "heldout_pentesting",
"base_prediction": "A",
"base_choice_probabilities": [
0.4917030334472656,
0.01250339113175869,
0.4076361358165741,
0.08815748989582062
],
"base_choice_token_mass": 0.07860322296619415,
"base_seconds": 7.241132199997082,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.6039285659790039,
0.07326484471559525,
0.1551016867160797,
0.16770489513874054
],
"adapter_choice_token_mass": 0.9821107983589172,
"adapter_seconds": 7.268766900000628
},
{
"id": "heldout_pentesting:pentest:619f17c35ef3a7bf9a299edf5113101d02541f539c9cddbfdf24e57942febcb0",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 86,
"suite": "heldout_pentesting",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0052198744378983974,
0.9947320222854614,
3.462595850578509e-05,
1.3559720173361711e-05
],
"adapter_choice_token_mass": 0.2983500063419342,
"adapter_seconds": 6.9111431000055745,
"base_prediction": "B",
"base_choice_probabilities": [
0.003482900792732835,
0.9963668584823608,
0.00010936416947515681,
4.086640547029674e-05
],
"base_choice_token_mass": 0.006205322686582804,
"base_seconds": 7.721905299986247
},
{
"id": "heldout_pentesting:pentest:c45ae177c4aaa2e838102ef1ee147c54fa0ae90d9e9b120b1e7008e480536695",
"answer_letter": "B",
"topics": [
"Pentesting"
],
"prompt_tokens": 124,
"suite": "heldout_pentesting",
"base_prediction": "B",
"base_choice_probabilities": [
0.01386524923145771,
0.528482973575592,
0.012624452821910381,
0.44502729177474976
],
"base_choice_token_mass": 0.1167023628950119,
"base_seconds": 7.1216046000190545,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.009163826704025269,
0.5941541790962219,
0.007026149891316891,
0.38965582847595215
],
"adapter_choice_token_mass": 0.8629553914070129,
"adapter_seconds": 7.047175400017295
},
{
"id": "heldout_pentesting:pentest:d12bb4eb3f2a38b814a9953fbc5c31f3aa70fed28a8f1171e92bd47de4290505",
"answer_letter": "C",
"topics": [
"Pentesting"
],
"prompt_tokens": 79,
"suite": "heldout_pentesting",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.5825219750404358,
0.013699604198336601,
0.376104474067688,
0.027673961594700813
],
"adapter_choice_token_mass": 0.5381636023521423,
"adapter_seconds": 6.914079500013031,
"base_prediction": "A",
"base_choice_probabilities": [
0.5044141411781311,
0.007423477713018656,
0.4738532304763794,
0.014309128746390343
],
"base_choice_token_mass": 0.01337301917374134,
"base_seconds": 7.919521999981953
},
{
"id": "seceval_200:80c18574-6ade-4e50-a37a-060d76ee1db4",
"answer_letter": "D",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 177,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.00012148632958997041,
8.481113036395982e-05,
0.4300585985183716,
0.5697351098060608
],
"base_choice_token_mass": 0.18058434128761292,
"base_seconds": 8.417344399989815,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00013598176883533597,
0.00014934675709810108,
0.11433589458465576,
0.8853787779808044
],
"adapter_choice_token_mass": 0.9526863694190979,
"adapter_seconds": 9.09619099998963
},
{
"id": "seceval_200:87631917-8bbb-4b9f-8cce-8b0d28f61a48",
"answer_letter": "D",
"topics": [
"Cryptography",
"SystemSecurity"
],
"prompt_tokens": 90,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0018937470158562064,
2.7438240067567676e-05,
0.0016453159041702747,
0.9964334964752197
],
"adapter_choice_token_mass": 0.9059353470802307,
"adapter_seconds": 7.2993985999783035,
"base_prediction": "D",
"base_choice_probabilities": [
0.0008335105958394706,
2.7860472982865758e-05,
0.01080913096666336,
0.9883295297622681
],
"base_choice_token_mass": 0.1322294920682907,
"base_seconds": 6.89748509999481
},
{
"id": "seceval_200:540afbd2-f362-4157-8770-71e34542958c",
"answer_letter": "D",
"topics": [
"MemorySafety",
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 168,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.39841359853744507,
0.11414739489555359,
0.4375717341899872,
0.049867309629917145
],
"base_choice_token_mass": 0.0035791860427707434,
"base_seconds": 7.827238000027137,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9325466752052307,
0.030448727309703827,
0.015073212794959545,
0.021931396797299385
],
"adapter_choice_token_mass": 0.9080199599266052,
"adapter_seconds": 8.701832799997646
},
{
"id": "seceval_200:92c2eb08-4bbf-47e4-badd-68c259e303ed",
"answer_letter": "A",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 154,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9965071082115173,
0.0012227853294461966,
0.0022490464616566896,
2.103918268403504e-05
],
"adapter_choice_token_mass": 0.9781624674797058,
"adapter_seconds": 7.993669899995439,
"base_prediction": "A",
"base_choice_probabilities": [
0.9491355419158936,
0.00347702344879508,
0.04725467413663864,
0.00013272867363411933
],
"base_choice_token_mass": 0.08933998644351959,
"base_seconds": 9.205804199998965
},
{
"id": "seceval_200:1786b19b-0206-4fa9-9727-a141b172cf66",
"answer_letter": "B",
"topics": [
"PenTest"
],
"prompt_tokens": 145,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.06992380321025848,
0.9210654497146606,
0.008221733383834362,
0.0007890157867223024
],
"base_choice_token_mass": 0.2279301881790161,
"base_seconds": 8.390699900017353,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.03304651752114296,
0.9657608270645142,
0.0008738065371289849,
0.0003189538838341832
],
"adapter_choice_token_mass": 0.9857684969902039,
"adapter_seconds": 9.096200600004522
},
{
"id": "seceval_200:f9ad54de-9c0c-42b4-bfc7-963c9807e85b",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 141,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9721705913543701,
0.02672985941171646,
0.0009290490997955203,
0.00017051938630174845
],
"adapter_choice_token_mass": 0.9592577815055847,
"adapter_seconds": 8.390631999995094,
"base_prediction": "A",
"base_choice_probabilities": [
0.9369689226150513,
0.05715547129511833,
0.00540000619366765,
0.00047554803313687444
],
"base_choice_token_mass": 0.05149294435977936,
"base_seconds": 9.196885699988343
},
{
"id": "seceval_200:5d0334d9-8c66-4218-9fc3-a46106d44a29",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"PenTest"
],
"prompt_tokens": 161,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.001242063008248806,
0.00027931539807468653,
0.9965245127677917,
0.0019540393259376287
],
"base_choice_token_mass": 0.47331351041793823,
"base_seconds": 8.032321200007573,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0003220491053070873,
0.0019727901089936495,
0.9904890060424805,
0.007216166239231825
],
"adapter_choice_token_mass": 0.9918640851974487,
"adapter_seconds": 8.659429599996656
},
{
"id": "seceval_200:56a9b6d1-dda9-4c22-a9c2-5e58f2bc56a0",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 142,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0015477617271244526,
0.997806966304779,
6.694960029562935e-05,
0.0005783562664873898
],
"adapter_choice_token_mass": 0.979659914970398,
"adapter_seconds": 8.04617529999814,
"base_prediction": "B",
"base_choice_probabilities": [
0.011285470798611641,
0.9846301674842834,
0.0003627708647400141,
0.0037215533666312695
],
"base_choice_token_mass": 0.3296656310558319,
"base_seconds": 8.809306300012395
},
{
"id": "seceval_200:a96ef1b8-a510-4c3f-aa1e-1b03043b9367",
"answer_letter": "B",
"topics": [
"Vulnerability"
],
"prompt_tokens": 145,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0014220277080312371,
0.9912420511245728,
0.006473447196185589,
0.0008625033660791814
],
"base_choice_token_mass": 0.22495055198669434,
"base_seconds": 8.082403500011424,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0007665101438760757,
0.9982140064239502,
0.0008032957557588816,
0.00021620412007905543
],
"adapter_choice_token_mass": 0.9823674559593201,
"adapter_seconds": 9.298061900015455
},
{
"id": "seceval_200:d76a3bf8-6514-4a18-9053-357d77355f0b",
"answer_letter": "D",
"topics": [
"WebSecurity"
],
"prompt_tokens": 139,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.25062134861946106,
0.0010733860544860363,
8.810889266896993e-05,
0.7482171654701233
],
"adapter_choice_token_mass": 0.8480190634727478,
"adapter_seconds": 8.221206999995047,
"base_prediction": "D",
"base_choice_probabilities": [
0.046718258410692215,
4.429859109222889e-05,
9.905154729494825e-05,
0.9531383514404297
],
"base_choice_token_mass": 0.33756712079048157,
"base_seconds": 9.11863959999755
},
{
"id": "seceval_200:b31a2b2a-15a2-4db5-aeb0-f6b644793e8c",
"answer_letter": "A",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 160,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9916689395904541,
0.0022034281864762306,
0.005805259570479393,
0.00032243275200016797
],
"base_choice_token_mass": 0.1519104242324829,
"base_seconds": 8.816643800004385,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9975989460945129,
0.0014311472186818719,
0.0008154430543072522,
0.00015441891446243972
],
"adapter_choice_token_mass": 0.9682502150535583,
"adapter_seconds": 9.541586200008169
},
{
"id": "seceval_200:5f9bc5fc-89b6-41bb-946d-fee2abb908bf",
"answer_letter": "A",
"topics": [
"Cryptography"
],
"prompt_tokens": 107,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8323091864585876,
0.16647231578826904,
0.0003585109661798924,
0.0008600231376476586
],
"adapter_choice_token_mass": 0.8627529740333557,
"adapter_seconds": 6.907035999989603,
"base_prediction": "A",
"base_choice_probabilities": [
0.7893845438957214,
0.1904478818178177,
0.005933585111051798,
0.01423393003642559
],
"base_choice_token_mass": 0.025929491966962814,
"base_seconds": 7.014087799994741
},
{
"id": "seceval_200:a12e0018-6465-426d-b6b2-d5993429caa4",
"answer_letter": "B",
"topics": [
"MemorySafety"
],
"prompt_tokens": 174,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.00042375162593089044,
0.9914858937263489,
0.001197761856019497,
0.0068926434032619
],
"base_choice_token_mass": 0.04086773470044136,
"base_seconds": 9.394019400002435,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00017120814300142229,
0.9992448091506958,
8.08729964774102e-05,
0.0005032087210565805
],
"adapter_choice_token_mass": 0.9591113924980164,
"adapter_seconds": 8.495438299985835
},
{
"id": "seceval_200:4023cd5b-05c9-4007-8129-db2d641d9a6b",
"answer_letter": "A",
"topics": [
"NetworkSecurity"
],
"prompt_tokens": 152,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999253749847412,
6.709174340358004e-05,
2.350195472899941e-06,
5.0933508646267e-06
],
"adapter_choice_token_mass": 0.9702264666557312,
"adapter_seconds": 9.276717399974586,
"base_prediction": "A",
"base_choice_probabilities": [
0.9995189905166626,
0.00016992240853141993,
0.00026942213298752904,
4.164128040429205e-05
],
"base_choice_token_mass": 0.06899407505989075,
"base_seconds": 8.576983900013147
},
{
"id": "seceval_200:a49b1c8f-f5e6-4c74-a362-6941b834bdae",
"answer_letter": "B",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 167,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0010000509209930897,
0.9985466599464417,
0.0003678981738630682,
8.535923552699387e-05
],
"base_choice_token_mass": 0.24565018713474274,
"base_seconds": 9.039733100013109,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0010986443376168609,
0.9988223910331726,
6.1020338762318715e-05,
1.803757913876325e-05
],
"adapter_choice_token_mass": 0.9930490851402283,
"adapter_seconds": 8.162724199995864
},
{
"id": "seceval_200:d9fba682-724f-40b9-982a-8fdf9c402406",
"answer_letter": "D",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 142,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00866437703371048,
0.00045919514377601445,
0.9859395027160645,
0.004936813376843929
],
"adapter_choice_token_mass": 0.9714486002922058,
"adapter_seconds": 9.137471299996832,
"base_prediction": "C",
"base_choice_probabilities": [
0.0032110894098877907,
0.00011788128176704049,
0.9932528138160706,
0.0034181869123131037
],
"base_choice_token_mass": 0.2186221182346344,
"base_seconds": 8.380384399992181
},
{
"id": "seceval_200:a3cf6caa-82c1-4a45-bb6f-94b7920ca8fb",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"MemorySafety",
"PenTest"
],
"prompt_tokens": 105,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
7.538022327935323e-05,
1.4613122402806766e-05,
0.9992221593856812,
0.0006877892301417887
],
"base_choice_token_mass": 0.28338032960891724,
"base_seconds": 7.668289799999911,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
4.641710620489903e-05,
2.6241854357067496e-05,
0.9987208247184753,
0.0012065020855516195
],
"adapter_choice_token_mass": 0.9895380139350891,
"adapter_seconds": 6.829234000004362
},
{
"id": "seceval_200:b98abfa0-e6e5-4f4c-a84b-6c640dcd554c",
"answer_letter": "A",
"topics": [
"Unspecified"
],
"prompt_tokens": 176,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9993864297866821,
1.2501320270530414e-05,
0.0005792717565782368,
2.176975795009639e-05
],
"adapter_choice_token_mass": 0.8196832537651062,
"adapter_seconds": 8.50207750001573,
"base_prediction": "A",
"base_choice_probabilities": [
0.9684484004974365,
9.603382932255045e-05,
0.031130723655223846,
0.00032487823045812547
],
"base_choice_token_mass": 0.013677936978638172,
"base_seconds": 9.363826400018297
},
{
"id": "seceval_200:6e1bbc2c-aff6-4734-9453-1c02e0c968de",
"answer_letter": "B",
"topics": [
"Vulnerability"
],
"prompt_tokens": 115,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.055657945573329926,
0.5709775686264038,
0.3686508238315582,
0.004713708069175482
],
"base_choice_token_mass": 0.19167999923229218,
"base_seconds": 7.203976099990541,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.050309278070926666,
0.8119506239891052,
0.13049234449863434,
0.007247738540172577
],
"adapter_choice_token_mass": 0.9778528213500977,
"adapter_seconds": 6.991471100016497
},
{
"id": "seceval_200:da4ece90-b3a0-4fe5-891d-76240553a154",
"answer_letter": "B",
"topics": [
"WebSecurity"
],
"prompt_tokens": 81,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.030235610902309418,
0.9554145932197571,
0.00839612539857626,
0.0059537445195019245
],
"adapter_choice_token_mass": 0.9352278113365173,
"adapter_seconds": 8.234769500006223,
"base_prediction": "B",
"base_choice_probabilities": [
0.009915555827319622,
0.9650981426239014,
0.023417411372065544,
0.0015688687562942505
],
"base_choice_token_mass": 0.3845612406730652,
"base_seconds": 7.498214899998857
},
{
"id": "seceval_200:1a24e3e9-7abb-4a95-bb13-e9861d435203",
"answer_letter": "B",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 72,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.5747036337852478,
0.08813366293907166,
0.24334482848644257,
0.09381779283285141
],
"base_choice_token_mass": 0.06907742470502853,
"base_seconds": 7.406713100004708,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.7125679850578308,
0.15653041005134583,
0.07055378705263138,
0.060347847640514374
],
"adapter_choice_token_mass": 0.951720118522644,
"adapter_seconds": 8.304909799975576
},
{
"id": "seceval_200:ca517bb5-0a29-4f6f-a946-b3d0247edc90",
"answer_letter": "D",
"topics": [
"Cryptography"
],
"prompt_tokens": 84,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.022360768169164658,
0.48562318086624146,
0.40893569588661194,
0.08308032900094986
],
"adapter_choice_token_mass": 0.9748769402503967,
"adapter_seconds": 7.568257500010077,
"base_prediction": "C",
"base_choice_probabilities": [
0.014067300595343113,
0.107243612408638,
0.8175835013389587,
0.06110557168722153
],
"base_choice_token_mass": 0.1625402420759201,
"base_seconds": 7.549231099983444
},
{
"id": "seceval_200:a8ada9ff-9197-41b7-8234-2752cb1478db",
"answer_letter": "D",
"topics": [
"MemorySafety"
],
"prompt_tokens": 219,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.137358158826828,
0.0115424245595932,
0.17499883472919464,
0.6761005520820618
],
"base_choice_token_mass": 8.34620077512227e-05,
"base_seconds": 10.645541100006085,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00553894555196166,
0.0003405278839636594,
0.0025359143037348986,
0.9915845394134521
],
"adapter_choice_token_mass": 0.11838402599096298,
"adapter_seconds": 10.703145900013624
},
{
"id": "seceval_200:588ec3ad-4efc-4c05-9cd0-0e51c024cda8",
"answer_letter": "A",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 109,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9995446801185608,
0.00037995586171746254,
4.8305970267392695e-05,
2.7097194106318057e-05
],
"adapter_choice_token_mass": 0.9937766790390015,
"adapter_seconds": 7.040288699994562,
"base_prediction": "A",
"base_choice_probabilities": [
0.999065101146698,
0.0003404268354643136,
0.0005110412021167576,
8.34251768537797e-05
],
"base_choice_token_mass": 0.22835445404052734,
"base_seconds": 7.968156299990369
},
{
"id": "seceval_200:5449601b-bc8d-420b-88ee-70903af8d16d",
"answer_letter": "B",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 102,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0008691216353327036,
0.9988481998443604,
0.00027348127332516015,
9.212931217916775e-06
],
"base_choice_token_mass": 0.36737388372421265,
"base_seconds": 7.092601599986665,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0006877945270389318,
0.999229907989502,
7.36346046323888e-05,
8.725961379241198e-06
],
"adapter_choice_token_mass": 0.9891436100006104,
"adapter_seconds": 7.015705500001786
},
{
"id": "seceval_200:22c281bb-e34e-4d52-b84d-0aac010d48a8",
"answer_letter": "D",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 161,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8288586139678955,
0.0009406347526237369,
0.1332097053527832,
0.036991000175476074
],
"adapter_choice_token_mass": 0.9307878017425537,
"adapter_seconds": 9.10932899999898,
"base_prediction": "C",
"base_choice_probabilities": [
0.13935495913028717,
0.0001618981041247025,
0.8536526560783386,
0.006830510217696428
],
"base_choice_token_mass": 0.0932198017835617,
"base_seconds": 8.392877400008729
},
{
"id": "seceval_200:fa76ffc9-73c4-415e-a1bc-cb9dbef0599c",
"answer_letter": "A",
"topics": [
"SystemSecurity"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9990963935852051,
4.6434561227215454e-05,
0.0008425908745266497,
1.4497576557914726e-05
],
"base_choice_token_mass": 0.24026839435100555,
"base_seconds": 9.700497999991057,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998815059661865,
6.302409747149795e-05,
4.198309761704877e-05,
1.3523846973839682e-05
],
"adapter_choice_token_mass": 0.979652464389801,
"adapter_seconds": 8.720912099990528
},
{
"id": "seceval_200:c0eb46ce-8669-4298-89c8-b071e95ffddb",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 158,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.16618278622627258,
0.8052986860275269,
0.008670818991959095,
0.019847702234983444
],
"adapter_choice_token_mass": 0.9532788991928101,
"adapter_seconds": 8.918758400017396,
"base_prediction": "B",
"base_choice_probabilities": [
0.029467040672898293,
0.9457917213439941,
0.01705421507358551,
0.007686936762183905
],
"base_choice_token_mass": 0.20329256355762482,
"base_seconds": 8.548344400012866
},
{
"id": "seceval_200:a374b7bf-874a-4574-a9ba-2751de339d25",
"answer_letter": "B",
"topics": [
"Vulnerability"
],
"prompt_tokens": 169,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0007204815046861768,
0.9987849593162537,
0.00023758960014674813,
0.0002568956115283072
],
"base_choice_token_mass": 0.1253361999988556,
"base_seconds": 9.112936399993487,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0011334286537021399,
0.9987427592277527,
5.469391544465907e-05,
6.913959077792242e-05
],
"adapter_choice_token_mass": 0.9809856414794922,
"adapter_seconds": 8.096364900004119
},
{
"id": "seceval_200:db12a597-7279-4c76-bad3-978864ed4444",
"answer_letter": "C",
"topics": [
"WebSecurity",
"SystemSecurity"
],
"prompt_tokens": 123,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.019768042489886284,
0.17619065940380096,
0.7653372883796692,
0.038703951984643936
],
"adapter_choice_token_mass": 0.9860938191413879,
"adapter_seconds": 7.370882699993672,
"base_prediction": "C",
"base_choice_probabilities": [
0.0012158164754509926,
0.05589868128299713,
0.9307966232299805,
0.012088937684893608
],
"base_choice_token_mass": 0.28321173787117004,
"base_seconds": 8.07018669997342
},
{
"id": "seceval_200:c1b2f20e-f4dd-493b-9146-3a612364d2d9",
"answer_letter": "D",
"topics": [
"ApplicationSecurity",
"SoftwareSecurity"
],
"prompt_tokens": 160,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0003681667731143534,
0.0002651805698405951,
9.093077824218199e-05,
0.9992757439613342
],
"base_choice_token_mass": 0.3863801956176758,
"base_seconds": 8.639797099982388,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00031498479074798524,
0.00016087849508039653,
1.0125170774699654e-05,
0.9995139837265015
],
"adapter_choice_token_mass": 0.9679714441299438,
"adapter_seconds": 9.402272399980575
},
{
"id": "seceval_200:4ac9808e-62b0-44ec-a026-fc19312cbe6b",
"answer_letter": "C",
"topics": [
"Cryptography"
],
"prompt_tokens": 167,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.06357455253601074,
0.1868560016155243,
0.705187201499939,
0.04438219219446182
],
"adapter_choice_token_mass": 0.9681117534637451,
"adapter_seconds": 8.086547999992035,
"base_prediction": "C",
"base_choice_probabilities": [
0.030154826119542122,
0.03416989743709564,
0.7420896291732788,
0.19358563423156738
],
"base_choice_token_mass": 0.040416888892650604,
"base_seconds": 8.940353899990441
},
{
"id": "seceval_200:ae28ad19-0e87-4a14-8fc5-836d5404288b",
"answer_letter": "C",
"topics": [
"MemorySafety"
],
"prompt_tokens": 97,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.00025323015870526433,
3.244710751459934e-05,
0.9922558069229126,
0.007458512205630541
],
"base_choice_token_mass": 0.0015111094107851386,
"base_seconds": 6.86079200002132,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
5.366438927012496e-05,
5.493700155057013e-05,
0.9647502899169922,
0.035141028463840485
],
"adapter_choice_token_mass": 0.7178167104721069,
"adapter_seconds": 6.9391334999818355
},
{
"id": "seceval_200:c93e6e77-3673-44e3-8cb5-2c3001a6614c",
"answer_letter": "A",
"topics": [
"NetworkSecurity"
],
"prompt_tokens": 121,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.23921425640583038,
0.0005398897337727249,
2.447408769512549e-05,
0.7602213621139526
],
"adapter_choice_token_mass": 0.8782636523246765,
"adapter_seconds": 8.51378099998692,
"base_prediction": "A",
"base_choice_probabilities": [
0.6070848107337952,
0.0006079993327148259,
0.0003437316627241671,
0.3919634222984314
],
"base_choice_token_mass": 0.05962859094142914,
"base_seconds": 7.504534500010777
},
{
"id": "seceval_200:b65e83f6-bda1-4e26-b387-8721a484b997",
"answer_letter": "D",
"topics": [
"PenTest"
],
"prompt_tokens": 168,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0011693809647113085,
1.8901284420280717e-05,
9.159299952443689e-05,
0.9987201690673828
],
"base_choice_token_mass": 0.43731117248535156,
"base_seconds": 8.459377399994992,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0009849375346675515,
4.827690281672403e-05,
2.3528284145868383e-05,
0.9989432692527771
],
"adapter_choice_token_mass": 0.9934772253036499,
"adapter_seconds": 9.21363330000895
},
{
"id": "seceval_200:ae5a93c3-a4c4-4eb7-b888-304d007318da",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 114,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9416212439537048,
0.021463485434651375,
0.03324329853057861,
0.003671969287097454
],
"adapter_choice_token_mass": 0.9857515692710876,
"adapter_seconds": 7.063250200008042,
"base_prediction": "A",
"base_choice_probabilities": [
0.7543998956680298,
0.03212622180581093,
0.20304398238658905,
0.010429858230054379
],
"base_choice_token_mass": 0.37403708696365356,
"base_seconds": 7.283881000010297
},
{
"id": "seceval_200:a684a5f8-bc60-4d08-bbdf-a8fdd340fe61",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"SoftwareSecurity"
],
"prompt_tokens": 134,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.023671923205256462,
6.913820107001811e-05,
0.9755873084068298,
0.0006715207709930837
],
"base_choice_token_mass": 0.2535673975944519,
"base_seconds": 9.216517799999565,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.18933822214603424,
0.0003030173829756677,
0.8096969127655029,
0.0006618508487008512
],
"adapter_choice_token_mass": 0.9573586583137512,
"adapter_seconds": 8.311018999986118
},
{
"id": "seceval_200:684b20be-e78c-48df-ba88-1e091e94115d",
"answer_letter": "C",
"topics": [
"Unspecified"
],
"prompt_tokens": 144,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.28175434470176697,
0.017732704058289528,
0.6758935451507568,
0.024619420990347862
],
"adapter_choice_token_mass": 0.9320669174194336,
"adapter_seconds": 9.407448799989652,
"base_prediction": "C",
"base_choice_probabilities": [
0.09957394003868103,
0.006365548819303513,
0.887492835521698,
0.00656761322170496
],
"base_choice_token_mass": 0.10176592320203781,
"base_seconds": 9.567233000008855
},
{
"id": "seceval_200:d0ba335d-82c1-4374-a2da-394b0175dda6",
"answer_letter": "A",
"topics": [
"Vulnerability"
],
"prompt_tokens": 180,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.06381868571043015,
0.016135890036821365,
0.9089553356170654,
0.011090023443102837
],
"base_choice_token_mass": 0.03165685012936592,
"base_seconds": 8.558240099984687,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.44245997071266174,
0.07452242821455002,
0.4494277238845825,
0.033589888364076614
],
"adapter_choice_token_mass": 0.9273906946182251,
"adapter_seconds": 9.306337899994105
},
{
"id": "seceval_200:09405153-85ae-4cef-9bc5-75628f68e199",
"answer_letter": "A",
"topics": [
"WebSecurity"
],
"prompt_tokens": 135,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9953083395957947,
0.003881328972056508,
0.0005419549415819347,
0.00026828714180737734
],
"adapter_choice_token_mass": 0.984130322933197,
"adapter_seconds": 8.246302099985769,
"base_prediction": "A",
"base_choice_probabilities": [
0.9820886254310608,
0.008235313929617405,
0.008766446262598038,
0.0009096517460420728
],
"base_choice_token_mass": 0.16186517477035522,
"base_seconds": 9.158014299988281
},
{
"id": "seceval_200:5cb80c47-fef9-4a9e-8c86-b764870d1ac5",
"answer_letter": "A",
"topics": [
"ApplicationSecurity",
"WebSecurity"
],
"prompt_tokens": 174,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.6537523865699768,
0.3236308991909027,
0.022370176389813423,
0.0002465762954670936
],
"base_choice_token_mass": 0.03992804139852524,
"base_seconds": 8.727080799988471,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8697661757469177,
0.12927919626235962,
0.0008575716055929661,
9.697151108412072e-05
],
"adapter_choice_token_mass": 0.9339871406555176,
"adapter_seconds": 9.326448999985587
},
{
"id": "seceval_200:a9ba3663-99af-47c8-9de8-f27362dc8315",
"answer_letter": "A",
"topics": [
"MemorySafety"
],
"prompt_tokens": 163,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9988698363304138,
0.00021133452537469566,
0.0004235852393321693,
0.0004952213494107127
],
"adapter_choice_token_mass": 0.9748601913452148,
"adapter_seconds": 8.17167330000666,
"base_prediction": "A",
"base_choice_probabilities": [
0.995610773563385,
0.0005506567540578544,
0.0029306707438081503,
0.0009078794973902404
],
"base_choice_token_mass": 0.11468112468719482,
"base_seconds": 9.08005120002781
},
{
"id": "seceval_200:c2dd663e-998e-44eb-b024-81ecdecf0dc4",
"answer_letter": "D",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 112,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0019946994725614786,
0.006747981067746878,
0.0650312751531601,
0.9262260794639587
],
"base_choice_token_mass": 0.1888645589351654,
"base_seconds": 6.992781699984334,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00022960748174227774,
0.004684414714574814,
0.014656237326562405,
0.9804297685623169
],
"adapter_choice_token_mass": 0.9904839992523193,
"adapter_seconds": 6.8952945000201
},
{
"id": "seceval_200:2e3be9fd-5288-4c02-9f46-e8ef419ab0a3",
"answer_letter": "A",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 129,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.7918269634246826,
0.18807539343833923,
0.007885044440627098,
0.012212596833705902
],
"adapter_choice_token_mass": 0.9847800135612488,
"adapter_seconds": 8.735310899995966,
"base_prediction": "A",
"base_choice_probabilities": [
0.8277818560600281,
0.06901850551366806,
0.0352512001991272,
0.06794847548007965
],
"base_choice_token_mass": 0.40304824709892273,
"base_seconds": 8.081096000008984
},
{
"id": "seceval_200:6b5b2a96-e297-41da-ac6f-79f7170969f4",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 197,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9959502220153809,
0.00012981885811313987,
0.0024687140248715878,
0.001451282063499093
],
"base_choice_token_mass": 0.49740490317344666,
"base_seconds": 9.689082999975653,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9984527826309204,
0.00032975079375319183,
0.0003349435864947736,
0.0008824588148854673
],
"adapter_choice_token_mass": 0.9827535152435303,
"adapter_seconds": 8.78290309998556
},
{
"id": "seceval_200:4afa2c75-50c0-4ce1-a326-feacf4a7511c",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 161,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.14782531559467316,
0.00010498174378881231,
0.8506759405136108,
0.001393709215335548
],
"adapter_choice_token_mass": 0.7831608057022095,
"adapter_seconds": 8.944218999997247,
"base_prediction": "C",
"base_choice_probabilities": [
0.14400111138820648,
3.1680498068453744e-05,
0.8549739718437195,
0.0009932812536135316
],
"base_choice_token_mass": 0.02763003297150135,
"base_seconds": 8.241539599985117
},
{
"id": "seceval_200:763e9c7b-13bf-4930-a115-5bd8966ce153",
"answer_letter": "D",
"topics": [
"Unspecified"
],
"prompt_tokens": 118,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.00172560999635607,
2.348732596146874e-05,
0.0010466353269293904,
0.9972042441368103
],
"base_choice_token_mass": 0.1488807499408722,
"base_seconds": 8.170475499995518,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00247213919647038,
3.526316868374124e-05,
0.00016052731371019036,
0.9973320960998535
],
"adapter_choice_token_mass": 0.750810444355011,
"adapter_seconds": 7.430279100022744
},
{
"id": "seceval_200:ed76d844-e5ab-4036-b792-0cad86ea465f",
"answer_letter": "A",
"topics": [
"Vulnerability"
],
"prompt_tokens": 129,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997233748435974,
0.00017672748072072864,
9.168500400846824e-05,
8.201330274459906e-06
],
"adapter_choice_token_mass": 0.9842617511749268,
"adapter_seconds": 8.034243300004164,
"base_prediction": "A",
"base_choice_probabilities": [
0.9861934781074524,
0.0032894734758883715,
0.010453925468027592,
6.31402144790627e-05
],
"base_choice_token_mass": 0.006342646665871143,
"base_seconds": 9.015736400004243
},
{
"id": "seceval_200:636b9144-b96d-4770-8700-7024355ba1ba",
"answer_letter": "D",
"topics": [
"WebSecurity"
],
"prompt_tokens": 137,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0006562626804225147,
7.479014311684296e-05,
9.307784057455137e-05,
0.9991758465766907
],
"base_choice_token_mass": 0.25647661089897156,
"base_seconds": 8.30086059999303,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0006262858514674008,
6.348123133648187e-05,
1.3621942343888804e-05,
0.9992965459823608
],
"adapter_choice_token_mass": 0.9759243726730347,
"adapter_seconds": 8.91092419999768
},
{
"id": "seceval_200:bf07b95b-90a0-4d3d-9527-5893e83a606b",
"answer_letter": "A",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 215,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9984924793243408,
0.0014778991462662816,
1.0517750524741132e-05,
1.9194560081814416e-05
],
"adapter_choice_token_mass": 0.9705485105514526,
"adapter_seconds": 9.513140499999281,
"base_prediction": "A",
"base_choice_probabilities": [
0.9952019453048706,
0.004397651646286249,
0.0002746200480032712,
0.00012573020649142563
],
"base_choice_token_mass": 0.1845591962337494,
"base_seconds": 10.449031600001035
},
{
"id": "seceval_200:3775b5e7-422b-45cd-8613-c3a5e85f5405",
"answer_letter": "D",
"topics": [
"MemorySafety"
],
"prompt_tokens": 113,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0017047786386683583,
0.0001324894546996802,
0.012996567413210869,
0.9851661324501038
],
"base_choice_token_mass": 0.25678500533103943,
"base_seconds": 7.927256399998441,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.013194148428738117,
0.0011350178392603993,
0.0010334454709663987,
0.9846373200416565
],
"adapter_choice_token_mass": 0.9938411116600037,
"adapter_seconds": 6.812005100015085
},
{
"id": "seceval_200:4f260853-18a9-4add-b1df-66c2859d9b9e",
"answer_letter": "A",
"topics": [
"NetworkSecurity"
],
"prompt_tokens": 129,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9320124387741089,
0.06543754786252975,
0.0006936562131159008,
0.0018563204212114215
],
"adapter_choice_token_mass": 0.9883477687835693,
"adapter_seconds": 7.927267999999458,
"base_prediction": "A",
"base_choice_probabilities": [
0.9222185611724854,
0.06892591714859009,
0.005314996000379324,
0.0035405505914241076
],
"base_choice_token_mass": 0.17349079251289368,
"base_seconds": 8.958381500007818
},
{
"id": "seceval_200:dd43f00b-919e-4629-8702-339e2097cafd",
"answer_letter": "D",
"topics": [
"PenTest"
],
"prompt_tokens": 85,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
9.038859570864588e-05,
4.2034604120999575e-05,
0.014280041679739952,
0.985587477684021
],
"base_choice_token_mass": 0.2889777719974518,
"base_seconds": 7.243506900005741,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00016315917309839278,
8.733281720196828e-05,
0.0017817500047385693,
0.9979677796363831
],
"adapter_choice_token_mass": 0.9857910871505737,
"adapter_seconds": 7.513780100009171
},
{
"id": "seceval_200:9fa2b7ab-123d-4eea-97fa-4436957e37df",
"answer_letter": "D",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 125,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0005690751131623983,
6.092462717788294e-05,
0.002114370698109269,
0.9972557425498962
],
"adapter_choice_token_mass": 0.9688875675201416,
"adapter_seconds": 8.240611499990337,
"base_prediction": "D",
"base_choice_probabilities": [
0.0004362567560747266,
3.087026561843231e-05,
0.002433244837448001,
0.9970996379852295
],
"base_choice_token_mass": 0.20040935277938843,
"base_seconds": 7.7052438000100665
},
{
"id": "seceval_200:715663af-a01e-40c5-be22-91e98f4f237e",
"answer_letter": "D",
"topics": [
"SystemSecurity",
"WebSecurity"
],
"prompt_tokens": 106,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.8375314474105835,
2.3243524992722087e-05,
8.24054513941519e-05,
0.16236288845539093
],
"base_choice_token_mass": 0.10600526630878448,
"base_seconds": 7.096514100005152,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8739297389984131,
0.00012807683378923684,
4.061724757775664e-05,
0.12590152025222778
],
"adapter_choice_token_mass": 0.954701840877533,
"adapter_seconds": 7.034364699997241
},
{
"id": "seceval_200:38625f1c-e6a9-4b27-a84a-ee2b21e28797",
"answer_letter": "D",
"topics": [
"Unspecified"
],
"prompt_tokens": 180,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.011492385528981686,
0.0003811488568317145,
0.0009886249899864197,
0.9871379137039185
],
"adapter_choice_token_mass": 0.9688838720321655,
"adapter_seconds": 9.194510100001935,
"base_prediction": "D",
"base_choice_probabilities": [
0.014215474016964436,
0.00018034990353044122,
0.0044730971567332745,
0.9811310768127441
],
"base_choice_token_mass": 0.1367606520652771,
"base_seconds": 8.453992599999765
},
{
"id": "seceval_200:c7fd5811-fc17-4d96-821e-30a6bd36718e",
"answer_letter": "B",
"topics": [
"Vulnerability"
],
"prompt_tokens": 146,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.03780693933367729,
0.9599369764328003,
0.0014774315059185028,
0.0007785516791045666
],
"base_choice_token_mass": 0.12345213443040848,
"base_seconds": 9.485190999985207,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0757400393486023,
0.9227025508880615,
0.0005265320069156587,
0.0010308997007086873
],
"adapter_choice_token_mass": 0.8827140927314758,
"adapter_seconds": 9.369385899975896
},
{
"id": "seceval_200:6b18e8e2-e7af-428c-a337-23a038898b7a",
"answer_letter": "C",
"topics": [
"WebSecurity"
],
"prompt_tokens": 107,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00010071206634165719,
3.966526037402218e-06,
0.9998800754547119,
1.520524801890133e-05
],
"adapter_choice_token_mass": 0.9950531721115112,
"adapter_seconds": 6.769933999981731,
"base_prediction": "C",
"base_choice_probabilities": [
5.144060924067162e-05,
8.185693900486513e-07,
0.9999173879623413,
3.0240375053836033e-05
],
"base_choice_token_mass": 0.6943159699440002,
"base_seconds": 6.614938799990341
},
{
"id": "seceval_200:5b245412-af4b-4202-b87c-5f46603d33f0",
"answer_letter": "A",
"topics": [
"ApplicationSecurity",
"WebSecurity"
],
"prompt_tokens": 148,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.7547689080238342,
0.16069957613945007,
0.08336983621120453,
0.0011616589035838842
],
"base_choice_token_mass": 0.025777021422982216,
"base_seconds": 8.012056500010658,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.6906999945640564,
0.287926584482193,
0.02118578739464283,
0.00018763965636026114
],
"adapter_choice_token_mass": 0.7040456533432007,
"adapter_seconds": 8.88562010001624
},
{
"id": "seceval_200:af9530ea-8f9c-4761-b1e9-f56a3c7abf26",
"answer_letter": "B",
"topics": [
"MemorySafety"
],
"prompt_tokens": 151,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.000616423087194562,
0.9990484118461609,
0.00019548741693142802,
0.00013970857253298163
],
"adapter_choice_token_mass": 0.9700025916099548,
"adapter_seconds": 7.96184629999334,
"base_prediction": "B",
"base_choice_probabilities": [
0.0024631484411656857,
0.9859719276428223,
0.009818348102271557,
0.0017466337885707617
],
"base_choice_token_mass": 0.08662812411785126,
"base_seconds": 9.14175010001054
},
{
"id": "seceval_200:e70a4db5-5f34-42aa-9aaa-236a9f989201",
"answer_letter": "D",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 137,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0008414956973865628,
0.00043656222987920046,
0.0009242022642865777,
0.9977978467941284
],
"base_choice_token_mass": 0.4192109704017639,
"base_seconds": 8.255622700002277,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0005437058862298727,
0.0006873086676932871,
0.0002450674364808947,
0.998524010181427
],
"adapter_choice_token_mass": 0.9948102831840515,
"adapter_seconds": 7.978799500007881
},
{
"id": "seceval_200:97a05d2f-9391-46c2-8acc-dd1d67b6b56c",
"answer_letter": "A",
"topics": [
"PenTest"
],
"prompt_tokens": 123,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9965180158615112,
0.0013429815880954266,
0.0007533444440923631,
0.0013856124132871628
],
"adapter_choice_token_mass": 0.9735502004623413,
"adapter_seconds": 8.017214499996044,
"base_prediction": "A",
"base_choice_probabilities": [
0.9943066239356995,
0.0009354716748930514,
0.0021750370506197214,
0.0025829202495515347
],
"base_choice_token_mass": 0.11215174198150635,
"base_seconds": 7.065468900022097
},
{
"id": "seceval_200:3ac6946b-cc31-4a19-939d-4bd94ade70ef",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"WebSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 126,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9873731732368469,
0.0032423490192741156,
0.008952411822974682,
0.000432001193985343
],
"base_choice_token_mass": 0.3124958276748657,
"base_seconds": 6.914200600003824,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9990893602371216,
0.00047264734166674316,
0.00041064308607019484,
2.7297279302729294e-05
],
"adapter_choice_token_mass": 0.9889851212501526,
"adapter_seconds": 7.07698369998252
},
{
"id": "seceval_200:31398b3a-babe-4b5f-a8d4-19f06ad2659f",
"answer_letter": "D",
"topics": [
"SystemSecurity",
"PenTest"
],
"prompt_tokens": 82,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.04873128980398178,
0.014181612990796566,
0.0031152868177741766,
0.9339718222618103
],
"adapter_choice_token_mass": 0.9751800894737244,
"adapter_seconds": 7.624412200006191,
"base_prediction": "D",
"base_choice_probabilities": [
0.006045456510037184,
0.002404716331511736,
0.006140658166259527,
0.9854092001914978
],
"base_choice_token_mass": 0.3157220482826233,
"base_seconds": 7.262775899987901
},
{
"id": "seceval_200:22b29a24-e05d-4e9a-ba76-ddeb6b2bccce",
"answer_letter": "D",
"topics": [
"Unspecified"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.016840308904647827,
0.0032646425534039736,
0.001145997317507863,
0.9787490963935852
],
"base_choice_token_mass": 0.08143036812543869,
"base_seconds": 8.521422299992992,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.008648576214909554,
0.007058762479573488,
0.0001511503360234201,
0.9841415286064148
],
"adapter_choice_token_mass": 0.9364737868309021,
"adapter_seconds": 8.682175900001312
},
{
"id": "seceval_200:7c74ff1d-9c67-4607-bd0e-8d6b63ca45fb",
"answer_letter": "A",
"topics": [
"Vulnerability"
],
"prompt_tokens": 130,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9928035140037537,
0.004597592167556286,
0.0023118145763874054,
0.00028710573678836226
],
"adapter_choice_token_mass": 0.8853816390037537,
"adapter_seconds": 7.826240599999437,
"base_prediction": "A",
"base_choice_probabilities": [
0.9712178707122803,
0.01130704302340746,
0.016196591779589653,
0.0012785649159923196
],
"base_choice_token_mass": 0.04240535944700241,
"base_seconds": 8.713552300003357
},
{
"id": "seceval_200:e46b299e-cc72-4515-b543-9036ad8c7e27",
"answer_letter": "A",
"topics": [
"WebSecurity"
],
"prompt_tokens": 132,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.999763548374176,
3.8521309761563316e-05,
0.000188132980838418,
9.739712368173059e-06
],
"base_choice_token_mass": 0.41184043884277344,
"base_seconds": 7.858522499998799,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.999914288520813,
6.709100125590339e-05,
1.428449741069926e-05,
4.356525550974766e-06
],
"adapter_choice_token_mass": 0.9918659925460815,
"adapter_seconds": 7.830006700009108
},
{
"id": "seceval_200:5e207899-3d03-432a-9822-2904db3324d0",
"answer_letter": "B",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 245,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0965927317738533,
0.902238130569458,
0.0007148042786866426,
0.00045435724314302206
],
"adapter_choice_token_mass": 0.9565227031707764,
"adapter_seconds": 10.08295119999093,
"base_prediction": "B",
"base_choice_probabilities": [
0.23477955162525177,
0.7461279034614563,
0.0175472479313612,
0.0015452868537977338
],
"base_choice_token_mass": 0.04435930773615837,
"base_seconds": 9.832181999983732
},
{
"id": "seceval_200:d45b6b4a-02d7-4f6a-9cf6-eeec8adb7c14",
"answer_letter": "C",
"topics": [
"MemorySafety",
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 133,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.001722593093290925,
0.000604687025770545,
0.9954608082771301,
0.0022118533961474895
],
"base_choice_token_mass": 0.2149791419506073,
"base_seconds": 7.556881100026658,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0016880567418411374,
0.004660874139517546,
0.9755027890205383,
0.01814831979572773
],
"adapter_choice_token_mass": 0.9918640851974487,
"adapter_seconds": 8.330948599992553
},
{
"id": "seceval_200:a6d7c4f6-7c95-487b-9052-2c89b0165f4b",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 145,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0010775950504466891,
0.00038422836223617196,
0.9951135516166687,
0.0034245902206748724
],
"adapter_choice_token_mass": 0.9893379211425781,
"adapter_seconds": 7.8136876999924425,
"base_prediction": "C",
"base_choice_probabilities": [
0.0014261823380365968,
0.00011083792196586728,
0.9941381216049194,
0.0043248445726931095
],
"base_choice_token_mass": 0.2320987731218338,
"base_seconds": 7.925603400013642
},
{
"id": "seceval_200:855c165f-b3b5-4eeb-8689-141511af8064",
"answer_letter": "C",
"topics": [
"PenTest"
],
"prompt_tokens": 81,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.014547944068908691,
0.8195043802261353,
0.004577713552862406,
0.16136997938156128
],
"base_choice_token_mass": 0.3309325575828552,
"base_seconds": 7.820991900021909,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0030078142881393433,
0.9303758144378662,
0.0012936446582898498,
0.0653226375579834
],
"adapter_choice_token_mass": 0.9930149912834167,
"adapter_seconds": 6.976667100010673
},
{
"id": "seceval_200:a6cb40b7-765b-4aab-999a-a615a8ed3843",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 161,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.21698535978794098,
0.7340520620346069,
0.014089853502810001,
0.03487272188067436
],
"adapter_choice_token_mass": 0.854468047618866,
"adapter_seconds": 7.862788800004637,
"base_prediction": "B",
"base_choice_probabilities": [
0.12442758679389954,
0.6220988035202026,
0.2183772176504135,
0.035096414387226105
],
"base_choice_token_mass": 0.022809097543358803,
"base_seconds": 8.609474600001704
},
{
"id": "seceval_200:73ea8b90-bd82-4e7f-8695-848ea4f1b2f6",
"answer_letter": "D",
"topics": [
"SystemSecurity",
"PenTest"
],
"prompt_tokens": 119,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.002006386872380972,
5.872355177416466e-05,
0.02157069370150566,
0.9763641357421875
],
"base_choice_token_mass": 0.1395745724439621,
"base_seconds": 7.013538800005335,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0004223985888529569,
0.0002858091611415148,
0.0032202021684497595,
0.9960715770721436
],
"adapter_choice_token_mass": 0.9920400381088257,
"adapter_seconds": 7.179032200016081
},
{
"id": "seceval_200:11ca3da8-2123-4329-9f4c-bbc511d41f07",
"answer_letter": "A",
"topics": [
"Unspecified"
],
"prompt_tokens": 117,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9940775632858276,
0.003367986995726824,
0.0024258699268102646,
0.00012856638932134956
],
"adapter_choice_token_mass": 0.9789071679115295,
"adapter_seconds": 7.930406300001778,
"base_prediction": "A",
"base_choice_probabilities": [
0.8039550185203552,
0.007520767860114574,
0.18508096039295197,
0.0034432585816830397
],
"base_choice_token_mass": 0.07209337502717972,
"base_seconds": 7.057283100002678
},
{
"id": "seceval_200:3a6cf71b-2c6b-4729-89b6-ff2a7128b81a",
"answer_letter": "B",
"topics": [
"Vulnerability",
"SoftwareSecurity",
"SystemSecurity"
],
"prompt_tokens": 148,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.00035688665229827166,
0.9916303753852844,
0.007690420374274254,
0.00032242023735307157
],
"base_choice_token_mass": 0.26757189631462097,
"base_seconds": 8.216054400021676,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00045779216452501714,
0.9984059929847717,
0.0009691459708847106,
0.00016710172349121422
],
"adapter_choice_token_mass": 0.9859734773635864,
"adapter_seconds": 8.990746800001943
},
{
"id": "seceval_200:0b79c7d5-d38d-420a-a6e1-b23d012fa303",
"answer_letter": "A",
"topics": [
"WebSecurity"
],
"prompt_tokens": 118,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999228715896606,
4.434482252690941e-05,
2.3005790353636257e-05,
9.741264875628985e-06
],
"adapter_choice_token_mass": 0.9498413801193237,
"adapter_seconds": 7.358461100026034,
"base_prediction": "A",
"base_choice_probabilities": [
0.9997883439064026,
7.601448305649683e-05,
0.00011411125160520896,
2.160903204639908e-05
],
"base_choice_token_mass": 0.16341404616832733,
"base_seconds": 7.1634026000101585
},
{
"id": "seceval_200:6f5decce-06fa-4883-a5fd-34a2dbea58bd",
"answer_letter": "A",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 130,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9276071190834045,
0.007082367781549692,
0.0548434779047966,
0.010467060841619968
],
"base_choice_token_mass": 0.009090796113014221,
"base_seconds": 8.21744919999037,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9784679412841797,
0.004675041418522596,
0.01139144878834486,
0.005465677008032799
],
"adapter_choice_token_mass": 0.8392991423606873,
"adapter_seconds": 7.562144099996658
},
{
"id": "seceval_200:765e4132-e4e9-4be9-810e-7a7fbb120c20",
"answer_letter": "A",
"topics": [
"MemorySafety"
],
"prompt_tokens": 220,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9973825812339783,
0.0010800522286444902,
0.0009681525989435613,
0.0005691475234925747
],
"adapter_choice_token_mass": 0.7974474430084229,
"adapter_seconds": 9.547472800011747,
"base_prediction": "A",
"base_choice_probabilities": [
0.9772363901138306,
0.0036648635286837816,
0.018466874957084656,
0.0006319016683846712
],
"base_choice_token_mass": 0.0037872083485126495,
"base_seconds": 8.633511099993484
},
{
"id": "seceval_200:9dd998bf-80e0-433c-9456-ec52ae9786d7",
"answer_letter": "A",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 122,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9991307854652405,
0.0003681133675854653,
0.00039802544051781297,
0.00010302312148269266
],
"base_choice_token_mass": 0.1838800013065338,
"base_seconds": 6.944479700003285,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9993118047714233,
0.0004075382021255791,
0.0002181394665967673,
6.249800935620442e-05
],
"adapter_choice_token_mass": 0.9409857988357544,
"adapter_seconds": 7.7453802000090946
},
{
"id": "seceval_200:35c9b34d-ad77-4037-94a1-22fa5ac0d7fc",
"answer_letter": "D",
"topics": [
"PenTest"
],
"prompt_tokens": 169,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.048854004591703415,
3.6932491639163345e-05,
4.0246744902106e-05,
0.9510687589645386
],
"adapter_choice_token_mass": 0.9659500122070312,
"adapter_seconds": 7.9221550000074785,
"base_prediction": "D",
"base_choice_probabilities": [
0.0255584716796875,
3.553787973942235e-05,
0.00022814332623966038,
0.9741777777671814
],
"base_choice_token_mass": 0.2586534023284912,
"base_seconds": 7.855952299985802
},
{
"id": "seceval_200:b8519767-7e16-4211-be00-7044d8a1f7d2",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"SystemSecurity"
],
"prompt_tokens": 195,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9976270794868469,
0.0017535275546833873,
0.0005063343560323119,
0.00011297847231617197
],
"base_choice_token_mass": 0.07667291164398193,
"base_seconds": 8.812194599973736,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9996709823608398,
0.0002571234945207834,
4.6461267629638314e-05,
2.5458672098466195e-05
],
"adapter_choice_token_mass": 0.9515441060066223,
"adapter_seconds": 7.995271599997068
},
{
"id": "seceval_200:05a61adf-de55-4abb-a96b-10c81b8f77b0",
"answer_letter": "C",
"topics": [
"SystemSecurity"
],
"prompt_tokens": 155,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.030763640999794006,
0.00034713500645011663,
0.9570288062095642,
0.01186043955385685
],
"adapter_choice_token_mass": 0.9463260173797607,
"adapter_seconds": 8.692937100015115,
"base_prediction": "C",
"base_choice_probabilities": [
0.04223724082112312,
0.0006721171084791422,
0.9464113712310791,
0.010679246857762337
],
"base_choice_token_mass": 0.14576099812984467,
"base_seconds": 7.982533800008241
},
{
"id": "seceval_200:3437a585-16fb-428f-8cff-40f2973f8c7d",
"answer_letter": "A",
"topics": [
"Unspecified"
],
"prompt_tokens": 146,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.4778496325016022,
0.15757808089256287,
0.28533703088760376,
0.07923524081707001
],
"base_choice_token_mass": 0.019416624680161476,
"base_seconds": 8.78614110001945,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8266106247901917,
0.152907595038414,
0.007732704281806946,
0.012749074958264828
],
"adapter_choice_token_mass": 0.9112470149993896,
"adapter_seconds": 8.124844400008442
},
{
"id": "seceval_200:55a9dce6-fd22-4fd0-be79-3cb24d826c4f",
"answer_letter": "D",
"topics": [
"Vulnerability"
],
"prompt_tokens": 147,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.005636346526443958,
0.43397459387779236,
0.04503144323825836,
0.5153576135635376
],
"adapter_choice_token_mass": 0.8833860754966736,
"adapter_seconds": 8.94882150000194,
"base_prediction": "D",
"base_choice_probabilities": [
0.014256568625569344,
0.4366328716278076,
0.09152322262525558,
0.457587331533432
],
"base_choice_token_mass": 0.06539395451545715,
"base_seconds": 7.9620982999913394
},
{
"id": "seceval_200:72cc36d2-2d0f-4b66-a7a1-2c055293ecd7",
"answer_letter": "B",
"topics": [
"WebSecurity",
"PenTest"
],
"prompt_tokens": 95,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.02473091520369053,
0.3991367816925049,
0.22389543056488037,
0.35223695635795593
],
"base_choice_token_mass": 0.1699853092432022,
"base_seconds": 7.009924099984346,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.034153204411268234,
0.6149134039878845,
0.1350785493850708,
0.21585489809513092
],
"adapter_choice_token_mass": 0.9338766932487488,
"adapter_seconds": 8.056877299997723
},
{
"id": "seceval_200:28355cb5-4b9a-44b8-8419-38fd17d335e4",
"answer_letter": "C",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 130,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0011476199142634869,
0.0006142767379060388,
0.9955697655677795,
0.002668296219781041
],
"adapter_choice_token_mass": 0.970815122127533,
"adapter_seconds": 7.7522788000060245,
"base_prediction": "C",
"base_choice_probabilities": [
0.00047077902127057314,
0.0017220380250364542,
0.9951400756835938,
0.002667144639417529
],
"base_choice_token_mass": 0.14421047270298004,
"base_seconds": 7.610082400002284
},
{
"id": "seceval_200:eb502d16-dbf2-44ea-a7e9-ad9f33fe3d0e",
"answer_letter": "D",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 139,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.18140169978141785,
0.5249031782150269,
0.24410265684127808,
0.04959246143698692
],
"base_choice_token_mass": 0.08092035353183746,
"base_seconds": 8.593963699997403,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.4200240969657898,
0.530960202217102,
0.036132294684648514,
0.012883367016911507
],
"adapter_choice_token_mass": 0.9346928596496582,
"adapter_seconds": 8.072429699997883
},
{
"id": "seceval_200:4ed72ff0-e601-46bb-a7f7-4cb0fdacba25",
"answer_letter": "A",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 96,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9971449971199036,
0.0016724192537367344,
0.0010797949507832527,
0.00010281836875947192
],
"adapter_choice_token_mass": 0.9950019121170044,
"adapter_seconds": 6.680767299985746,
"base_prediction": "A",
"base_choice_probabilities": [
0.9576176404953003,
0.004037794657051563,
0.03771559149026871,
0.0006289670709520578
],
"base_choice_token_mass": 0.36929604411125183,
"base_seconds": 7.491517899994506
},
{
"id": "seceval_200:b35ebd47-4fc7-44b4-9021-4878e8b98360",
"answer_letter": "C",
"topics": [
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 155,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.0006357323727570474,
0.00029105952125974,
0.9986431002616882,
0.0004301580775063485
],
"base_choice_token_mass": 0.01853560283780098,
"base_seconds": 8.05125560000306,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.000867833208758384,
0.00035615803790278733,
0.9973674416542053,
0.0014086323790252209
],
"adapter_choice_token_mass": 0.9642841219902039,
"adapter_seconds": 7.980741300008958
},
{
"id": "seceval_200:6ae2a0a6-d13c-42a8-b0b3-a167974f3fb2",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 199,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0009507746435701847,
0.008342782035470009,
0.9794799089431763,
0.011226440779864788
],
"adapter_choice_token_mass": 0.6275278925895691,
"adapter_seconds": 8.985108599998057,
"base_prediction": "C",
"base_choice_probabilities": [
0.0005625940393656492,
0.007295830175280571,
0.9630597829818726,
0.029081886634230614
],
"base_choice_token_mass": 0.025251656770706177,
"base_seconds": 8.28895069999271
},
{
"id": "seceval_200:0f7736e8-3a11-4334-b383-e5e811813e47",
"answer_letter": "A",
"topics": [
"Unspecified"
],
"prompt_tokens": 147,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9998575448989868,
2.4487935661454685e-05,
2.1109921362949535e-05,
9.68516687862575e-05
],
"base_choice_token_mass": 0.47279560565948486,
"base_seconds": 8.64362240000628,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999504089355469,
2.5070978153962642e-05,
5.6379667512374e-06,
1.8924567484646104e-05
],
"adapter_choice_token_mass": 0.9464343190193176,
"adapter_seconds": 7.844958600006066
},
{
"id": "seceval_200:1b56cd95-e2f7-40b9-b323-69aa26e91a60",
"answer_letter": "C",
"topics": [
"Vulnerability"
],
"prompt_tokens": 171,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0001660255074966699,
1.0950569048873149e-05,
0.9997559189796448,
6.708037108182907e-05
],
"adapter_choice_token_mass": 0.9911681413650513,
"adapter_seconds": 8.487512499996228,
"base_prediction": "C",
"base_choice_probabilities": [
0.0004728185886051506,
4.527972123469226e-06,
0.9994513392448425,
7.138492946978658e-05
],
"base_choice_token_mass": 0.4328482747077942,
"base_seconds": 7.751885300007416
},
{
"id": "seceval_200:bc0798bf-20e4-4e51-84b9-83afb887212a",
"answer_letter": "A",
"topics": [
"WebSecurity",
"SoftwareSecurity"
],
"prompt_tokens": 90,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9939215183258057,
0.0003282542747911066,
0.005728239193558693,
2.1991663743392564e-05
],
"base_choice_token_mass": 0.10958217084407806,
"base_seconds": 6.976166200009175,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9997358918190002,
0.0001559634110890329,
0.0001055301254382357,
2.6626155431586085e-06
],
"adapter_choice_token_mass": 0.9911719560623169,
"adapter_seconds": 7.703248599980725
},
{
"id": "seceval_200:bc023b01-d17a-438c-83cb-fabcbd04e188",
"answer_letter": "A",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 147,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999500513076782,
1.0129588190466166e-05,
3.373668005224317e-05,
6.048653176549124e-06
],
"adapter_choice_token_mass": 0.9907580018043518,
"adapter_seconds": 8.61547900000005,
"base_prediction": "A",
"base_choice_probabilities": [
0.9995203018188477,
1.0125234439328779e-05,
0.0004511977604124695,
1.8334429114474915e-05
],
"base_choice_token_mass": 0.14432741701602936,
"base_seconds": 9.293064800003776
},
{
"id": "seceval_200:917a3891-25ad-4f06-91d5-a5d6c0ef6836",
"answer_letter": "D",
"topics": [
"NetworkSecurity"
],
"prompt_tokens": 102,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0011878464138135314,
9.069319276022725e-06,
4.064587119501084e-05,
0.9987624883651733
],
"base_choice_token_mass": 0.18117877840995789,
"base_seconds": 6.888358399999561,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0025507451500743628,
5.091096318210475e-05,
1.4700636711495463e-05,
0.9973836541175842
],
"adapter_choice_token_mass": 0.9504013657569885,
"adapter_seconds": 7.039947899989784
},
{
"id": "seceval_200:543cbc91-4cd7-4c8c-b9b3-000022a600fe",
"answer_letter": "A",
"topics": [
"PenTest",
"ApplicationSecurity"
],
"prompt_tokens": 139,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8675245046615601,
0.0858965590596199,
0.03752542659640312,
0.009053430519998074
],
"adapter_choice_token_mass": 0.954490602016449,
"adapter_seconds": 8.280704800010426,
"base_prediction": "A",
"base_choice_probabilities": [
0.49954622983932495,
0.12434688955545425,
0.2982926666736603,
0.07781429588794708
],
"base_choice_token_mass": 0.10523912310600281,
"base_seconds": 9.113113500003237
},
{
"id": "seceval_200:211cf668-247f-4d34-a0cb-ba9bb69c316f",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 199,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0025461630430072546,
0.9801567792892456,
0.016345661133527756,
0.000951431633438915
],
"base_choice_token_mass": 0.008376973681151867,
"base_seconds": 9.92953709998983,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00095389160560444,
0.9981661438941956,
0.0007313740206882358,
0.0001485876418882981
],
"adapter_choice_token_mass": 0.7335159182548523,
"adapter_seconds": 9.097164800012251
},
{
"id": "seceval_200:e0f49000-1b7c-4ae0-8e19-513c09417db5",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 172,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0002523678704164922,
0.002509309444576502,
0.996632993221283,
0.0006053990218788385
],
"adapter_choice_token_mass": 0.9884495735168457,
"adapter_seconds": 8.879013199999463,
"base_prediction": "C",
"base_choice_probabilities": [
0.0013633827911689878,
0.0015692440792918205,
0.9959718585014343,
0.001095508923754096
],
"base_choice_token_mass": 0.07692325115203857,
"base_seconds": 8.104941400000826
},
{
"id": "seceval_200:1d0968b5-fd80-4cdf-9ef1-5dd9c55c5794",
"answer_letter": "C",
"topics": [
"Unspecified"
],
"prompt_tokens": 182,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.11588482558727264,
0.6988748908042908,
0.029300272464752197,
0.15594005584716797
],
"base_choice_token_mass": 0.03961074724793434,
"base_seconds": 9.033553100016434,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.18182417750358582,
0.5428267121315002,
0.002398671815171838,
0.2729504108428955
],
"adapter_choice_token_mass": 0.8699857592582703,
"adapter_seconds": 8.176552399992943
},
{
"id": "seceval_200:8907298f-92a2-4d46-9774-73f291ba7e16",
"answer_letter": "D",
"topics": [
"Vulnerability"
],
"prompt_tokens": 150,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.049813006073236465,
0.89686119556427,
0.004285131115466356,
0.04904072359204292
],
"adapter_choice_token_mass": 0.9210745096206665,
"adapter_seconds": 9.14660130001721,
"base_prediction": "B",
"base_choice_probabilities": [
0.18466097116470337,
0.6969032883644104,
0.027021871879696846,
0.09141381084918976
],
"base_choice_token_mass": 0.018081240355968475,
"base_seconds": 8.398061399988364
},
{
"id": "seceval_200:a1c55e1a-4c9e-4434-9841-7354aeda3586",
"answer_letter": "B",
"topics": [
"WebSecurity",
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 183,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0003073571133427322,
0.9984384179115295,
0.0008034763741306961,
0.00045070936903357506
],
"base_choice_token_mass": 0.23910582065582275,
"base_seconds": 9.242196999985026,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0005109705962240696,
0.9989270567893982,
0.00032990743056870997,
0.0002321188512723893
],
"adapter_choice_token_mass": 0.9529426097869873,
"adapter_seconds": 8.356669000000693
},
{
"id": "seceval_200:cb7e8e3e-4eda-4c81-bb29-277d1e00f51f",
"answer_letter": "C",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 152,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0006260513328015804,
0.00038569900789298117,
0.9989223480224609,
6.598531763302162e-05
],
"adapter_choice_token_mass": 0.9911095499992371,
"adapter_seconds": 9.166353000007803,
"base_prediction": "C",
"base_choice_probabilities": [
0.0001660305861150846,
7.644963261554949e-06,
0.9997865557670593,
3.974502033088356e-05
],
"base_choice_token_mass": 0.5153011083602905,
"base_seconds": 8.373801099980483
},
{
"id": "seceval_200:2f29bfc1-ec35-4c65-a115-127765d2bec0",
"answer_letter": "B",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 143,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0005969602498225868,
0.9982165694236755,
0.0010641966946423054,
0.0001222310384036973
],
"base_choice_token_mass": 0.22906333208084106,
"base_seconds": 9.052269100007834,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00029129558242857456,
0.9994531273841858,
0.0002164725010516122,
3.911578460247256e-05
],
"adapter_choice_token_mass": 0.9863572120666504,
"adapter_seconds": 8.275446900021052
},
{
"id": "seceval_200:2b52de5c-3403-4963-81e5-8c1bb8b8e93c",
"answer_letter": "A",
"topics": [
"PenTest"
],
"prompt_tokens": 157,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9994214773178101,
0.0003625117533374578,
0.00016086360847111791,
5.516034798347391e-05
],
"adapter_choice_token_mass": 0.9957271814346313,
"adapter_seconds": 8.687138500012225,
"base_prediction": "A",
"base_choice_probabilities": [
0.9929143190383911,
0.0011093445355072618,
0.005460385698825121,
0.0005158931598998606
],
"base_choice_token_mass": 0.16775868833065033,
"base_seconds": 8.660012399981497
},
{
"id": "seceval_200:31e2b5b9-f5cc-411b-91ba-03918199ebd5",
"answer_letter": "D",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"WebSecurity"
],
"prompt_tokens": 179,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.000133312598336488,
0.000133312598336488,
0.0006717486539855599,
0.999061644077301
],
"base_choice_token_mass": 0.028412869200110435,
"base_seconds": 9.236366099998122,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
6.301858957158402e-05,
7.085369725245982e-05,
7.19694871804677e-05,
0.9997941851615906
],
"adapter_choice_token_mass": 0.9305464029312134,
"adapter_seconds": 8.386441300011938
},
{
"id": "seceval_200:8793c56e-832c-4fa0-822a-5712af3012e4",
"answer_letter": "B",
"topics": [
"SystemSecurity"
],
"prompt_tokens": 112,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.48438888788223267,
0.0939030721783638,
0.3656359016895294,
0.056072086095809937
],
"adapter_choice_token_mass": 0.938655436038971,
"adapter_seconds": 6.702695299987681,
"base_prediction": "C",
"base_choice_probabilities": [
0.12565940618515015,
0.014775198884308338,
0.8194022178649902,
0.040163155645132065
],
"base_choice_token_mass": 0.10114514082670212,
"base_seconds": 7.617107600002782
},
{
"id": "seceval_200:e064003d-c2a9-45ee-a3a6-eb1f8c39e022",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 150,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.026380030438303947,
0.9594472050666809,
0.013685779646039009,
0.0004869566182605922
],
"base_choice_token_mass": 0.16935425996780396,
"base_seconds": 8.427796999982093,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.01530180312693119,
0.9767401814460754,
0.007574941962957382,
0.00038307314389385283
],
"adapter_choice_token_mass": 0.9216122627258301,
"adapter_seconds": 9.119909100001678
},
{
"id": "seceval_200:881ba8de-aada-4511-80a5-7cd595abd301",
"answer_letter": "D",
"topics": [
"Vulnerability"
],
"prompt_tokens": 144,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0004954892210662365,
7.422552153002471e-05,
2.0134213627898134e-05,
0.9994101524353027
],
"adapter_choice_token_mass": 0.9751744866371155,
"adapter_seconds": 8.219945199991344,
"base_prediction": "D",
"base_choice_probabilities": [
0.0009394373628310859,
8.806663390714675e-05,
0.00045074697118252516,
0.9985217452049255
],
"base_choice_token_mass": 0.06725713610649109,
"base_seconds": 8.662163200002396
},
{
"id": "seceval_200:0e7e958e-18ed-4c54-bfdf-77fbfd6c7bcf",
"answer_letter": "C",
"topics": [
"WebSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 92,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0046638986095786095,
0.6204696297645569,
0.09079476445913315,
0.2840717136859894
],
"base_choice_token_mass": 0.01301146112382412,
"base_seconds": 6.955972299998393,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.004825360141694546,
0.3080109655857086,
0.014405864290893078,
0.6727578043937683
],
"adapter_choice_token_mass": 0.9618098139762878,
"adapter_seconds": 6.873669900000095
},
{
"id": "seceval_200:9a3242a9-fee3-4e87-842c-4533388282c3",
"answer_letter": "C",
"topics": [
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 166,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.21160976588726044,
0.0015416739042848349,
0.7862256765365601,
0.0006228926358744502
],
"adapter_choice_token_mass": 0.9577111601829529,
"adapter_seconds": 8.630676600005245,
"base_prediction": "C",
"base_choice_probabilities": [
0.09929485619068146,
0.0004933217423968017,
0.898942232131958,
0.0012696214253082871
],
"base_choice_token_mass": 0.1064140722155571,
"base_seconds": 8.055580199987162
},
{
"id": "seceval_200:977186bb-ac66-419f-bf6d-620202754434",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 174,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.010324751026928425,
2.945678716059774e-05,
0.9893459677696228,
0.0002998363634105772
],
"base_choice_token_mass": 0.24167296290397644,
"base_seconds": 9.243242800002918,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0004043416993226856,
0.0001546744751976803,
0.9992499947547913,
0.00019099749624729156
],
"adapter_choice_token_mass": 0.9923409819602966,
"adapter_seconds": 8.333450699981768
},
{
"id": "seceval_200:41b922f3-d410-46af-ae65-980401ea54ad",
"answer_letter": "B",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 132,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0007659393595531583,
0.9974707365036011,
0.001547240186482668,
0.000216043132240884
],
"adapter_choice_token_mass": 0.9362844228744507,
"adapter_seconds": 7.95302570000058,
"base_prediction": "B",
"base_choice_probabilities": [
0.0013838323066011071,
0.9952378273010254,
0.0029757071752101183,
0.00040271817124448717
],
"base_choice_token_mass": 0.0843537226319313,
"base_seconds": 8.88145389998681
},
{
"id": "seceval_200:de1c290f-0dc4-4cf5-967a-443788c16ae8",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 289,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.004435897804796696,
0.7230511903762817,
0.2618717849254608,
0.010641166009008884
],
"base_choice_token_mass": 0.010201267898082733,
"base_seconds": 16.0629031000135,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0009221059735864401,
0.9724730253219604,
0.02632364071905613,
0.00028122644289396703
],
"adapter_choice_token_mass": 0.7800703644752502,
"adapter_seconds": 15.16263850001269
},
{
"id": "seceval_200:e80b794a-28f1-45c2-a20a-0d9f1d5814fc",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"ApplicationSecurity",
"PenTest"
],
"prompt_tokens": 160,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00027345074340701103,
3.5312838008394465e-05,
0.9987367987632751,
0.0009544369531795382
],
"adapter_choice_token_mass": 0.9925075173377991,
"adapter_seconds": 8.787668899982236,
"base_prediction": "C",
"base_choice_probabilities": [
8.748021355131641e-05,
3.236646080040373e-06,
0.9996521472930908,
0.0002571186632849276
],
"base_choice_token_mass": 0.5154337882995605,
"base_seconds": 8.152890299999854
},
{
"id": "seceval_200:7360ae18-72ad-4319-af55-85d06d5626d3",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 146,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.07447825372219086,
0.9216192364692688,
0.0029795051086694,
0.0009230076684616506
],
"base_choice_token_mass": 0.19809490442276,
"base_seconds": 8.724586500000441,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0046796612441539764,
0.9948587417602539,
0.00034975435119122267,
0.00011178819841006771
],
"adapter_choice_token_mass": 0.9717914462089539,
"adapter_seconds": 7.826427699998021
},
{
"id": "seceval_200:0f1c2b65-1831-4b26-8f1d-95ecc6ef4eb6",
"answer_letter": "D",
"topics": [
"Vulnerability"
],
"prompt_tokens": 152,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0004727263003587723,
0.9837642312049866,
0.00128500338178128,
0.014478087425231934
],
"adapter_choice_token_mass": 0.976468026638031,
"adapter_seconds": 9.13868029997684,
"base_prediction": "B",
"base_choice_probabilities": [
0.0014648562064394355,
0.9743368625640869,
0.018126629292964935,
0.006071660667657852
],
"base_choice_token_mass": 0.033930789679288864,
"base_seconds": 7.9039536000054795
},
{
"id": "seceval_200:84bdc80d-820e-47a2-aad4-1a06531f6310",
"answer_letter": "C",
"topics": [
"WebSecurity"
],
"prompt_tokens": 194,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.00029531135805882514,
0.001986799528822303,
0.99752277135849,
0.00019518888439051807
],
"base_choice_token_mass": 0.05994755029678345,
"base_seconds": 8.214590399991721,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00011149089550599456,
0.007575638592243195,
0.9922128319740295,
9.993978892453015e-05
],
"adapter_choice_token_mass": 0.9611368179321289,
"adapter_seconds": 9.160910300008254
},
{
"id": "seceval_200:b047f025-46db-4ea2-adb5-f40cbd3024de",
"answer_letter": "B",
"topics": [
"ApplicationSecurity",
"PenTest"
],
"prompt_tokens": 119,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.001364817377179861,
0.9970198273658752,
0.0005689405370503664,
0.001046441844664514
],
"adapter_choice_token_mass": 0.9380433559417725,
"adapter_seconds": 7.151373500004411,
"base_prediction": "B",
"base_choice_probabilities": [
0.05675150826573372,
0.915922999382019,
0.010017236694693565,
0.017308233305811882
],
"base_choice_token_mass": 0.01787969097495079,
"base_seconds": 7.12882409998565
},
{
"id": "seceval_200:7a1db287-9f9b-4621-ba2c-9667a765bbab",
"answer_letter": "A",
"topics": [
"NetworkSecurity"
],
"prompt_tokens": 122,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9985149502754211,
0.0013456727610900998,
6.647573172813281e-05,
7.300930155906826e-05
],
"base_choice_token_mass": 0.2839473485946655,
"base_seconds": 7.122440400009509,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9994685053825378,
0.0005112475482746959,
1.1929822903766762e-05,
8.328358489961829e-06
],
"adapter_choice_token_mass": 0.9699988961219788,
"adapter_seconds": 7.229528900003061
},
{
"id": "seceval_200:a65068b5-093a-4654-9e28-43ce4d04e764",
"answer_letter": "D",
"topics": [
"PenTest",
"NetworkSecurity"
],
"prompt_tokens": 124,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.12573416531085968,
0.02362474799156189,
0.00472524669021368,
0.8459157943725586
],
"adapter_choice_token_mass": 0.9823225140571594,
"adapter_seconds": 7.090219600009732,
"base_prediction": "D",
"base_choice_probabilities": [
0.12862950563430786,
0.003427860327064991,
0.002547369571402669,
0.865395188331604
],
"base_choice_token_mass": 0.15783023834228516,
"base_seconds": 7.870997599995462
},
{
"id": "seceval_200:7c7607cc-6038-450b-9eeb-101589d71f50",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 156,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.060192231088876724,
0.9269673824310303,
0.0015306130517274141,
0.011309785768389702
],
"base_choice_token_mass": 0.01965191215276718,
"base_seconds": 7.678123699995922,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.043889615684747696,
0.9531790018081665,
8.14810482552275e-05,
0.002849953481927514
],
"adapter_choice_token_mass": 0.7300112247467041,
"adapter_seconds": 8.258750199980568
},
{
"id": "seceval_200:a4242b47-3b29-407d-9459-8f1d87f0470e",
"answer_letter": "D",
"topics": [
"SystemSecurity",
"NetworkSecurity"
],
"prompt_tokens": 119,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.09291768819093704,
0.01640094630420208,
0.8679109215736389,
0.022770460695028305
],
"adapter_choice_token_mass": 0.9846128821372986,
"adapter_seconds": 7.170657000009669,
"base_prediction": "C",
"base_choice_probabilities": [
0.1387777328491211,
0.004602610133588314,
0.8369367718696594,
0.01968286745250225
],
"base_choice_token_mass": 0.15583325922489166,
"base_seconds": 7.228580599999987
},
{
"id": "seceval_200:0742ca88-dfd7-4b40-b042-8eec1b094f1a",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 109,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.011102911084890366,
0.48709410429000854,
0.48709410429000854,
0.014708967879414558
],
"base_choice_token_mass": 0.011038430035114288,
"base_seconds": 6.645305599988205,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0016932773869484663,
0.9939291477203369,
0.0041259280405938625,
0.00025168320280499756
],
"adapter_choice_token_mass": 0.7778091430664062,
"adapter_seconds": 7.393950700003188
},
{
"id": "seceval_200:93dc670a-9816-4c4a-a8b9-1d2e77b88550",
"answer_letter": "D",
"topics": [
"Vulnerability"
],
"prompt_tokens": 108,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.07442768663167953,
0.017134224995970726,
0.0017232343088835478,
0.9067147970199585
],
"adapter_choice_token_mass": 0.8597943782806396,
"adapter_seconds": 6.6779060999979265,
"base_prediction": "A",
"base_choice_probabilities": [
0.47726938128471375,
0.027780409902334213,
0.053548142313957214,
0.44140201807022095
],
"base_choice_token_mass": 0.002712306333705783,
"base_seconds": 6.7596425999945495
},
{
"id": "seceval_200:b99d911a-1326-44b7-8063-0dc5a56d10ca",
"answer_letter": "C",
"topics": [
"WebSecurity",
"Vulnerability"
],
"prompt_tokens": 134,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.004134672228246927,
0.9211824536323547,
0.07103396207094193,
0.0036488354671746492
],
"base_choice_token_mass": 0.13530586659908295,
"base_seconds": 7.91177449998213,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0006699852528981864,
0.9886847138404846,
0.009248861111700535,
0.0013963693054392934
],
"adapter_choice_token_mass": 0.9918054342269897,
"adapter_seconds": 8.766469900001539
},
{
"id": "seceval_200:c431eb4c-c92d-4f6c-9805-759179e27363",
"answer_letter": "D",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 124,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0006460238946601748,
0.00024520300212316215,
3.2415900932392105e-05,
0.9990763664245605
],
"adapter_choice_token_mass": 0.9775320887565613,
"adapter_seconds": 7.299498400010634,
"base_prediction": "D",
"base_choice_probabilities": [
0.00026727511431090534,
0.0002065341832349077,
0.00019554262689780444,
0.9993306398391724
],
"base_choice_token_mass": 0.3200089931488037,
"base_seconds": 7.065661199972965
},
{
"id": "seceval_200:00d8980f-9175-4392-8464-e070450e42ff",
"answer_letter": "D",
"topics": [
"NetworkSecurity",
"SystemSecurity"
],
"prompt_tokens": 149,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.009975372813642025,
0.03322306275367737,
0.04470651596784592,
0.9120951294898987
],
"base_choice_token_mass": 0.18800663948059082,
"base_seconds": 8.82449140000972,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.004002512898296118,
0.10650292038917542,
0.011581655591726303,
0.8779129385948181
],
"adapter_choice_token_mass": 0.8973435759544373,
"adapter_seconds": 8.337178299989318
},
{
"id": "seceval_200:c2dc2a1d-45c9-4a49-a513-a41b8d257c07",
"answer_letter": "A",
"topics": [
"PenTest"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.995984673500061,
0.0019529807614162564,
0.0009667956037446856,
0.0010955228935927153
],
"adapter_choice_token_mass": 0.9817568063735962,
"adapter_seconds": 8.11591199997929,
"base_prediction": "A",
"base_choice_probabilities": [
0.9918481111526489,
0.002829768694937229,
0.004247985780239105,
0.0010740590514615178
],
"base_choice_token_mass": 0.20217196643352509,
"base_seconds": 8.07427650000318
},
{
"id": "seceval_200:fb409d74-45c5-4518-bc5e-c762a4e028cc",
"answer_letter": "D",
"topics": [
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 210,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.03717895597219467,
0.007672311272472143,
0.040200039744377136,
0.9149486422538757
],
"base_choice_token_mass": 0.1290159672498703,
"base_seconds": 9.524892899993574,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.10030551999807358,
0.02919057011604309,
0.017430484294891357,
0.8530735373497009
],
"adapter_choice_token_mass": 0.9684127569198608,
"adapter_seconds": 9.900577699998394
},
{
"id": "seceval_200:b908bc08-64af-48b8-b596-8236c488193d",
"answer_letter": "B",
"topics": [
"SystemSecurity",
"PenTest"
],
"prompt_tokens": 158,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.002223432995378971,
0.9851582646369934,
0.012019763700664043,
0.0005984288291074336
],
"adapter_choice_token_mass": 0.9887682199478149,
"adapter_seconds": 7.847697499993956,
"base_prediction": "B",
"base_choice_probabilities": [
0.01820400170981884,
0.904960572719574,
0.0754534900188446,
0.0013819787418469787
],
"base_choice_token_mass": 0.15385977923870087,
"base_seconds": 8.329853600007482
},
{
"id": "seceval_200:57cd7b7e-122a-4fba-959a-9e848de2be3e",
"answer_letter": "C",
"topics": [
"Unspecified"
],
"prompt_tokens": 105,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.0003458034188952297,
0.0001333191030425951,
0.9991102814674377,
0.00041065170080401003
],
"base_choice_token_mass": 0.7975067496299744,
"base_seconds": 7.815487999992911,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.001095897052437067,
0.0006857948610559106,
0.9963247179985046,
0.0018935402622446418
],
"adapter_choice_token_mass": 0.9951480627059937,
"adapter_seconds": 6.606248699972639
},
{
"id": "seceval_200:7f62d9d0-e532-4583-924d-b891555f8348",
"answer_letter": "B",
"topics": [
"Vulnerability",
"PenTest"
],
"prompt_tokens": 123,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00017673432012088597,
0.9997621178627014,
4.197808084427379e-05,
1.9218967281631194e-05
],
"adapter_choice_token_mass": 0.9852177500724792,
"adapter_seconds": 7.114711200003512,
"base_prediction": "B",
"base_choice_probabilities": [
0.0005789906135760248,
0.9989013671875,
0.00042359859799034894,
9.600606426829472e-05
],
"base_choice_token_mass": 0.10699450969696045,
"base_seconds": 6.915462100005243
},
{
"id": "seceval_200:c46aa73a-1298-43ba-b4ab-fd69127df8dd",
"answer_letter": "D",
"topics": [
"WebSecurity",
"PenTest"
],
"prompt_tokens": 138,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.007666794117540121,
0.1218174397945404,
0.5545452833175659,
0.31597042083740234
],
"base_choice_token_mass": 0.07608900964260101,
"base_seconds": 8.452237499994226,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0156809464097023,
0.14877666532993317,
0.26522308588027954,
0.5703192949295044
],
"adapter_choice_token_mass": 0.9723254442214966,
"adapter_seconds": 7.722047699993709
},
{
"id": "seceval_200:400d0595-809e-4174-9f5e-bfd275471f09",
"answer_letter": "C",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 96,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.10921130329370499,
0.0034025819040834904,
0.8862814903259277,
0.001104656606912613
],
"adapter_choice_token_mass": 0.9902743697166443,
"adapter_seconds": 6.5931832999922335,
"base_prediction": "C",
"base_choice_probabilities": [
0.004975967574864626,
0.00047752910177223384,
0.9937591552734375,
0.0007873123977333307
],
"base_choice_token_mass": 0.3318997025489807,
"base_seconds": 6.552847799990559
},
{
"id": "seceval_200:a7def17b-3864-4a6f-b341-44289f82ee31",
"answer_letter": "D",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 127,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.12864841520786285,
0.10337191075086594,
0.17311535775661469,
0.5948642492294312
],
"base_choice_token_mass": 0.16957242786884308,
"base_seconds": 7.024894600006519,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.24870650470256805,
0.27745217084884644,
0.07015088945627213,
0.40369051694869995
],
"adapter_choice_token_mass": 0.9736820459365845,
"adapter_seconds": 7.079950300016208
},
{
"id": "seceval_200:d150584b-1899-4a9d-b73c-e81544bae767",
"answer_letter": "C",
"topics": [
"PenTest"
],
"prompt_tokens": 125,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0012902499875053763,
0.006060007028281689,
0.9724668264389038,
0.020182905718684196
],
"adapter_choice_token_mass": 0.9876768589019775,
"adapter_seconds": 8.004433399997652,
"base_prediction": "C",
"base_choice_probabilities": [
0.00262288935482502,
0.0009499475709162652,
0.9940390586853027,
0.002388167893514037
],
"base_choice_token_mass": 0.18920816481113434,
"base_seconds": 7.047543199994834
},
{
"id": "seceval_200:e82f3efd-9d1b-4ba6-8b38-fb479c19ef3e",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 128,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.45793452858924866,
0.5353797078132629,
0.006532085128128529,
0.000153619927004911
],
"base_choice_token_mass": 0.028122853487730026,
"base_seconds": 7.059097599994857,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.5229865908622742,
0.4761847257614136,
0.0007620873511768878,
6.659044447587803e-05
],
"adapter_choice_token_mass": 0.6622758507728577,
"adapter_seconds": 7.165312699973583
},
{
"id": "seceval_200:21c388dd-5877-46a8-a61b-7eeb0c041e65",
"answer_letter": "A",
"topics": [
"SystemSecurity"
],
"prompt_tokens": 143,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9996050000190735,
0.0003353301144670695,
4.906963658868335e-05,
1.0612053301883861e-05
],
"adapter_choice_token_mass": 0.988264799118042,
"adapter_seconds": 8.875363700004527,
"base_prediction": "A",
"base_choice_probabilities": [
0.998944103717804,
0.00036804459523409605,
0.000656110409181565,
3.166079113725573e-05
],
"base_choice_token_mass": 0.5543599724769592,
"base_seconds": 7.935242000006838
},
{
"id": "seceval_200:f19c34e2-39b7-4745-839c-f4a075c2fff9",
"answer_letter": "C",
"topics": [
"Unspecified"
],
"prompt_tokens": 180,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.00047268313937820494,
4.003184585599229e-05,
0.9991650581359863,
0.00032234188984148204
],
"base_choice_token_mass": 0.536560595035553,
"base_seconds": 8.994315100018866,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0002867029106710106,
0.00024909179774113,
0.9991863369941711,
0.00027788200532086194
],
"adapter_choice_token_mass": 0.9928293824195862,
"adapter_seconds": 7.926965999999084
},
{
"id": "seceval_200:731ccd08-13e2-4030-a3b0-210858d3199f",
"answer_letter": "C",
"topics": [
"Vulnerability"
],
"prompt_tokens": 145,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.11324185132980347,
0.061568524688482285,
0.786054253578186,
0.03913533315062523
],
"adapter_choice_token_mass": 0.9788792133331299,
"adapter_seconds": 8.004760500014527,
"base_prediction": "C",
"base_choice_probabilities": [
0.07389325648546219,
0.012445654720067978,
0.9002041220664978,
0.013456962071359158
],
"base_choice_token_mass": 0.44236648082733154,
"base_seconds": 8.644432800007053
},
{
"id": "seceval_200:3720deea-6d3b-4810-b52d-c85d3e28589d",
"answer_letter": "B",
"topics": [
"WebSecurity"
],
"prompt_tokens": 189,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.00012894651445094496,
0.002756998175755143,
0.9970166683197021,
9.733394108479843e-05
],
"base_choice_token_mass": 0.12472356110811234,
"base_seconds": 8.446079599991208,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00016842811601236463,
0.0012445247266441584,
0.9984996318817139,
8.73793542268686e-05
],
"adapter_choice_token_mass": 0.9770660996437073,
"adapter_seconds": 9.227273800002877
},
{
"id": "seceval_200:192acf52-327f-49b5-8911-1023319d540f",
"answer_letter": "B",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 152,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0028444956988096237,
0.9970099925994873,
8.260569302365184e-05,
6.284310074988753e-05
],
"adapter_choice_token_mass": 0.9395366907119751,
"adapter_seconds": 8.370423399988795,
"base_prediction": "B",
"base_choice_probabilities": [
0.0015230602584779263,
0.9973449110984802,
0.0009383301949128509,
0.00019363538012839854
],
"base_choice_token_mass": 0.08939419686794281,
"base_seconds": 9.449785400007386
},
{
"id": "seceval_200:6b91f5be-5700-464e-90db-c7adc842f649",
"answer_letter": "A",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 144,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9867226481437683,
0.007077275309711695,
0.005097570363432169,
0.0011024269042536616
],
"base_choice_token_mass": 0.29807528853416443,
"base_seconds": 8.30310869999812,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9974308609962463,
0.00218187109567225,
0.0002177288697566837,
0.00016956742911133915
],
"adapter_choice_token_mass": 0.9943625926971436,
"adapter_seconds": 9.065643999987515
},
{
"id": "seceval_200:fffea5ee-2add-4699-85c9-a704226fb28f",
"answer_letter": "D",
"topics": [
"PenTest"
],
"prompt_tokens": 237,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
9.679083450464532e-05,
0.0005613613757304847,
0.0001122793328249827,
0.9992295503616333
],
"adapter_choice_token_mass": 0.9638795256614685,
"adapter_seconds": 10.484130500000902,
"base_prediction": "D",
"base_choice_probabilities": [
0.0005583789316006005,
0.0044611538760364056,
0.0010596169158816338,
0.9939208030700684
],
"base_choice_token_mass": 0.06095537915825844,
"base_seconds": 9.632282300008228
},
{
"id": "seceval_200:8f574b93-a7c9-4579-b790-42164beeffea",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 172,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9996733665466309,
3.143733920296654e-05,
9.025905455928296e-05,
0.00020499720994848758
],
"base_choice_token_mass": 0.42374011874198914,
"base_seconds": 8.082970699993894,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9998902082443237,
4.133256879867986e-05,
7.352852207986871e-06,
6.108557136030868e-05
],
"adapter_choice_token_mass": 0.9944327473640442,
"adapter_seconds": 7.364544600015506
},
{
"id": "seceval_200:ae6ef008-0353-4dfa-acbc-d8d551dbbe55",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 160,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.997680127620697,
0.00014061144611332566,
0.002148581203073263,
3.0647865060018376e-05
],
"adapter_choice_token_mass": 0.9630801677703857,
"adapter_seconds": 7.736438899999484,
"base_prediction": "A",
"base_choice_probabilities": [
0.9615395069122314,
0.0003102034388575703,
0.037870053201913834,
0.00028024546918459237
],
"base_choice_token_mass": 0.06380056589841843,
"base_seconds": 7.824576599989086
},
{
"id": "seceval_200:7d53d175-59d3-4002-b8ff-bdfcd5681167",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 94,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.00059601099928841,
0.9811780452728271,
0.014216153882443905,
0.004009850323200226
],
"base_choice_token_mass": 0.5022545456886292,
"base_seconds": 6.476945399976103,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00102854217402637,
0.9953977465629578,
0.0029300437308847904,
0.0006436451803892851
],
"adapter_choice_token_mass": 0.9902195930480957,
"adapter_seconds": 6.457874199986691
},
{
"id": "seceval_200:9d9e61a9-c660-4b2f-b060-52c69cd32589",
"answer_letter": "D",
"topics": [
"Vulnerability"
],
"prompt_tokens": 150,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0006554831052199006,
0.0011325732339173555,
0.0002230168756796047,
0.9979889392852783
],
"adapter_choice_token_mass": 0.9869481325149536,
"adapter_seconds": 7.596728699980304,
"base_prediction": "D",
"base_choice_probabilities": [
0.00029576951055787504,
0.0002510163467377424,
0.00038275422411970794,
0.9990704655647278
],
"base_choice_token_mass": 0.39342185854911804,
"base_seconds": 7.677153199998429
},
{
"id": "seceval_200:56cc3621-aafb-47cc-a864-e3d53f8b08a7",
"answer_letter": "A",
"topics": [
"WebSecurity",
"PenTest"
],
"prompt_tokens": 169,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.994469165802002,
0.004193178843706846,
0.0012788473395630717,
5.8885176258627325e-05
],
"base_choice_token_mass": 0.15659213066101074,
"base_seconds": 7.339900299994042,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9993106126785278,
0.0006461753509938717,
3.6740639188792557e-05,
6.53597135169548e-06
],
"adapter_choice_token_mass": 0.9884382486343384,
"adapter_seconds": 7.358022700005677
},
{
"id": "seceval_200:dcabb8e6-1568-4a4a-ba81-45e68dd35d33",
"answer_letter": "D",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 200,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.02125665545463562,
6.16008328506723e-05,
0.00138029211666435,
0.9773014187812805
],
"adapter_choice_token_mass": 0.9351297616958618,
"adapter_seconds": 8.281989900016924,
"base_prediction": "D",
"base_choice_probabilities": [
0.005976399406790733,
8.864468691172078e-05,
0.004441282711923122,
0.9894936084747314
],
"base_choice_token_mass": 0.09157034009695053,
"base_seconds": 8.378174300014507
},
{
"id": "seceval_200:34ea72d4-4a71-4096-9fd8-6807cfabf3b4",
"answer_letter": "D",
"topics": [
"NetworkSecurity",
"SystemSecurity",
"PenTest"
],
"prompt_tokens": 151,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0003740279353223741,
4.187664217170095e-06,
0.00017667829524725676,
0.9994451403617859
],
"base_choice_token_mass": 0.43551403284072876,
"base_seconds": 7.645805799984373,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0004582976398523897,
5.377407433115877e-06,
2.795633554342203e-05,
0.9995083808898926
],
"adapter_choice_token_mass": 0.9846222400665283,
"adapter_seconds": 7.348778299987316
},
{
"id": "seceval_200:2e4273a6-0178-4c96-a29c-de5954857e3f",
"answer_letter": "C",
"topics": [
"PenTest",
"NetworkSecurity"
],
"prompt_tokens": 118,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0002842290559783578,
0.0008033920894376934,
0.998333752155304,
0.0005786615656688809
],
"adapter_choice_token_mass": 0.9882327914237976,
"adapter_seconds": 6.398428199987393,
"base_prediction": "C",
"base_choice_probabilities": [
0.0016174949705600739,
0.002390502719208598,
0.9950108528137207,
0.0009810602059587836
],
"base_choice_token_mass": 0.11774156987667084,
"base_seconds": 6.315662100008922
},
{
"id": "seceval_200:4d43f415-22c1-48f4-9d0c-7e4c41f3e014",
"answer_letter": "D",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.14328227937221527,
0.0043948860839009285,
0.0016167882131412625,
0.8507060408592224
],
"base_choice_token_mass": 0.3086961507797241,
"base_seconds": 7.52811769998516,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.037219833582639694,
0.002613242482766509,
0.0002547345357015729,
0.9599121809005737
],
"adapter_choice_token_mass": 0.9873340725898743,
"adapter_seconds": 7.507408600009512
},
{
"id": "seceval_200:c03adffe-3d27-4939-8a7e-b164c705f9ea",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"Vulnerability",
"PenTest"
],
"prompt_tokens": 121,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00658672908321023,
0.00028049678076058626,
0.9929515719413757,
0.00018110232485923916
],
"adapter_choice_token_mass": 0.9859527945518494,
"adapter_seconds": 7.246752500010189,
"base_prediction": "C",
"base_choice_probabilities": [
0.01262310054153204,
8.373514720005915e-05,
0.9872304797172546,
6.271472375374287e-05
],
"base_choice_token_mass": 0.16573625802993774,
"base_seconds": 6.453801899973769
},
{
"id": "seceval_200:31030e6c-998e-4e75-98db-56b1b8956a5d",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 124,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.7825506329536438,
0.15170437097549438,
0.034924328327178955,
0.030820610001683235
],
"base_choice_token_mass": 0.080006442964077,
"base_seconds": 7.314175000006799,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8171826601028442,
0.1442413032054901,
0.02429419942200184,
0.014281824231147766
],
"adapter_choice_token_mass": 0.9366166591644287,
"adapter_seconds": 6.341958200006047
},
{
"id": "seceval_200:8250f916-c99c-4ab2-8ad6-418303077550",
"answer_letter": "D",
"topics": [
"WebSecurity",
"Vulnerability"
],
"prompt_tokens": 144,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.001000386429950595,
9.089491504710168e-05,
2.7079217034042813e-05,
0.9988816380500793
],
"adapter_choice_token_mass": 0.9550533294677734,
"adapter_seconds": 7.824715599999763,
"base_prediction": "D",
"base_choice_probabilities": [
0.00034592344309203327,
4.830173202208243e-05,
0.0001487798144808039,
0.999457061290741
],
"base_choice_token_mass": 0.16000166535377502,
"base_seconds": 7.962258399988059
},
{
"id": "seceval_200:75040e11-74b6-4665-8350-ba02a0ba5199",
"answer_letter": "B",
"topics": [
"ApplicationSecurity",
"WebSecurity"
],
"prompt_tokens": 216,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.667803168296814,
0.3305865228176117,
0.0005332211148925126,
0.0010771362576633692
],
"base_choice_token_mass": 0.10781311988830566,
"base_seconds": 8.866227199992863,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8280457854270935,
0.1708769053220749,
0.00015221000649034977,
0.0009251433657482266
],
"adapter_choice_token_mass": 0.8950735330581665,
"adapter_seconds": 9.659759400004987
},
{
"id": "seceval_200:2cb27832-f74a-4625-9f2c-31bb9a7b5504",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 129,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0014529199106618762,
0.0003760672698263079,
0.9970742464065552,
0.0010967213893309236
],
"adapter_choice_token_mass": 0.9672331809997559,
"adapter_seconds": 6.702177400002256,
"base_prediction": "C",
"base_choice_probabilities": [
0.005128714721649885,
0.00048078628606162965,
0.9927512407302856,
0.0016392356483265758
],
"base_choice_token_mass": 0.18027150630950928,
"base_seconds": 6.74035229999572
},
{
"id": "seceval_200:f725cb3b-8b30-45aa-bca1-681980dd839d",
"answer_letter": "A",
"topics": [
"PenTest"
],
"prompt_tokens": 132,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9878631830215454,
0.003193665063008666,
0.006975606083869934,
0.0019675600342452526
],
"base_choice_token_mass": 0.0492437407374382,
"base_seconds": 7.8217164000088815,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9982529282569885,
0.0010314923711121082,
0.0005186669295653701,
0.00019686378072947264
],
"adapter_choice_token_mass": 0.9305073618888855,
"adapter_seconds": 7.567528700019466
},
{
"id": "seceval_200:9edf17f6-1347-4c8e-8ede-5152ae97a410",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 180,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0045050666667521,
0.988143265247345,
0.007312444970011711,
3.9282160287257284e-05
],
"adapter_choice_token_mass": 0.8608823418617249,
"adapter_seconds": 7.91005760000553,
"base_prediction": "B",
"base_choice_probabilities": [
0.012737344950437546,
0.9358106851577759,
0.05117049813270569,
0.0002814045874401927
],
"base_choice_token_mass": 0.033993106335401535,
"base_seconds": 8.041136299987556
},
{
"id": "seceval_200:d7764627-cf8e-406b-8c06-5fc6955e0c04",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"PenTest"
],
"prompt_tokens": 149,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.002115397248417139,
8.511114174325485e-06,
0.9977399110794067,
0.0001362934708595276
],
"base_choice_token_mass": 0.21519246697425842,
"base_seconds": 8.4694438999868,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0005357471527531743,
2.2637319489149377e-05,
0.9994019269943237,
3.972973354393616e-05
],
"adapter_choice_token_mass": 0.9895058870315552,
"adapter_seconds": 7.407846999994945
},
{
"id": "seceval_200:ea569ad5-39e2-4506-8baa-58c16e265e72",
"answer_letter": "C",
"topics": [
"Unspecified"
],
"prompt_tokens": 116,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0004371801041997969,
0.0002130645589204505,
0.999210000038147,
0.00013973115710541606
],
"adapter_choice_token_mass": 0.9804113507270813,
"adapter_seconds": 7.8672252999967895,
"base_prediction": "C",
"base_choice_probabilities": [
0.001131535042077303,
0.00017352652503177524,
0.9970741271972656,
0.0016208491288125515
],
"base_choice_token_mass": 0.5859129428863525,
"base_seconds": 7.003865900012897
},
{
"id": "seceval_200:f9c70ae5-becb-4539-bcc6-c41bbc18d6e5",
"answer_letter": "B",
"topics": [
"WebSecurity"
],
"prompt_tokens": 149,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.021171821281313896,
0.002129306085407734,
0.9734009504318237,
0.0032979336101561785
],
"base_choice_token_mass": 0.08531550318002701,
"base_seconds": 7.6503648000070825,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.006523707415908575,
0.006422566715627909,
0.9834510087966919,
0.0036027333699166775
],
"adapter_choice_token_mass": 0.9502255320549011,
"adapter_seconds": 7.581351900007576
},
{
"id": "seceval_200:759c61b5-b5e3-44eb-8990-54a8ef08a949",
"answer_letter": "B",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 105,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
9.451770893065259e-05,
0.9531591534614563,
0.046719279140233994,
2.70797754637897e-05
],
"adapter_choice_token_mass": 0.9423795342445374,
"adapter_seconds": 6.005808899994008,
"base_prediction": "B",
"base_choice_probabilities": [
8.507537859259173e-05,
0.9204335808753967,
0.07917968928813934,
0.000301618390949443
],
"base_choice_token_mass": 0.012223510071635246,
"base_seconds": 6.075049699982628
},
{
"id": "seceval_200:096e7d32-54e4-4460-a4b9-8cd1d67bf990",
"answer_letter": "A",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 155,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9999079704284668,
2.264878094138112e-05,
5.390878868638538e-05,
1.544512815598864e-05
],
"base_choice_token_mass": 0.47721049189567566,
"base_seconds": 7.752161799988244,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999728202819824,
1.8342730982112698e-05,
5.908670573262498e-06,
2.879656221921323e-06
],
"adapter_choice_token_mass": 0.9822419285774231,
"adapter_seconds": 7.619580099999439
},
{
"id": "seceval_200:99d6c43e-098e-4a31-81b3-dccaa953a708",
"answer_letter": "A",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 155,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9857181310653687,
0.007181407883763313,
0.004288214258849621,
0.0028122796211391687
],
"adapter_choice_token_mass": 0.9812551140785217,
"adapter_seconds": 7.64943009999115,
"base_prediction": "A",
"base_choice_probabilities": [
0.8443208336830139,
0.03430899605154991,
0.077316515147686,
0.04405362531542778
],
"base_choice_token_mass": 0.1126912459731102,
"base_seconds": 7.750500699999975
},
{
"id": "seceval_200:db7eb7d2-91c0-4f79-8d52-61bcf7030e12",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 140,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.00018954603001475334,
0.9994339346885681,
0.0003028936916962266,
7.364964403677732e-05
],
"base_choice_token_mass": 0.06160101294517517,
"base_seconds": 7.173257199989166,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0003199804632458836,
0.999624490737915,
3.9122493035392836e-05,
1.630868064239621e-05
],
"adapter_choice_token_mass": 0.9739030599594116,
"adapter_seconds": 7.301168799982406
},
{
"id": "seceval_200:c79ab447-186f-453e-bcc4-37486a93fe92",
"answer_letter": "D",
"topics": [
"SystemSecurity"
],
"prompt_tokens": 140,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0013035372830927372,
0.39080414175987244,
0.011987105943262577,
0.5959051251411438
],
"adapter_choice_token_mass": 0.9101682901382446,
"adapter_seconds": 7.32169910002267,
"base_prediction": "D",
"base_choice_probabilities": [
0.004858961794525385,
0.3741193115711212,
0.023180808871984482,
0.5978409647941589
],
"base_choice_token_mass": 0.06845585256814957,
"base_seconds": 7.176336400007131
},
{
"id": "seceval_200:b2e3ad40-91da-4903-aa8c-4a54d26e7ed0",
"answer_letter": "A",
"topics": [
"Unspecified"
],
"prompt_tokens": 130,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.010593120940029621,
0.0026575173251330853,
0.9461417198181152,
0.04060758650302887
],
"base_choice_token_mass": 0.003192039905115962,
"base_seconds": 6.943262400018284,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0035670369397848845,
0.0014188692439347506,
0.9890404343605042,
0.0059736622497439384
],
"adapter_choice_token_mass": 0.8430084586143494,
"adapter_seconds": 7.137999099999433
},
{
"id": "seceval_200:19c9e763-ce52-4980-8bf4-a548ae8c9192",
"answer_letter": "D",
"topics": [
"WebSecurity",
"Vulnerability"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00029324745992198586,
0.00912266131490469,
3.290145468781702e-05,
0.990551233291626
],
"adapter_choice_token_mass": 0.9706040620803833,
"adapter_seconds": 7.448562400008086,
"base_prediction": "D",
"base_choice_probabilities": [
0.00019562823581509292,
0.0076931919902563095,
0.00012338117812760174,
0.9919878244400024
],
"base_choice_token_mass": 0.16186393797397614,
"base_seconds": 7.585138700000243
},
{
"id": "seceval_200:9cbbdc44-9115-48a4-a8b8-6abd14759a04",
"answer_letter": "B",
"topics": [
"ApplicationSecurity",
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 131,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.0025892730336636305,
0.9967522621154785,
0.00064452103106305,
1.4018564797879662e-05
],
"base_choice_token_mass": 0.23943166434764862,
"base_seconds": 7.088148500013631,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0009396013338118792,
0.9986960291862488,
0.00035663245944306254,
7.696519787714351e-06
],
"adapter_choice_token_mass": 0.9740164279937744,
"adapter_seconds": 7.041286299994681
},
{
"id": "seceval_200:2e5dcce8-94fc-4524-a0eb-a69787b77cfc",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 144,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.1616598218679428,
0.0022003562189638615,
0.8339048027992249,
0.0022350067738443613
],
"adapter_choice_token_mass": 0.9650550484657288,
"adapter_seconds": 7.328910699987318,
"base_prediction": "C",
"base_choice_probabilities": [
0.12554019689559937,
0.0003498724545352161,
0.8714216351509094,
0.0026882111560553312
],
"base_choice_token_mass": 0.14791075885295868,
"base_seconds": 7.403253799973754
},
{
"id": "seceval_200:451e0b47-acbf-4ae8-8272-9be2f70ed2e1",
"answer_letter": "D",
"topics": [
"PenTest"
],
"prompt_tokens": 142,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.008427645079791546,
0.00018912300583906472,
0.001940153888426721,
0.9894431829452515
],
"base_choice_token_mass": 0.19017164409160614,
"base_seconds": 7.424585100001423,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0018653874285519123,
0.0005514074582606554,
0.0006151394918560982,
0.9969680905342102
],
"adapter_choice_token_mass": 0.9849039912223816,
"adapter_seconds": 7.108259700005874
},
{
"id": "seceval_200:f725c3e7-99fe-4eb2-b589-97d7c43de1b4",
"answer_letter": "B",
"topics": [
"SoftwareSecurity",
"SystemSecurity",
"Vulnerability"
],
"prompt_tokens": 151,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.6436144113540649,
0.3499270975589752,
0.003948555793613195,
0.002509854733943939
],
"adapter_choice_token_mass": 0.81431645154953,
"adapter_seconds": 7.280444800009718,
"base_prediction": "A",
"base_choice_probabilities": [
0.7947272658348083,
0.17732761800289154,
0.015494714491069317,
0.012450353242456913
],
"base_choice_token_mass": 0.02934880554676056,
"base_seconds": 7.342537099990295
},
{
"id": "seceval_200:5a1f53b4-4828-4f77-aee3-6310ceb717b2",
"answer_letter": "B",
"topics": [
"SystemSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 160,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.00042908851173706353,
0.9961600303649902,
0.0032204880844801664,
0.00019040687766391784
],
"base_choice_token_mass": 0.23747868835926056,
"base_seconds": 7.526094100001501,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00024336650676559657,
0.9993707537651062,
0.00031989923445507884,
6.601493805646896e-05
],
"adapter_choice_token_mass": 0.9809033274650574,
"adapter_seconds": 7.68522430001758
},
{
"id": "seceval_200:502af749-bc98-421e-88f7-81503a2410f8",
"answer_letter": "B",
"topics": [
"Unspecified"
],
"prompt_tokens": 128,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0005787737318314612,
0.9985271692276001,
0.0007202955894172192,
0.0001737794082146138
],
"adapter_choice_token_mass": 0.9918413758277893,
"adapter_seconds": 6.380524000007426,
"base_prediction": "B",
"base_choice_probabilities": [
0.0007511369185522199,
0.9935979843139648,
0.005296031013131142,
0.0003548119857441634
],
"base_choice_token_mass": 0.2717881202697754,
"base_seconds": 6.4101808999839704
},
{
"id": "seceval_200:97b6b195-1575-42ed-b774-487e3d4cf290",
"answer_letter": "C",
"topics": [
"WebSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 122,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.0075749862007796764,
5.0248556362930685e-05,
0.9921274185180664,
0.00024733206373639405
],
"base_choice_token_mass": 0.3781675696372986,
"base_seconds": 6.281558599992422,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.030211426317691803,
3.428560376050882e-05,
0.9696839451789856,
7.034948794171214e-05
],
"adapter_choice_token_mass": 0.9780542850494385,
"adapter_seconds": 6.514287199999671
},
{
"id": "seceval_200:cfd5b92e-3b75-4b3f-bb89-574fa8644611",
"answer_letter": "A",
"topics": [
"ApplicationSecurity"
],
"prompt_tokens": 128,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.8757056593894958,
3.764114808291197e-05,
5.519709884538315e-05,
0.12420147657394409
],
"adapter_choice_token_mass": 0.9250693321228027,
"adapter_seconds": 6.4765008999966085,
"base_prediction": "A",
"base_choice_probabilities": [
0.885195255279541,
6.372016650857404e-05,
0.0004286972980480641,
0.11431219428777695
],
"base_choice_token_mass": 0.04011957719922066,
"base_seconds": 6.476490800007014
},
{
"id": "seceval_200:6f705495-73e4-45ea-ad41-1aa86f6592ab",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 108,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.00043013240792788565,
1.707355295366142e-05,
0.9985835552215576,
0.0009693183819763362
],
"base_choice_token_mass": 0.3168344497680664,
"base_seconds": 6.086877900001127,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00032992122578434646,
0.0001312335953116417,
0.9989688396453857,
0.00057005271082744
],
"adapter_choice_token_mass": 0.9947438836097717,
"adapter_seconds": 5.9955288999772165
},
{
"id": "seceval_200:90f2b0f4-d6ab-4284-8876-d95630371162",
"answer_letter": "B",
"topics": [
"PenTest"
],
"prompt_tokens": 98,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.18626521527767181,
0.7600783705711365,
0.04156138747930527,
0.012095053680241108
],
"adapter_choice_token_mass": 0.962968111038208,
"adapter_seconds": 5.900176299997838,
"base_prediction": "B",
"base_choice_probabilities": [
0.1645432859659195,
0.6820129752159119,
0.12615971267223358,
0.02728394977748394
],
"base_choice_token_mass": 0.14270254969596863,
"base_seconds": 6.721592599991709
},
{
"id": "seceval_200:dda9364e-c19c-4674-991c-14b089338275",
"answer_letter": "C",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 151,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.0005976217216812074,
1.1561093742784578e-05,
0.9993226528167725,
6.810719787608832e-05
],
"base_choice_token_mass": 0.6883230209350586,
"base_seconds": 8.18747670002631,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0005883504054509103,
3.3192460250575095e-05,
0.9993124008178711,
6.601108179893345e-05
],
"adapter_choice_token_mass": 0.9955296516418457,
"adapter_seconds": 7.47376580000855
},
{
"id": "seceval_200:bac3e3e0-a11e-403c-8dfe-8a6efd9b1d17",
"answer_letter": "C",
"topics": [
"SystemSecurity",
"WebSecurity"
],
"prompt_tokens": 112,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00020961774862371385,
0.00036789060686714947,
0.9985260963439941,
0.0008964213775470853
],
"adapter_choice_token_mass": 0.9876938462257385,
"adapter_seconds": 6.0231471999723,
"base_prediction": "C",
"base_choice_probabilities": [
0.0003946899378206581,
6.100238169892691e-05,
0.9985284805297852,
0.001015780959278345
],
"base_choice_token_mass": 0.541433572769165,
"base_seconds": 6.01406269997824
},
{
"id": "seceval_200:e39f13f5-a2c8-4a10-aa31-b81fc0cd0f38",
"answer_letter": "A",
"topics": [
"Unspecified"
],
"prompt_tokens": 158,
"suite": "seceval_200",
"base_prediction": "A",
"base_choice_probabilities": [
0.9044545292854309,
0.014392497949302197,
0.0790303573012352,
0.0021226059179753065
],
"base_choice_token_mass": 0.14329057931900024,
"base_seconds": 7.496931900008349,
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9936880469322205,
0.00482252798974514,
0.0011454508639872074,
0.00034392671659588814
],
"adapter_choice_token_mass": 0.9879840016365051,
"adapter_seconds": 7.072288399998797
},
{
"id": "seceval_200:179996e9-39cc-4221-9ee7-9c3ecb78a77c",
"answer_letter": "C",
"topics": [
"WebSecurity"
],
"prompt_tokens": 156,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.00026105414144694805,
5.601756492978893e-05,
0.9992175102233887,
0.00046537930029444396
],
"adapter_choice_token_mass": 0.9683001041412354,
"adapter_seconds": 8.36092100001406,
"base_prediction": "C",
"base_choice_probabilities": [
0.000882804743014276,
3.90919522033073e-05,
0.9988441467285156,
0.00023391997092403471
],
"base_choice_token_mass": 0.16359056532382965,
"base_seconds": 7.584910599980503
},
{
"id": "seceval_200:94c13d6d-3918-4e23-afe9-c60e6a04acaf",
"answer_letter": "C",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 198,
"suite": "seceval_200",
"base_prediction": "C",
"base_choice_probabilities": [
0.0019263396970927715,
2.5812967578531243e-05,
0.9978686571121216,
0.0001791774993762374
],
"base_choice_token_mass": 0.0654556006193161,
"base_seconds": 7.7218585000082385,
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0014323851792141795,
3.214385287719779e-05,
0.9984619617462158,
7.35780195100233e-05
],
"adapter_choice_token_mass": 0.9480459094047546,
"adapter_seconds": 7.556377300003078
},
{
"id": "seceval_200:28307634-cb9d-4021-8918-0fa8eaa52f80",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 97,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.001497961115092039,
0.0005510690971277654,
0.9963563680648804,
0.0015945712802931666
],
"adapter_choice_token_mass": 0.9881272315979004,
"adapter_seconds": 5.775181800010614,
"base_prediction": "C",
"base_choice_probabilities": [
0.002246810356155038,
0.0002540622663218528,
0.9955163598060608,
0.001982803223654628
],
"base_choice_token_mass": 0.16184604167938232,
"base_seconds": 5.806275899987668
},
{
"id": "seceval_200:a1a3d520-8217-43af-adf6-a67c97ff04c2",
"answer_letter": "B",
"topics": [
"PenTest"
],
"prompt_tokens": 154,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.019389908760786057,
0.9790940880775452,
0.0014491870533674955,
6.672855670331046e-05
],
"base_choice_token_mass": 0.11989424377679825,
"base_seconds": 7.263193999999203,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.0009848934132605791,
0.9988985061645508,
0.00010878880857490003,
7.75845728639979e-06
],
"adapter_choice_token_mass": 0.9910377264022827,
"adapter_seconds": 7.441917899996042
},
{
"id": "seceval_200:6a83b92a-6fbc-49be-8a66-ad438bd79733",
"answer_letter": "C",
"topics": [
"SoftwareSecurity",
"Vulnerability"
],
"prompt_tokens": 154,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.005356478039175272,
0.0018803005805239081,
0.9893584251403809,
0.0034047847148030996
],
"adapter_choice_token_mass": 0.9579943418502808,
"adapter_seconds": 7.563793900015298,
"base_prediction": "C",
"base_choice_probabilities": [
0.0038073239848017693,
0.00043389684287831187,
0.9917058944702148,
0.004052875563502312
],
"base_choice_token_mass": 0.08194202184677124,
"base_seconds": 7.514028000005055
},
{
"id": "seceval_200:ee7713cb-f0c3-4e77-a625-8f28a96ebc17",
"answer_letter": "D",
"topics": [
"SystemSecurity",
"ApplicationSecurity"
],
"prompt_tokens": 134,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.004903774242848158,
8.842330134939402e-05,
0.0002441445249132812,
0.9947636127471924
],
"base_choice_token_mass": 0.19644613564014435,
"base_seconds": 7.213480100006564,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.03619705140590668,
0.00041813126881606877,
0.00021692365407943726,
0.9631677865982056
],
"adapter_choice_token_mass": 0.9040538668632507,
"adapter_seconds": 7.003278599993791
},
{
"id": "seceval_200:0fca90cc-51a1-4111-a112-2a37b63e168c",
"answer_letter": "C",
"topics": [
"WebSecurity"
],
"prompt_tokens": 145,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.0010476420866325498,
0.0005026641301810741,
0.9981634020805359,
0.00028640939854085445
],
"adapter_choice_token_mass": 0.9725164771080017,
"adapter_seconds": 7.0634388999897055,
"base_prediction": "C",
"base_choice_probabilities": [
0.0006763790734112263,
0.0004948495188727975,
0.9981198906898499,
0.0007088392158038914
],
"base_choice_token_mass": 0.20121636986732483,
"base_seconds": 7.2109872000000905
},
{
"id": "seceval_200:bfda1ac7-8ea2-419a-a993-8a559eb5cae3",
"answer_letter": "B",
"topics": [
"ApplicationSecurity",
"SystemSecurity"
],
"prompt_tokens": 121,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.1241338849067688,
0.8752291202545166,
0.0006024417234584689,
3.452266173553653e-05
],
"base_choice_token_mass": 0.1990060806274414,
"base_seconds": 6.281870200007688,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.060085151344537735,
0.9398899674415588,
2.047373891400639e-05,
4.532757884589955e-06
],
"adapter_choice_token_mass": 0.9621034264564514,
"adapter_seconds": 7.204165400005877
},
{
"id": "seceval_200:46605378-381d-478f-9f77-cd66355ecded",
"answer_letter": "B",
"topics": [
"NetworkSecurity"
],
"prompt_tokens": 139,
"suite": "seceval_200",
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.00011772828293032944,
0.9997437596321106,
1.3105955986247864e-05,
0.00012532110849861056
],
"adapter_choice_token_mass": 0.9712522029876709,
"adapter_seconds": 7.088246600003913,
"base_prediction": "B",
"base_choice_probabilities": [
0.0014198996359482408,
0.9975214600563049,
0.00016308620979543775,
0.0008955194498412311
],
"base_choice_token_mass": 0.10339512676000595,
"base_seconds": 7.250452600012068
},
{
"id": "seceval_200:e6d248a2-00f2-4cae-8a2e-e5c5666deab3",
"answer_letter": "C",
"topics": [
"PenTest"
],
"prompt_tokens": 106,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.10268999636173248,
0.6192998886108398,
0.24252131581306458,
0.035488713532686234
],
"base_choice_token_mass": 0.24918189644813538,
"base_seconds": 5.9919071000185795,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.19713230431079865,
0.6463719010353088,
0.1253049075603485,
0.031190859153866768
],
"adapter_choice_token_mass": 0.9418566226959229,
"adapter_seconds": 6.003425500006415
},
{
"id": "seceval_200:21ddfd4a-6da1-4361-b983-4e30e68cfae0",
"answer_letter": "C",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 176,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.05814556032419205,
0.0026773419231176376,
0.9384218454360962,
0.0007551790331490338
],
"adapter_choice_token_mass": 0.9573130011558533,
"adapter_seconds": 8.29752299998654,
"base_prediction": "C",
"base_choice_probabilities": [
0.026721162721514702,
0.0006955991848371923,
0.9718542695045471,
0.0007289817440323532
],
"base_choice_token_mass": 0.08955888450145721,
"base_seconds": 9.206401500006905
},
{
"id": "seceval_200:05914d87-9141-4505-8bdc-273bf001948d",
"answer_letter": "B",
"topics": [
"SystemSecurity"
],
"prompt_tokens": 110,
"suite": "seceval_200",
"base_prediction": "B",
"base_choice_probabilities": [
0.03853354603052139,
0.6027660965919495,
0.26332899928092957,
0.09537144005298615
],
"base_choice_token_mass": 0.1296212524175644,
"base_seconds": 6.087040999991586,
"adapter_prediction": "B",
"adapter_choice_probabilities": [
0.04260627180337906,
0.8968391418457031,
0.038793452084064484,
0.021761154755949974
],
"adapter_choice_token_mass": 0.9897192120552063,
"adapter_seconds": 6.074671300011687
},
{
"id": "seceval_200:445f2a45-ce45-4c8e-9a97-0abc80aa58dd",
"answer_letter": "B",
"topics": [
"WebSecurity",
"MemorySafety"
],
"prompt_tokens": 87,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.08779919892549515,
0.25604796409606934,
0.5253763198852539,
0.130776509642601
],
"adapter_choice_token_mass": 0.038486555218696594,
"adapter_seconds": 6.298932999983663,
"base_prediction": "C",
"base_choice_probabilities": [
0.19306573271751404,
0.10334064066410065,
0.6844774484634399,
0.019116096198558807
],
"base_choice_token_mass": 0.000660346238873899,
"base_seconds": 6.438448500004597
},
{
"id": "seceval_200:bb5ac0b2-b3d2-4024-a045-403fc90cd9cc",
"answer_letter": "D",
"topics": [
"ApplicationSecurity",
"WebSecurity"
],
"prompt_tokens": 124,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
7.310188811970875e-05,
1.6439167666248977e-05,
0.00012930405500810593,
0.9997811913490295
],
"base_choice_token_mass": 0.2533020079135895,
"base_seconds": 7.415867800009437,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.00010889161785598844,
2.627132744237315e-05,
2.2296186216408387e-05,
0.9998425245285034
],
"adapter_choice_token_mass": 0.9612504839897156,
"adapter_seconds": 6.490303099999437
},
{
"id": "seceval_200:59e85d9a-477e-4eb6-b184-e6f73408e807",
"answer_letter": "C",
"topics": [
"NetworkSecurity",
"PenTest"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"adapter_prediction": "C",
"adapter_choice_probabilities": [
0.004844231531023979,
0.0019269020995125175,
0.9231470823287964,
0.07008183002471924
],
"adapter_choice_token_mass": 0.9694846868515015,
"adapter_seconds": 7.530378399998881,
"base_prediction": "C",
"base_choice_probabilities": [
0.0039606778882443905,
0.0005236026481725276,
0.9844077229499817,
0.011107996106147766
],
"base_choice_token_mass": 0.2367577701807022,
"base_seconds": 7.393246199993882
},
{
"id": "seceval_200:2df4501f-b222-4cee-95db-0abfa5fc6567",
"answer_letter": "D",
"topics": [
"PenTest",
"SystemSecurity"
],
"prompt_tokens": 116,
"suite": "seceval_200",
"base_prediction": "D",
"base_choice_probabilities": [
0.0011334187583997846,
5.331426109478343e-06,
0.00012716605851892382,
0.9987340569496155
],
"base_choice_token_mass": 0.3162488043308258,
"base_seconds": 6.24548430001596,
"adapter_prediction": "D",
"adapter_choice_probabilities": [
0.0005112296785227954,
2.409806074865628e-05,
3.118520908174105e-05,
0.9994334578514099
],
"adapter_choice_token_mass": 0.9887437224388123,
"adapter_seconds": 7.026152899983572
},
{
"id": "seceval_200:aeb8ff51-2c9f-47cd-8793-815b330a8e2c",
"answer_letter": "A",
"topics": [
"SoftwareSecurity",
"ApplicationSecurity",
"Vulnerability"
],
"prompt_tokens": 153,
"suite": "seceval_200",
"adapter_prediction": "A",
"adapter_choice_probabilities": [
0.9999027252197266,
7.48445454519242e-05,
1.657011489442084e-05,
5.8166574490314815e-06
],
"adapter_choice_token_mass": 0.9752302765846252,
"adapter_seconds": 8.224242099997355,
"base_prediction": "A",
"base_choice_probabilities": [
0.999450147151947,
0.00012430932838469744,
0.000395054230466485,
3.0463310395134613e-05
],
"base_choice_token_mass": 0.06542652100324631,
"base_seconds": 7.464495899999747
}
]
}
{
"status": "completed",
"fingerprint": {
"suite_sha256": "02eee659949261311227ff35f2dca6f784b9ed4a73981554e87cd004005211b1",
"base_model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
"base_revision": "1df8507178afcc1bef68cd8c393f61a886323761",
"adapter_sha256": "b28f7107ec6c5fb05d105c0d6372f6c95b0e29aece2ae7a8188e6dfe72563ea5",
"adapter_config_sha256": "73e8430f28b5ed39b11c41e13169c2825f2f4ffbf4895f18bd4f21f6619a8c02",
"scorer_sha256": "ff9247eb8e1741ac3cb0a8a37cb813257869d7c93a9585d04bddb518c0d4869a",
"metrics_sha256": "ea012aa1520f35c8d20685953b792b80e8c78ac2f677162ae80386eb977dd5c6",
"formatting_sha256": "851524de30bf094997673594ed0bb9f02bcb9a8d930e0f92b8fd58bd714755ed",
"protocol": "balanced-letter-ranking-v1",
"quantization": "NF4 double quantization, FP16 compute",
"packages": {
"torch": "2.13.0+rocm10.0.0",
"transformers": "5.22.2",
"peft": "0.21.2",
"bitsandbytes": "0.50.2"
},
"gpu": "AMD Radeon RX 6800 XT"
},
"started_utc": "2026-11-09T17:54:28.512942+00:00",
"completed_utc": "2026-11-09T19:37:48.249156+00:00",
"total_questions": 376,
"method": "Argmax of next-token A/B/C/D logits after a closed reasoning prefix; no generated reasoning. Paired base and adapter in the same NF4 runtime.",
"comparison": "Adapted subsets, not official SecEval leaderboard scores or generated-answer pass@1.",
"suites": {
"heldout_pentesting": {
"overall": {
"base": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"adapter": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"delta_percentage_points": 0.0,
"corrected": 2,
"regressed": 2,
"both_correct": 150,
"both_wrong": 22,
"correct_answer_labels": {
"A": 44,
"D": 44,
"C": 44,
"B": 44
},
"base_prediction_labels": {
"A": 55,
"D": 38,
"B": 39,
"C": 44
},
"adapter_prediction_labels": {
"A": 57,
"D": 37,
"B": 42,
"C": 40
}
},
"by_topic": {
"Pentesting": {
"base": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"adapter": {
"correct": 152,
"total": 176,
"accuracy": 0.8636363636363636,
"wilson_95": [
0.8051156864385216,
0.9066223123486148
]
},
"delta_percentage_points": 0.0,
"corrected": 2,
"regressed": 2,
"both_correct": 150,
"both_wrong": 22,
"correct_answer_labels": {
"A": 44,
"D": 44,
"C": 44,