| { |
| "status": "integration_smoke_only", |
| "trained": false, |
| "calibration_validated": false, |
| "model": "models/gemma-4-e2b-it-4bit", |
| "load_seconds": 3.297944833000656, |
| "results": [ |
| { |
| "modality": "text", |
| "question": "category", |
| "expected": "cat", |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "choice", |
| "choice": "cat", |
| "probabilities": { |
| "cat": 0.9999999980978347, |
| "dog": 1.6052280521322024e-09, |
| "other": 2.9693733225389006e-10 |
| }, |
| "selected_probability": 0.9999999980978347, |
| "confidence": 0.9999999627511029, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "cat": 10.8125, |
| "dog": -9.4375, |
| "other": -11.125 |
| }, |
| "allowed_token_mass": 0.9996110200881958, |
| "input_tokens": 139 |
| } |
| }, |
| "actual": "cat", |
| "correct": true, |
| "seconds": 0.24791974999243394 |
| }, |
| { |
| "modality": "text", |
| "question": "dog", |
| "expected": false, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "noul", |
| "noul": 2.0176444634455703e-08, |
| "diagnostics": { |
| "logits": { |
| "true": -4.90625, |
| "false": 12.8125 |
| }, |
| "allowed_token_mass": 0.9999866485595703, |
| "input_tokens": 126 |
| } |
| }, |
| "actual": false, |
| "correct": true, |
| "seconds": 0.09814774998812936 |
| }, |
| { |
| "modality": "text", |
| "question": "grade", |
| "expected": 1, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "score", |
| "score": 0.0001815971183056414, |
| "legend": { |
| "0": "No animals are present", |
| "1": "One kind of animal is present", |
| "2": "Two or more kinds of animals are present" |
| }, |
| "probabilities": { |
| "0": 0.9998194375490497, |
| "1": 0.00017952778359504813, |
| "2": 1.0346673552966412e-06 |
| }, |
| "confidence": 0.9984132119080671, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "0": 16.375, |
| "1": 7.75, |
| "2": 2.59375 |
| }, |
| "allowed_token_mass": 0.9999465346336365, |
| "input_tokens": 152 |
| } |
| }, |
| "actual": 0, |
| "correct": false, |
| "seconds": 0.10193237499333918 |
| }, |
| { |
| "modality": "image", |
| "question": "category", |
| "expected": "red", |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "choice", |
| "choice": "red", |
| "probabilities": { |
| "red": 0.9999999999997546, |
| "green": 3.4424771084691324e-14, |
| "blue": 2.1087728885901501e-13 |
| }, |
| "selected_probability": 0.9999999999997546, |
| "confidence": 0.9999999999932028, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "red": 14.25, |
| "green": -16.75, |
| "blue": -14.9375 |
| }, |
| "allowed_token_mass": 0.9999980330467224, |
| "input_tokens": 380 |
| } |
| }, |
| "actual": "red", |
| "correct": true, |
| "seconds": 1.7971077499969397 |
| }, |
| { |
| "modality": "image", |
| "question": "red", |
| "expected": true, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "noul", |
| "noul": 0.9999999999821676, |
| "diagnostics": { |
| "logits": { |
| "true": 12.3125, |
| "false": -12.4375 |
| }, |
| "allowed_token_mass": 0.9999980330467224, |
| "input_tokens": 372 |
| } |
| }, |
| "actual": true, |
| "correct": true, |
| "seconds": 0.8673579579917714 |
| }, |
| { |
| "modality": "image", |
| "question": "grade", |
| "expected": 2, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "score", |
| "score": 1.9999580095263223, |
| "legend": { |
| "0": "No red", |
| "1": "Red covers some but not most of the image", |
| "2": "Red covers most or all of the image" |
| }, |
| "probabilities": { |
| "0": 1.4285325940992285e-05, |
| "1": 1.341982179551636e-05, |
| "2": 0.9999722948522634 |
| }, |
| "confidence": 0.9996926760893615, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "0": -3.09375, |
| "1": -3.15625, |
| "2": 8.0625 |
| }, |
| "allowed_token_mass": 0.9998178482055664, |
| "input_tokens": 398 |
| } |
| }, |
| "actual": 2, |
| "correct": true, |
| "seconds": 0.5219861660152674 |
| }, |
| { |
| "modality": "speech", |
| "question": "category", |
| "expected": "dog", |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "choice", |
| "choice": "dog", |
| "probabilities": { |
| "cat": 1.53817296877965e-07, |
| "dog": 0.9999996772474515, |
| "other": 1.6893525148338628e-07 |
| }, |
| "selected_probability": 0.9999996772474515, |
| "confidence": 0.9999951119284041, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "cat": -2.75, |
| "dog": 12.9375, |
| "other": -2.65625 |
| }, |
| "allowed_token_mass": 0.9996881484985352, |
| "input_tokens": 179 |
| } |
| }, |
| "actual": "dog", |
| "correct": true, |
| "seconds": 2.3318575419834815 |
| }, |
| { |
| "modality": "speech", |
| "question": "dog", |
| "expected": true, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "noul", |
| "noul": 0.9999997300422225, |
| "diagnostics": { |
| "logits": { |
| "true": 18.0, |
| "false": 2.875 |
| }, |
| "allowed_token_mass": 0.9999866485595703, |
| "input_tokens": 166 |
| } |
| }, |
| "actual": true, |
| "correct": true, |
| "seconds": 0.21468862501205876 |
| }, |
| { |
| "modality": "speech", |
| "question": "grade", |
| "expected": 1, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "score", |
| "score": 0.9993982862177304, |
| "legend": { |
| "0": "No animals are present", |
| "1": "One kind of animal is present", |
| "2": "Two or more kinds of animals are present" |
| }, |
| "probabilities": { |
| "0": 0.0019241866090907301, |
| "1": 0.9967533405640879, |
| "2": 0.0013224728268212142 |
| }, |
| "confidence": 0.9781183131643107, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "0": 14.875, |
| "1": 21.125, |
| "2": 14.5 |
| }, |
| "allowed_token_mass": 0.999580442905426, |
| "input_tokens": 192 |
| } |
| }, |
| "actual": 1, |
| "correct": true, |
| "seconds": 0.559575000021141 |
| }, |
| { |
| "modality": "video", |
| "question": "category", |
| "expected": "red_blue", |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "choice", |
| "choice": "red_blue", |
| "probabilities": { |
| "red_blue": 0.9999992049220983, |
| "blue_red": 6.893609726439972e-07, |
| "unchanging": 1.0571692910400784e-07 |
| }, |
| "selected_probability": 0.9999992049220983, |
| "confidence": 0.9999888282105366, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "red_blue": 8.0, |
| "blue_red": -6.1875, |
| "unchanging": -8.0625 |
| }, |
| "allowed_token_mass": 0.9999522566795349, |
| "input_tokens": 431 |
| } |
| }, |
| "actual": "red_blue", |
| "correct": true, |
| "seconds": 7.703662458981853 |
| }, |
| { |
| "modality": "video", |
| "question": "both", |
| "expected": true, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "noul", |
| "noul": 0.9982108408384048, |
| "diagnostics": { |
| "logits": { |
| "true": 5.5, |
| "false": -0.82421875 |
| }, |
| "allowed_token_mass": 0.9994622468948364, |
| "input_tokens": 408 |
| } |
| }, |
| "actual": true, |
| "correct": true, |
| "seconds": 0.4792473329871427 |
| }, |
| { |
| "modality": "video", |
| "question": "grade", |
| "expected": 2, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "score", |
| "score": 1.8834621966517164, |
| "legend": { |
| "0": "Neither red nor blue", |
| "1": "Only one of red or blue", |
| "2": "Both red and blue" |
| }, |
| "probabilities": { |
| "0": 0.0005184191907773904, |
| "1": 0.11550096496672892, |
| "2": 0.8839806158424938 |
| }, |
| "confidence": 0.670274557602579, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "0": -6.84375, |
| "1": -1.4375, |
| "2": 0.59765625 |
| }, |
| "allowed_token_mass": 0.9960461258888245, |
| "input_tokens": 429 |
| } |
| }, |
| "actual": 2, |
| "correct": true, |
| "seconds": 0.4473298749944661 |
| }, |
| { |
| "modality": "video_with_speech", |
| "question": "animal", |
| "expected": "dog", |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "choice", |
| "choice": "dog", |
| "probabilities": { |
| "cat": 4.5536814821725125e-11, |
| "dog": 0.9999999998306812, |
| "other": 1.237818962557999e-10 |
| }, |
| "selected_probability": 0.9999999998306812, |
| "confidence": 0.9999999962885554, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "cat": -13.1875, |
| "dog": 10.625, |
| "other": -12.1875 |
| }, |
| "allowed_token_mass": 1.0, |
| "input_tokens": 525 |
| } |
| }, |
| "actual": "dog", |
| "correct": true, |
| "seconds": 1.1345192909939215 |
| }, |
| { |
| "modality": "video_with_speech", |
| "question": "dog", |
| "expected": true, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "noul", |
| "noul": 0.0029810327298548972, |
| "diagnostics": { |
| "logits": { |
| "true": 14.3125, |
| "false": 20.125 |
| }, |
| "allowed_token_mass": 0.9998398423194885, |
| "input_tokens": 509 |
| } |
| }, |
| "actual": false, |
| "correct": false, |
| "seconds": 0.7891140409919899 |
| }, |
| { |
| "modality": "video_with_speech", |
| "question": "grade", |
| "expected": 2, |
| "answer": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_next_token_logits", |
| "type": "score", |
| "score": 1.9770646011535025, |
| "legend": { |
| "0": "Neither red nor blue", |
| "1": "Only one of red or blue", |
| "2": "Both red and blue" |
| }, |
| "probabilities": { |
| "0": 0.00015485148939468261, |
| "1": 0.02262569586770833, |
| "2": 0.9772194526428971 |
| }, |
| "confidence": 0.900238809496441, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "diagnostics": { |
| "logits": { |
| "0": -3.8125, |
| "1": 1.171875, |
| "2": 4.9375 |
| }, |
| "allowed_token_mass": 0.9957465529441833, |
| "input_tokens": 532 |
| } |
| }, |
| "actual": 2, |
| "correct": true, |
| "seconds": 0.8423217090021353 |
| } |
| ], |
| "attempted": 15, |
| "completed": 15, |
| "correct": 13 |
| } |
|
|
