{ "model": "vela", "variant": "3-fast", "scope": "diagnostic", "status": "diagnostic-passed", "source_revision": "fa60e3aa41132c60bf05a84cabfdced39cb6d4f7", "versions": { "torch": "2.14.0", "numpy": "2.3.5", "transformers": "5.17.0", "safetensors": "0.8.0", "coreai-core": "1.0.0b3", "coreai-torch": "0.4.3" }, "cases_sha256": "184b5ab437c84bea6c1517d84e145ecd68ba7e4fbb7b61d7f628b9d474076e17", "evaluation_config": { "scope": "diagnostic", "cases": [ "cases/decisions.json" ], "atol": 0.02, "rtol": 0.02, "require_decision_agreement": true }, "capability_coverage": { "choice": "exercised-in-diagnostic", "noul": "exercised-in-diagnostic", "router_choice": "untested", "router_set": "untested", "span": "untested", "relation": "untested", "ner": "untested", "span_sets": "untested", "evidence": "untested", "router_task_adapters": "untested", "structured_state": "exercised-in-diagnostic", "multiple_questions": "untested", "variable_sequence_length": "exercised-in-diagnostic", "variable_options": "exercised-in-diagnostic", "system_one_response": "untested", "domain_adapter": "untested", "factcheck_adapter": "untested", "feedback_adapter": "untested", "feedback_df_adapter": "untested" }, "reference_cases": [ { "id": "coding-en", "capability": "choice", "inference_ms": 51.31262500071898, "status": "evaluated", "probabilities": [ 0.9964854118964818, 0.003514588103518185 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "cooking-zh", "capability": "choice", "inference_ms": 68.58845800161362, "status": "evaluated", "probabilities": [ 0.22242194924412478, 0.7775780507558753 ], "decision": "cooking", "expected": "cooking", "correct": true }, { "id": "noul-positive", "capability": "noul", "inference_ms": 48.34995899000205, "status": "evaluated", "probabilities": [ 0.04283435292391378, 0.9571656470760861 ], "decision": "yes", "expected": "yes", "correct": true }, { "id": "noul-negative", "capability": "noul", "inference_ms": 52.418874998693354, "status": "evaluated", "probabilities": [ 0.9887530718985025, 0.011246928101497617 ], "decision": "no", "expected": "no", "correct": true }, { "id": "coding-zh", "capability": "choice", "inference_ms": 70.75237500248477, "status": "evaluated", "probabilities": [ 0.9965400461728955, 0.0034599538271045662 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "cooking-en", "capability": "choice", "inference_ms": 51.56712500320282, "status": "evaluated", "probabilities": [ 0.0008233419586070937, 0.9991766580413929 ], "decision": "cooking", "expected": "cooking", "correct": true }, { "id": "three-options", "capability": "variable_options", "inference_ms": 53.129124993574806, "status": "evaluated", "probabilities": [ 0.9929379587678295, 0.0028259221336674827, 0.004236119098503003 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "one-option", "capability": "variable_options", "inference_ms": 49.474542000098154, "status": "evaluated", "probabilities": [ 1.0 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "structured-json", "capability": "structured_state", "inference_ms": 69.43170799058862, "status": "evaluated", "probabilities": [ 0.9955305388600278, 0.0022821065546790487, 0.002187354585293287 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "exact-129-tokens", "capability": "variable_sequence_length", "inference_ms": 141.63116700365208, "status": "evaluated", "probabilities": [ 0.6347567731488714, 0.13812266903655834, 0.22712055781457036 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "exact-1024-tokens", "capability": "variable_sequence_length", "inference_ms": 968.6068749870174, "status": "evaluated", "probabilities": [ 0.4407633906736511, 0.37238073879955924, 0.18685587052678965 ], "decision": "coding", "expected": "coding", "correct": true }, { "id": "exact-2048-tokens", "capability": "variable_sequence_length", "inference_ms": 2569.3836659920635, "status": "evaluated", "probabilities": [ 0.5216577502831695, 0.34918566249456723, 0.12915658722226325 ], "decision": "coding", "expected": "coding", "correct": true } ], "candidate": { "runtime": "coreai", "status": "diagnostic-passed", "cases": [ { "id": "coding-en", "capability": "choice", "reference_ms": 51.31262500071898, "converted_ms": 2374.499250014196, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 5.427491678311469e-09, "mean_absolute_error": 5.4274916770104265e-09, "reference_probabilities": [ 0.9964854118964818, 0.003514588103518185 ], "converted_probabilities": [ 0.9964854064689901, 0.0035145935310098605 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "cooking-zh", "capability": "choice", "reference_ms": 68.58845800161362, "converted_ms": 91.65208299236838, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 4.7419820070615515e-07, "mean_absolute_error": 4.74198200650644e-07, "reference_probabilities": [ 0.22242194924412478, 0.7775780507558753 ], "converted_probabilities": [ 0.22242242344232538, 0.7775775765576746 ], "reference_decision": "cooking", "converted_decision": "cooking", "expected": "cooking", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "noul-positive", "capability": "noul", "reference_ms": 48.34995899000205, "converted_ms": 47.826540991081856, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 1.3685062416612936e-07, "mean_absolute_error": 1.3685062412102655e-07, "reference_probabilities": [ 0.04283435292391378, 0.9571656470760861 ], "converted_probabilities": [ 0.0428342160732897, 0.9571657839267103 ], "reference_decision": "yes", "converted_decision": "yes", "expected": "yes", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "noul-negative", "capability": "noul", "reference_ms": 52.418874998693354, "converted_ms": 80.2467909961706, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 2.6513179342962934e-09, "mean_absolute_error": 2.6513178276107996e-09, "reference_probabilities": [ 0.9887530718985025, 0.011246928101497617 ], "converted_probabilities": [ 0.9887530745498202, 0.011246925450179683 ], "reference_decision": "no", "converted_decision": "no", "expected": "no", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "coding-zh", "capability": "choice", "reference_ms": 70.75237500248477, "converted_ms": 79.13508299679961, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 8.426156995235135e-09, "mean_absolute_error": 8.426156994801454e-09, "reference_probabilities": [ 0.9965400461728955, 0.0034599538271045662 ], "converted_probabilities": [ 0.9965400377467385, 0.0034599622532615615 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "cooking-en", "capability": "choice", "reference_ms": 51.56712500320282, "converted_ms": 60.23716600611806, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 1.5691086767584406e-09, "mean_absolute_error": 1.5691086487860245e-09, "reference_probabilities": [ 0.0008233419586070937, 0.9991766580413929 ], "converted_probabilities": [ 0.0008233435277157704, 0.9991766564722843 ], "reference_decision": "cooking", "converted_decision": "cooking", "expected": "cooking", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "three-options", "capability": "variable_options", "reference_ms": 53.129124993574806, "converted_ms": 59.11033399752341, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 3.460630049190172e-08, "mean_absolute_error": 2.3070867004286687e-08, "reference_probabilities": [ 0.9929379587678295, 0.0028259221336674827, 0.004236119098503003 ], "converted_probabilities": [ 0.99293799337413, 0.002825909094017193, 0.004236097531852771 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "one-option", "capability": "variable_options", "reference_ms": 49.474542000098154, "converted_ms": 56.663583003683016, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 0.0, "mean_absolute_error": 0.0, "reference_probabilities": [ 1.0 ], "converted_probabilities": [ 1.0 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "structured-json", "capability": "structured_state", "reference_ms": 69.43170799058862, "converted_ms": 75.03466600610409, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 1.4141390947131072e-08, "mean_absolute_error": 9.427594002917964e-09, "reference_probabilities": [ 0.9955305388600278, 0.0022821065546790487, 0.002187354585293287 ], "converted_probabilities": [ 0.9955305530014188, 0.002282103594566704, 0.0021873434040145696 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "exact-129-tokens", "capability": "variable_sequence_length", "reference_ms": 141.63116700365208, "converted_ms": 134.83975001145154, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 7.795353781370196e-07, "mean_absolute_error": 5.196902520913463e-07, "reference_probabilities": [ 0.6347567731488714, 0.13812266903655834, 0.22712055781457036 ], "converted_probabilities": [ 0.6347575526842495, 0.13812251758558605, 0.2271199297301645 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "exact-1024-tokens", "capability": "variable_sequence_length", "reference_ms": 968.6068749870174, "converted_ms": 978.0740829883143, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 1.5057525626360757e-07, "mean_absolute_error": 1.0038350412947909e-07, "reference_probabilities": [ 0.4407633906736511, 0.37238073879955924, 0.18685587052678965 ], "converted_probabilities": [ 0.4407635277058288, 0.372380588224303, 0.1868558840698681 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" }, { "id": "exact-2048-tokens", "capability": "variable_sequence_length", "reference_ms": 2569.3836659920635, "converted_ms": 2785.3902079950785, "passed": true, "numeric_passed": true, "decision_agreement": true, "max_absolute_error": 2.872474891835175e-07, "mean_absolute_error": 1.914983261038413e-07, "reference_probabilities": [ 0.5216577502831695, 0.34918566249456723, 0.12915658722226325 ], "converted_probabilities": [ 0.5216574953587729, 0.3491859497420564, 0.12915655489917072 ], "reference_decision": "coding", "converted_decision": "coding", "expected": "coding", "reference_correct": true, "converted_correct": true, "status": "passed" } ] } }