vela3-fast-CoreAI / evaluation.json
devilyouwei's picture
Replace fixed graph with dynamic sequence and option dimensions
6aaa9cc verified
Raw History Blame Contribute Delete
15.1 kB
{
"model": "vela",
"variant": "3-fast",
"scope": "diagnostic",
"status": "diagnostic-passed",
"source_revision": "fa60e3aa41132c60bf05a84cabfdced39cb6d4f7",
"versions": {
"torch": "2.14.0",
"numpy": "2.3.5",
"transformers": "5.17.0",
"safetensors": "0.8.0",
"coreai-core": "1.0.0b3",
"coreai-torch": "0.4.3"
},
"cases_sha256": "184b5ab437c84bea6c1517d84e145ecd68ba7e4fbb7b61d7f628b9d474076e17",
"evaluation_config": {
"scope": "diagnostic",
"cases": [
"cases/decisions.json"
],
"atol": 0.02,
"rtol": 0.02,
"require_decision_agreement": true
},
"capability_coverage": {
"choice": "exercised-in-diagnostic",
"noul": "exercised-in-diagnostic",
"router_choice": "untested",
"router_set": "untested",
"span": "untested",
"relation": "untested",
"ner": "untested",
"span_sets": "untested",
"evidence": "untested",
"router_task_adapters": "untested",
"structured_state": "exercised-in-diagnostic",
"multiple_questions": "untested",
"variable_sequence_length": "exercised-in-diagnostic",
"variable_options": "exercised-in-diagnostic",
"system_one_response": "untested",
"domain_adapter": "untested",
"factcheck_adapter": "untested",
"feedback_adapter": "untested",
"feedback_df_adapter": "untested"
},
"reference_cases": [
{
"id": "coding-en",
"capability": "choice",
"inference_ms": 51.31262500071898,
"status": "evaluated",
"probabilities": [
0.9964854118964818,
0.003514588103518185
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "cooking-zh",
"capability": "choice",
"inference_ms": 68.58845800161362,
"status": "evaluated",
"probabilities": [
0.22242194924412478,
0.7775780507558753
],
"decision": "cooking",
"expected": "cooking",
"correct": true
},
{
"id": "noul-positive",
"capability": "noul",
"inference_ms": 48.34995899000205,
"status": "evaluated",
"probabilities": [
0.04283435292391378,
0.9571656470760861
],
"decision": "yes",
"expected": "yes",
"correct": true
},
{
"id": "noul-negative",
"capability": "noul",
"inference_ms": 52.418874998693354,
"status": "evaluated",
"probabilities": [
0.9887530718985025,
0.011246928101497617
],
"decision": "no",
"expected": "no",
"correct": true
},
{
"id": "coding-zh",
"capability": "choice",
"inference_ms": 70.75237500248477,
"status": "evaluated",
"probabilities": [
0.9965400461728955,
0.0034599538271045662
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "cooking-en",
"capability": "choice",
"inference_ms": 51.56712500320282,
"status": "evaluated",
"probabilities": [
0.0008233419586070937,
0.9991766580413929
],
"decision": "cooking",
"expected": "cooking",
"correct": true
},
{
"id": "three-options",
"capability": "variable_options",
"inference_ms": 53.129124993574806,
"status": "evaluated",
"probabilities": [
0.9929379587678295,
0.0028259221336674827,
0.004236119098503003
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "one-option",
"capability": "variable_options",
"inference_ms": 49.474542000098154,
"status": "evaluated",
"probabilities": [
1.0
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "structured-json",
"capability": "structured_state",
"inference_ms": 69.43170799058862,
"status": "evaluated",
"probabilities": [
0.9955305388600278,
0.0022821065546790487,
0.002187354585293287
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "exact-129-tokens",
"capability": "variable_sequence_length",
"inference_ms": 141.63116700365208,
"status": "evaluated",
"probabilities": [
0.6347567731488714,
0.13812266903655834,
0.22712055781457036
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "exact-1024-tokens",
"capability": "variable_sequence_length",
"inference_ms": 968.6068749870174,
"status": "evaluated",
"probabilities": [
0.4407633906736511,
0.37238073879955924,
0.18685587052678965
],
"decision": "coding",
"expected": "coding",
"correct": true
},
{
"id": "exact-2048-tokens",
"capability": "variable_sequence_length",
"inference_ms": 2569.3836659920635,
"status": "evaluated",
"probabilities": [
0.5216577502831695,
0.34918566249456723,
0.12915658722226325
],
"decision": "coding",
"expected": "coding",
"correct": true
}
],
"candidate": {
"runtime": "coreai",
"status": "diagnostic-passed",
"cases": [
{
"id": "coding-en",
"capability": "choice",
"reference_ms": 51.31262500071898,
"converted_ms": 2374.499250014196,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 5.427491678311469e-09,
"mean_absolute_error": 5.4274916770104265e-09,
"reference_probabilities": [
0.9964854118964818,
0.003514588103518185
],
"converted_probabilities": [
0.9964854064689901,
0.0035145935310098605
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "cooking-zh",
"capability": "choice",
"reference_ms": 68.58845800161362,
"converted_ms": 91.65208299236838,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 4.7419820070615515e-07,
"mean_absolute_error": 4.74198200650644e-07,
"reference_probabilities": [
0.22242194924412478,
0.7775780507558753
],
"converted_probabilities": [
0.22242242344232538,
0.7775775765576746
],
"reference_decision": "cooking",
"converted_decision": "cooking",
"expected": "cooking",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "noul-positive",
"capability": "noul",
"reference_ms": 48.34995899000205,
"converted_ms": 47.826540991081856,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 1.3685062416612936e-07,
"mean_absolute_error": 1.3685062412102655e-07,
"reference_probabilities": [
0.04283435292391378,
0.9571656470760861
],
"converted_probabilities": [
0.0428342160732897,
0.9571657839267103
],
"reference_decision": "yes",
"converted_decision": "yes",
"expected": "yes",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "noul-negative",
"capability": "noul",
"reference_ms": 52.418874998693354,
"converted_ms": 80.2467909961706,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 2.6513179342962934e-09,
"mean_absolute_error": 2.6513178276107996e-09,
"reference_probabilities": [
0.9887530718985025,
0.011246928101497617
],
"converted_probabilities": [
0.9887530745498202,
0.011246925450179683
],
"reference_decision": "no",
"converted_decision": "no",
"expected": "no",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "coding-zh",
"capability": "choice",
"reference_ms": 70.75237500248477,
"converted_ms": 79.13508299679961,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 8.426156995235135e-09,
"mean_absolute_error": 8.426156994801454e-09,
"reference_probabilities": [
0.9965400461728955,
0.0034599538271045662
],
"converted_probabilities": [
0.9965400377467385,
0.0034599622532615615
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "cooking-en",
"capability": "choice",
"reference_ms": 51.56712500320282,
"converted_ms": 60.23716600611806,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 1.5691086767584406e-09,
"mean_absolute_error": 1.5691086487860245e-09,
"reference_probabilities": [
0.0008233419586070937,
0.9991766580413929
],
"converted_probabilities": [
0.0008233435277157704,
0.9991766564722843
],
"reference_decision": "cooking",
"converted_decision": "cooking",
"expected": "cooking",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "three-options",
"capability": "variable_options",
"reference_ms": 53.129124993574806,
"converted_ms": 59.11033399752341,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 3.460630049190172e-08,
"mean_absolute_error": 2.3070867004286687e-08,
"reference_probabilities": [
0.9929379587678295,
0.0028259221336674827,
0.004236119098503003
],
"converted_probabilities": [
0.99293799337413,
0.002825909094017193,
0.004236097531852771
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "one-option",
"capability": "variable_options",
"reference_ms": 49.474542000098154,
"converted_ms": 56.663583003683016,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 0.0,
"mean_absolute_error": 0.0,
"reference_probabilities": [
1.0
],
"converted_probabilities": [
1.0
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "structured-json",
"capability": "structured_state",
"reference_ms": 69.43170799058862,
"converted_ms": 75.03466600610409,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 1.4141390947131072e-08,
"mean_absolute_error": 9.427594002917964e-09,
"reference_probabilities": [
0.9955305388600278,
0.0022821065546790487,
0.002187354585293287
],
"converted_probabilities": [
0.9955305530014188,
0.002282103594566704,
0.0021873434040145696
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "exact-129-tokens",
"capability": "variable_sequence_length",
"reference_ms": 141.63116700365208,
"converted_ms": 134.83975001145154,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 7.795353781370196e-07,
"mean_absolute_error": 5.196902520913463e-07,
"reference_probabilities": [
0.6347567731488714,
0.13812266903655834,
0.22712055781457036
],
"converted_probabilities": [
0.6347575526842495,
0.13812251758558605,
0.2271199297301645
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "exact-1024-tokens",
"capability": "variable_sequence_length",
"reference_ms": 968.6068749870174,
"converted_ms": 978.0740829883143,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 1.5057525626360757e-07,
"mean_absolute_error": 1.0038350412947909e-07,
"reference_probabilities": [
0.4407633906736511,
0.37238073879955924,
0.18685587052678965
],
"converted_probabilities": [
0.4407635277058288,
0.372380588224303,
0.1868558840698681
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
},
{
"id": "exact-2048-tokens",
"capability": "variable_sequence_length",
"reference_ms": 2569.3836659920635,
"converted_ms": 2785.3902079950785,
"passed": true,
"numeric_passed": true,
"decision_agreement": true,
"max_absolute_error": 2.872474891835175e-07,
"mean_absolute_error": 1.914983261038413e-07,
"reference_probabilities": [
0.5216577502831695,
0.34918566249456723,
0.12915658722226325
],
"converted_probabilities": [
0.5216574953587729,
0.3491859497420564,
0.12915655489917072
],
"reference_decision": "coding",
"converted_decision": "coding",
"expected": "coding",
"reference_correct": true,
"converted_correct": true,
"status": "passed"
}
]
}
}