Download evaluation.json from hugging-mac/vela3-fast-CoreAI: direct link, hf CLI and curl.
- Browser
- Download file 15.1 kB
-
https://huggingface.co/hugging-mac/vela3-fast-CoreAI/resolve/main/evaluation.json
- Command line
-
hf download hf://hugging-mac/vela3-fast-CoreAI/evaluation.json
-
curl -L -o evaluation.json https://huggingface.co/hugging-mac/vela3-fast-CoreAI/resolve/main/evaluation.json
15.1 kB
| { | |
| "model": "vela", | |
| "variant": "3-fast", | |
| "scope": "diagnostic", | |
| "status": "diagnostic-passed", | |
| "source_revision": "fa60e3aa41132c60bf05a84cabfdced39cb6d4f7", | |
| "versions": { | |
| "torch": "2.14.0", | |
| "numpy": "2.3.5", | |
| "transformers": "5.17.0", | |
| "safetensors": "0.8.0", | |
| "coreai-core": "1.0.0b3", | |
| "coreai-torch": "0.4.3" | |
| }, | |
| "cases_sha256": "184b5ab437c84bea6c1517d84e145ecd68ba7e4fbb7b61d7f628b9d474076e17", | |
| "evaluation_config": { | |
| "scope": "diagnostic", | |
| "cases": [ | |
| "cases/decisions.json" | |
| ], | |
| "atol": 0.02, | |
| "rtol": 0.02, | |
| "require_decision_agreement": true | |
| }, | |
| "capability_coverage": { | |
| "choice": "exercised-in-diagnostic", | |
| "noul": "exercised-in-diagnostic", | |
| "router_choice": "untested", | |
| "router_set": "untested", | |
| "span": "untested", | |
| "relation": "untested", | |
| "ner": "untested", | |
| "span_sets": "untested", | |
| "evidence": "untested", | |
| "router_task_adapters": "untested", | |
| "structured_state": "exercised-in-diagnostic", | |
| "multiple_questions": "untested", | |
| "variable_sequence_length": "exercised-in-diagnostic", | |
| "variable_options": "exercised-in-diagnostic", | |
| "system_one_response": "untested", | |
| "domain_adapter": "untested", | |
| "factcheck_adapter": "untested", | |
| "feedback_adapter": "untested", | |
| "feedback_df_adapter": "untested" | |
| }, | |
| "reference_cases": [ | |
| { | |
| "id": "coding-en", | |
| "capability": "choice", | |
| "inference_ms": 51.31262500071898, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.9964854118964818, | |
| 0.003514588103518185 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "cooking-zh", | |
| "capability": "choice", | |
| "inference_ms": 68.58845800161362, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.22242194924412478, | |
| 0.7775780507558753 | |
| ], | |
| "decision": "cooking", | |
| "expected": "cooking", | |
| "correct": true | |
| }, | |
| { | |
| "id": "noul-positive", | |
| "capability": "noul", | |
| "inference_ms": 48.34995899000205, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.04283435292391378, | |
| 0.9571656470760861 | |
| ], | |
| "decision": "yes", | |
| "expected": "yes", | |
| "correct": true | |
| }, | |
| { | |
| "id": "noul-negative", | |
| "capability": "noul", | |
| "inference_ms": 52.418874998693354, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.9887530718985025, | |
| 0.011246928101497617 | |
| ], | |
| "decision": "no", | |
| "expected": "no", | |
| "correct": true | |
| }, | |
| { | |
| "id": "coding-zh", | |
| "capability": "choice", | |
| "inference_ms": 70.75237500248477, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.9965400461728955, | |
| 0.0034599538271045662 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "cooking-en", | |
| "capability": "choice", | |
| "inference_ms": 51.56712500320282, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.0008233419586070937, | |
| 0.9991766580413929 | |
| ], | |
| "decision": "cooking", | |
| "expected": "cooking", | |
| "correct": true | |
| }, | |
| { | |
| "id": "three-options", | |
| "capability": "variable_options", | |
| "inference_ms": 53.129124993574806, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.9929379587678295, | |
| 0.0028259221336674827, | |
| 0.004236119098503003 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "one-option", | |
| "capability": "variable_options", | |
| "inference_ms": 49.474542000098154, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 1.0 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "structured-json", | |
| "capability": "structured_state", | |
| "inference_ms": 69.43170799058862, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.9955305388600278, | |
| 0.0022821065546790487, | |
| 0.002187354585293287 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "exact-129-tokens", | |
| "capability": "variable_sequence_length", | |
| "inference_ms": 141.63116700365208, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.6347567731488714, | |
| 0.13812266903655834, | |
| 0.22712055781457036 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "exact-1024-tokens", | |
| "capability": "variable_sequence_length", | |
| "inference_ms": 968.6068749870174, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.4407633906736511, | |
| 0.37238073879955924, | |
| 0.18685587052678965 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| }, | |
| { | |
| "id": "exact-2048-tokens", | |
| "capability": "variable_sequence_length", | |
| "inference_ms": 2569.3836659920635, | |
| "status": "evaluated", | |
| "probabilities": [ | |
| 0.5216577502831695, | |
| 0.34918566249456723, | |
| 0.12915658722226325 | |
| ], | |
| "decision": "coding", | |
| "expected": "coding", | |
| "correct": true | |
| } | |
| ], | |
| "candidate": { | |
| "runtime": "coreai", | |
| "status": "diagnostic-passed", | |
| "cases": [ | |
| { | |
| "id": "coding-en", | |
| "capability": "choice", | |
| "reference_ms": 51.31262500071898, | |
| "converted_ms": 2374.499250014196, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 5.427491678311469e-09, | |
| "mean_absolute_error": 5.4274916770104265e-09, | |
| "reference_probabilities": [ | |
| 0.9964854118964818, | |
| 0.003514588103518185 | |
| ], | |
| "converted_probabilities": [ | |
| 0.9964854064689901, | |
| 0.0035145935310098605 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "cooking-zh", | |
| "capability": "choice", | |
| "reference_ms": 68.58845800161362, | |
| "converted_ms": 91.65208299236838, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 4.7419820070615515e-07, | |
| "mean_absolute_error": 4.74198200650644e-07, | |
| "reference_probabilities": [ | |
| 0.22242194924412478, | |
| 0.7775780507558753 | |
| ], | |
| "converted_probabilities": [ | |
| 0.22242242344232538, | |
| 0.7775775765576746 | |
| ], | |
| "reference_decision": "cooking", | |
| "converted_decision": "cooking", | |
| "expected": "cooking", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "noul-positive", | |
| "capability": "noul", | |
| "reference_ms": 48.34995899000205, | |
| "converted_ms": 47.826540991081856, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 1.3685062416612936e-07, | |
| "mean_absolute_error": 1.3685062412102655e-07, | |
| "reference_probabilities": [ | |
| 0.04283435292391378, | |
| 0.9571656470760861 | |
| ], | |
| "converted_probabilities": [ | |
| 0.0428342160732897, | |
| 0.9571657839267103 | |
| ], | |
| "reference_decision": "yes", | |
| "converted_decision": "yes", | |
| "expected": "yes", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "noul-negative", | |
| "capability": "noul", | |
| "reference_ms": 52.418874998693354, | |
| "converted_ms": 80.2467909961706, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 2.6513179342962934e-09, | |
| "mean_absolute_error": 2.6513178276107996e-09, | |
| "reference_probabilities": [ | |
| 0.9887530718985025, | |
| 0.011246928101497617 | |
| ], | |
| "converted_probabilities": [ | |
| 0.9887530745498202, | |
| 0.011246925450179683 | |
| ], | |
| "reference_decision": "no", | |
| "converted_decision": "no", | |
| "expected": "no", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "coding-zh", | |
| "capability": "choice", | |
| "reference_ms": 70.75237500248477, | |
| "converted_ms": 79.13508299679961, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 8.426156995235135e-09, | |
| "mean_absolute_error": 8.426156994801454e-09, | |
| "reference_probabilities": [ | |
| 0.9965400461728955, | |
| 0.0034599538271045662 | |
| ], | |
| "converted_probabilities": [ | |
| 0.9965400377467385, | |
| 0.0034599622532615615 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "cooking-en", | |
| "capability": "choice", | |
| "reference_ms": 51.56712500320282, | |
| "converted_ms": 60.23716600611806, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 1.5691086767584406e-09, | |
| "mean_absolute_error": 1.5691086487860245e-09, | |
| "reference_probabilities": [ | |
| 0.0008233419586070937, | |
| 0.9991766580413929 | |
| ], | |
| "converted_probabilities": [ | |
| 0.0008233435277157704, | |
| 0.9991766564722843 | |
| ], | |
| "reference_decision": "cooking", | |
| "converted_decision": "cooking", | |
| "expected": "cooking", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "three-options", | |
| "capability": "variable_options", | |
| "reference_ms": 53.129124993574806, | |
| "converted_ms": 59.11033399752341, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 3.460630049190172e-08, | |
| "mean_absolute_error": 2.3070867004286687e-08, | |
| "reference_probabilities": [ | |
| 0.9929379587678295, | |
| 0.0028259221336674827, | |
| 0.004236119098503003 | |
| ], | |
| "converted_probabilities": [ | |
| 0.99293799337413, | |
| 0.002825909094017193, | |
| 0.004236097531852771 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "one-option", | |
| "capability": "variable_options", | |
| "reference_ms": 49.474542000098154, | |
| "converted_ms": 56.663583003683016, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 0.0, | |
| "mean_absolute_error": 0.0, | |
| "reference_probabilities": [ | |
| 1.0 | |
| ], | |
| "converted_probabilities": [ | |
| 1.0 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "structured-json", | |
| "capability": "structured_state", | |
| "reference_ms": 69.43170799058862, | |
| "converted_ms": 75.03466600610409, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 1.4141390947131072e-08, | |
| "mean_absolute_error": 9.427594002917964e-09, | |
| "reference_probabilities": [ | |
| 0.9955305388600278, | |
| 0.0022821065546790487, | |
| 0.002187354585293287 | |
| ], | |
| "converted_probabilities": [ | |
| 0.9955305530014188, | |
| 0.002282103594566704, | |
| 0.0021873434040145696 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "exact-129-tokens", | |
| "capability": "variable_sequence_length", | |
| "reference_ms": 141.63116700365208, | |
| "converted_ms": 134.83975001145154, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 7.795353781370196e-07, | |
| "mean_absolute_error": 5.196902520913463e-07, | |
| "reference_probabilities": [ | |
| 0.6347567731488714, | |
| 0.13812266903655834, | |
| 0.22712055781457036 | |
| ], | |
| "converted_probabilities": [ | |
| 0.6347575526842495, | |
| 0.13812251758558605, | |
| 0.2271199297301645 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "exact-1024-tokens", | |
| "capability": "variable_sequence_length", | |
| "reference_ms": 968.6068749870174, | |
| "converted_ms": 978.0740829883143, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 1.5057525626360757e-07, | |
| "mean_absolute_error": 1.0038350412947909e-07, | |
| "reference_probabilities": [ | |
| 0.4407633906736511, | |
| 0.37238073879955924, | |
| 0.18685587052678965 | |
| ], | |
| "converted_probabilities": [ | |
| 0.4407635277058288, | |
| 0.372380588224303, | |
| 0.1868558840698681 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| }, | |
| { | |
| "id": "exact-2048-tokens", | |
| "capability": "variable_sequence_length", | |
| "reference_ms": 2569.3836659920635, | |
| "converted_ms": 2785.3902079950785, | |
| "passed": true, | |
| "numeric_passed": true, | |
| "decision_agreement": true, | |
| "max_absolute_error": 2.872474891835175e-07, | |
| "mean_absolute_error": 1.914983261038413e-07, | |
| "reference_probabilities": [ | |
| 0.5216577502831695, | |
| 0.34918566249456723, | |
| 0.12915658722226325 | |
| ], | |
| "converted_probabilities": [ | |
| 0.5216574953587729, | |
| 0.3491859497420564, | |
| 0.12915655489917072 | |
| ], | |
| "reference_decision": "coding", | |
| "converted_decision": "coding", | |
| "expected": "coding", | |
| "reference_correct": true, | |
| "converted_correct": true, | |
| "status": "passed" | |
| } | |
| ] | |
| } | |
| } | |