| { |
| "schema": "kapi-hf-eval-summary/1", |
| "kind": "val_v2c", |
| "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.", |
| "model": "esilva/KapiCoder-Mongo-Nano", |
| "split": "val", |
| "n": 2000, |
| "seed": 20261008, |
| "contract": "editor-context-v1", |
| "dataset": { |
| "sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f", |
| "sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7", |
| "manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb", |
| "stratify_by": [ |
| "cut_kind", |
| "domain" |
| ], |
| "cut_kind": { |
| "multiline": 161, |
| "key": 319, |
| "value": 280, |
| "rest": 241, |
| "operator": 268, |
| "member": 201, |
| "stage": 251, |
| "partial": 200, |
| "stmt_start": 79 |
| }, |
| "domain": { |
| "commerce": 168, |
| "content": 161, |
| "energy": 171, |
| "finance": 175, |
| "gaming": 163, |
| "healthcare-synthetic": 173, |
| "hr": 168, |
| "iot": 168, |
| "logistics": 157, |
| "saas": 165, |
| "travel": 162, |
| "workspace": 169 |
| } |
| }, |
| "decoding": "greedy (argmax), padding do vocab mascarado", |
| "max_new_tokens": 96, |
| "context_tokens": 2048, |
| "load": "bf16", |
| "this_model": { |
| "headline": { |
| "n": 2000, |
| "exact_match": 0.405, |
| "normalized_match": 0.405, |
| "prefix_match": 0.4105, |
| "edit_similarity": 0.6499, |
| "chars_accepted": 7.9775, |
| "tab_chars": 6.343, |
| "accepted_prefix_tokens": 2.62, |
| "syntax_valid": 0.9955, |
| "mongo_valid": 0.992, |
| "exec_valid": 0.5945, |
| "exec_valid_incl_empty": 0.5945, |
| "nonexistent_fields_rate": 0.0265, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.997, |
| "mean_generated_tokens": 15.292 |
| }, |
| "headline_ide32": { |
| "n": 2000, |
| "exact_match": 0.4035, |
| "normalized_match": 0.4035, |
| "prefix_match": 0.4105, |
| "edit_similarity": 0.6421, |
| "chars_accepted": 7.9575, |
| "tab_chars": 6.323, |
| "accepted_prefix_tokens": 2.6125, |
| "syntax_valid": 0.836, |
| "mongo_valid": 0.833, |
| "exec_valid": 0.5055, |
| "exec_valid_incl_empty": 0.5055, |
| "nonexistent_fields_rate": 0.0165, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": null, |
| "mean_generated_tokens": null |
| }, |
| "reference_ceiling": { |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6255, |
| "nonexistent_fields_rate": 0.0 |
| }, |
| "exec_stage_distribution": { |
| "ok": 1189, |
| "api": 483, |
| "syntax": 248, |
| "fields": 75, |
| "exec": 2, |
| "type": 3 |
| }, |
| "by_cut_kind": { |
| "key": { |
| "n": 319, |
| "exact_match": 0.4639, |
| "normalized_match": 0.4639, |
| "prefix_match": 0.4639, |
| "edit_similarity": 0.6428, |
| "chars_accepted": 6.2351, |
| "tab_chars": 5.7335, |
| "accepted_prefix_tokens": 1.9154, |
| "syntax_valid": 0.9969, |
| "mongo_valid": 0.9937, |
| "exec_valid": 0.5392, |
| "exec_valid_incl_empty": 0.5392, |
| "nonexistent_fields_rate": 0.0376, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 8.7147 |
| }, |
| "member": { |
| "n": 201, |
| "exact_match": 0.7711, |
| "normalized_match": 0.7711, |
| "prefix_match": 0.7761, |
| "edit_similarity": 0.8523, |
| "chars_accepted": 7.9254, |
| "tab_chars": 7.6368, |
| "accepted_prefix_tokens": 1.6269, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.995, |
| "exec_valid": 0.6368, |
| "exec_valid_incl_empty": 0.6368, |
| "nonexistent_fields_rate": 0.0, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 5.209 |
| }, |
| "multiline": { |
| "n": 161, |
| "exact_match": 0.0, |
| "normalized_match": 0.0, |
| "prefix_match": 0.0, |
| "edit_similarity": 0.4178, |
| "chars_accepted": 11.5155, |
| "tab_chars": 7.1491, |
| "accepted_prefix_tokens": 3.5839, |
| "syntax_valid": 0.9938, |
| "mongo_valid": 0.9938, |
| "exec_valid": 0.5901, |
| "exec_valid_incl_empty": 0.5901, |
| "nonexistent_fields_rate": 0.087, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9876, |
| "mean_generated_tokens": 43.7578 |
| }, |
| "operator": { |
| "n": 268, |
| "exact_match": 0.6045, |
| "normalized_match": 0.6045, |
| "prefix_match": 0.6157, |
| "edit_similarity": 0.7894, |
| "chars_accepted": 7.6194, |
| "tab_chars": 6.4701, |
| "accepted_prefix_tokens": 3.0261, |
| "syntax_valid": 0.9963, |
| "mongo_valid": 0.9963, |
| "exec_valid": 0.6306, |
| "exec_valid_incl_empty": 0.6306, |
| "nonexistent_fields_rate": 0.0187, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9963, |
| "mean_generated_tokens": 9.8881 |
| }, |
| "partial": { |
| "n": 200, |
| "exact_match": 0.695, |
| "normalized_match": 0.695, |
| "prefix_match": 0.695, |
| "edit_similarity": 0.8264, |
| "chars_accepted": 6.34, |
| "tab_chars": 5.425, |
| "accepted_prefix_tokens": 2.405, |
| "syntax_valid": 0.995, |
| "mongo_valid": 0.98, |
| "exec_valid": 0.57, |
| "exec_valid_incl_empty": 0.57, |
| "nonexistent_fields_rate": 0.05, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 7.9 |
| }, |
| "rest": { |
| "n": 241, |
| "exact_match": 0.3444, |
| "normalized_match": 0.3444, |
| "prefix_match": 0.3485, |
| "edit_similarity": 0.6293, |
| "chars_accepted": 9.4855, |
| "tab_chars": 7.3361, |
| "accepted_prefix_tokens": 2.9793, |
| "syntax_valid": 0.9959, |
| "mongo_valid": 0.9876, |
| "exec_valid": 0.5809, |
| "exec_valid_incl_empty": 0.5809, |
| "nonexistent_fields_rate": 0.0124, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 16.7054 |
| }, |
| "stage": { |
| "n": 251, |
| "exact_match": 0.012, |
| "normalized_match": 0.012, |
| "prefix_match": 0.012, |
| "edit_similarity": 0.4255, |
| "chars_accepted": 11.4861, |
| "tab_chars": 7.8247, |
| "accepted_prefix_tokens": 4.243, |
| "syntax_valid": 0.992, |
| "mongo_valid": 0.992, |
| "exec_valid": 0.5817, |
| "exec_valid_incl_empty": 0.5817, |
| "nonexistent_fields_rate": 0.0239, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.992, |
| "mean_generated_tokens": 30.7131 |
| }, |
| "stmt_start": { |
| "n": 79, |
| "exact_match": 0.3671, |
| "normalized_match": 0.3671, |
| "prefix_match": 0.3671, |
| "edit_similarity": 0.6448, |
| "chars_accepted": 16.0633, |
| "tab_chars": 13.8734, |
| "accepted_prefix_tokens": 3.5316, |
| "syntax_valid": 0.9873, |
| "mongo_valid": 0.9873, |
| "exec_valid": 0.7468, |
| "exec_valid_incl_empty": 0.7468, |
| "nonexistent_fields_rate": 0.038, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9873, |
| "mean_generated_tokens": 16.3924 |
| }, |
| "value": { |
| "n": 280, |
| "exact_match": 0.325, |
| "normalized_match": 0.325, |
| "prefix_match": 0.3464, |
| "edit_similarity": 0.6071, |
| "chars_accepted": 2.7536, |
| "tab_chars": 1.8714, |
| "accepted_prefix_tokens": 1.325, |
| "syntax_valid": 0.9964, |
| "mongo_valid": 0.9964, |
| "exec_valid": 0.5929, |
| "exec_valid_incl_empty": 0.5929, |
| "nonexistent_fields_rate": 0.0, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 8.7571 |
| } |
| }, |
| "by_domain": { |
| "commerce": { |
| "n": 168, |
| "exact_match": 0.375, |
| "normalized_match": 0.375, |
| "prefix_match": 0.381, |
| "edit_similarity": 0.623, |
| "chars_accepted": 6.9881, |
| "tab_chars": 5.3214, |
| "accepted_prefix_tokens": 2.25, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.994, |
| "exec_valid": 0.6548, |
| "exec_valid_incl_empty": 0.6548, |
| "nonexistent_fields_rate": 0.0298, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 14.1786 |
| }, |
| "content": { |
| "n": 161, |
| "exact_match": 0.3913, |
| "normalized_match": 0.3913, |
| "prefix_match": 0.4099, |
| "edit_similarity": 0.6281, |
| "chars_accepted": 8.0497, |
| "tab_chars": 6.5652, |
| "accepted_prefix_tokens": 2.7019, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.5901, |
| "exec_valid_incl_empty": 0.5901, |
| "nonexistent_fields_rate": 0.0062, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 14.8012 |
| }, |
| "energy": { |
| "n": 171, |
| "exact_match": 0.4269, |
| "normalized_match": 0.4269, |
| "prefix_match": 0.4327, |
| "edit_similarity": 0.6494, |
| "chars_accepted": 7.7953, |
| "tab_chars": 6.5146, |
| "accepted_prefix_tokens": 2.3509, |
| "syntax_valid": 0.9942, |
| "mongo_valid": 0.9942, |
| "exec_valid": 0.614, |
| "exec_valid_incl_empty": 0.614, |
| "nonexistent_fields_rate": 0.0351, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9942, |
| "mean_generated_tokens": 16.6842 |
| }, |
| "finance": { |
| "n": 175, |
| "exact_match": 0.3486, |
| "normalized_match": 0.3486, |
| "prefix_match": 0.3543, |
| "edit_similarity": 0.6135, |
| "chars_accepted": 8.1257, |
| "tab_chars": 6.6286, |
| "accepted_prefix_tokens": 2.88, |
| "syntax_valid": 0.9943, |
| "mongo_valid": 0.9943, |
| "exec_valid": 0.5943, |
| "exec_valid_incl_empty": 0.5943, |
| "nonexistent_fields_rate": 0.04, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 18.5257 |
| }, |
| "gaming": { |
| "n": 163, |
| "exact_match": 0.3681, |
| "normalized_match": 0.3681, |
| "prefix_match": 0.3804, |
| "edit_similarity": 0.6296, |
| "chars_accepted": 7.6196, |
| "tab_chars": 6.0184, |
| "accepted_prefix_tokens": 2.5092, |
| "syntax_valid": 0.9877, |
| "mongo_valid": 0.9755, |
| "exec_valid": 0.5951, |
| "exec_valid_incl_empty": 0.5951, |
| "nonexistent_fields_rate": 0.0245, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 14.8712 |
| }, |
| "healthcare-synthetic": { |
| "n": 173, |
| "exact_match": 0.4104, |
| "normalized_match": 0.4104, |
| "prefix_match": 0.4104, |
| "edit_similarity": 0.6618, |
| "chars_accepted": 8.3815, |
| "tab_chars": 6.4624, |
| "accepted_prefix_tokens": 2.7341, |
| "syntax_valid": 0.9942, |
| "mongo_valid": 0.9884, |
| "exec_valid": 0.5607, |
| "exec_valid_incl_empty": 0.5607, |
| "nonexistent_fields_rate": 0.0231, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9942, |
| "mean_generated_tokens": 15.0636 |
| }, |
| "hr": { |
| "n": 168, |
| "exact_match": 0.4643, |
| "normalized_match": 0.4643, |
| "prefix_match": 0.4762, |
| "edit_similarity": 0.6899, |
| "chars_accepted": 7.8155, |
| "tab_chars": 6.3393, |
| "accepted_prefix_tokens": 2.5357, |
| "syntax_valid": 0.994, |
| "mongo_valid": 0.994, |
| "exec_valid": 0.5833, |
| "exec_valid_incl_empty": 0.5833, |
| "nonexistent_fields_rate": 0.0119, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.994, |
| "mean_generated_tokens": 12.7738 |
| }, |
| "iot": { |
| "n": 168, |
| "exact_match": 0.4107, |
| "normalized_match": 0.4107, |
| "prefix_match": 0.4107, |
| "edit_similarity": 0.6606, |
| "chars_accepted": 8.4821, |
| "tab_chars": 6.4643, |
| "accepted_prefix_tokens": 2.869, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6369, |
| "exec_valid_incl_empty": 0.6369, |
| "nonexistent_fields_rate": 0.0536, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 15.3155 |
| }, |
| "logistics": { |
| "n": 157, |
| "exact_match": 0.4076, |
| "normalized_match": 0.4076, |
| "prefix_match": 0.4076, |
| "edit_similarity": 0.6721, |
| "chars_accepted": 8.9554, |
| "tab_chars": 6.8408, |
| "accepted_prefix_tokens": 2.8535, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9936, |
| "exec_valid": 0.5987, |
| "exec_valid_incl_empty": 0.5987, |
| "nonexistent_fields_rate": 0.0255, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 15.2102 |
| }, |
| "saas": { |
| "n": 165, |
| "exact_match": 0.4788, |
| "normalized_match": 0.4788, |
| "prefix_match": 0.4788, |
| "edit_similarity": 0.7021, |
| "chars_accepted": 7.0242, |
| "tab_chars": 5.5636, |
| "accepted_prefix_tokens": 2.3758, |
| "syntax_valid": 0.9939, |
| "mongo_valid": 0.9939, |
| "exec_valid": 0.4848, |
| "exec_valid_incl_empty": 0.4848, |
| "nonexistent_fields_rate": 0.0364, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 12.4909 |
| }, |
| "travel": { |
| "n": 162, |
| "exact_match": 0.4136, |
| "normalized_match": 0.4136, |
| "prefix_match": 0.4198, |
| "edit_similarity": 0.6412, |
| "chars_accepted": 8.4506, |
| "tab_chars": 6.6667, |
| "accepted_prefix_tokens": 2.7901, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6543, |
| "exec_valid_incl_empty": 0.6543, |
| "nonexistent_fields_rate": 0.0123, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 16.5123 |
| }, |
| "workspace": { |
| "n": 169, |
| "exact_match": 0.3669, |
| "normalized_match": 0.3669, |
| "prefix_match": 0.3669, |
| "edit_similarity": 0.6296, |
| "chars_accepted": 8.0828, |
| "tab_chars": 6.7396, |
| "accepted_prefix_tokens": 2.5976, |
| "syntax_valid": 0.9882, |
| "mongo_valid": 0.9763, |
| "exec_valid": 0.568, |
| "exec_valid_incl_empty": 0.568, |
| "nonexistent_fields_rate": 0.0178, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9822, |
| "mean_generated_tokens": 16.8698 |
| } |
| }, |
| "by_context_kind": { |
| "full": { |
| "n": 1647, |
| "exact_match": 0.4141, |
| "normalized_match": 0.4141, |
| "prefix_match": 0.4196, |
| "edit_similarity": 0.6598, |
| "chars_accepted": 8.1488, |
| "tab_chars": 6.4584, |
| "accepted_prefix_tokens": 2.6752, |
| "syntax_valid": 0.9951, |
| "mongo_valid": 0.9927, |
| "exec_valid": 0.6169, |
| "exec_valid_incl_empty": 0.6169, |
| "nonexistent_fields_rate": 0.0134, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.997, |
| "mean_generated_tokens": 15.4244 |
| }, |
| "irrelevant": { |
| "n": 198, |
| "exact_match": 0.3687, |
| "normalized_match": 0.3687, |
| "prefix_match": 0.3737, |
| "edit_similarity": 0.5935, |
| "chars_accepted": 7.0707, |
| "tab_chars": 5.8081, |
| "accepted_prefix_tokens": 2.2374, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9949, |
| "exec_valid": 0.5051, |
| "exec_valid_incl_empty": 0.5051, |
| "nonexistent_fields_rate": 0.0859, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9949, |
| "mean_generated_tokens": 14.5505 |
| }, |
| "none": { |
| "n": 155, |
| "exact_match": 0.3548, |
| "normalized_match": 0.3548, |
| "prefix_match": 0.3613, |
| "edit_similarity": 0.6167, |
| "chars_accepted": 7.3161, |
| "tab_chars": 5.8, |
| "accepted_prefix_tokens": 2.5226, |
| "syntax_valid": 0.9935, |
| "mongo_valid": 0.9806, |
| "exec_valid": 0.471, |
| "exec_valid_incl_empty": 0.471, |
| "nonexistent_fields_rate": 0.0903, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 14.8323 |
| } |
| } |
| }, |
| "metrics_by_model": { |
| "this_model_v3": { |
| "syntax_valid": 0.9955, |
| "mongodb_valid": 0.992, |
| "nonexistent_fields_rate": 2.65, |
| "conversational_suggestion_rate": 0.0, |
| "fim_acc": 0.405, |
| "exact_match": 0.405, |
| "apt_mean": 2.62, |
| "exec_valid_autocomplete": 0.5945, |
| "empty_rate": 0.0, |
| "reference_ceiling_exec_valid": 0.6255, |
| "exec_valid_vs_ceiling": 0.9504, |
| "nonexistent_fields_full_rate_pct": 1.34, |
| "n_context_full": 1647, |
| "exec_valid_full": 0.6169, |
| "nonexistent_fields_irrelevant_rate_pct": 8.59, |
| "n_context_irrelevant": 198, |
| "exec_valid_irrelevant": 0.5051, |
| "nonexistent_fields_none_rate_pct": 9.03, |
| "n_context_none": 155, |
| "exec_valid_none": 0.471, |
| "info_fim_acc_gain_vs_zero_shot_base_pp": 32.9, |
| "info_apt_delta_vs_zero_shot_base": 1.4155, |
| "info_exec_valid_delta_vs_zero_shot_base_pp": 38.15, |
| "generic_js_fim_acc": 0.53, |
| "generic_js_fim_delta_pp_vs_zero_shot_base": 19.0, |
| "write_or_destructive_executions": 0, |
| "train_eval_leak_count": 0, |
| "decontamination_approved": true, |
| "auto_report_attached": true |
| }, |
| "previous_internal_checkpoint_v1_not_released": { |
| "syntax_valid": 0.995, |
| "mongodb_valid": 0.994, |
| "nonexistent_fields_rate": 3.8, |
| "conversational_suggestion_rate": 0.0, |
| "fim_acc": 0.403, |
| "exact_match": 0.403, |
| "apt_mean": 2.6605, |
| "exec_valid_autocomplete": 0.582, |
| "empty_rate": 0.0, |
| "reference_ceiling_exec_valid": 0.6255, |
| "exec_valid_vs_ceiling": 0.9305, |
| "nonexistent_fields_full_rate_pct": 2.06, |
| "n_context_full": 1647, |
| "exec_valid_full": 0.6084, |
| "nonexistent_fields_irrelevant_rate_pct": 11.11, |
| "n_context_irrelevant": 198, |
| "exec_valid_irrelevant": 0.4798, |
| "nonexistent_fields_none_rate_pct": 12.9, |
| "n_context_none": 155, |
| "exec_valid_none": 0.4323, |
| "info_fim_acc_gain_vs_zero_shot_base_pp": 32.7, |
| "info_apt_delta_vs_zero_shot_base": 1.456, |
| "info_exec_valid_delta_vs_zero_shot_base_pp": 36.9, |
| "generic_js_fim_acc": 0.518, |
| "generic_js_fim_delta_pp_vs_zero_shot_base": 17.8, |
| "write_or_destructive_executions": 0, |
| "train_eval_leak_count": 0, |
| "decontamination_approved": true, |
| "auto_report_attached": true |
| }, |
| "zero_shot_base": { |
| "syntax_valid": 0.498, |
| "mongodb_valid": 0.4275, |
| "nonexistent_fields_rate": 4.85, |
| "conversational_suggestion_rate": 0.0, |
| "fim_acc": 0.076, |
| "exact_match": 0.076, |
| "apt_mean": 1.2045, |
| "exec_valid_autocomplete": 0.213, |
| "empty_rate": 0.0055, |
| "reference_ceiling_exec_valid": 0.6255, |
| "exec_valid_vs_ceiling": 0.3405, |
| "nonexistent_fields_full_rate_pct": 4.13, |
| "n_context_full": 1647, |
| "exec_valid_full": 0.2198, |
| "nonexistent_fields_irrelevant_rate_pct": 8.59, |
| "n_context_irrelevant": 198, |
| "exec_valid_irrelevant": 0.1667, |
| "nonexistent_fields_none_rate_pct": 7.74, |
| "n_context_none": 155, |
| "exec_valid_none": 0.2, |
| "generic_js_fim_acc": 0.34, |
| "write_or_destructive_executions": 0, |
| "train_eval_leak_count": 0, |
| "decontamination_approved": true, |
| "auto_report_attached": true |
| } |
| }, |
| "metric_definitions": { |
| "exact_match": "pred.strip() == middle.strip()", |
| "normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)", |
| "chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle", |
| "accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)", |
| "syntax_valid": "node vm.Script sobre prefix+pred+suffix", |
| "mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido", |
| "exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido", |
| "nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos", |
| "reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)" |
| } |
| } |
|
|
