| { |
| "schema": "kapi-hf-eval-summary/1", |
| "kind": "val_v2c", |
| "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.", |
| "model": "esilva/KapiCoder-Mongo-Small", |
| "split": "val", |
| "n": 2000, |
| "seed": 20261008, |
| "contract": "editor-context-v1", |
| "dataset": { |
| "sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f", |
| "sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7", |
| "manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb", |
| "stratify_by": [ |
| "cut_kind", |
| "domain" |
| ], |
| "cut_kind": { |
| "multiline": 161, |
| "key": 319, |
| "value": 280, |
| "rest": 241, |
| "operator": 268, |
| "member": 201, |
| "stage": 251, |
| "partial": 200, |
| "stmt_start": 79 |
| }, |
| "domain": { |
| "commerce": 168, |
| "content": 161, |
| "energy": 171, |
| "finance": 175, |
| "gaming": 163, |
| "healthcare-synthetic": 173, |
| "hr": 168, |
| "iot": 168, |
| "logistics": 157, |
| "saas": 165, |
| "travel": 162, |
| "workspace": 169 |
| } |
| }, |
| "decoding": "greedy (argmax), padding do vocab mascarado", |
| "max_new_tokens": 96, |
| "context_tokens": 2048, |
| "load": "bf16", |
| "this_model": { |
| "headline": { |
| "n": 2000, |
| "exact_match": 0.41, |
| "normalized_match": 0.41, |
| "prefix_match": 0.416, |
| "edit_similarity": 0.6536, |
| "chars_accepted": 8.214, |
| "tab_chars": 6.5425, |
| "accepted_prefix_tokens": 2.698, |
| "syntax_valid": 0.998, |
| "mongo_valid": 0.9975, |
| "exec_valid": 0.6015, |
| "exec_valid_incl_empty": 0.6015, |
| "nonexistent_fields_rate": 0.021, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9975, |
| "mean_generated_tokens": 15.3875 |
| }, |
| "headline_ide32": { |
| "n": 2000, |
| "exact_match": 0.409, |
| "normalized_match": 0.409, |
| "prefix_match": 0.4165, |
| "edit_similarity": 0.6463, |
| "chars_accepted": 8.2, |
| "tab_chars": 6.5285, |
| "accepted_prefix_tokens": 2.693, |
| "syntax_valid": 0.836, |
| "mongo_valid": 0.8355, |
| "exec_valid": 0.5115, |
| "exec_valid_incl_empty": 0.5115, |
| "nonexistent_fields_rate": 0.012, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": null, |
| "mean_generated_tokens": null |
| }, |
| "reference_ceiling": { |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6255, |
| "nonexistent_fields_rate": 0.0 |
| }, |
| "exec_stage_distribution": { |
| "ok": 1203, |
| "api": 490, |
| "syntax": 243, |
| "type": 3, |
| "fields": 59, |
| "exec": 2 |
| }, |
| "by_cut_kind": { |
| "key": { |
| "n": 319, |
| "exact_match": 0.4514, |
| "normalized_match": 0.4514, |
| "prefix_match": 0.4545, |
| "edit_similarity": 0.6407, |
| "chars_accepted": 6.4639, |
| "tab_chars": 5.8119, |
| "accepted_prefix_tokens": 2.0, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.5361, |
| "exec_valid_incl_empty": 0.5361, |
| "nonexistent_fields_rate": 0.0408, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 9.1442 |
| }, |
| "member": { |
| "n": 201, |
| "exact_match": 0.801, |
| "normalized_match": 0.801, |
| "prefix_match": 0.801, |
| "edit_similarity": 0.8754, |
| "chars_accepted": 8.6965, |
| "tab_chars": 8.3134, |
| "accepted_prefix_tokens": 1.8259, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6517, |
| "exec_valid_incl_empty": 0.6517, |
| "nonexistent_fields_rate": 0.0, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 5.1045 |
| }, |
| "multiline": { |
| "n": 161, |
| "exact_match": 0.0, |
| "normalized_match": 0.0, |
| "prefix_match": 0.0, |
| "edit_similarity": 0.4159, |
| "chars_accepted": 11.5652, |
| "tab_chars": 7.2671, |
| "accepted_prefix_tokens": 3.677, |
| "syntax_valid": 0.9876, |
| "mongo_valid": 0.9876, |
| "exec_valid": 0.6149, |
| "exec_valid_incl_empty": 0.6149, |
| "nonexistent_fields_rate": 0.0807, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9814, |
| "mean_generated_tokens": 44.3665 |
| }, |
| "operator": { |
| "n": 268, |
| "exact_match": 0.6119, |
| "normalized_match": 0.6119, |
| "prefix_match": 0.6231, |
| "edit_similarity": 0.7941, |
| "chars_accepted": 7.7052, |
| "tab_chars": 6.5522, |
| "accepted_prefix_tokens": 3.0597, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6306, |
| "exec_valid_incl_empty": 0.6306, |
| "nonexistent_fields_rate": 0.0037, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 9.8134 |
| }, |
| "partial": { |
| "n": 200, |
| "exact_match": 0.72, |
| "normalized_match": 0.72, |
| "prefix_match": 0.72, |
| "edit_similarity": 0.8414, |
| "chars_accepted": 6.265, |
| "tab_chars": 5.435, |
| "accepted_prefix_tokens": 2.36, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.59, |
| "exec_valid_incl_empty": 0.59, |
| "nonexistent_fields_rate": 0.035, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 7.66 |
| }, |
| "rest": { |
| "n": 241, |
| "exact_match": 0.3444, |
| "normalized_match": 0.3444, |
| "prefix_match": 0.3485, |
| "edit_similarity": 0.6199, |
| "chars_accepted": 9.8714, |
| "tab_chars": 7.7344, |
| "accepted_prefix_tokens": 3.1286, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9959, |
| "exec_valid": 0.5892, |
| "exec_valid_incl_empty": 0.5892, |
| "nonexistent_fields_rate": 0.0124, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 17.5021 |
| }, |
| "stage": { |
| "n": 251, |
| "exact_match": 0.0239, |
| "normalized_match": 0.0239, |
| "prefix_match": 0.0239, |
| "edit_similarity": 0.4418, |
| "chars_accepted": 11.8367, |
| "tab_chars": 8.1474, |
| "accepted_prefix_tokens": 4.3347, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.5976, |
| "exec_valid_incl_empty": 0.5976, |
| "nonexistent_fields_rate": 0.012, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 30.0478 |
| }, |
| "stmt_start": { |
| "n": 79, |
| "exact_match": 0.3671, |
| "normalized_match": 0.3671, |
| "prefix_match": 0.3671, |
| "edit_similarity": 0.6481, |
| "chars_accepted": 16.2785, |
| "tab_chars": 13.9747, |
| "accepted_prefix_tokens": 3.6076, |
| "syntax_valid": 0.9873, |
| "mongo_valid": 0.9873, |
| "exec_valid": 0.7089, |
| "exec_valid_incl_empty": 0.7089, |
| "nonexistent_fields_rate": 0.0253, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9873, |
| "mean_generated_tokens": 16.3924 |
| }, |
| "value": { |
| "n": 280, |
| "exact_match": 0.3179, |
| "normalized_match": 0.3179, |
| "prefix_match": 0.3429, |
| "edit_similarity": 0.5979, |
| "chars_accepted": 2.8643, |
| "tab_chars": 1.9071, |
| "accepted_prefix_tokens": 1.3571, |
| "syntax_valid": 0.9964, |
| "mongo_valid": 0.9964, |
| "exec_valid": 0.5964, |
| "exec_valid_incl_empty": 0.5964, |
| "nonexistent_fields_rate": 0.0, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9964, |
| "mean_generated_tokens": 8.8286 |
| } |
| }, |
| "by_domain": { |
| "commerce": { |
| "n": 168, |
| "exact_match": 0.381, |
| "normalized_match": 0.381, |
| "prefix_match": 0.3869, |
| "edit_similarity": 0.6404, |
| "chars_accepted": 7.1667, |
| "tab_chars": 5.4464, |
| "accepted_prefix_tokens": 2.2917, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6607, |
| "exec_valid_incl_empty": 0.6607, |
| "nonexistent_fields_rate": 0.0119, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 15.1488 |
| }, |
| "content": { |
| "n": 161, |
| "exact_match": 0.3851, |
| "normalized_match": 0.3851, |
| "prefix_match": 0.4037, |
| "edit_similarity": 0.6223, |
| "chars_accepted": 8.1366, |
| "tab_chars": 6.5093, |
| "accepted_prefix_tokens": 2.6894, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6087, |
| "exec_valid_incl_empty": 0.6087, |
| "nonexistent_fields_rate": 0.0062, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 15.1491 |
| }, |
| "energy": { |
| "n": 171, |
| "exact_match": 0.4386, |
| "normalized_match": 0.4386, |
| "prefix_match": 0.4444, |
| "edit_similarity": 0.6549, |
| "chars_accepted": 8.1637, |
| "tab_chars": 6.8129, |
| "accepted_prefix_tokens": 2.4737, |
| "syntax_valid": 0.9942, |
| "mongo_valid": 0.9942, |
| "exec_valid": 0.6023, |
| "exec_valid_incl_empty": 0.6023, |
| "nonexistent_fields_rate": 0.0468, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9942, |
| "mean_generated_tokens": 16.0526 |
| }, |
| "finance": { |
| "n": 175, |
| "exact_match": 0.3429, |
| "normalized_match": 0.3429, |
| "prefix_match": 0.3486, |
| "edit_similarity": 0.6153, |
| "chars_accepted": 8.2457, |
| "tab_chars": 6.5314, |
| "accepted_prefix_tokens": 2.9486, |
| "syntax_valid": 0.9943, |
| "mongo_valid": 0.9943, |
| "exec_valid": 0.5943, |
| "exec_valid_incl_empty": 0.5943, |
| "nonexistent_fields_rate": 0.0286, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9943, |
| "mean_generated_tokens": 18.6514 |
| }, |
| "gaming": { |
| "n": 163, |
| "exact_match": 0.3742, |
| "normalized_match": 0.3742, |
| "prefix_match": 0.3804, |
| "edit_similarity": 0.6392, |
| "chars_accepted": 7.9264, |
| "tab_chars": 6.1902, |
| "accepted_prefix_tokens": 2.5644, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9939, |
| "exec_valid": 0.6012, |
| "exec_valid_incl_empty": 0.6012, |
| "nonexistent_fields_rate": 0.0184, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 15.0184 |
| }, |
| "healthcare-synthetic": { |
| "n": 173, |
| "exact_match": 0.4046, |
| "normalized_match": 0.4046, |
| "prefix_match": 0.4046, |
| "edit_similarity": 0.6652, |
| "chars_accepted": 8.5896, |
| "tab_chars": 6.4913, |
| "accepted_prefix_tokens": 2.8497, |
| "syntax_valid": 0.9942, |
| "mongo_valid": 0.9942, |
| "exec_valid": 0.5838, |
| "exec_valid_incl_empty": 0.5838, |
| "nonexistent_fields_rate": 0.0116, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9942, |
| "mean_generated_tokens": 14.9827 |
| }, |
| "hr": { |
| "n": 168, |
| "exact_match": 0.4762, |
| "normalized_match": 0.4762, |
| "prefix_match": 0.4881, |
| "edit_similarity": 0.7032, |
| "chars_accepted": 8.8333, |
| "tab_chars": 7.5119, |
| "accepted_prefix_tokens": 2.8512, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6012, |
| "exec_valid_incl_empty": 0.6012, |
| "nonexistent_fields_rate": 0.0119, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 12.8869 |
| }, |
| "iot": { |
| "n": 168, |
| "exact_match": 0.4345, |
| "normalized_match": 0.4345, |
| "prefix_match": 0.4405, |
| "edit_similarity": 0.6659, |
| "chars_accepted": 8.8393, |
| "tab_chars": 6.9583, |
| "accepted_prefix_tokens": 2.9702, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6429, |
| "exec_valid_incl_empty": 0.6429, |
| "nonexistent_fields_rate": 0.0357, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 15.6071 |
| }, |
| "logistics": { |
| "n": 157, |
| "exact_match": 0.4331, |
| "normalized_match": 0.4331, |
| "prefix_match": 0.4331, |
| "edit_similarity": 0.6879, |
| "chars_accepted": 8.9172, |
| "tab_chars": 6.8726, |
| "accepted_prefix_tokens": 2.8917, |
| "syntax_valid": 0.9936, |
| "mongo_valid": 0.9936, |
| "exec_valid": 0.586, |
| "exec_valid_incl_empty": 0.586, |
| "nonexistent_fields_rate": 0.0318, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9936, |
| "mean_generated_tokens": 14.5669 |
| }, |
| "saas": { |
| "n": 165, |
| "exact_match": 0.4788, |
| "normalized_match": 0.4788, |
| "prefix_match": 0.4848, |
| "edit_similarity": 0.6938, |
| "chars_accepted": 7.2424, |
| "tab_chars": 5.6788, |
| "accepted_prefix_tokens": 2.4909, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.5091, |
| "exec_valid_incl_empty": 0.5091, |
| "nonexistent_fields_rate": 0.0303, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 12.6061 |
| }, |
| "travel": { |
| "n": 162, |
| "exact_match": 0.4074, |
| "normalized_match": 0.4074, |
| "prefix_match": 0.4136, |
| "edit_similarity": 0.6385, |
| "chars_accepted": 8.8086, |
| "tab_chars": 7.0864, |
| "accepted_prefix_tokens": 2.9136, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.6605, |
| "exec_valid_incl_empty": 0.6605, |
| "nonexistent_fields_rate": 0.0062, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 16.5309 |
| }, |
| "workspace": { |
| "n": 169, |
| "exact_match": 0.3669, |
| "normalized_match": 0.3669, |
| "prefix_match": 0.3669, |
| "edit_similarity": 0.6191, |
| "chars_accepted": 7.7278, |
| "tab_chars": 6.432, |
| "accepted_prefix_tokens": 2.4438, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.568, |
| "exec_valid_incl_empty": 0.568, |
| "nonexistent_fields_rate": 0.0118, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9941, |
| "mean_generated_tokens": 17.2189 |
| } |
| }, |
| "by_context_kind": { |
| "full": { |
| "n": 1647, |
| "exact_match": 0.4183, |
| "normalized_match": 0.4183, |
| "prefix_match": 0.4238, |
| "edit_similarity": 0.6613, |
| "chars_accepted": 8.3734, |
| "tab_chars": 6.6758, |
| "accepted_prefix_tokens": 2.759, |
| "syntax_valid": 0.9976, |
| "mongo_valid": 0.997, |
| "exec_valid": 0.6211, |
| "exec_valid_incl_empty": 0.6211, |
| "nonexistent_fields_rate": 0.0121, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9976, |
| "mean_generated_tokens": 15.4857 |
| }, |
| "irrelevant": { |
| "n": 198, |
| "exact_match": 0.3838, |
| "normalized_match": 0.3838, |
| "prefix_match": 0.3889, |
| "edit_similarity": 0.6152, |
| "chars_accepted": 7.5657, |
| "tab_chars": 6.2121, |
| "accepted_prefix_tokens": 2.3333, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.5303, |
| "exec_valid_incl_empty": 0.5303, |
| "nonexistent_fields_rate": 0.0556, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 0.9949, |
| "mean_generated_tokens": 15.0 |
| }, |
| "none": { |
| "n": 155, |
| "exact_match": 0.3548, |
| "normalized_match": 0.3548, |
| "prefix_match": 0.3677, |
| "edit_similarity": 0.6217, |
| "chars_accepted": 7.3484, |
| "tab_chars": 5.5484, |
| "accepted_prefix_tokens": 2.5161, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.4839, |
| "exec_valid_incl_empty": 0.4839, |
| "nonexistent_fields_rate": 0.071, |
| "empty_rate": 0.0, |
| "eos_or_suffix_stop_rate": 1.0, |
| "mean_generated_tokens": 14.8387 |
| } |
| } |
| }, |
| "metrics_by_model": { |
| "this_model_v3": { |
| "syntax_valid": 0.998, |
| "mongodb_valid": 0.9975, |
| "nonexistent_fields_rate": 2.1, |
| "conversational_suggestion_rate": 0.0, |
| "fim_acc": 0.41, |
| "exact_match": 0.41, |
| "apt_mean": 2.698, |
| "exec_valid_autocomplete": 0.6015, |
| "empty_rate": 0.0, |
| "reference_ceiling_exec_valid": 0.6255, |
| "exec_valid_vs_ceiling": 0.9616, |
| "nonexistent_fields_full_rate_pct": 1.21, |
| "n_context_full": 1647, |
| "exec_valid_full": 0.6211, |
| "nonexistent_fields_irrelevant_rate_pct": 5.56, |
| "n_context_irrelevant": 198, |
| "exec_valid_irrelevant": 0.5303, |
| "nonexistent_fields_none_rate_pct": 7.1, |
| "n_context_none": 155, |
| "exec_valid_none": 0.4839, |
| "info_fim_acc_gain_vs_zero_shot_base_pp": 26.8, |
| "info_apt_delta_vs_zero_shot_base": 1.259, |
| "info_exec_valid_delta_vs_zero_shot_base_pp": 28.55, |
| "generic_js_fim_acc": 0.606, |
| "generic_js_fim_delta_pp_vs_zero_shot_base": 20.0, |
| "write_or_destructive_executions": 0, |
| "train_eval_leak_count": 0, |
| "decontamination_approved": true, |
| "auto_report_attached": true |
| }, |
| "previous_internal_checkpoint_v1_not_released": { |
| "syntax_valid": 0.9955, |
| "mongodb_valid": 0.9945, |
| "nonexistent_fields_rate": 2.9, |
| "conversational_suggestion_rate": 0.0, |
| "fim_acc": 0.4105, |
| "exact_match": 0.4105, |
| "apt_mean": 2.6665, |
| "exec_valid_autocomplete": 0.595, |
| "empty_rate": 0.0, |
| "reference_ceiling_exec_valid": 0.6255, |
| "exec_valid_vs_ceiling": 0.9512, |
| "nonexistent_fields_full_rate_pct": 1.58, |
| "n_context_full": 1647, |
| "exec_valid_full": 0.6199, |
| "nonexistent_fields_irrelevant_rate_pct": 7.58, |
| "n_context_irrelevant": 198, |
| "exec_valid_irrelevant": 0.5051, |
| "nonexistent_fields_none_rate_pct": 10.97, |
| "n_context_none": 155, |
| "exec_valid_none": 0.4452, |
| "info_fim_acc_gain_vs_zero_shot_base_pp": 26.85, |
| "info_apt_delta_vs_zero_shot_base": 1.2275, |
| "info_exec_valid_delta_vs_zero_shot_base_pp": 27.9, |
| "generic_js_fim_acc": 0.632, |
| "generic_js_fim_delta_pp_vs_zero_shot_base": 22.6, |
| "write_or_destructive_executions": 0, |
| "train_eval_leak_count": 0, |
| "decontamination_approved": true, |
| "auto_report_attached": true |
| }, |
| "zero_shot_base": { |
| "syntax_valid": 0.667, |
| "mongodb_valid": 0.595, |
| "nonexistent_fields_rate": 5.1, |
| "conversational_suggestion_rate": 0.0, |
| "fim_acc": 0.142, |
| "exact_match": 0.142, |
| "apt_mean": 1.439, |
| "exec_valid_autocomplete": 0.316, |
| "empty_rate": 0.0095, |
| "reference_ceiling_exec_valid": 0.6255, |
| "exec_valid_vs_ceiling": 0.5052, |
| "nonexistent_fields_full_rate_pct": 3.76, |
| "n_context_full": 1647, |
| "exec_valid_full": 0.3364, |
| "nonexistent_fields_irrelevant_rate_pct": 10.61, |
| "n_context_irrelevant": 198, |
| "exec_valid_irrelevant": 0.2172, |
| "nonexistent_fields_none_rate_pct": 12.26, |
| "n_context_none": 155, |
| "exec_valid_none": 0.2258, |
| "generic_js_fim_acc": 0.406, |
| "write_or_destructive_executions": 0, |
| "train_eval_leak_count": 0, |
| "decontamination_approved": true, |
| "auto_report_attached": true |
| } |
| }, |
| "metric_definitions": { |
| "exact_match": "pred.strip() == middle.strip()", |
| "normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)", |
| "chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle", |
| "accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)", |
| "syntax_valid": "node vm.Script sobre prefix+pred+suffix", |
| "mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido", |
| "exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido", |
| "nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos", |
| "reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)" |
| } |
| } |
|
|
