{
  "schema": "kapi-hf-eval-summary/1",
  "kind": "val_v2c",
  "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.",
  "model": "esilva/KapiCoder-Mongo-Nano",
  "split": "val",
  "n": 2000,
  "seed": 20261008,
  "contract": "editor-context-v1",
  "dataset": {
    "sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f",
    "sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7",
    "manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb",
    "stratify_by": [
      "cut_kind",
      "domain"
    ],
    "cut_kind": {
      "multiline": 161,
      "key": 319,
      "value": 280,
      "rest": 241,
      "operator": 268,
      "member": 201,
      "stage": 251,
      "partial": 200,
      "stmt_start": 79
    },
    "domain": {
      "commerce": 168,
      "content": 161,
      "energy": 171,
      "finance": 175,
      "gaming": 163,
      "healthcare-synthetic": 173,
      "hr": 168,
      "iot": 168,
      "logistics": 157,
      "saas": 165,
      "travel": 162,
      "workspace": 169
    }
  },
  "decoding": "greedy (argmax), padding do vocab mascarado",
  "max_new_tokens": 96,
  "context_tokens": 2048,
  "load": "bf16",
  "this_model": {
    "headline": {
      "n": 2000,
      "exact_match": 0.405,
      "normalized_match": 0.405,
      "prefix_match": 0.4105,
      "edit_similarity": 0.6499,
      "chars_accepted": 7.9775,
      "tab_chars": 6.343,
      "accepted_prefix_tokens": 2.62,
      "syntax_valid": 0.9955,
      "mongo_valid": 0.992,
      "exec_valid": 0.5945,
      "exec_valid_incl_empty": 0.5945,
      "nonexistent_fields_rate": 0.0265,
      "empty_rate": 0.0,
      "eos_or_suffix_stop_rate": 0.997,
      "mean_generated_tokens": 15.292
    },
    "headline_ide32": {
      "n": 2000,
      "exact_match": 0.4035,
      "normalized_match": 0.4035,
      "prefix_match": 0.4105,
      "edit_similarity": 0.6421,
      "chars_accepted": 7.9575,
      "tab_chars": 6.323,
      "accepted_prefix_tokens": 2.6125,
      "syntax_valid": 0.836,
      "mongo_valid": 0.833,
      "exec_valid": 0.5055,
      "exec_valid_incl_empty": 0.5055,
      "nonexistent_fields_rate": 0.0165,
      "empty_rate": 0.0,
      "eos_or_suffix_stop_rate": null,
      "mean_generated_tokens": null
    },
    "reference_ceiling": {
      "syntax_valid": 1.0,
      "mongo_valid": 1.0,
      "exec_valid": 0.6255,
      "nonexistent_fields_rate": 0.0
    },
    "exec_stage_distribution": {
      "ok": 1189,
      "api": 483,
      "syntax": 248,
      "fields": 75,
      "exec": 2,
      "type": 3
    },
    "by_cut_kind": {
      "key": {
        "n": 319,
        "exact_match": 0.4639,
        "normalized_match": 0.4639,
        "prefix_match": 0.4639,
        "edit_similarity": 0.6428,
        "chars_accepted": 6.2351,
        "tab_chars": 5.7335,
        "accepted_prefix_tokens": 1.9154,
        "syntax_valid": 0.9969,
        "mongo_valid": 0.9937,
        "exec_valid": 0.5392,
        "exec_valid_incl_empty": 0.5392,
        "nonexistent_fields_rate": 0.0376,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 8.7147
      },
      "member": {
        "n": 201,
        "exact_match": 0.7711,
        "normalized_match": 0.7711,
        "prefix_match": 0.7761,
        "edit_similarity": 0.8523,
        "chars_accepted": 7.9254,
        "tab_chars": 7.6368,
        "accepted_prefix_tokens": 1.6269,
        "syntax_valid": 1.0,
        "mongo_valid": 0.995,
        "exec_valid": 0.6368,
        "exec_valid_incl_empty": 0.6368,
        "nonexistent_fields_rate": 0.0,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 5.209
      },
      "multiline": {
        "n": 161,
        "exact_match": 0.0,
        "normalized_match": 0.0,
        "prefix_match": 0.0,
        "edit_similarity": 0.4178,
        "chars_accepted": 11.5155,
        "tab_chars": 7.1491,
        "accepted_prefix_tokens": 3.5839,
        "syntax_valid": 0.9938,
        "mongo_valid": 0.9938,
        "exec_valid": 0.5901,
        "exec_valid_incl_empty": 0.5901,
        "nonexistent_fields_rate": 0.087,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9876,
        "mean_generated_tokens": 43.7578
      },
      "operator": {
        "n": 268,
        "exact_match": 0.6045,
        "normalized_match": 0.6045,
        "prefix_match": 0.6157,
        "edit_similarity": 0.7894,
        "chars_accepted": 7.6194,
        "tab_chars": 6.4701,
        "accepted_prefix_tokens": 3.0261,
        "syntax_valid": 0.9963,
        "mongo_valid": 0.9963,
        "exec_valid": 0.6306,
        "exec_valid_incl_empty": 0.6306,
        "nonexistent_fields_rate": 0.0187,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9963,
        "mean_generated_tokens": 9.8881
      },
      "partial": {
        "n": 200,
        "exact_match": 0.695,
        "normalized_match": 0.695,
        "prefix_match": 0.695,
        "edit_similarity": 0.8264,
        "chars_accepted": 6.34,
        "tab_chars": 5.425,
        "accepted_prefix_tokens": 2.405,
        "syntax_valid": 0.995,
        "mongo_valid": 0.98,
        "exec_valid": 0.57,
        "exec_valid_incl_empty": 0.57,
        "nonexistent_fields_rate": 0.05,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 7.9
      },
      "rest": {
        "n": 241,
        "exact_match": 0.3444,
        "normalized_match": 0.3444,
        "prefix_match": 0.3485,
        "edit_similarity": 0.6293,
        "chars_accepted": 9.4855,
        "tab_chars": 7.3361,
        "accepted_prefix_tokens": 2.9793,
        "syntax_valid": 0.9959,
        "mongo_valid": 0.9876,
        "exec_valid": 0.5809,
        "exec_valid_incl_empty": 0.5809,
        "nonexistent_fields_rate": 0.0124,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 16.7054
      },
      "stage": {
        "n": 251,
        "exact_match": 0.012,
        "normalized_match": 0.012,
        "prefix_match": 0.012,
        "edit_similarity": 0.4255,
        "chars_accepted": 11.4861,
        "tab_chars": 7.8247,
        "accepted_prefix_tokens": 4.243,
        "syntax_valid": 0.992,
        "mongo_valid": 0.992,
        "exec_valid": 0.5817,
        "exec_valid_incl_empty": 0.5817,
        "nonexistent_fields_rate": 0.0239,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.992,
        "mean_generated_tokens": 30.7131
      },
      "stmt_start": {
        "n": 79,
        "exact_match": 0.3671,
        "normalized_match": 0.3671,
        "prefix_match": 0.3671,
        "edit_similarity": 0.6448,
        "chars_accepted": 16.0633,
        "tab_chars": 13.8734,
        "accepted_prefix_tokens": 3.5316,
        "syntax_valid": 0.9873,
        "mongo_valid": 0.9873,
        "exec_valid": 0.7468,
        "exec_valid_incl_empty": 0.7468,
        "nonexistent_fields_rate": 0.038,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9873,
        "mean_generated_tokens": 16.3924
      },
      "value": {
        "n": 280,
        "exact_match": 0.325,
        "normalized_match": 0.325,
        "prefix_match": 0.3464,
        "edit_similarity": 0.6071,
        "chars_accepted": 2.7536,
        "tab_chars": 1.8714,
        "accepted_prefix_tokens": 1.325,
        "syntax_valid": 0.9964,
        "mongo_valid": 0.9964,
        "exec_valid": 0.5929,
        "exec_valid_incl_empty": 0.5929,
        "nonexistent_fields_rate": 0.0,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 8.7571
      }
    },
    "by_domain": {
      "commerce": {
        "n": 168,
        "exact_match": 0.375,
        "normalized_match": 0.375,
        "prefix_match": 0.381,
        "edit_similarity": 0.623,
        "chars_accepted": 6.9881,
        "tab_chars": 5.3214,
        "accepted_prefix_tokens": 2.25,
        "syntax_valid": 1.0,
        "mongo_valid": 0.994,
        "exec_valid": 0.6548,
        "exec_valid_incl_empty": 0.6548,
        "nonexistent_fields_rate": 0.0298,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 14.1786
      },
      "content": {
        "n": 161,
        "exact_match": 0.3913,
        "normalized_match": 0.3913,
        "prefix_match": 0.4099,
        "edit_similarity": 0.6281,
        "chars_accepted": 8.0497,
        "tab_chars": 6.5652,
        "accepted_prefix_tokens": 2.7019,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.5901,
        "exec_valid_incl_empty": 0.5901,
        "nonexistent_fields_rate": 0.0062,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 14.8012
      },
      "energy": {
        "n": 171,
        "exact_match": 0.4269,
        "normalized_match": 0.4269,
        "prefix_match": 0.4327,
        "edit_similarity": 0.6494,
        "chars_accepted": 7.7953,
        "tab_chars": 6.5146,
        "accepted_prefix_tokens": 2.3509,
        "syntax_valid": 0.9942,
        "mongo_valid": 0.9942,
        "exec_valid": 0.614,
        "exec_valid_incl_empty": 0.614,
        "nonexistent_fields_rate": 0.0351,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9942,
        "mean_generated_tokens": 16.6842
      },
      "finance": {
        "n": 175,
        "exact_match": 0.3486,
        "normalized_match": 0.3486,
        "prefix_match": 0.3543,
        "edit_similarity": 0.6135,
        "chars_accepted": 8.1257,
        "tab_chars": 6.6286,
        "accepted_prefix_tokens": 2.88,
        "syntax_valid": 0.9943,
        "mongo_valid": 0.9943,
        "exec_valid": 0.5943,
        "exec_valid_incl_empty": 0.5943,
        "nonexistent_fields_rate": 0.04,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 18.5257
      },
      "gaming": {
        "n": 163,
        "exact_match": 0.3681,
        "normalized_match": 0.3681,
        "prefix_match": 0.3804,
        "edit_similarity": 0.6296,
        "chars_accepted": 7.6196,
        "tab_chars": 6.0184,
        "accepted_prefix_tokens": 2.5092,
        "syntax_valid": 0.9877,
        "mongo_valid": 0.9755,
        "exec_valid": 0.5951,
        "exec_valid_incl_empty": 0.5951,
        "nonexistent_fields_rate": 0.0245,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 14.8712
      },
      "healthcare-synthetic": {
        "n": 173,
        "exact_match": 0.4104,
        "normalized_match": 0.4104,
        "prefix_match": 0.4104,
        "edit_similarity": 0.6618,
        "chars_accepted": 8.3815,
        "tab_chars": 6.4624,
        "accepted_prefix_tokens": 2.7341,
        "syntax_valid": 0.9942,
        "mongo_valid": 0.9884,
        "exec_valid": 0.5607,
        "exec_valid_incl_empty": 0.5607,
        "nonexistent_fields_rate": 0.0231,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9942,
        "mean_generated_tokens": 15.0636
      },
      "hr": {
        "n": 168,
        "exact_match": 0.4643,
        "normalized_match": 0.4643,
        "prefix_match": 0.4762,
        "edit_similarity": 0.6899,
        "chars_accepted": 7.8155,
        "tab_chars": 6.3393,
        "accepted_prefix_tokens": 2.5357,
        "syntax_valid": 0.994,
        "mongo_valid": 0.994,
        "exec_valid": 0.5833,
        "exec_valid_incl_empty": 0.5833,
        "nonexistent_fields_rate": 0.0119,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.994,
        "mean_generated_tokens": 12.7738
      },
      "iot": {
        "n": 168,
        "exact_match": 0.4107,
        "normalized_match": 0.4107,
        "prefix_match": 0.4107,
        "edit_similarity": 0.6606,
        "chars_accepted": 8.4821,
        "tab_chars": 6.4643,
        "accepted_prefix_tokens": 2.869,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6369,
        "exec_valid_incl_empty": 0.6369,
        "nonexistent_fields_rate": 0.0536,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 15.3155
      },
      "logistics": {
        "n": 157,
        "exact_match": 0.4076,
        "normalized_match": 0.4076,
        "prefix_match": 0.4076,
        "edit_similarity": 0.6721,
        "chars_accepted": 8.9554,
        "tab_chars": 6.8408,
        "accepted_prefix_tokens": 2.8535,
        "syntax_valid": 1.0,
        "mongo_valid": 0.9936,
        "exec_valid": 0.5987,
        "exec_valid_incl_empty": 0.5987,
        "nonexistent_fields_rate": 0.0255,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 15.2102
      },
      "saas": {
        "n": 165,
        "exact_match": 0.4788,
        "normalized_match": 0.4788,
        "prefix_match": 0.4788,
        "edit_similarity": 0.7021,
        "chars_accepted": 7.0242,
        "tab_chars": 5.5636,
        "accepted_prefix_tokens": 2.3758,
        "syntax_valid": 0.9939,
        "mongo_valid": 0.9939,
        "exec_valid": 0.4848,
        "exec_valid_incl_empty": 0.4848,
        "nonexistent_fields_rate": 0.0364,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 12.4909
      },
      "travel": {
        "n": 162,
        "exact_match": 0.4136,
        "normalized_match": 0.4136,
        "prefix_match": 0.4198,
        "edit_similarity": 0.6412,
        "chars_accepted": 8.4506,
        "tab_chars": 6.6667,
        "accepted_prefix_tokens": 2.7901,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6543,
        "exec_valid_incl_empty": 0.6543,
        "nonexistent_fields_rate": 0.0123,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 16.5123
      },
      "workspace": {
        "n": 169,
        "exact_match": 0.3669,
        "normalized_match": 0.3669,
        "prefix_match": 0.3669,
        "edit_similarity": 0.6296,
        "chars_accepted": 8.0828,
        "tab_chars": 6.7396,
        "accepted_prefix_tokens": 2.5976,
        "syntax_valid": 0.9882,
        "mongo_valid": 0.9763,
        "exec_valid": 0.568,
        "exec_valid_incl_empty": 0.568,
        "nonexistent_fields_rate": 0.0178,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9822,
        "mean_generated_tokens": 16.8698
      }
    },
    "by_context_kind": {
      "full": {
        "n": 1647,
        "exact_match": 0.4141,
        "normalized_match": 0.4141,
        "prefix_match": 0.4196,
        "edit_similarity": 0.6598,
        "chars_accepted": 8.1488,
        "tab_chars": 6.4584,
        "accepted_prefix_tokens": 2.6752,
        "syntax_valid": 0.9951,
        "mongo_valid": 0.9927,
        "exec_valid": 0.6169,
        "exec_valid_incl_empty": 0.6169,
        "nonexistent_fields_rate": 0.0134,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.997,
        "mean_generated_tokens": 15.4244
      },
      "irrelevant": {
        "n": 198,
        "exact_match": 0.3687,
        "normalized_match": 0.3687,
        "prefix_match": 0.3737,
        "edit_similarity": 0.5935,
        "chars_accepted": 7.0707,
        "tab_chars": 5.8081,
        "accepted_prefix_tokens": 2.2374,
        "syntax_valid": 1.0,
        "mongo_valid": 0.9949,
        "exec_valid": 0.5051,
        "exec_valid_incl_empty": 0.5051,
        "nonexistent_fields_rate": 0.0859,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9949,
        "mean_generated_tokens": 14.5505
      },
      "none": {
        "n": 155,
        "exact_match": 0.3548,
        "normalized_match": 0.3548,
        "prefix_match": 0.3613,
        "edit_similarity": 0.6167,
        "chars_accepted": 7.3161,
        "tab_chars": 5.8,
        "accepted_prefix_tokens": 2.5226,
        "syntax_valid": 0.9935,
        "mongo_valid": 0.9806,
        "exec_valid": 0.471,
        "exec_valid_incl_empty": 0.471,
        "nonexistent_fields_rate": 0.0903,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 14.8323
      }
    }
  },
  "metrics_by_model": {
    "this_model_v3": {
      "syntax_valid": 0.9955,
      "mongodb_valid": 0.992,
      "nonexistent_fields_rate": 2.65,
      "conversational_suggestion_rate": 0.0,
      "fim_acc": 0.405,
      "exact_match": 0.405,
      "apt_mean": 2.62,
      "exec_valid_autocomplete": 0.5945,
      "empty_rate": 0.0,
      "reference_ceiling_exec_valid": 0.6255,
      "exec_valid_vs_ceiling": 0.9504,
      "nonexistent_fields_full_rate_pct": 1.34,
      "n_context_full": 1647,
      "exec_valid_full": 0.6169,
      "nonexistent_fields_irrelevant_rate_pct": 8.59,
      "n_context_irrelevant": 198,
      "exec_valid_irrelevant": 0.5051,
      "nonexistent_fields_none_rate_pct": 9.03,
      "n_context_none": 155,
      "exec_valid_none": 0.471,
      "info_fim_acc_gain_vs_zero_shot_base_pp": 32.9,
      "info_apt_delta_vs_zero_shot_base": 1.4155,
      "info_exec_valid_delta_vs_zero_shot_base_pp": 38.15,
      "generic_js_fim_acc": 0.53,
      "generic_js_fim_delta_pp_vs_zero_shot_base": 19.0,
      "write_or_destructive_executions": 0,
      "train_eval_leak_count": 0,
      "decontamination_approved": true,
      "auto_report_attached": true
    },
    "previous_internal_checkpoint_v1_not_released": {
      "syntax_valid": 0.995,
      "mongodb_valid": 0.994,
      "nonexistent_fields_rate": 3.8,
      "conversational_suggestion_rate": 0.0,
      "fim_acc": 0.403,
      "exact_match": 0.403,
      "apt_mean": 2.6605,
      "exec_valid_autocomplete": 0.582,
      "empty_rate": 0.0,
      "reference_ceiling_exec_valid": 0.6255,
      "exec_valid_vs_ceiling": 0.9305,
      "nonexistent_fields_full_rate_pct": 2.06,
      "n_context_full": 1647,
      "exec_valid_full": 0.6084,
      "nonexistent_fields_irrelevant_rate_pct": 11.11,
      "n_context_irrelevant": 198,
      "exec_valid_irrelevant": 0.4798,
      "nonexistent_fields_none_rate_pct": 12.9,
      "n_context_none": 155,
      "exec_valid_none": 0.4323,
      "info_fim_acc_gain_vs_zero_shot_base_pp": 32.7,
      "info_apt_delta_vs_zero_shot_base": 1.456,
      "info_exec_valid_delta_vs_zero_shot_base_pp": 36.9,
      "generic_js_fim_acc": 0.518,
      "generic_js_fim_delta_pp_vs_zero_shot_base": 17.8,
      "write_or_destructive_executions": 0,
      "train_eval_leak_count": 0,
      "decontamination_approved": true,
      "auto_report_attached": true
    },
    "zero_shot_base": {
      "syntax_valid": 0.498,
      "mongodb_valid": 0.4275,
      "nonexistent_fields_rate": 4.85,
      "conversational_suggestion_rate": 0.0,
      "fim_acc": 0.076,
      "exact_match": 0.076,
      "apt_mean": 1.2045,
      "exec_valid_autocomplete": 0.213,
      "empty_rate": 0.0055,
      "reference_ceiling_exec_valid": 0.6255,
      "exec_valid_vs_ceiling": 0.3405,
      "nonexistent_fields_full_rate_pct": 4.13,
      "n_context_full": 1647,
      "exec_valid_full": 0.2198,
      "nonexistent_fields_irrelevant_rate_pct": 8.59,
      "n_context_irrelevant": 198,
      "exec_valid_irrelevant": 0.1667,
      "nonexistent_fields_none_rate_pct": 7.74,
      "n_context_none": 155,
      "exec_valid_none": 0.2,
      "generic_js_fim_acc": 0.34,
      "write_or_destructive_executions": 0,
      "train_eval_leak_count": 0,
      "decontamination_approved": true,
      "auto_report_attached": true
    }
  },
  "metric_definitions": {
    "exact_match": "pred.strip() == middle.strip()",
    "normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)",
    "chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle",
    "accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)",
    "syntax_valid": "node vm.Script sobre prefix+pred+suffix",
    "mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido",
    "exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido",
    "nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos",
    "reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)"
  }
}