{
  "schema": "kapi-hf-eval-summary/1",
  "kind": "val_v2c",
  "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.",
  "model": "esilva/KapiCoder-Mongo-Small",
  "split": "val",
  "n": 2000,
  "seed": 20261008,
  "contract": "editor-context-v1",
  "dataset": {
    "sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f",
    "sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7",
    "manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb",
    "stratify_by": [
      "cut_kind",
      "domain"
    ],
    "cut_kind": {
      "multiline": 161,
      "key": 319,
      "value": 280,
      "rest": 241,
      "operator": 268,
      "member": 201,
      "stage": 251,
      "partial": 200,
      "stmt_start": 79
    },
    "domain": {
      "commerce": 168,
      "content": 161,
      "energy": 171,
      "finance": 175,
      "gaming": 163,
      "healthcare-synthetic": 173,
      "hr": 168,
      "iot": 168,
      "logistics": 157,
      "saas": 165,
      "travel": 162,
      "workspace": 169
    }
  },
  "decoding": "greedy (argmax), padding do vocab mascarado",
  "max_new_tokens": 96,
  "context_tokens": 2048,
  "load": "bf16",
  "this_model": {
    "headline": {
      "n": 2000,
      "exact_match": 0.41,
      "normalized_match": 0.41,
      "prefix_match": 0.416,
      "edit_similarity": 0.6536,
      "chars_accepted": 8.214,
      "tab_chars": 6.5425,
      "accepted_prefix_tokens": 2.698,
      "syntax_valid": 0.998,
      "mongo_valid": 0.9975,
      "exec_valid": 0.6015,
      "exec_valid_incl_empty": 0.6015,
      "nonexistent_fields_rate": 0.021,
      "empty_rate": 0.0,
      "eos_or_suffix_stop_rate": 0.9975,
      "mean_generated_tokens": 15.3875
    },
    "headline_ide32": {
      "n": 2000,
      "exact_match": 0.409,
      "normalized_match": 0.409,
      "prefix_match": 0.4165,
      "edit_similarity": 0.6463,
      "chars_accepted": 8.2,
      "tab_chars": 6.5285,
      "accepted_prefix_tokens": 2.693,
      "syntax_valid": 0.836,
      "mongo_valid": 0.8355,
      "exec_valid": 0.5115,
      "exec_valid_incl_empty": 0.5115,
      "nonexistent_fields_rate": 0.012,
      "empty_rate": 0.0,
      "eos_or_suffix_stop_rate": null,
      "mean_generated_tokens": null
    },
    "reference_ceiling": {
      "syntax_valid": 1.0,
      "mongo_valid": 1.0,
      "exec_valid": 0.6255,
      "nonexistent_fields_rate": 0.0
    },
    "exec_stage_distribution": {
      "ok": 1203,
      "api": 490,
      "syntax": 243,
      "type": 3,
      "fields": 59,
      "exec": 2
    },
    "by_cut_kind": {
      "key": {
        "n": 319,
        "exact_match": 0.4514,
        "normalized_match": 0.4514,
        "prefix_match": 0.4545,
        "edit_similarity": 0.6407,
        "chars_accepted": 6.4639,
        "tab_chars": 5.8119,
        "accepted_prefix_tokens": 2.0,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.5361,
        "exec_valid_incl_empty": 0.5361,
        "nonexistent_fields_rate": 0.0408,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 9.1442
      },
      "member": {
        "n": 201,
        "exact_match": 0.801,
        "normalized_match": 0.801,
        "prefix_match": 0.801,
        "edit_similarity": 0.8754,
        "chars_accepted": 8.6965,
        "tab_chars": 8.3134,
        "accepted_prefix_tokens": 1.8259,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6517,
        "exec_valid_incl_empty": 0.6517,
        "nonexistent_fields_rate": 0.0,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 5.1045
      },
      "multiline": {
        "n": 161,
        "exact_match": 0.0,
        "normalized_match": 0.0,
        "prefix_match": 0.0,
        "edit_similarity": 0.4159,
        "chars_accepted": 11.5652,
        "tab_chars": 7.2671,
        "accepted_prefix_tokens": 3.677,
        "syntax_valid": 0.9876,
        "mongo_valid": 0.9876,
        "exec_valid": 0.6149,
        "exec_valid_incl_empty": 0.6149,
        "nonexistent_fields_rate": 0.0807,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9814,
        "mean_generated_tokens": 44.3665
      },
      "operator": {
        "n": 268,
        "exact_match": 0.6119,
        "normalized_match": 0.6119,
        "prefix_match": 0.6231,
        "edit_similarity": 0.7941,
        "chars_accepted": 7.7052,
        "tab_chars": 6.5522,
        "accepted_prefix_tokens": 3.0597,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6306,
        "exec_valid_incl_empty": 0.6306,
        "nonexistent_fields_rate": 0.0037,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 9.8134
      },
      "partial": {
        "n": 200,
        "exact_match": 0.72,
        "normalized_match": 0.72,
        "prefix_match": 0.72,
        "edit_similarity": 0.8414,
        "chars_accepted": 6.265,
        "tab_chars": 5.435,
        "accepted_prefix_tokens": 2.36,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.59,
        "exec_valid_incl_empty": 0.59,
        "nonexistent_fields_rate": 0.035,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 7.66
      },
      "rest": {
        "n": 241,
        "exact_match": 0.3444,
        "normalized_match": 0.3444,
        "prefix_match": 0.3485,
        "edit_similarity": 0.6199,
        "chars_accepted": 9.8714,
        "tab_chars": 7.7344,
        "accepted_prefix_tokens": 3.1286,
        "syntax_valid": 1.0,
        "mongo_valid": 0.9959,
        "exec_valid": 0.5892,
        "exec_valid_incl_empty": 0.5892,
        "nonexistent_fields_rate": 0.0124,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 17.5021
      },
      "stage": {
        "n": 251,
        "exact_match": 0.0239,
        "normalized_match": 0.0239,
        "prefix_match": 0.0239,
        "edit_similarity": 0.4418,
        "chars_accepted": 11.8367,
        "tab_chars": 8.1474,
        "accepted_prefix_tokens": 4.3347,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.5976,
        "exec_valid_incl_empty": 0.5976,
        "nonexistent_fields_rate": 0.012,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 30.0478
      },
      "stmt_start": {
        "n": 79,
        "exact_match": 0.3671,
        "normalized_match": 0.3671,
        "prefix_match": 0.3671,
        "edit_similarity": 0.6481,
        "chars_accepted": 16.2785,
        "tab_chars": 13.9747,
        "accepted_prefix_tokens": 3.6076,
        "syntax_valid": 0.9873,
        "mongo_valid": 0.9873,
        "exec_valid": 0.7089,
        "exec_valid_incl_empty": 0.7089,
        "nonexistent_fields_rate": 0.0253,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9873,
        "mean_generated_tokens": 16.3924
      },
      "value": {
        "n": 280,
        "exact_match": 0.3179,
        "normalized_match": 0.3179,
        "prefix_match": 0.3429,
        "edit_similarity": 0.5979,
        "chars_accepted": 2.8643,
        "tab_chars": 1.9071,
        "accepted_prefix_tokens": 1.3571,
        "syntax_valid": 0.9964,
        "mongo_valid": 0.9964,
        "exec_valid": 0.5964,
        "exec_valid_incl_empty": 0.5964,
        "nonexistent_fields_rate": 0.0,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9964,
        "mean_generated_tokens": 8.8286
      }
    },
    "by_domain": {
      "commerce": {
        "n": 168,
        "exact_match": 0.381,
        "normalized_match": 0.381,
        "prefix_match": 0.3869,
        "edit_similarity": 0.6404,
        "chars_accepted": 7.1667,
        "tab_chars": 5.4464,
        "accepted_prefix_tokens": 2.2917,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6607,
        "exec_valid_incl_empty": 0.6607,
        "nonexistent_fields_rate": 0.0119,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 15.1488
      },
      "content": {
        "n": 161,
        "exact_match": 0.3851,
        "normalized_match": 0.3851,
        "prefix_match": 0.4037,
        "edit_similarity": 0.6223,
        "chars_accepted": 8.1366,
        "tab_chars": 6.5093,
        "accepted_prefix_tokens": 2.6894,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6087,
        "exec_valid_incl_empty": 0.6087,
        "nonexistent_fields_rate": 0.0062,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 15.1491
      },
      "energy": {
        "n": 171,
        "exact_match": 0.4386,
        "normalized_match": 0.4386,
        "prefix_match": 0.4444,
        "edit_similarity": 0.6549,
        "chars_accepted": 8.1637,
        "tab_chars": 6.8129,
        "accepted_prefix_tokens": 2.4737,
        "syntax_valid": 0.9942,
        "mongo_valid": 0.9942,
        "exec_valid": 0.6023,
        "exec_valid_incl_empty": 0.6023,
        "nonexistent_fields_rate": 0.0468,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9942,
        "mean_generated_tokens": 16.0526
      },
      "finance": {
        "n": 175,
        "exact_match": 0.3429,
        "normalized_match": 0.3429,
        "prefix_match": 0.3486,
        "edit_similarity": 0.6153,
        "chars_accepted": 8.2457,
        "tab_chars": 6.5314,
        "accepted_prefix_tokens": 2.9486,
        "syntax_valid": 0.9943,
        "mongo_valid": 0.9943,
        "exec_valid": 0.5943,
        "exec_valid_incl_empty": 0.5943,
        "nonexistent_fields_rate": 0.0286,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9943,
        "mean_generated_tokens": 18.6514
      },
      "gaming": {
        "n": 163,
        "exact_match": 0.3742,
        "normalized_match": 0.3742,
        "prefix_match": 0.3804,
        "edit_similarity": 0.6392,
        "chars_accepted": 7.9264,
        "tab_chars": 6.1902,
        "accepted_prefix_tokens": 2.5644,
        "syntax_valid": 1.0,
        "mongo_valid": 0.9939,
        "exec_valid": 0.6012,
        "exec_valid_incl_empty": 0.6012,
        "nonexistent_fields_rate": 0.0184,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 15.0184
      },
      "healthcare-synthetic": {
        "n": 173,
        "exact_match": 0.4046,
        "normalized_match": 0.4046,
        "prefix_match": 0.4046,
        "edit_similarity": 0.6652,
        "chars_accepted": 8.5896,
        "tab_chars": 6.4913,
        "accepted_prefix_tokens": 2.8497,
        "syntax_valid": 0.9942,
        "mongo_valid": 0.9942,
        "exec_valid": 0.5838,
        "exec_valid_incl_empty": 0.5838,
        "nonexistent_fields_rate": 0.0116,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9942,
        "mean_generated_tokens": 14.9827
      },
      "hr": {
        "n": 168,
        "exact_match": 0.4762,
        "normalized_match": 0.4762,
        "prefix_match": 0.4881,
        "edit_similarity": 0.7032,
        "chars_accepted": 8.8333,
        "tab_chars": 7.5119,
        "accepted_prefix_tokens": 2.8512,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6012,
        "exec_valid_incl_empty": 0.6012,
        "nonexistent_fields_rate": 0.0119,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 12.8869
      },
      "iot": {
        "n": 168,
        "exact_match": 0.4345,
        "normalized_match": 0.4345,
        "prefix_match": 0.4405,
        "edit_similarity": 0.6659,
        "chars_accepted": 8.8393,
        "tab_chars": 6.9583,
        "accepted_prefix_tokens": 2.9702,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6429,
        "exec_valid_incl_empty": 0.6429,
        "nonexistent_fields_rate": 0.0357,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 15.6071
      },
      "logistics": {
        "n": 157,
        "exact_match": 0.4331,
        "normalized_match": 0.4331,
        "prefix_match": 0.4331,
        "edit_similarity": 0.6879,
        "chars_accepted": 8.9172,
        "tab_chars": 6.8726,
        "accepted_prefix_tokens": 2.8917,
        "syntax_valid": 0.9936,
        "mongo_valid": 0.9936,
        "exec_valid": 0.586,
        "exec_valid_incl_empty": 0.586,
        "nonexistent_fields_rate": 0.0318,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9936,
        "mean_generated_tokens": 14.5669
      },
      "saas": {
        "n": 165,
        "exact_match": 0.4788,
        "normalized_match": 0.4788,
        "prefix_match": 0.4848,
        "edit_similarity": 0.6938,
        "chars_accepted": 7.2424,
        "tab_chars": 5.6788,
        "accepted_prefix_tokens": 2.4909,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.5091,
        "exec_valid_incl_empty": 0.5091,
        "nonexistent_fields_rate": 0.0303,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 12.6061
      },
      "travel": {
        "n": 162,
        "exact_match": 0.4074,
        "normalized_match": 0.4074,
        "prefix_match": 0.4136,
        "edit_similarity": 0.6385,
        "chars_accepted": 8.8086,
        "tab_chars": 7.0864,
        "accepted_prefix_tokens": 2.9136,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.6605,
        "exec_valid_incl_empty": 0.6605,
        "nonexistent_fields_rate": 0.0062,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 16.5309
      },
      "workspace": {
        "n": 169,
        "exact_match": 0.3669,
        "normalized_match": 0.3669,
        "prefix_match": 0.3669,
        "edit_similarity": 0.6191,
        "chars_accepted": 7.7278,
        "tab_chars": 6.432,
        "accepted_prefix_tokens": 2.4438,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.568,
        "exec_valid_incl_empty": 0.568,
        "nonexistent_fields_rate": 0.0118,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9941,
        "mean_generated_tokens": 17.2189
      }
    },
    "by_context_kind": {
      "full": {
        "n": 1647,
        "exact_match": 0.4183,
        "normalized_match": 0.4183,
        "prefix_match": 0.4238,
        "edit_similarity": 0.6613,
        "chars_accepted": 8.3734,
        "tab_chars": 6.6758,
        "accepted_prefix_tokens": 2.759,
        "syntax_valid": 0.9976,
        "mongo_valid": 0.997,
        "exec_valid": 0.6211,
        "exec_valid_incl_empty": 0.6211,
        "nonexistent_fields_rate": 0.0121,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9976,
        "mean_generated_tokens": 15.4857
      },
      "irrelevant": {
        "n": 198,
        "exact_match": 0.3838,
        "normalized_match": 0.3838,
        "prefix_match": 0.3889,
        "edit_similarity": 0.6152,
        "chars_accepted": 7.5657,
        "tab_chars": 6.2121,
        "accepted_prefix_tokens": 2.3333,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.5303,
        "exec_valid_incl_empty": 0.5303,
        "nonexistent_fields_rate": 0.0556,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 0.9949,
        "mean_generated_tokens": 15.0
      },
      "none": {
        "n": 155,
        "exact_match": 0.3548,
        "normalized_match": 0.3548,
        "prefix_match": 0.3677,
        "edit_similarity": 0.6217,
        "chars_accepted": 7.3484,
        "tab_chars": 5.5484,
        "accepted_prefix_tokens": 2.5161,
        "syntax_valid": 1.0,
        "mongo_valid": 1.0,
        "exec_valid": 0.4839,
        "exec_valid_incl_empty": 0.4839,
        "nonexistent_fields_rate": 0.071,
        "empty_rate": 0.0,
        "eos_or_suffix_stop_rate": 1.0,
        "mean_generated_tokens": 14.8387
      }
    }
  },
  "metrics_by_model": {
    "this_model_v3": {
      "syntax_valid": 0.998,
      "mongodb_valid": 0.9975,
      "nonexistent_fields_rate": 2.1,
      "conversational_suggestion_rate": 0.0,
      "fim_acc": 0.41,
      "exact_match": 0.41,
      "apt_mean": 2.698,
      "exec_valid_autocomplete": 0.6015,
      "empty_rate": 0.0,
      "reference_ceiling_exec_valid": 0.6255,
      "exec_valid_vs_ceiling": 0.9616,
      "nonexistent_fields_full_rate_pct": 1.21,
      "n_context_full": 1647,
      "exec_valid_full": 0.6211,
      "nonexistent_fields_irrelevant_rate_pct": 5.56,
      "n_context_irrelevant": 198,
      "exec_valid_irrelevant": 0.5303,
      "nonexistent_fields_none_rate_pct": 7.1,
      "n_context_none": 155,
      "exec_valid_none": 0.4839,
      "info_fim_acc_gain_vs_zero_shot_base_pp": 26.8,
      "info_apt_delta_vs_zero_shot_base": 1.259,
      "info_exec_valid_delta_vs_zero_shot_base_pp": 28.55,
      "generic_js_fim_acc": 0.606,
      "generic_js_fim_delta_pp_vs_zero_shot_base": 20.0,
      "write_or_destructive_executions": 0,
      "train_eval_leak_count": 0,
      "decontamination_approved": true,
      "auto_report_attached": true
    },
    "previous_internal_checkpoint_v1_not_released": {
      "syntax_valid": 0.9955,
      "mongodb_valid": 0.9945,
      "nonexistent_fields_rate": 2.9,
      "conversational_suggestion_rate": 0.0,
      "fim_acc": 0.4105,
      "exact_match": 0.4105,
      "apt_mean": 2.6665,
      "exec_valid_autocomplete": 0.595,
      "empty_rate": 0.0,
      "reference_ceiling_exec_valid": 0.6255,
      "exec_valid_vs_ceiling": 0.9512,
      "nonexistent_fields_full_rate_pct": 1.58,
      "n_context_full": 1647,
      "exec_valid_full": 0.6199,
      "nonexistent_fields_irrelevant_rate_pct": 7.58,
      "n_context_irrelevant": 198,
      "exec_valid_irrelevant": 0.5051,
      "nonexistent_fields_none_rate_pct": 10.97,
      "n_context_none": 155,
      "exec_valid_none": 0.4452,
      "info_fim_acc_gain_vs_zero_shot_base_pp": 26.85,
      "info_apt_delta_vs_zero_shot_base": 1.2275,
      "info_exec_valid_delta_vs_zero_shot_base_pp": 27.9,
      "generic_js_fim_acc": 0.632,
      "generic_js_fim_delta_pp_vs_zero_shot_base": 22.6,
      "write_or_destructive_executions": 0,
      "train_eval_leak_count": 0,
      "decontamination_approved": true,
      "auto_report_attached": true
    },
    "zero_shot_base": {
      "syntax_valid": 0.667,
      "mongodb_valid": 0.595,
      "nonexistent_fields_rate": 5.1,
      "conversational_suggestion_rate": 0.0,
      "fim_acc": 0.142,
      "exact_match": 0.142,
      "apt_mean": 1.439,
      "exec_valid_autocomplete": 0.316,
      "empty_rate": 0.0095,
      "reference_ceiling_exec_valid": 0.6255,
      "exec_valid_vs_ceiling": 0.5052,
      "nonexistent_fields_full_rate_pct": 3.76,
      "n_context_full": 1647,
      "exec_valid_full": 0.3364,
      "nonexistent_fields_irrelevant_rate_pct": 10.61,
      "n_context_irrelevant": 198,
      "exec_valid_irrelevant": 0.2172,
      "nonexistent_fields_none_rate_pct": 12.26,
      "n_context_none": 155,
      "exec_valid_none": 0.2258,
      "generic_js_fim_acc": 0.406,
      "write_or_destructive_executions": 0,
      "train_eval_leak_count": 0,
      "decontamination_approved": true,
      "auto_report_attached": true
    }
  },
  "metric_definitions": {
    "exact_match": "pred.strip() == middle.strip()",
    "normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)",
    "chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle",
    "accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)",
    "syntax_valid": "node vm.Script sobre prefix+pred+suffix",
    "mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido",
    "exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido",
    "nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos",
    "reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)"
  }
}