validation.json

{
  "environment": {
    "created_at": "2026-09-20T03:20:32.385614+00:00",
    "chip": "Apple M3 Max",
    "memory_bytes": 137438953472,
    "os": "macOS-27.2-arm64-arm-64bit",
    "python": "3.12.13",
    "packages": {
      "coremltools": "9.0",
      "numpy": "2.1.3",
      "torch": "2.7.0",
      "tokenizers": "0.23.2",
      "mlx": "0.32.2",
      "laya-mlx": "0.1.0"
    },
    "source_sha256": "48f7d107ba44feedfa8f643e4c7808b1e540652d4e8275840336e8adca28f4f2"
  },
  "repository": "aac6fef/laya-multilingual-coreml-ane",
  "format": "laya-coreml-ane",
  "shape": {
    "batch_size": 1,
    "max_length": 96,
    "min_length": 96,
    "max_options": 32,
    "flexible": false,
    "lengths": null
  },
  "compute_units": "cpu_ne",
  "load_seconds": 19.706000457998016,
  "source_weights_sha256": "9d628fd971b700382ac6f65920a86f149777b2e748e0c955fb3b19695aa8f204",
  "package_sha256": "53ba84d9b002997599dafc89cde611e6edac450572c63c4896e7b9f082b53651",
  "reference_sha256": "27642bf5f16e1881514a65f91d2ba986fce786b832e60ed1a35fa44d1bfdd682",
  "bundle_files_verified": true,
  "cases": [
    {
      "name": "email",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0002779567730952781,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "en",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0003270161033079866,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "zh",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0003505306841716446,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "de",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.00045834381210041464,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "fr",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.001640766152765205,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "es",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0029248531754093587,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "hi",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0005735619059983321,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "ja",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0004844246714378331,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "ru",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0009792158013824936,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "empty_state",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0016108414481667,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "long",
      "skipped": "outside bundle capacity"
    },
    {
      "name": "conversation",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0004194883099918334,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "mask_literals",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.001467615478298795,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "many_questions",
      "questions": 20,
      "argmax_agreements": 20,
      "probability_max_abs_error": 0.00027793535504919836,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "structured",
      "questions": 3,
      "argmax_agreements": 3,
      "probability_max_abs_error": 0.0016406952452901885,
      "action_probability_max_abs_error": 0.0
    },
    {
      "name": "twenty_options",
      "skipped": "outside bundle capacity"
    }
  ],
  "passed": true,
  "questions": 59,
  "argmax_agreements": 59,
  "probability_max_abs_error": 0.0029248531754093587,
  "action_probability_max_abs_error": 0.0,
  "stability": {
    "calls": 100,
    "identical_rounded_results": true
  },
  "gate": {
    "all_argmax_agree": true,
    "max_probability_error": 0.02,
    "max_action_probability_error": 0.02,
    "finite": true,
    "stable": true
  }
}