{
  "passed": true,
  "method": "Clean wheel installation served from a directory outside the source checkout; temporary API reused existing pinned native GPU server; API stopped afterward.",
  "version": "0.1.1",
  "source_matches_benchmark": true,
  "source_sha256": {
    "__init__.py": "b6d87a4431c3d0de11cc824d2e3f2e6b4e267fe3c2aa07bf6b39098b85771d47",
    "app.py": "949cb10750883a7b0ff80f58c856151d7ce2390a745342b40037461a2159c153",
    "core.py": "1b8b4cce7260bcfbd4f8d8d0daf8ebfacabc9c202911f652674691fc30c8a1fd",
    "models.py": "8a61790f307bc21decc4c768d853f34dca5edc0ba2f5c9c585a77339396338d3"
  },
  "examples": {
    "support-request.json": {
      "model": "bev-bonsai-27b",
      "backend": "prism_llama_cpp",
      "backend_build": "b1-9a9394a",
      "mode": "llama_full_vocab_categorical_scoring",
      "strategy": "batch",
      "requested_strategy": "auto",
      "strategy_implementation": "bounded_concurrent_native_requests",
      "parsed_json": {
        "route": "billing"
      },
      "fields": {
        "route": {
          "value": "billing",
          "selected_value": "billing",
          "selected_label": "A",
          "selected_token_id": 32,
          "probability": 0.9989061260210375,
          "abstained": false,
          "candidate_probability_mass": 0.9086667884355636,
          "candidates": [
            {
              "value": "billing",
              "label": "A",
              "token_id": 32,
              "probability": 0.9989061260210375,
              "logprob": -0.09687129408121109
            },
            {
              "value": "technical",
              "label": "B",
              "token_id": 33,
              "probability": 0.0008516050271616897,
              "logprob": -7.1641645431518555
            },
            {
              "value": "unknown",
              "label": "C",
              "token_id": 34,
              "probability": 0.0002422689518008115,
              "logprob": -8.421238899230957
            }
          ],
          "elapsed_ms": 408.331,
          "queue_ms": 0.003,
          "backend_requests": 1,
          "score_retrieval": "selected_token_logprobs",
          "requested_top_n": 3,
          "prompt_tokens": 142,
          "backend_timings": {
            "cache_n": 0,
            "prompt_n": 142,
            "prompt_ms": 284.805,
            "prompt_per_token_ms": 2.005669014084507,
            "prompt_per_second": 498.58675233931984,
            "predicted_n": 1,
            "predicted_ms": 0.001,
            "predicted_per_token_ms": 0.0,
            "predicted_per_second": 0.0
          },
          "usage": {
            "prompt_tokens": 142,
            "completion_tokens": 1,
            "total_tokens": 143
          }
        }
      },
      "scores_are_calibrated": false,
      "score_source": "pre_sampling_full_vocabulary_logprobs",
      "selected_token_logprobs": true,
      "score_temperature": 1.0,
      "abstained_fields": [],
      "elapsed_ms": 412.751,
      "prompt_preparation_ms": 4.37,
      "backend_requests": 1,
      "scored_fields": 1,
      "max_prompt_tokens": 142,
      "max_model_len": 16384,
      "backend_slots": 2,
      "cache_block_tokens": null,
      "usage": {
        "prompt_tokens": 142,
        "completion_tokens": 1,
        "total_tokens": 143
      }
    },
    "systemone-request.json": {
      "model": "bev-bonsai-27b",
      "answers": {
        "queue": {
          "type": "choice",
          "choice": "billing",
          "probabilities": {
            "billing": 0.9981141129355442,
            "technical": 0.0017793923442547745,
            "general": 0.00010649472020099965
          },
          "confidence": 0.9981141129355442
        },
        "outage": {
          "type": "noul",
          "noul": 0.002480204845362905
        },
        "priority": {
          "type": "score",
          "score": 1.0002383132357986,
          "probabilities": {
            "0": 0.0007058337315065528,
            "1": 0.9983500193011885,
            "2": 0.0009441469673050024
          },
          "legend": {
            "0": "General question",
            "1": "Billing issue",
            "2": "Outage"
          },
          "confidence": 0.9983500193011885
        }
      },
      "usage": {
        "input_tokens": 547,
        "output_tokens": 3,
        "prompt_tokens": 547,
        "completion_tokens": 3,
        "total_tokens": 550
      },
      "usage_accounting": "logical_prompt_and_scoring_tokens_including_retries",
      "scores_are_calibrated": false,
      "confidence_definition": "maximum_candidate_relative_probability",
      "evaluation_trace": {
        "model": "bev-bonsai-27b",
        "backend": "prism_llama_cpp",
        "backend_build": "b1-9a9394a",
        "mode": "llama_full_vocab_categorical_scoring",
        "strategy": "batch",
        "requested_strategy": "auto",
        "strategy_implementation": "bounded_concurrent_native_requests",
        "parsed_json": {
          "queue": "billing",
          "outage": false,
          "priority": "1"
        },
        "fields": {
          "queue": {
            "value": "billing",
            "selected_value": "billing",
            "selected_label": "A",
            "selected_token_id": 32,
            "probability": 0.9981141129355442,
            "abstained": false,
            "candidate_probability_mass": 0.8587864055938211,
            "candidates": [
              {
                "value": "billing",
                "label": "A",
                "token_id": 32,
                "probability": 0.9981141129355442,
                "logprob": -0.1541227102279663
              },
              {
                "value": "technical",
                "label": "B",
                "token_id": 33,
                "probability": 0.0017793923442547745,
                "logprob": -6.483718395233154
              },
              {
                "value": "general",
                "label": "C",
                "token_id": 34,
                "probability": 0.00010649472020099965,
                "logprob": -9.299650192260742
              }
            ],
            "elapsed_ms": 668.761,
            "queue_ms": 0.002,
            "backend_requests": 1,
            "score_retrieval": "selected_token_logprobs",
            "requested_top_n": 3,
            "prompt_tokens": 190,
            "backend_timings": {
              "cache_n": 0,
              "prompt_n": 190,
              "prompt_ms": 580.518,
              "prompt_per_token_ms": 3.0553578947368423,
              "prompt_per_second": 327.2938995862316,
              "predicted_n": 1,
              "predicted_ms": 0.001,
              "predicted_per_token_ms": 0.0,
              "predicted_per_second": 0.0
            },
            "usage": {
              "prompt_tokens": 190,
              "completion_tokens": 1,
              "total_tokens": 191
            }
          },
          "outage": {
            "value": false,
            "selected_value": false,
            "selected_label": "B",
            "selected_token_id": 33,
            "probability": 0.9975197951546371,
            "abstained": false,
            "candidate_probability_mass": 0.9867459180461733,
            "candidates": [
              {
                "value": true,
                "label": "A",
                "token_id": 32,
                "probability": 0.002480204845362905,
                "logprob": -6.012756824493408
              },
              {
                "value": false,
                "label": "B",
                "token_id": 33,
                "probability": 0.9975197951546371,
                "logprob": -0.015825986862182617
              }
            ],
            "elapsed_ms": 666.695,
            "queue_ms": 0.001,
            "backend_requests": 1,
            "score_retrieval": "selected_token_logprobs",
            "requested_top_n": 2,
            "prompt_tokens": 164,
            "backend_timings": {
              "cache_n": 0,
              "prompt_n": 164,
              "prompt_ms": 583.47,
              "prompt_per_token_ms": 3.5577439024390247,
              "prompt_per_second": 281.07700481601455,
              "predicted_n": 1,
              "predicted_ms": 0.001,
              "predicted_per_token_ms": 0.0,
              "predicted_per_second": 0.0
            },
            "usage": {
              "prompt_tokens": 164,
              "completion_tokens": 1,
              "total_tokens": 165
            }
          },
          "priority": {
            "value": "1",
            "selected_value": "1",
            "selected_label": "B",
            "selected_token_id": 33,
            "probability": 0.9983500193011885,
            "abstained": false,
            "candidate_probability_mass": 0.9648334149708054,
            "candidates": [
              {
                "value": "0",
                "label": "A",
                "token_id": 32,
                "probability": 0.0007058337315065528,
                "logprob": -7.291930675506592
              },
              {
                "value": "1",
                "label": "B",
                "token_id": 33,
                "probability": 0.9983500193011885,
                "logprob": -0.037451162934303284
              },
              {
                "value": "2",
                "label": "C",
                "token_id": 34,
                "probability": 0.0009441469673050024,
                "logprob": -7.001028537750244
              }
            ],
            "elapsed_ms": 1076.032,
            "queue_ms": 666.574,
            "backend_requests": 1,
            "score_retrieval": "selected_token_logprobs",
            "requested_top_n": 3,
            "prompt_tokens": 193,
            "backend_timings": {
              "cache_n": 0,
              "prompt_n": 193,
              "prompt_ms": 322.763,
              "prompt_per_token_ms": 1.6723471502590672,
              "prompt_per_second": 597.9619720971734,
              "predicted_n": 1,
              "predicted_ms": 0.001,
              "predicted_per_token_ms": 0.0,
              "predicted_per_second": 0.0
            },
            "usage": {
              "prompt_tokens": 193,
              "completion_tokens": 1,
              "total_tokens": 194
            }
          }
        },
        "scores_are_calibrated": false,
        "score_source": "pre_sampling_full_vocabulary_logprobs",
        "selected_token_logprobs": true,
        "score_temperature": 1.0,
        "abstained_fields": [],
        "elapsed_ms": 1083.211,
        "prompt_preparation_ms": 6.793,
        "backend_requests": 3,
        "scored_fields": 3,
        "max_prompt_tokens": 193,
        "max_model_len": 16384,
        "backend_slots": 2,
        "cache_block_tokens": null,
        "usage": {
          "prompt_tokens": 547,
          "completion_tokens": 3,
          "total_tokens": 550
        }
      }
    }
  }
}