| { |
| "schema": "automaticity.v9.needle_v1_vs_needle2_ab/v1", |
| "status": "completed", |
| "benchmark": { |
| "version": "automaticity-v9-benchmark-v1.1", |
| "dataset": "/var/home/turnercore/automaticity-v9/benchmark-v1.1/retrieval-friendly/benchmark.jsonl", |
| "dataset_sha256": "c24db4269886018a400996d4d5e7c6f5ad9fad31ead5c6aab2a90217f691c9bb", |
| "rows": 1050, |
| "sealed_benchmark_modified": false, |
| "matching_row_set": true, |
| "metric_contract": "functiongemma_spine_benchmark aggregate scorer; both summaries carry the same fields and dataset hash" |
| }, |
| "needle_v1_needle_point": { |
| "model": "needle-v9", |
| "model_sha256": "665c019772577175affc1bed5744c8e0deabf4ae3eca136466b899e8ce9d030a", |
| "base_model": "Cactus-Compute/needle@5f89b4307696d669c3df1d38ae057e6e1728b107", |
| "runtime": "Needle Point persistent PyTorch ROCm direct constrained runtime", |
| "summary": "/var/home/turnercore/automaticity-v9/benchmark-v1.1/needle-final-benchmark/summary.json", |
| "summary_sha256": "6ee6b777d243e3b754c2b084d8ce1f11e8ec13e6407059ee4b153758285783ac", |
| "predictions": "/var/home/turnercore/automaticity-v9/benchmark-v1.1/needle-final-benchmark/predictions.jsonl", |
| "predictions_sha256": "91b4620ce5a87e1901a0d47026c75e34880efcdd7e412be74cc54c08a4a37e04", |
| "execution": "pinned completed reference; no model or evaluation setting changed" |
| }, |
| "needle2_official_engine": { |
| "model": "Cactus-Compute/needle2 + AUTOMATICITY_V9 LoRA", |
| "base_model_revision": "98fbd955b0347e78059be0c253cc1ffa09b87bc7", |
| "source_revision": "571fcd68f48b6649c91f353eb91d0a0c9b9135ce", |
| "artifact_sha256": "4f19e7fb27e8c11780df8f2497b26dd96aef68b112c1a158a5e2fec68f4d7c70", |
| "runtime": "official Cactus Needle engine 2.0.3 on Bosbitch CPU, native byte grammar", |
| "summary": "/var/home/turnercore/automaticity-v9/needle2-v9-linux-boy-transfer/benchmark-trained/summary.json", |
| "summary_sha256": "d12c964c8f47c87b801081a6091eddecbc6357c23dc8a596dcb9bc4089324ddb", |
| "predictions": "/var/home/turnercore/automaticity-v9/needle2-v9-linux-boy-transfer/benchmark-trained/predictions.jsonl", |
| "predictions_sha256": "b50ae95c695f37d64019bb78bd6697c73567e789df324b278583a77bda06bbf1", |
| "harness_exit_status": 0, |
| "confidence": "unavailable by upstream design for fine-tuned archives" |
| }, |
| "metrics": { |
| "full_exact_match_accuracy": {"v1": 0.960952380952381, "v2": 0.7038095238095238, "delta_v2_minus_v1": -0.25714285714285723}, |
| "routing_accuracy": {"v1": 0.9895238095238095, "v2": 0.7076190476190476, "delta_v2_minus_v1": -0.2819047619047619}, |
| "capability_accuracy": {"v1": 1.0, "v2": 1.0, "delta_v2_minus_v1": 0.0}, |
| "contract_match_accuracy": {"v1": 0.9895238095238095, "v2": 0.7076190476190476, "delta_v2_minus_v1": -0.2819047619047619}, |
| "arguments_accuracy": {"v1": 0.9619047619047619, "v2": 0.7495238095238095, "delta_v2_minus_v1": -0.21238095238095245}, |
| "arguments_schema_valid_rate": {"v1": 1.0, "v2": 1.0, "delta_v2_minus_v1": 0.0}, |
| "noop_precision": {"v1": 0.9946091644204852, "v2": 0.7068965517241379, "delta_v2_minus_v1": -0.2877126126963473}, |
| "noop_recall": {"v1": 1.0, "v2": 1.0, "delta_v2_minus_v1": 0.0}, |
| "end_to_end_action_exact_accuracy": {"v1": 0.8685897435897436, "v2": 0.003205128205128205, "delta_v2_minus_v1": -0.8653846153846154}, |
| "end_to_end_exact_accuracy": {"v1": 0.960952380952381, "v2": 0.7038095238095238, "delta_v2_minus_v1": -0.25714285714285723}, |
| "failure_counts": {"v1": {"ok": 1009, "wrong_arguments": 30, "wrong_tool": 11}, "v2": {"ok": 739, "wrong_arguments": 4, "wrong_tool": 307}} |
| }, |
| "runtime_latency_ms": { |
| "v1": {"avg": 324.81173395807366, "p50": 51.34388000078616, "p95": 1453.3176269997057}, |
| "v2": {"avg": 875.296740870302, "p50": 823.4355130698532, "p95": 1433.258183998987}, |
| "interpretation": "Runtime-specific operational measurements only; not a pure model-speed comparison. V1 is a persistent PyTorch ROCm runtime and v2 uses the official CPU engine with per-row agent binding." |
| }, |
| "confounders": [ |
| "Needle v1 and Needle 2 have different architectures, pretraining, fine-tuning objectives (full model versus LoRA), and native prompt/rendering contracts.", |
| "Needle Point token-level constrained decoding differs from Needle 2's native byte grammar and tool-retrieval engine.", |
| "Needle 2 deliberately omits the V9 instructional preamble because its system turn accepts factual state rather than steering instructions.", |
| "Needle 2 fine-tuning does not update the confidence head; confidence is null for its tuned archive.", |
| "Quality metrics use the same sealed rows and scorer fields; latency remains runtime-specific evidence." |
| ], |
| "conclusion": "Needle 2 satisfies structural validity but is not a replacement candidate for ln-point-v9 on Automaticity V9: its action exact rate is 0.32% versus v1's 86.86%, despite perfect schema validity and no-tool recall." |
| } |
|
|
