{
  "objective": -0.8137446728855184,
  "paired_flip": null,
  "unknowable": null,
  "clean": {
    "n": 175,
    "nll": 0.8137446728855184,
    "acc": 0.9028571428571428,
    "ece": 0.09141608822222473,
    "brier": 0.17599452427593348,
    "mean_conf": 0.9942732310793674,
    "confident_error_rate": 0.08,
    "coverage_at_0_9": 0.9828571428571429,
    "accuracy_at_0_9": 0.9186046511627907,
    "coverage_at_5pct_error": 0.9142857142857143,
    "coverage_at_1pct_error": 0.011428571428571429,
    "aurc": 0.06143423339655433,
    "error_rate_at_0_9": 0.08139534883720931,
    "confidence_bias": 0.09141608822222458,
    "top_bins": {
      "0.9": {
        "n": 172,
        "errors": 14,
        "error_rate": 0.08139534883720931
      },
      "0.95": {
        "n": 172,
        "errors": 14,
        "error_rate": 0.08139534883720931
      },
      "0.99": {
        "n": 168,
        "errors": 11,
        "error_rate": 0.06547619047619048
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5028571428571429,
        "accuracy": 0.9431818181818182,
        "confidence_cutoff": 0.9999983862993777
      },
      "0.8": {
        "coverage": 0.8,
        "accuracy": 0.9571428571428572,
        "confidence_cutoff": 0.9999883543164546
      }
    }
  },
  "tasks": {
    "sev_swarmtraces_response_policy": {
      "n": 175,
      "nll": 0.8137446728855184,
      "acc": 0.9028571428571428,
      "ece": 0.09141608822222473,
      "brier": 0.17599452427593348,
      "mean_conf": 0.9942732310793674,
      "confident_error_rate": 0.08,
      "coverage_at_0_9": 0.9828571428571429,
      "accuracy_at_0_9": 0.9186046511627907,
      "coverage_at_5pct_error": 0.9142857142857143,
      "coverage_at_1pct_error": 0.011428571428571429,
      "aurc": 0.06143423339655433,
      "error_rate_at_0_9": 0.08139534883720931,
      "confidence_bias": 0.09141608822222458,
      "top_bins": {
        "0.9": {
          "n": 172,
          "errors": 14,
          "error_rate": 0.08139534883720931
        },
        "0.95": {
          "n": 172,
          "errors": 14,
          "error_rate": 0.08139534883720931
        },
        "0.99": {
          "n": 168,
          "errors": 11,
          "error_rate": 0.06547619047619048
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5028571428571429,
          "accuracy": 0.9431818181818182,
          "confidence_cutoff": 0.9999983862993777
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9571428571428572,
          "confidence_cutoff": 0.9999883543164546
        }
      }
    }
  },
  "variants": {
    "clean": {
      "n": 175,
      "nll": 0.8137446728855184,
      "acc": 0.9028571428571428,
      "ece": 0.09141608822222473,
      "brier": 0.17599452427593348,
      "mean_conf": 0.9942732310793674,
      "confident_error_rate": 0.08,
      "coverage_at_0_9": 0.9828571428571429,
      "accuracy_at_0_9": 0.9186046511627907,
      "coverage_at_5pct_error": 0.9142857142857143,
      "coverage_at_1pct_error": 0.011428571428571429,
      "aurc": 0.06143423339655433,
      "error_rate_at_0_9": 0.08139534883720931,
      "confidence_bias": 0.09141608822222458,
      "top_bins": {
        "0.9": {
          "n": 172,
          "errors": 14,
          "error_rate": 0.08139534883720931
        },
        "0.95": {
          "n": 172,
          "errors": 14,
          "error_rate": 0.08139534883720931
        },
        "0.99": {
          "n": 168,
          "errors": 11,
          "error_rate": 0.06547619047619048
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5028571428571429,
          "accuracy": 0.9431818181818182,
          "confidence_cutoff": 0.9999983862993777
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9571428571428572,
          "confidence_cutoff": 0.9999883543164546
        }
      }
    }
  },
  "heldout_tasks": {},
  "permutation": {
    "n": 0,
    "mean_max_delta": null,
    "flip_rate": null
  },
  "temperature": 4.0,
  "calibrated_clean": {
    "n": 175,
    "nll": 0.26348233904029333,
    "acc": 0.9028571428571428,
    "ece": 0.04076488602051544,
    "brier": 0.1426360448938433,
    "mean_conf": 0.9436220288776582,
    "confident_error_rate": 0.045714285714285714,
    "coverage_at_0_9": 0.8971428571428571,
    "accuracy_at_0_9": 0.9490445859872612,
    "coverage_at_5pct_error": 0.9142857142857143,
    "coverage_at_1pct_error": 0.011428571428571429,
    "aurc": 0.06143423339655433,
    "error_rate_at_0_9": 0.050955414012738856,
    "confidence_bias": 0.04076488602051542,
    "top_bins": {
      "0.9": {
        "n": 157,
        "errors": 8,
        "error_rate": 0.050955414012738856
      },
      "0.95": {
        "n": 131,
        "errors": 5,
        "error_rate": 0.03816793893129771
      },
      "0.99": {
        "n": 0,
        "errors": 0,
        "error_rate": null
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5028571428571429,
        "accuracy": 0.9431818181818182,
        "confidence_cutoff": 0.9655851033024723
      },
      "0.8": {
        "coverage": 0.8,
        "accuracy": 0.9571428571428572,
        "confidence_cutoff": 0.9448068121971666
      }
    }
  },
  "metric_policy": {
    "version": 2,
    "selective_ties": "whole_confidence_groups",
    "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
    "aurc": "right-step integral over whole confidence groups",
    "confident_error_rate": "high-confidence errors divided by all questions",
    "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
    "nll": "exact from logits when recorded; otherwise from floored probabilities",
    "nll_floor": 1e-09,
    "renormalize_returned_probabilities": true,
    "raw_sums_outside_1e_5": 0,
    "returned_zeros": 0
  },
  "coverage": {
    "requested_records": 35,
    "requested_questions": 175,
    "evaluated_records": 35,
    "evaluated_questions": 175,
    "rejected_records": 0,
    "truncated_records": 0
  },
  "latency_ms": {
    "median": 240.12951999998222,
    "p95": 301.12752519999043
  },
  "calibration": {
    "inference_temperature": 1.0,
    "additional_temperature": 4.0,
    "logits_recorded": true
  },
  "transfer": null,
  "provenance": {
    "config": {
      "epochs": 5,
      "seed": 4,
      "lr": 1e-05,
      "lora": 16,
      "accum": 2,
      "batch": 4,
      "perm_kl": 0,
      "perm_frac": 0.3,
      "ord_w": 0.0,
      "p_none": 0,
      "p_none_distract": 0,
      "p_distract": 0,
      "p_none_pair": 0,
      "synthetic_repeat": 1,
      "public_frac": 1,
      "head_lr": 0.0,
      "weight_decay": 0.01,
      "anchor_w": 0.0,
      "label_smoothing": 0.0,
      "brier_w": 0.0,
      "focal_gamma": 0.0,
      "base": "Qwen/Qwen3.5-4B-Base",
      "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b",
      "init_from": "macmacmacmac/Sev-4B@v0.3.0-response-policy-research",
      "lora_targets": "all",
      "head_dim": 256,
      "weights_dtype": "fp32",
      "dtype": "bf16",
      "checkpointing": 1,
      "option_isolation": 0,
      "special_embeddings": 0
    },
    "config_sha256": "31d8cf5af214ae4ed0012d1e4fea9fd73e207209cfce1f1112107e4aad65cc9f",
    "suite_sha256": "40d3a8b3bd416a3754ecc30e9791a9ebb14a26c8f2a4422057f769a2dbce4dee",
    "source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
      "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
      "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
      "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f",
      "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
      "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59",
      "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
      "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c",
      "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
      "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
      "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0",
      "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
      "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b",
      "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
      "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e",
      "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
      "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3",
      "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
      "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735",
      "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
      "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db",
      "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9",
      "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
      "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb",
      "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30",
      "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
    },
    "git_commit": "fcd6756a329f42a515b4b48f83fb8bde9776232f",
    "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36",
    "torch": "2.8.0+cu128",
    "device": "cuda",
    "gpu": "NVIDIA H100 80GB HBM3",
    "legacy_checkpoint": false,
    "measured_checkpoint": {
      "requested": "/runs/sev-html-contrast-4b-v1/00-trial-0/checkpoint",
      "resolved": "/runs/sev-html-contrast-4b-v1/00-trial-0/checkpoint",
      "head_sha256": "2724bebcc54d4846194130594b98d3e3b9c632935c0f1d5077e46e23c48b2dc8",
      "adapter_sha256": "b58b3963175e6c40bd022db5893ebefb6a6f1e1bd1ec11efc6692a089e1dcf51",
      "inference_temperature": 1.0
    }
  },
  "mechanism_checks": {
    "n": 40,
    "packed_max_delta": 0.0015369057655334473,
    "sibling_max_delta": 0.0007038116455078125,
    "tolerance": 0.001,
    "passed": false
  },
  "gates": {
    "passed": false,
    "checks": {
      "complete_coverage": true,
      "isolation_and_packing": false
    },
    "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
  },
  "calibration_fit": {
    "temperature": 4.0,
    "aggregation": "micro",
    "objective": "raw-logit NLL",
    "split": "calibration",
    "rows_sha256": "725dc3c48e167e782825490bebc3b3f07b5933660fdca7ac65ad37f13481eb6e",
    "suite_sha256": "40d3a8b3bd416a3754ecc30e9791a9ebb14a26c8f2a4422057f769a2dbce4dee",
    "n": 140
  },
  "wall_seconds": 366.39384709,
  "promotable": false,
  "test_evaluated": false,
  "training_resources": {
    "wall_seconds": 232.53715324401855,
    "records_seen": 615,
    "requested_records": 615,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 80,
    "forward_tokens": 209595,
    "peak_device_bytes": 20358582784,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 4,
    "peak_rss_bytes": 31542779904
  }
}