| { |
| "objective": -0.8137446728855184, |
| "paired_flip": null, |
| "unknowable": null, |
| "clean": { |
| "n": 175, |
| "nll": 0.8137446728855184, |
| "acc": 0.9028571428571428, |
| "ece": 0.09141608822222473, |
| "brier": 0.17599452427593348, |
| "mean_conf": 0.9942732310793674, |
| "confident_error_rate": 0.08, |
| "coverage_at_0_9": 0.9828571428571429, |
| "accuracy_at_0_9": 0.9186046511627907, |
| "coverage_at_5pct_error": 0.9142857142857143, |
| "coverage_at_1pct_error": 0.011428571428571429, |
| "aurc": 0.06143423339655433, |
| "error_rate_at_0_9": 0.08139534883720931, |
| "confidence_bias": 0.09141608822222458, |
| "top_bins": { |
| "0.9": { |
| "n": 172, |
| "errors": 14, |
| "error_rate": 0.08139534883720931 |
| }, |
| "0.95": { |
| "n": 172, |
| "errors": 14, |
| "error_rate": 0.08139534883720931 |
| }, |
| "0.99": { |
| "n": 168, |
| "errors": 11, |
| "error_rate": 0.06547619047619048 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9431818181818182, |
| "confidence_cutoff": 0.9999983862993777 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9571428571428572, |
| "confidence_cutoff": 0.9999883543164546 |
| } |
| } |
| }, |
| "tasks": { |
| "sev_swarmtraces_response_policy": { |
| "n": 175, |
| "nll": 0.8137446728855184, |
| "acc": 0.9028571428571428, |
| "ece": 0.09141608822222473, |
| "brier": 0.17599452427593348, |
| "mean_conf": 0.9942732310793674, |
| "confident_error_rate": 0.08, |
| "coverage_at_0_9": 0.9828571428571429, |
| "accuracy_at_0_9": 0.9186046511627907, |
| "coverage_at_5pct_error": 0.9142857142857143, |
| "coverage_at_1pct_error": 0.011428571428571429, |
| "aurc": 0.06143423339655433, |
| "error_rate_at_0_9": 0.08139534883720931, |
| "confidence_bias": 0.09141608822222458, |
| "top_bins": { |
| "0.9": { |
| "n": 172, |
| "errors": 14, |
| "error_rate": 0.08139534883720931 |
| }, |
| "0.95": { |
| "n": 172, |
| "errors": 14, |
| "error_rate": 0.08139534883720931 |
| }, |
| "0.99": { |
| "n": 168, |
| "errors": 11, |
| "error_rate": 0.06547619047619048 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9431818181818182, |
| "confidence_cutoff": 0.9999983862993777 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9571428571428572, |
| "confidence_cutoff": 0.9999883543164546 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 175, |
| "nll": 0.8137446728855184, |
| "acc": 0.9028571428571428, |
| "ece": 0.09141608822222473, |
| "brier": 0.17599452427593348, |
| "mean_conf": 0.9942732310793674, |
| "confident_error_rate": 0.08, |
| "coverage_at_0_9": 0.9828571428571429, |
| "accuracy_at_0_9": 0.9186046511627907, |
| "coverage_at_5pct_error": 0.9142857142857143, |
| "coverage_at_1pct_error": 0.011428571428571429, |
| "aurc": 0.06143423339655433, |
| "error_rate_at_0_9": 0.08139534883720931, |
| "confidence_bias": 0.09141608822222458, |
| "top_bins": { |
| "0.9": { |
| "n": 172, |
| "errors": 14, |
| "error_rate": 0.08139534883720931 |
| }, |
| "0.95": { |
| "n": 172, |
| "errors": 14, |
| "error_rate": 0.08139534883720931 |
| }, |
| "0.99": { |
| "n": 168, |
| "errors": 11, |
| "error_rate": 0.06547619047619048 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9431818181818182, |
| "confidence_cutoff": 0.9999983862993777 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9571428571428572, |
| "confidence_cutoff": 0.9999883543164546 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": {}, |
| "permutation": { |
| "n": 0, |
| "mean_max_delta": null, |
| "flip_rate": null |
| }, |
| "temperature": 4.0, |
| "calibrated_clean": { |
| "n": 175, |
| "nll": 0.26348233904029333, |
| "acc": 0.9028571428571428, |
| "ece": 0.04076488602051544, |
| "brier": 0.1426360448938433, |
| "mean_conf": 0.9436220288776582, |
| "confident_error_rate": 0.045714285714285714, |
| "coverage_at_0_9": 0.8971428571428571, |
| "accuracy_at_0_9": 0.9490445859872612, |
| "coverage_at_5pct_error": 0.9142857142857143, |
| "coverage_at_1pct_error": 0.011428571428571429, |
| "aurc": 0.06143423339655433, |
| "error_rate_at_0_9": 0.050955414012738856, |
| "confidence_bias": 0.04076488602051542, |
| "top_bins": { |
| "0.9": { |
| "n": 157, |
| "errors": 8, |
| "error_rate": 0.050955414012738856 |
| }, |
| "0.95": { |
| "n": 131, |
| "errors": 5, |
| "error_rate": 0.03816793893129771 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9431818181818182, |
| "confidence_cutoff": 0.9655851033024723 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9571428571428572, |
| "confidence_cutoff": 0.9448068121971666 |
| } |
| } |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 35, |
| "requested_questions": 175, |
| "evaluated_records": 35, |
| "evaluated_questions": 175, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 240.12951999998222, |
| "p95": 301.12752519999043 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 4.0, |
| "logits_recorded": true |
| }, |
| "transfer": null, |
| "provenance": { |
| "config": { |
| "epochs": 5, |
| "seed": 4, |
| "lr": 1e-05, |
| "lora": 16, |
| "accum": 2, |
| "batch": 4, |
| "perm_kl": 0, |
| "perm_frac": 0.3, |
| "ord_w": 0.0, |
| "p_none": 0, |
| "p_none_distract": 0, |
| "p_distract": 0, |
| "p_none_pair": 0, |
| "synthetic_repeat": 1, |
| "public_frac": 1, |
| "head_lr": 0.0, |
| "weight_decay": 0.01, |
| "anchor_w": 0.0, |
| "label_smoothing": 0.0, |
| "brier_w": 0.0, |
| "focal_gamma": 0.0, |
| "base": "Qwen/Qwen3.5-4B-Base", |
| "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", |
| "init_from": "macmacmacmac/Sev-4B@v0.3.0-response-policy-research", |
| "lora_targets": "all", |
| "head_dim": 256, |
| "weights_dtype": "fp32", |
| "dtype": "bf16", |
| "checkpointing": 1, |
| "option_isolation": 0, |
| "special_embeddings": 0 |
| }, |
| "config_sha256": "31d8cf5af214ae4ed0012d1e4fea9fd73e207209cfce1f1112107e4aad65cc9f", |
| "suite_sha256": "40d3a8b3bd416a3754ecc30e9791a9ebb14a26c8f2a4422057f769a2dbce4dee", |
| "source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db", |
| "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb", |
| "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "git_commit": "fcd6756a329f42a515b4b48f83fb8bde9776232f", |
| "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", |
| "torch": "2.8.0+cu128", |
| "device": "cuda", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "legacy_checkpoint": false, |
| "measured_checkpoint": { |
| "requested": "/runs/sev-html-contrast-4b-v1/00-trial-0/checkpoint", |
| "resolved": "/runs/sev-html-contrast-4b-v1/00-trial-0/checkpoint", |
| "head_sha256": "2724bebcc54d4846194130594b98d3e3b9c632935c0f1d5077e46e23c48b2dc8", |
| "adapter_sha256": "b58b3963175e6c40bd022db5893ebefb6a6f1e1bd1ec11efc6692a089e1dcf51", |
| "inference_temperature": 1.0 |
| } |
| }, |
| "mechanism_checks": { |
| "n": 40, |
| "packed_max_delta": 0.0015369057655334473, |
| "sibling_max_delta": 0.0007038116455078125, |
| "tolerance": 0.001, |
| "passed": false |
| }, |
| "gates": { |
| "passed": false, |
| "checks": { |
| "complete_coverage": true, |
| "isolation_and_packing": false |
| }, |
| "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." |
| }, |
| "calibration_fit": { |
| "temperature": 4.0, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "725dc3c48e167e782825490bebc3b3f07b5933660fdca7ac65ad37f13481eb6e", |
| "suite_sha256": "40d3a8b3bd416a3754ecc30e9791a9ebb14a26c8f2a4422057f769a2dbce4dee", |
| "n": 140 |
| }, |
| "wall_seconds": 366.39384709, |
| "promotable": false, |
| "test_evaluated": false, |
| "training_resources": { |
| "wall_seconds": 232.53715324401855, |
| "records_seen": 615, |
| "requested_records": 615, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 80, |
| "forward_tokens": 209595, |
| "peak_device_bytes": 20358582784, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 4, |
| "peak_rss_bytes": 31542779904 |
| } |
| } |
|
|
