| { |
| "arm": "shared", |
| "backward_called": false, |
| "buffers_unchanged": true, |
| "elapsed_seconds": 740.7935931779211, |
| "gradients_absent": true, |
| "load_receipt": { |
| "arm": "shared", |
| "backbone_parameters": 110465280, |
| "baseline": { |
| "ancestor_receipt_sha256": "8851a1a4690ef7d7b119db914f6ecf3cf4a6877731f49019a14d5f836bce915e", |
| "artifact_manifest_sha256": "1a72c590e35aa8dd61a0ffa526e2e26d62ab737d517859c1558925129dc911ff", |
| "backbone_and_buffers_bit_equal": true, |
| "backbone_parameters": 110465280, |
| "continuation_from_step": 6000, |
| "device": "cpu", |
| "dtype": "float32", |
| "export_verification_sha256": "5f740ed2ebcfc88b894543049406b37f8dfea6fd638bbc44a289199f1729afe2", |
| "final_checkpoint_receipt_sha256": "5de189db4e6d7cac839afb4bc421124e523ab1df19c5b3640b11d63cd2823837", |
| "gate_parameters": 1771776, |
| "initialized_gate_tensors": [ |
| "model.layers.0.attn.melt_gate.bias", |
| "model.layers.0.attn.melt_gate.input_weight", |
| "model.layers.0.attn.melt_gate.memory_weight", |
| "model.layers.1.attn.melt_gate.bias", |
| "model.layers.1.attn.melt_gate.input_weight", |
| "model.layers.1.attn.melt_gate.memory_weight", |
| "model.layers.2.attn.melt_gate.bias", |
| "model.layers.2.attn.melt_gate.input_weight", |
| "model.layers.2.attn.melt_gate.memory_weight", |
| "model.layers.3.attn.melt_gate.bias", |
| "model.layers.3.attn.melt_gate.input_weight", |
| "model.layers.3.attn.melt_gate.memory_weight", |
| "model.layers.4.attn.melt_gate.bias", |
| "model.layers.4.attn.melt_gate.input_weight", |
| "model.layers.4.attn.melt_gate.memory_weight", |
| "model.layers.5.attn.melt_gate.bias", |
| "model.layers.5.attn.melt_gate.input_weight", |
| "model.layers.5.attn.melt_gate.memory_weight" |
| ], |
| "loader_sha256": "082f1e59d0ab8e0240e5902ba85a9720e3f645836a7322da40fa6b79f496a3a3", |
| "mode": "completed_scratch", |
| "optimizer_created": false, |
| "origin_manifest_sha256": "efb3804850e4fa6fb4b6f88b8f71c55a329b9d57e40368a3df9b52a56cbbd4cf", |
| "original_origin_manifest_sha256": "42cf14afaae1d23201cc4ca56de3441c9257a7f22ffd882359b1806c76d12c93", |
| "parent_receipt_sha256": "6476628a108b6a87380726f074fae9b84070ae7873d206bcf2a03f92accf3d9e", |
| "retrieval_index_sha256": "ab8b46387f547e1a6b162cfba3d04efe94c7633882846d0d416901b7115323e2", |
| "run_manifest_sha256": "af9c994f4a162f05870bd191f84a4322865366fb34ba72bbd6878e9816e34e17", |
| "source_integrity_sha256": "9074c5ea1d86c3c20892d099a8aedc2b561d08c07dda3d9887ac3631568d4713", |
| "source_step": 7217, |
| "trained_provenance": true, |
| "trained_run": "/home/moloch/Projects/mistral-application/runs/scratch-cloud-continuation-6000-v1", |
| "unique_parameters": 112237056 |
| }, |
| "cuda_initialized": false, |
| "gates_executed": true, |
| "model_fingerprint": "18c3eac60d7b51d7211cd943eb646bf1cb50d0dacfb587bd182cd9fd9d88b935", |
| "pair_bundle_manifest_sha256": "809a3836e2ad6a62c0bc95ae95431b4a18ba50bc839d25998a03bfa62eadd915", |
| "pair_summary_sha256": "306b576f59993d0caa6f08f458cb289bebe805a1eb395191de1d5dfd541fbe3b", |
| "parameter_tensors": 92, |
| "source_files": { |
| "research/probes/evaluate_loom_finish.py": "cfce8bc406b4bfae28a504731d06b6e6c013e66e8361981afdf553939d05ba18", |
| "research/probes/evaluate_loom_melt_pair_v2.py": "d155251dad16b797d26449e08df9ad5d9b9de993f3b6557a0c99fabb9b5d93d1", |
| "research/probes/load_loom_melt_v3.py": "082f1e59d0ab8e0240e5902ba85a9720e3f645836a7322da40fa6b79f496a3a3", |
| "research/probes/loom_melt_cuda.py": "7dad5d3355bbd71c5db6df5a7ad4632f29f7dcc1f139010da21f97b167c73e81", |
| "research/probes/loom_melt_cuda_v2.py": "a20865b78c47f5d2aadb31f8d573f5623bca419fa3c6c6d60a439ff03b4cd9d7", |
| "research/probes/loom_melt_inference.py": "07d5f127e1dd8d78416ac2488b77577b98efc27220ee443dad01d6f6d3b52b25", |
| "research/probes/pack_loom_adaptation.py": "fd8f4c0814777cb677afd134354ad360a46cfb7ebef5f4e53c36f5cf312d4f3a", |
| "research/probes/prepare_loom_finish_eval.py": "b684178e8ff35bff3322de585b5926122b28460785e54cc20f1121273f250e57", |
| "research/probes/train_loom_melt.py": "7cf0da22488108b5e5e35410b26a31bb2a33b9d4a3a55efdced42880112f3880", |
| "research/probes/train_loom_melt_v2.py": "cf52f529563d40edc10723433820e7a21d9b3083a6c0d6f7d835948591e6a609" |
| }, |
| "storage_parameters": 112237056, |
| "trained_adaptation_updates": 32 |
| }, |
| "memory": { |
| "peak_allocated_bytes": 936485888, |
| "peak_reserved_bytes": 1090519040 |
| }, |
| "mode": "score", |
| "model_fingerprint": "18c3eac60d7b51d7211cd943eb646bf1cb50d0dacfb587bd182cd9fd9d88b935", |
| "new_chunks": 10242, |
| "new_groups": 948, |
| "new_microbatches": 62, |
| "new_raw_targets": 992326, |
| "optimizer_created": false, |
| "pack_receipt_sha256": "06b0e290a4a02f5565ec4508e68deadbae19b939ca3fa995dd57718315614546", |
| "precision": { |
| "attention_backend": "math", |
| "batch": 1, |
| "buffers_preserved_fp32": true, |
| "chunks": 3, |
| "compute_contract": { |
| "autocast": false, |
| "buffers": "float32", |
| "cast_scope": "one independent unit through backward", |
| "parameter_views": "bfloat16", |
| "source_policy": "FSDP2 param_dtype=bfloat16/reduce_dtype=float32" |
| }, |
| "length": 144, |
| "live_kv_bytes": 1124352, |
| "loss_dtype": "torch.float32", |
| "master_dtype": "torch.float32", |
| "master_parameter_tensors": 92, |
| "observed_dtypes": { |
| "embedding_core": [ |
| "torch.bfloat16" |
| ], |
| "embedding_source": [ |
| "torch.bfloat16" |
| ], |
| "gate_input": [ |
| "torch.bfloat16" |
| ], |
| "key": [ |
| "torch.bfloat16" |
| ], |
| "loop_memory": [ |
| "torch.bfloat16" |
| ], |
| "parameter_view": [ |
| "torch.bfloat16" |
| ], |
| "query": [ |
| "torch.bfloat16" |
| ], |
| "route_scores": [ |
| "torch.bfloat16" |
| ], |
| "routed_compute": [ |
| "torch.bfloat16" |
| ], |
| "routed_input": [ |
| "torch.bfloat16" |
| ], |
| "router_logits": [ |
| "torch.float32" |
| ], |
| "router_matmul_input": [ |
| "torch.float32" |
| ], |
| "router_matmul_weight": [ |
| "torch.float32" |
| ], |
| "value": [ |
| "torch.bfloat16" |
| ] |
| }, |
| "precision": "bf16", |
| "readout_dtypes": [ |
| "torch.bfloat16" |
| ], |
| "referenced_kv_storage_bytes": 1124352, |
| "routed_compute_basis": "source_grouped_swiglu_uses_input_dtype", |
| "router_compute_dtype": "torch.float32", |
| "routing_adapter": "original_route_no_autocast_with_actual_logits_observation", |
| "storage_is_total_training_memory": false |
| }, |
| "primary_ce": 3.785574605887762, |
| "primary_targets": 1021448, |
| "raw_ce": 3.786990511118591, |
| "raw_targets": 1024890, |
| "rng_preserved": true, |
| "routing_stats_clear": true, |
| "rows_file": { |
| "bytes": 604527, |
| "path": "unit-losses.jsonl", |
| "sha256": "9c4a51b4ff1036dd92fb05d5d909d173fd89ca3c1b1bbcec71fb5b169ddf0f37" |
| }, |
| "runtime": { |
| "bf16_matmul": true, |
| "capability": [ |
| 8, |
| 6 |
| ], |
| "device": "cuda", |
| "gpu": "NVIDIA A40", |
| "memory_bytes": 50897289216 |
| }, |
| "seconds_forward": 714.6343508122955, |
| "source_files": { |
| "research/probes/evaluate_loom_finish.py": "cfce8bc406b4bfae28a504731d06b6e6c013e66e8361981afdf553939d05ba18", |
| "research/probes/evaluate_loom_melt_pair_v2.py": "d155251dad16b797d26449e08df9ad5d9b9de993f3b6557a0c99fabb9b5d93d1", |
| "research/probes/load_loom_melt_v3.py": "082f1e59d0ab8e0240e5902ba85a9720e3f645836a7322da40fa6b79f496a3a3", |
| "research/probes/loom_melt_cuda.py": "7dad5d3355bbd71c5db6df5a7ad4632f29f7dcc1f139010da21f97b167c73e81", |
| "research/probes/loom_melt_cuda_v2.py": "a20865b78c47f5d2aadb31f8d573f5623bca419fa3c6c6d60a439ff03b4cd9d7", |
| "research/probes/loom_melt_inference.py": "07d5f127e1dd8d78416ac2488b77577b98efc27220ee443dad01d6f6d3b52b25", |
| "research/probes/pack_loom_adaptation.py": "fd8f4c0814777cb677afd134354ad360a46cfb7ebef5f4e53c36f5cf312d4f3a", |
| "research/probes/prepare_loom_finish_eval.py": "b684178e8ff35bff3322de585b5926122b28460785e54cc20f1121273f250e57", |
| "research/probes/train_loom_melt.py": "7cf0da22488108b5e5e35410b26a31bb2a33b9d4a3a55efdced42880112f3880", |
| "research/probes/train_loom_melt_v2.py": "cf52f529563d40edc10723433820e7a21d9b3083a6c0d6f7d835948591e6a609" |
| }, |
| "state_unchanged": true, |
| "status": "complete", |
| "tasks": { |
| "accuracy": 0.65625, |
| "backward_called": false, |
| "buffers_unchanged": true, |
| "cases": [ |
| { |
| "candidates": [ |
| { |
| "answer_targets": 16, |
| "loss_sum": 68.5475298166275, |
| "mean_answer_nll": 4.284220613539219 |
| }, |
| { |
| "answer_targets": 16, |
| "loss_sum": 65.64686036109924, |
| "mean_answer_nll": 4.102928772568703 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:513", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 0 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 36, |
| "loss_sum": 126.57257795333862, |
| "mean_answer_nll": 3.515904943148295 |
| }, |
| { |
| "answer_targets": 36, |
| "loss_sum": 125.6378527879715, |
| "mean_answer_nll": 3.4899403552214303 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1634", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 1 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 6, |
| "loss_sum": 20.711718440055847, |
| "mean_answer_nll": 3.4519530733426413 |
| }, |
| { |
| "answer_targets": 5, |
| "loss_sum": 25.911905527114868, |
| "mean_answer_nll": 5.182381105422974 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1089", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 2 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 21, |
| "loss_sum": 87.10708074644208, |
| "mean_answer_nll": 4.147956226021051 |
| }, |
| { |
| "answer_targets": 21, |
| "loss_sum": 84.81375490874052, |
| "mean_answer_nll": 4.038750233749549 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:411", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 3 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 60, |
| "loss_sum": 257.73825839161873, |
| "mean_answer_nll": 4.295637639860312 |
| }, |
| { |
| "answer_targets": 107, |
| "loss_sum": 442.0394263714552, |
| "mean_answer_nll": 4.131209592256591 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:771", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 4 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 35, |
| "loss_sum": 91.77772015333176, |
| "mean_answer_nll": 2.6222205758094788 |
| }, |
| { |
| "answer_targets": 40, |
| "loss_sum": 125.31447923183441, |
| "mean_answer_nll": 3.13286198079586 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1742", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 5 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 32, |
| "loss_sum": 156.62766665406525, |
| "mean_answer_nll": 4.894614582939539 |
| }, |
| { |
| "answer_targets": 32, |
| "loss_sum": 157.38724090717733, |
| "mean_answer_nll": 4.9183512783492915 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1194", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 6 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 3, |
| "loss_sum": 21.68057870864868, |
| "mean_answer_nll": 7.2268595695495605 |
| }, |
| { |
| "answer_targets": 5, |
| "loss_sum": 15.259928695857525, |
| "mean_answer_nll": 3.051985739171505 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:919", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 7 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 27, |
| "loss_sum": 131.91240333020687, |
| "mean_answer_nll": 4.88564456778544 |
| }, |
| { |
| "answer_targets": 25, |
| "loss_sum": 128.79841957986355, |
| "mean_answer_nll": 5.151936783194542 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:613", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 8 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 27, |
| "loss_sum": 112.20432268828154, |
| "mean_answer_nll": 4.155715655121538 |
| }, |
| { |
| "answer_targets": 28, |
| "loss_sum": 113.8412959203124, |
| "mean_answer_nll": 4.0657605685825855 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1138", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 9 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 17, |
| "loss_sum": 72.6776533126831, |
| "mean_answer_nll": 4.2751560772166535 |
| }, |
| { |
| "answer_targets": 17, |
| "loss_sum": 67.08504620194435, |
| "mean_answer_nll": 3.9461791883496677 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:356", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 10 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 46, |
| "loss_sum": 214.16678887605667, |
| "mean_answer_nll": 4.655799758175145 |
| }, |
| { |
| "answer_targets": 45, |
| "loss_sum": 200.59760043025017, |
| "mean_answer_nll": 4.457724454005559 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:208", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 11 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 62, |
| "loss_sum": 262.97786244750023, |
| "mean_answer_nll": 4.241578426572584 |
| }, |
| { |
| "answer_targets": 62, |
| "loss_sum": 251.71729150414467, |
| "mean_answer_nll": 4.059956314582979 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1162", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 12 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 14, |
| "loss_sum": 51.7253004014492, |
| "mean_answer_nll": 3.694664314389229 |
| }, |
| { |
| "answer_targets": 13, |
| "loss_sum": 40.81349352002144, |
| "mean_answer_nll": 3.1394995015401106 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:485", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 13 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 12, |
| "loss_sum": 51.268762826919556, |
| "mean_answer_nll": 4.272396902243297 |
| }, |
| { |
| "answer_targets": 6, |
| "loss_sum": 28.0722878575325, |
| "mean_answer_nll": 4.678714642922084 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1250", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 14 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 14, |
| "loss_sum": 47.090190559625626, |
| "mean_answer_nll": 3.363585039973259 |
| }, |
| { |
| "answer_targets": 14, |
| "loss_sum": 51.64306902885437, |
| "mean_answer_nll": 3.688790644918169 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:992", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 15 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 3, |
| "loss_sum": 30.38958168029785, |
| "mean_answer_nll": 10.129860560099283 |
| }, |
| { |
| "answer_targets": 3, |
| "loss_sum": 26.16910409927368, |
| "mean_answer_nll": 8.723034699757894 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:479", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 16 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 14, |
| "loss_sum": 50.62457346916199, |
| "mean_answer_nll": 3.616040962082999 |
| }, |
| { |
| "answer_targets": 12, |
| "loss_sum": 45.61664766073227, |
| "mean_answer_nll": 3.8013873050610223 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:242", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 17 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 23, |
| "loss_sum": 84.15969115495682, |
| "mean_answer_nll": 3.659117006737253 |
| }, |
| { |
| "answer_targets": 18, |
| "loss_sum": 84.8076197206974, |
| "mean_answer_nll": 4.711534428927633 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1215", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 18 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 132, |
| "loss_sum": 443.08306596428156, |
| "mean_answer_nll": 3.3566898936687997 |
| }, |
| { |
| "answer_targets": 133, |
| "loss_sum": 450.8020041026175, |
| "mean_answer_nll": 3.3894887526512596 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:569", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 19 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 5, |
| "loss_sum": 26.740032196044922, |
| "mean_answer_nll": 5.3480064392089846 |
| }, |
| { |
| "answer_targets": 7, |
| "loss_sum": 41.806596755981445, |
| "mean_answer_nll": 5.972370965140207 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1655", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 20 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 23, |
| "loss_sum": 85.59374976158142, |
| "mean_answer_nll": 3.7214673809383227 |
| }, |
| { |
| "answer_targets": 24, |
| "loss_sum": 89.42527055740356, |
| "mean_answer_nll": 3.726052939891815 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1495", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 21 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 16, |
| "loss_sum": 59.04576522111893, |
| "mean_answer_nll": 3.690360326319933 |
| }, |
| { |
| "answer_targets": 19, |
| "loss_sum": 71.80738918483257, |
| "mean_answer_nll": 3.7793362728859248 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:838", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 22 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 10, |
| "loss_sum": 59.85038113594055, |
| "mean_answer_nll": 5.985038113594055 |
| }, |
| { |
| "answer_targets": 11, |
| "loss_sum": 68.15844941139221, |
| "mean_answer_nll": 6.196222673762929 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1517", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 23 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 17, |
| "loss_sum": 64.74342477321625, |
| "mean_answer_nll": 3.8084367513656616 |
| }, |
| { |
| "answer_targets": 17, |
| "loss_sum": 68.32545977830887, |
| "mean_answer_nll": 4.0191446928416985 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1305", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 24 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 39, |
| "loss_sum": 138.43891184031963, |
| "mean_answer_nll": 3.549715688213324 |
| }, |
| { |
| "answer_targets": 39, |
| "loss_sum": 142.69705894589424, |
| "mean_answer_nll": 3.6588989473306217 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:824", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 25 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 14, |
| "loss_sum": 59.271815955638885, |
| "mean_answer_nll": 4.233701139688492 |
| }, |
| { |
| "answer_targets": 14, |
| "loss_sum": 58.68761947751045, |
| "mean_answer_nll": 4.191972819822175 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:404", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 26 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 8, |
| "loss_sum": 59.27084457874298, |
| "mean_answer_nll": 7.408855572342873 |
| }, |
| { |
| "answer_targets": 9, |
| "loss_sum": 75.27413439750671, |
| "mean_answer_nll": 8.36379271083408 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:376", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 27 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 145, |
| "loss_sum": 594.6252560764551, |
| "mean_answer_nll": 4.100863835010036 |
| }, |
| { |
| "answer_targets": 144, |
| "loss_sum": 596.8856801390648, |
| "mean_answer_nll": 4.145039445410172 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:476", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 28 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 78, |
| "loss_sum": 504.5745495259762, |
| "mean_answer_nll": 6.468904481102259 |
| }, |
| { |
| "answer_targets": 78, |
| "loss_sum": 503.8303242921829, |
| "mean_answer_nll": 6.459363131951063 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1353", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 29 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 38, |
| "loss_sum": 152.18538855481893, |
| "mean_answer_nll": 4.004878646179446 |
| }, |
| { |
| "answer_targets": 27, |
| "loss_sum": 115.69065392017365, |
| "mean_answer_nll": 4.284839034080505 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:979", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 30 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 7, |
| "loss_sum": 41.740835309028625, |
| "mean_answer_nll": 5.962976472718375 |
| }, |
| { |
| "answer_targets": 7, |
| "loss_sum": 45.72141253948212, |
| "mean_answer_nll": 6.53163036278316 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1209", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 31 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 3, |
| "loss_sum": 27.53316593170166, |
| "mean_answer_nll": 9.177721977233887 |
| }, |
| { |
| "answer_targets": 3, |
| "loss_sum": 27.291183471679688, |
| "mean_answer_nll": 9.097061157226562 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:905", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 32 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 51, |
| "loss_sum": 203.88678428903222, |
| "mean_answer_nll": 3.997780084098671 |
| }, |
| { |
| "answer_targets": 51, |
| "loss_sum": 179.6838759109378, |
| "mean_answer_nll": 3.523213253155643 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:6", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 33 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 5, |
| "loss_sum": 34.197272062301636, |
| "mean_answer_nll": 6.839454412460327 |
| }, |
| { |
| "answer_targets": 4, |
| "loss_sum": 19.360405921936035, |
| "mean_answer_nll": 4.840101480484009 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1011", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 34 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 13, |
| "loss_sum": 49.02774082124233, |
| "mean_answer_nll": 3.7713646785571027 |
| }, |
| { |
| "answer_targets": 13, |
| "loss_sum": 46.08056040108204, |
| "mean_answer_nll": 3.544658492390926 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1016", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 35 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 16, |
| "loss_sum": 79.49679970741272, |
| "mean_answer_nll": 4.968549981713295 |
| }, |
| { |
| "answer_targets": 18, |
| "loss_sum": 78.04774770140648, |
| "mean_answer_nll": 4.335985983411471 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1796", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 36 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 7, |
| "loss_sum": 30.05105686187744, |
| "mean_answer_nll": 4.2930081231253485 |
| }, |
| { |
| "answer_targets": 6, |
| "loss_sum": 30.051896631717682, |
| "mean_answer_nll": 5.008649438619614 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1606", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 37 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 31, |
| "loss_sum": 124.32921782135963, |
| "mean_answer_nll": 4.010619929721279 |
| }, |
| { |
| "answer_targets": 30, |
| "loss_sum": 117.08171704411507, |
| "mean_answer_nll": 3.902723901470502 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:310", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 38 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 9, |
| "loss_sum": 32.239086359739304, |
| "mean_answer_nll": 3.5821207066377005 |
| }, |
| { |
| "answer_targets": 9, |
| "loss_sum": 35.75029677152634, |
| "mean_answer_nll": 3.97225519683626 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1256", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 39 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 38, |
| "loss_sum": 152.84790597856045, |
| "mean_answer_nll": 4.022313315225275 |
| }, |
| { |
| "answer_targets": 38, |
| "loss_sum": 151.73591051250696, |
| "mean_answer_nll": 3.99305027664492 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:801", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 40 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 14, |
| "loss_sum": 62.53528136014938, |
| "mean_answer_nll": 4.466805811439242 |
| }, |
| { |
| "answer_targets": 13, |
| "loss_sum": 53.655186370015144, |
| "mean_answer_nll": 4.127322028462704 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:608", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 41 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 8, |
| "loss_sum": 28.815916180610657, |
| "mean_answer_nll": 3.601989522576332 |
| }, |
| { |
| "answer_targets": 9, |
| "loss_sum": 40.550236999988556, |
| "mean_answer_nll": 4.505581888887617 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1048", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 42 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 20, |
| "loss_sum": 73.49589157104492, |
| "mean_answer_nll": 3.674794578552246 |
| }, |
| { |
| "answer_targets": 20, |
| "loss_sum": 67.57065208256245, |
| "mean_answer_nll": 3.3785326041281225 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:519", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 43 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 25, |
| "loss_sum": 92.55776011943817, |
| "mean_answer_nll": 3.702310404777527 |
| }, |
| { |
| "answer_targets": 25, |
| "loss_sum": 90.88231357932091, |
| "mean_answer_nll": 3.6352925431728362 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:1143", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 44 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 45, |
| "loss_sum": 157.98248773161322, |
| "mean_answer_nll": 3.510721949591405 |
| }, |
| { |
| "answer_targets": 45, |
| "loss_sum": 181.00169798731804, |
| "mean_answer_nll": 4.022259955273734 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1594", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 45 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 39, |
| "loss_sum": 157.74828431010246, |
| "mean_answer_nll": 4.04482780282314 |
| }, |
| { |
| "answer_targets": 37, |
| "loss_sum": 135.8927779495716, |
| "mean_answer_nll": 3.6727777824208543 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:737", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 46 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 41, |
| "loss_sum": 158.3344525396824, |
| "mean_answer_nll": 3.8618159156020093 |
| }, |
| { |
| "answer_targets": 42, |
| "loss_sum": 159.10637986660004, |
| "mean_answer_nll": 3.788247139680953 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:915", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 47 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 17, |
| "loss_sum": 60.27305355668068, |
| "mean_answer_nll": 3.5454737386282753 |
| }, |
| { |
| "answer_targets": 15, |
| "loss_sum": 63.66574463248253, |
| "mean_answer_nll": 4.244382975498835 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1773", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 48 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 31, |
| "loss_sum": 244.4357331059873, |
| "mean_answer_nll": 7.885023648580236 |
| }, |
| { |
| "answer_targets": 34, |
| "loss_sum": 262.4472395963967, |
| "mean_answer_nll": 7.719036458717549 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1223", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 49 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 6, |
| "loss_sum": 35.47548735141754, |
| "mean_answer_nll": 5.912581225236257 |
| }, |
| { |
| "answer_targets": 7, |
| "loss_sum": 37.37843841314316, |
| "mean_answer_nll": 5.339776916163308 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:297", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 50 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 56, |
| "loss_sum": 206.1907831672579, |
| "mean_answer_nll": 3.6819782708438913 |
| }, |
| { |
| "answer_targets": 62, |
| "loss_sum": 221.96145190298557, |
| "mean_answer_nll": 3.58002341779009 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1202", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 51 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 8, |
| "loss_sum": 48.21444845199585, |
| "mean_answer_nll": 6.026806056499481 |
| }, |
| { |
| "answer_targets": 9, |
| "loss_sum": 54.28672158718109, |
| "mean_answer_nll": 6.031857954131232 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:489", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 52 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 9, |
| "loss_sum": 31.84696339070797, |
| "mean_answer_nll": 3.538551487856441 |
| }, |
| { |
| "answer_targets": 8, |
| "loss_sum": 21.101733058691025, |
| "mean_answer_nll": 2.637716632336378 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:335", |
| "label": 0, |
| "predicted": 1, |
| "source_index": 53 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 15, |
| "loss_sum": 66.23619198799133, |
| "mean_answer_nll": 4.415746132532756 |
| }, |
| { |
| "answer_targets": 9, |
| "loss_sum": 38.3056583404541, |
| "mean_answer_nll": 4.256184260050456 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1230", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 54 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 25, |
| "loss_sum": 93.76409621536732, |
| "mean_answer_nll": 3.750563848614693 |
| }, |
| { |
| "answer_targets": 25, |
| "loss_sum": 97.005739107728, |
| "mean_answer_nll": 3.8802295643091202 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:40", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 55 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 4, |
| "loss_sum": 26.139243125915527, |
| "mean_answer_nll": 6.534810781478882 |
| }, |
| { |
| "answer_targets": 4, |
| "loss_sum": 29.49397325515747, |
| "mean_answer_nll": 7.373493313789368 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:848", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 56 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 26, |
| "loss_sum": 87.83841823041439, |
| "mean_answer_nll": 3.378400701169784 |
| }, |
| { |
| "answer_targets": 13, |
| "loss_sum": 58.2717467546463, |
| "mean_answer_nll": 4.482442058049715 |
| } |
| ], |
| "correct": false, |
| "id": "piqa:validation:255", |
| "label": 1, |
| "predicted": 0, |
| "source_index": 57 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 6, |
| "loss_sum": 31.52444678544998, |
| "mean_answer_nll": 5.254074464241664 |
| }, |
| { |
| "answer_targets": 6, |
| "loss_sum": 36.8781356215477, |
| "mean_answer_nll": 6.146355936924617 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:433", |
| "label": 0, |
| "predicted": 0, |
| "source_index": 58 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 4, |
| "loss_sum": 22.5718355178833, |
| "mean_answer_nll": 5.642958879470825 |
| }, |
| { |
| "answer_targets": 4, |
| "loss_sum": 18.861961603164673, |
| "mean_answer_nll": 4.715490400791168 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:370", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 59 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 41, |
| "loss_sum": 195.100503385067, |
| "mean_answer_nll": 4.7585488630504145 |
| }, |
| { |
| "answer_targets": 42, |
| "loss_sum": 198.83656460046768, |
| "mean_answer_nll": 4.734203919058754 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1403", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 60 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 13, |
| "loss_sum": 40.40865534543991, |
| "mean_answer_nll": 3.1083581034953776 |
| }, |
| { |
| "answer_targets": 13, |
| "loss_sum": 31.075763165950775, |
| "mean_answer_nll": 2.3904433204577518 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:861", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 61 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 6, |
| "loss_sum": 46.91595816612244, |
| "mean_answer_nll": 7.819326361020406 |
| }, |
| { |
| "answer_targets": 8, |
| "loss_sum": 48.481138467788696, |
| "mean_answer_nll": 6.060142308473587 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:161", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 62 |
| }, |
| { |
| "candidates": [ |
| { |
| "answer_targets": 29, |
| "loss_sum": 99.11024868488312, |
| "mean_answer_nll": 3.417594782237349 |
| }, |
| { |
| "answer_targets": 29, |
| "loss_sum": 98.31966036558151, |
| "mean_answer_nll": 3.390333116054535 |
| } |
| ], |
| "correct": true, |
| "id": "piqa:validation:1785", |
| "label": 1, |
| "predicted": 1, |
| "source_index": 63 |
| } |
| ], |
| "correct": 42, |
| "count": 64, |
| "optimizer_created": false, |
| "protocol": { |
| "answer_prefix": " ", |
| "eos": false, |
| "maximum_input_tokens": 1023, |
| "prompt": "Goal: {goal}\nSolution:", |
| "prompt_supervised": false, |
| "scoring": "mean answer-token negative log-likelihood", |
| "selection": "first 64 saved development rows, original order", |
| "tie_break": 0, |
| "tokenization": "prompt and complete candidate separately, then concatenate" |
| }, |
| "protocol_sha256": "52820e53fdd0e9d42fcc8b35fa538bb8379a807ce7510925e1234bafb321f2e7", |
| "rng_preserved": true, |
| "routing_stats_clear": true, |
| "scope": "Fixed 64-case development diagnostic, not a complete PIQA reproduction", |
| "seconds": 11.764678032021038, |
| "source_sha256": "15a6e4494731c4e7cff583391674f1ca2a80a3249ab4757ed0c1dee88c4eaa73", |
| "state_unchanged": true, |
| "status": "complete", |
| "task": "piqa-development64-v1" |
| }, |
| "total_microbatches": 64, |
| "unit_rows": 1492 |
| } |
|
|
