Publish bounded-workspace replacement (TensorCode 0.4.0a4) with evaluation against the previous revision
6fc386f verified | { | |
| "candidate": "/home/brandonin/tensorcode-runs/cognition-20260921/artifacts/replacement-20260924/model-final", | |
| "complete_model_sha256": "91cc67bf6e9adbb9b24531034ed3636e8391443d8fe020342a2529342bd62fd0", | |
| "gates_local_scope": { | |
| "generator_test_exact_ge_30_of_128": { | |
| "value": 36, | |
| "pass": true | |
| }, | |
| "generator_test_token_f1_ge_0.8291862": { | |
| "value": 0.8413782072760705, | |
| "pass": true | |
| }, | |
| "realizer_dev_statement_exact_64_of_64": { | |
| "value": 64.0, | |
| "pass": true | |
| }, | |
| "realizer_dev_verbatim_ge_62_of_64": { | |
| "value": 62.0, | |
| "pass": true | |
| }, | |
| "realizer_dev_token_f1_eq_1": { | |
| "value": 1.0, | |
| "pass": true | |
| }, | |
| "reload_generation_equal_both": { | |
| "value": true, | |
| "pass": true | |
| }, | |
| "memory_update_bounded_both": { | |
| "value": "relative_rms_bounded", | |
| "pass": true | |
| }, | |
| "unchanged_component_weights": { | |
| "value": { | |
| "verifier": "2a97c54e", | |
| "ranker": "f34d470b", | |
| "retrieval": "53aa5117" | |
| }, | |
| "pass": true | |
| }, | |
| "policy_and_memory_frozen": { | |
| "value": { | |
| "max_contradiction": 0.2, | |
| "max_unknown": 0.3, | |
| "min_support": 0.7, | |
| "memory": { | |
| "capacity": 256, | |
| "top_k": 5 | |
| }, | |
| "max_records": 1024 | |
| }, | |
| "pass": true | |
| }, | |
| "reviewed_correct_ge_1": { | |
| "value": 2, | |
| "pass": true | |
| }, | |
| "bad_nonabstentions_le_1": { | |
| "value": 1, | |
| "pass": true | |
| }, | |
| "failed_calls_0_including_controls": { | |
| "value": [ | |
| 0, | |
| 0, | |
| 0 | |
| ], | |
| "pass": true | |
| }, | |
| "generation_truncations_0": { | |
| "value": [ | |
| 0, | |
| 0, | |
| 0 | |
| ], | |
| "pass": true | |
| }, | |
| "contradiction_veto_violations_0": { | |
| "value": 0, | |
| "pass": true | |
| }, | |
| "uncalibrated_verifier_records_0": { | |
| "value": 0, | |
| "pass": true | |
| }, | |
| "every_nonabstained_preserves_selection": { | |
| "value": "3/3", | |
| "pass": true | |
| }, | |
| "omission_controls_abstain_8_of_8": { | |
| "value": 8.0, | |
| "pass": true | |
| }, | |
| "replacement_controls_abstain_8_of_8": { | |
| "value": 8.0, | |
| "pass": true | |
| }, | |
| "authored_conflict_abstains": { | |
| "value": 0, | |
| "pass": true | |
| }, | |
| "episodic_learned_hit_at_1_and_5_32_of_32": { | |
| "value": { | |
| "learned_hit_at_1": 1.0, | |
| "learned_hit_at_5": 1.0, | |
| "lexical_hit_at_1": 0.96875, | |
| "lexical_hit_at_5": 1.0 | |
| }, | |
| "pass": true | |
| }, | |
| "roundtrip_state_dict_bitwise_and_canonical_config": { | |
| "value": { | |
| "resave_state_dict_bitwise_equal": true, | |
| "resave_configuration_bytes_equal": true, | |
| "configuration_equal": true | |
| }, | |
| "pass": true | |
| }, | |
| "fresh_process_replay_correct_cases": { | |
| "value": [ | |
| "5ae048a255429924de1b708e", | |
| "5a78db8055429970f5fffdb2" | |
| ], | |
| "pass": true | |
| } | |
| }, | |
| "all_local_gates_pass": true, | |
| "not_evaluated_here": [ | |
| "independent second-assistant review of the 3 non-abstained responses (pending)", | |
| "Hub push, loading from Hub main, LFS hash and receipt check on Hub (publishing not authorized for this agent)", | |
| "old pin 8836ba5 loading with runtime 6607a8b", | |
| "repository docs/records update" | |
| ], | |
| "generator_selection": { | |
| "v1_dev_rule_choice": "g3 (fails test F1 0.82257)", | |
| "v2_extended_validation_choice": "../artifacts/replacement-20260924/sweep/g5", | |
| "disclosure": "v2 rule written after all variants' test scores were visible; see generator-selection-v2.json" | |
| }, | |
| "comparison_with_original": { | |
| "primary": { | |
| "original": { | |
| "correct": 1, | |
| "bad_nonabstentions": "1 circular non-answer", | |
| "abstentions": 30 | |
| }, | |
| "replacement": { | |
| "correct": 2, | |
| "bad_nonabstentions": "1 incorrect factual answer", | |
| "abstentions": 29 | |
| } | |
| }, | |
| "metrics": { | |
| "answer_wholeword_containment_not_accuracy": { | |
| "original": 0.03125, | |
| "replacement": 0.0625 | |
| }, | |
| "candidate_answer_substring_coverage": { | |
| "original": 0.59375, | |
| "replacement": 0.59375 | |
| }, | |
| "cognitive_abstention_rate": { | |
| "original": 0.9375, | |
| "replacement": 0.90625 | |
| }, | |
| "selected_text_preserved_rate": { | |
| "original": 0.0625, | |
| "replacement": 0.09375 | |
| }, | |
| "realization_nli_support_mean_on_selected": { | |
| "original": 0.8267748653888702, | |
| "replacement": 0.8741148312886556 | |
| }, | |
| "short_answer_exact_match_format_sensitive": { | |
| "original": 0.0, | |
| "replacement": 0.0 | |
| } | |
| }, | |
| "episodic": { | |
| "original": { | |
| "learned_hit_at_1": 1.0, | |
| "learned_hit_at_5": 1.0, | |
| "lexical_hit_at_1": 0.96875, | |
| "lexical_hit_at_5": 1.0 | |
| }, | |
| "replacement": { | |
| "learned_hit_at_1": 1.0, | |
| "learned_hit_at_5": 1.0, | |
| "lexical_hit_at_1": 0.96875, | |
| "lexical_hit_at_5": 1.0 | |
| } | |
| }, | |
| "generator_test": { | |
| "original": { | |
| "exact": 30, | |
| "token_f1": 0.8291862, | |
| "nli_support": 0.2265625 | |
| }, | |
| "replacement": { | |
| "exact": 36, | |
| "token_f1": 0.8413782072760705, | |
| "nli_support": 0.234375 | |
| } | |
| }, | |
| "generator_dev": { | |
| "original": { | |
| "exact": 33, | |
| "token_f1": 0.8329 | |
| }, | |
| "replacement": { | |
| "exact": 39, | |
| "token_f1": 0.8470289000479485 | |
| } | |
| }, | |
| "realizer_dev": { | |
| "original": { | |
| "statement_exact": 64, | |
| "verbatim": 62, | |
| "token_f1": 1.0 | |
| }, | |
| "replacement": { | |
| "statement_exact": 64.0, | |
| "verbatim": 62.0, | |
| "token_f1": 1.0 | |
| } | |
| }, | |
| "repository_smoke": { | |
| "original": "abstained", | |
| "replacement": "The preferred model host for TensorCode is nugging Face . (corrupted proper noun, not abstained)", | |
| "save_load_snapshot_exact": true | |
| } | |
| }, | |
| "protocol_differences": [ | |
| "Current source routes bounded prior dialogue (conversation_context_tokens=128) into the same-session replacement control; failed_calls were 0 for all control calls.", | |
| "evaluate_memory uses investigator.new_cognitive_session + ingest/remember (examples/evaluate_cognition.py at HEAD 090ebc4) instead of the removed runtime.LearnedEpisodicMemory.", | |
| "The 32 cases were reused as development data by cognitive-coverage-development and the XL experiments; this is a fixed-configuration re-run on known cases, not a held-out test.", | |
| "Ranker loaded from the Hub snapshot jacob-valdez/tensorcode-investigator-hotpot-001@1bc2259 (weights byte-identical to the original ranker).", | |
| "Repository smoke used the current docs/pretrained.md, whose first paragraph differs from the original run.", | |
| "Assembly and evaluation were separate processes: the frozen model-final was loaded with Chatbot.from_pretrained for evaluation." | |
| ] | |
| } | |