{ "real_data": { "metrics": { "count": 32, "failed_calls": 0, "short_answer_exact_match_format_sensitive": 0.0, "answer_wholeword_containment_not_accuracy": 0.0625, "candidate_answer_substring_coverage": 0.59375, "cognitive_abstention_rate": 0.90625, "final_enforced_abstention_rate": 0.90625, "default_abstention_text_rate": 0.90625, "selected_text_preserved_rate": 0.09375, "selected_contradiction_veto_violations": 0, "generation_truncations": 0, "realization_nli_support_mean_on_selected": 0.8741148312886556, "uncalibrated_verifier_records": 0 }, "records": [ { "id": "5ae048a255429924de1b708e", "question": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?", "target": "Catwoman", "answer": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .", "answer_exact_match": false, "answer_wholeword_containment": true, "candidate_answer_substring_coverage": true, "selected_text_preserved": true, "abstained": false, "abstention_enforced": false, "default_abstention_text": false, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .", "cognition": { "question": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?", "selected_id": "hypothesis-e280cfd29c5891d526e2", "abstained": false, "candidates": [ { "id": "hypothesis-d500d71343afee730525", "text": "Pitof directed Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.6109209060668945, "probability": 0.29777318239212036, "verifications": [ { "source_id": "Catwoman (film)", "distribution": { "contradiction": 0.027872735634446144, "support": 0.901282548904419, "unknown": 0.07084470987319946 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Catwoman (video game)", "distribution": { "contradiction": 0.03385399281978607, "support": 0.02250676229596138, "unknown": 0.9436392188072205 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-e280cfd29c5891d526e2", "text": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.660310745239258, "probability": 0.31284940242767334, "verifications": [ { "source_id": "Catwoman (film)", "distribution": { "contradiction": 0.026821404695510864, "support": 0.9117333889007568, "unknown": 0.0614452064037323 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Catwoman (video game)", "distribution": { "contradiction": 0.04225413501262665, "support": 0.02814735658466816, "unknown": 0.9295985698699951 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-0d605dfd3bb00e14b9fa", "text": "Pitof directs Catwoman , a 2004 American action superhero film loosely based on the DC Comics character of the same name .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.879137992858887, "probability": 0.3893774449825287, "verifications": [ { "source_id": "Catwoman (film)", "distribution": { "contradiction": 0.18035893142223358, "support": 0.6398777365684509, "unknown": 0.17976327240467072 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Catwoman (video game)", "distribution": { "contradiction": 0.019533880054950714, "support": 0.02135777287185192, "unknown": 0.9591083526611328 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Catwoman (film)\nCatwoman is a 2004 American action superhero film loosely based on the DC Comics character of the same name directed by Pitof, produced by Denise Di Novi and Edward McDonnell, and written by John Rogers, John Brancato and Michael Ferris, with music by Klaus Badelt. It stars Halle Berry, Sharon Stone, Benjamin Bratt, Lambert Wilson, Frances Conroy, and Alex Borstein.", "source_id": "Catwoman (film)" }, { "id": "doc-5", "text": "Catwoman (video game)\nCatwoman is an action-adventure tie-in video game based on the 2004 film of the same name based on the fictional character. It features the likeness of the film's lead actress, Halle Berry, though the character's voice is provided by actress Jennifer Hale.", "source_id": "Catwoman (video game)" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-5" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": false, "realization_verifications": [ { "source_id": "doc-2", "distribution": { "contradiction": 0.026821404695510864, "support": 0.9117333889007568, "unknown": 0.0614452064037323 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false }, { "source_id": "doc-5", "distribution": { "contradiction": 0.04225413501262665, "support": 0.02814735658466816, "unknown": 0.9295985698699951 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Catwoman (film)\nCatwoman is a 2004 American action superhero film loosely based on the DC Comics character of the same name directed by Pitof, produced by Denise Di Novi and Edward McDonnell, and written by John Rogers, John Brancato and Michael Ferris, with music by Klaus Badelt. It stars Halle Berry, Sharon Stone, Benjamin Bratt, Lambert Wilson, Frances Conroy, and Alex Borstein.", "source_id": "Catwoman (film)" }, { "id": "doc-5", "text": "Catwoman (video game)\nCatwoman is an action-adventure tie-in video game based on the 2004 film of the same name based on the fictional character. It features the likeness of the film's lead actress, Halle Berry, though the character's voice is provided by actress Jennifer Hale.", "source_id": "Catwoman (video game)" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false }, "realization_nli": [ { "source_id": "hypothesis-e280cfd29c5891d526e2", "distribution": { "contradiction": 0.023340655490756035, "support": 0.9193984270095825, "unknown": 0.05726095288991928 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ] }, { "id": "5a9068a95542990a98493621", "question": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?", "target": "7 miles", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-1ccbd8554450537017a7", "text": "On 16 January 1995 , First Knight was being filmed at a studio located approximately 20 mi west of central London .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-6" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.157319068908691, "probability": 0.31627392768859863, "verifications": [ { "source_id": "Filming of James Bond in the 1990s", "distribution": { "contradiction": 0.0200054794549942, "support": 0.010994340293109417, "unknown": 0.9690001606941223 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Pinewood Studios", "distribution": { "contradiction": 0.01134956069290638, "support": 0.009057464078068733, "unknown": 0.97959303855896 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-362d611c89aacc0ffd9c", "text": "On 16 January 1995 First Knight '' was being filmed at a studio located approximately 20 mi west of central London .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-6" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.2785539627075195, "probability": 0.3570385277271271, "verifications": [ { "source_id": "Filming of James Bond in the 1990s", "distribution": { "contradiction": 0.022342246025800705, "support": 0.010831733234226704, "unknown": 0.9668260216712952 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Pinewood Studios", "distribution": { "contradiction": 0.01023869775235653, "support": 0.010796387679874897, "unknown": 0.9789648652076721 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-02cc309f4f0218070891", "text": "On 16 January 1995 , First Knight was being filmed at a studio located approximately 20 mi west of Windsor .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-6" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.189714431762695, "probability": 0.3266875445842743, "verifications": [ { "source_id": "Filming of James Bond in the 1990s", "distribution": { "contradiction": 0.020037850365042686, "support": 0.010284077376127243, "unknown": 0.9696780443191528 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Pinewood Studios", "distribution": { "contradiction": 0.012723266147077084, "support": 0.008630764670670033, "unknown": 0.9786459803581238 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Filming of James Bond in the 1990s\nPrincipal photography for the film began on 16 January 1995 and continued until 6 June. The producers were unable to film at Pinewood Studios, the usual location for Bond films, because it had been reserved for \"First Knight\". Instead, an old Rolls-Royce factory at the Leavesden Aerodrome in Hertfordshire was converted into a new studio. The producers later said Pinewood would have been too small.", "source_id": "Filming of James Bond in the 1990s" }, { "id": "doc-6", "text": "Pinewood Studios\nPinewood Studios is a British film and television studio located approximately 20 mi west of central London, and 7 miles from Windsor.", "source_id": "Pinewood Studios" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-6" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Filming of James Bond in the 1990s\nPrincipal photography for the film began on 16 January 1995 and continued until 6 June. The producers were unable to film at Pinewood Studios, the usual location for Bond films, because it had been reserved for \"First Knight\". Instead, an old Rolls-Royce factory at the Leavesden Aerodrome in Hertfordshire was converted into a new studio. The producers later said Pinewood would have been too small.", "source_id": "Filming of James Bond in the 1990s" }, { "id": "doc-6", "text": "Pinewood Studios\nPinewood Studios is a British film and television studio located approximately 20 mi west of central London, and 7 miles from Windsor.", "source_id": "Pinewood Studios" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5adf9dc3554299025d62a2e6", "question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "target": "Laura Ann Osnes", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-4281dbd6594787c1f845", "text": "Laura Osnes is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Ginderella '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.004489898681641, "probability": 0.3463187515735626, "verifications": [ { "source_id": "Laura Osnes", "distribution": { "contradiction": 0.02550291270017624, "support": 0.013737461529672146, "unknown": 0.9607595801353455 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Grease: The New Broadway Cast Recording", "distribution": { "contradiction": 0.0225700493901968, "support": 0.03176889941096306, "unknown": 0.9456610083580017 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-b3c6572522288e86d204", "text": "Laura Osnes is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Ginderella .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.6558427810668945, "probability": 0.24437709152698517, "verifications": [ { "source_id": "Laura Osnes", "distribution": { "contradiction": 0.02933584526181221, "support": 0.012845129705965519, "unknown": 0.9578190445899963 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Grease: The New Broadway Cast Recording", "distribution": { "contradiction": 0.026930175721645355, "support": 0.026760367676615715, "unknown": 0.9463093876838684 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-38987edf2360e79c401a", "text": "Laura Osnes is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Glenn & Clyde '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.171588897705078, "probability": 0.4093041718006134, "verifications": [ { "source_id": "Laura Osnes", "distribution": { "contradiction": 0.022445805370807648, "support": 0.015732070431113243, "unknown": 0.9618220329284668 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Grease: The New Broadway Cast Recording", "distribution": { "contradiction": 0.02185141108930111, "support": 0.028515229001641273, "unknown": 0.9496333599090576 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Laura Osnes\nLaura Ann Osnes (born November 19, 1985) is an American actress and singer known for her work on the Broadway stage. She has played starring roles in \"Grease\" as Sandy, \"South Pacific\" as Nellie Forbush, \"Anything Goes\" as Hope Harcourt, and \"Bonnie and Clyde\" as Bonnie Parker, for which she received a Tony Award nomination for Best Actress in a Musical. She also starred in the title role of Rodgers & Hammerstein's \"Cinderella\" on Broadway, for which she received a Drama Desk Award and her second Tony Award nomination for Best Actress in a Musical.", "source_id": "Laura Osnes" }, { "id": "doc-8", "text": "Grease: The New Broadway Cast Recording\nGrease: The New Broadway Cast Recording is the cast album for the 2007 Broadway production of the hit musical, \"Grease\". The show, directed by Tony Award-winner Kathleen Marshall, played at the Brooks Atkinson Theatre in New York City. This recording features performances from the cast of \"Grease\" including Jenny Powers, Matthew Saldivar, and \"\" winners, Laura Osnes and Max Crumm as Sandy Dumbrowski and Danny Zuko, respectively.", "source_id": "Grease: The New Broadway Cast Recording" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-8" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Laura Osnes\nLaura Ann Osnes (born November 19, 1985) is an American actress and singer known for her work on the Broadway stage. She has played starring roles in \"Grease\" as Sandy, \"South Pacific\" as Nellie Forbush, \"Anything Goes\" as Hope Harcourt, and \"Bonnie and Clyde\" as Bonnie Parker, for which she received a Tony Award nomination for Best Actress in a Musical. She also starred in the title role of Rodgers & Hammerstein's \"Cinderella\" on Broadway, for which she received a Drama Desk Award and her second Tony Award nomination for Best Actress in a Musical.", "source_id": "Laura Osnes" }, { "id": "doc-8", "text": "Grease: The New Broadway Cast Recording\nGrease: The New Broadway Cast Recording is the cast album for the 2007 Broadway production of the hit musical, \"Grease\". The show, directed by Tony Award-winner Kathleen Marshall, played at the Brooks Atkinson Theatre in New York City. This recording features performances from the cast of \"Grease\" including Jenny Powers, Matthew Saldivar, and \"\" winners, Laura Osnes and Max Crumm as Sandy Dumbrowski and Danny Zuko, respectively.", "source_id": "Grease: The New Broadway Cast Recording" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ae406245542995dadf242e2", "question": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?", "target": "20", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-4422a9254d024cf66934", "text": "Marion Greene was a health policy analyst for St. Judt Medical company , which had 20 principal operations worldwide .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.198635101318359, "probability": 0.49619540572166443, "verifications": [ { "source_id": "St. Jude Medical", "distribution": { "contradiction": 0.023653332144021988, "support": 0.008523043245077133, "unknown": 0.9678236842155457 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Marion Greene", "distribution": { "contradiction": 0.05239468440413475, "support": 0.008243249729275703, "unknown": 0.939362108707428 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-b0b5bc90170b0a3a8edf", "text": "Marion Greene was a health policy analyst for St. Judt Medical company , which had more than 20 principal operations worldwide .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.21385383605957, "probability": 0.503804624080658, "verifications": [ { "source_id": "St. Jude Medical", "distribution": { "contradiction": 0.02782052382826805, "support": 0.008606739342212677, "unknown": 0.9635727405548096 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Marion Greene", "distribution": { "contradiction": 0.036528073251247406, "support": 0.008758832700550556, "unknown": 0.9547130465507507 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "St. Jude Medical\nSt. Jude Medical, Inc. was an American global medical device company headquartered in Little Canada, Minnesota, U.S., a suburb of Saint Paul. The company had more than 20 principal operations and manufacturing facilities worldwide with products sold in more than 100 countries. Its major markets include the United States, Europe, Latin America and Asia-Pacific. The company was named after Jude the Apostle, the patron saint of lost causes.", "source_id": "St. Jude Medical" }, { "id": "doc-5", "text": "Marion Greene\nMarion C. Greene (born July 16, 1970) is a Hennepin County commissioner (District 3) and a former member of the Minnesota House of Representatives who represented District 60A, which included portions of the city of Minneapolis in Hennepin County, which is in the Twin Cities metropolitan area. A Democrat, she was a health policy analyst for St. Jude Medical in Little Canada.", "source_id": "Marion Greene" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-5" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "St. Jude Medical\nSt. Jude Medical, Inc. was an American global medical device company headquartered in Little Canada, Minnesota, U.S., a suburb of Saint Paul. The company had more than 20 principal operations and manufacturing facilities worldwide with products sold in more than 100 countries. Its major markets include the United States, Europe, Latin America and Asia-Pacific. The company was named after Jude the Apostle, the patron saint of lost causes.", "source_id": "St. Jude Medical" }, { "id": "doc-5", "text": "Marion Greene\nMarion C. Greene (born July 16, 1970) is a Hennepin County commissioner (District 3) and a former member of the Minnesota House of Representatives who represented District 60A, which included portions of the city of Minneapolis in Hennepin County, which is in the Twin Cities metropolitan area. A Democrat, she was a health policy analyst for St. Jude Medical in Little Canada.", "source_id": "Marion Greene" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ab6369655429953192ad2a1", "question": "Which Genus has more species Eucryphia or Lepidozamia ?", "target": "Eucryphia", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which Genus has more species Eucryphia or Lepidozamia ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-7e4ba5c8e6b261ef2eb7", "text": "Eucryphia or Lepidozamia have more species Eucryphia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-1" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.380141258239746, "probability": 0.33240941166877747, "verifications": [ { "source_id": "Eucryphia", "distribution": { "contradiction": 0.02957230620086193, "support": 0.011799771338701248, "unknown": 0.9586278200149536 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Lepidozamia", "distribution": { "contradiction": 0.020354993641376495, "support": 0.013211431913077831, "unknown": 0.9664335250854492 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-1c4e9485ba669f788b08", "text": "Eucryphia or Lepidozamia have more species .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-1" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.421391487121582, "probability": 0.3464081287384033, "verifications": [ { "source_id": "Eucryphia", "distribution": { "contradiction": 0.03068607859313488, "support": 0.011532524600625038, "unknown": 0.9577814340591431 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Lepidozamia", "distribution": { "contradiction": 0.02372654713690281, "support": 0.012992643751204014, "unknown": 0.9632807374000549 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-c08f510533af8273b2b7", "text": "Eucryphia or Lepidozamia has more species Eucryphia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-1" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.345783233642578, "probability": 0.3211824595928192, "verifications": [ { "source_id": "Eucryphia", "distribution": { "contradiction": 0.03318432718515396, "support": 0.011994807980954647, "unknown": 0.9548208117485046 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Lepidozamia", "distribution": { "contradiction": 0.023349309340119362, "support": 0.012667030096054077, "unknown": 0.9639836549758911 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "Eucryphia\nEucryphia is a small genus of trees and large shrubs native to the south temperate regions of South America and coastal eastern Australia. Sometimes placed in a family of their own, the Eucryphiaceae, more recent classifications place them in the Cunoniaceae. There are seven species, two in South America and five in Australia, and several named hybrids. They are mostly evergreen though one species (\"E. glutinosa\") is usually deciduous.", "source_id": "Eucryphia" }, { "id": "doc-1", "text": "Lepidozamia\nLepidozamia is a genus of two species of cycad, both endemic to Australia. The name, derived from the Greek word lepidos, meaning scaly, refers to the scale-like structure of the stem and leaf bases. They are native to rainforest climates in eastern Queensland and eastern New South Wales. They have a chromosome number of 2n = 18.", "source_id": "Lepidozamia" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-1" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Eucryphia\nEucryphia is a small genus of trees and large shrubs native to the south temperate regions of South America and coastal eastern Australia. Sometimes placed in a family of their own, the Eucryphiaceae, more recent classifications place them in the Cunoniaceae. There are seven species, two in South America and five in Australia, and several named hybrids. They are mostly evergreen though one species (\"E. glutinosa\") is usually deciduous.", "source_id": "Eucryphia" }, { "id": "doc-1", "text": "Lepidozamia\nLepidozamia is a genus of two species of cycad, both endemic to Australia. The name, derived from the Greek word lepidos, meaning scaly, refers to the scale-like structure of the stem and leaf bases. They are native to rainforest climates in eastern Queensland and eastern New South Wales. They have a chromosome number of 2n = 18.", "source_id": "Lepidozamia" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ae628f55542995703ce8b32", "question": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?", "target": "Venice", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-9f3f3e93fe7ff1fb4e0f", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5, was born in Venice .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.518367767333984, "probability": 0.3933103084564209, "verifications": [ { "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)", "distribution": { "contradiction": 0.01021115854382515, "support": 0.012494650669395924, "unknown": 0.9772941470146179 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Antonio Vivaldi", "distribution": { "contradiction": 0.015412329696118832, "support": 0.007666146848350763, "unknown": 0.9769216179847717 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-a048517c6a2866f7ba5b", "text": "The Italian Baroque composer who composed a set of sonatas , titled Op. 5, was born in Venice .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.349864959716797, "probability": 0.3323192596435547, "verifications": [ { "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)", "distribution": { "contradiction": 0.01044224202632904, "support": 0.012235049158334732, "unknown": 0.9773226976394653 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Antonio Vivaldi", "distribution": { "contradiction": 0.01609073206782341, "support": 0.0077773816883563995, "unknown": 0.9761318564414978 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-93ad31fa0f3f7b2be842", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5 was born in Venice .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.158247947692871, "probability": 0.274370402097702, "verifications": [ { "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)", "distribution": { "contradiction": 0.011795050464570522, "support": 0.010710539296269417, "unknown": 0.9774944186210632 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Antonio Vivaldi", "distribution": { "contradiction": 0.015616467222571373, "support": 0.0076646870002150536, "unknown": 0.9767187833786011 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-1", "text": "Six Violin Sonatas, Op. 5 (Vivaldi)\nAntonio Vivaldi wrote a set of sonatas, Op. 5, in 1716.", "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)" }, { "id": "doc-7", "text": "Antonio Vivaldi\nAntonio Lucio Vivaldi (] ; 4 March 1678 \u2013 28 July 1741) was an Italian Baroque composer, virtuoso violinist, teacher and cleric. Born in Venice, he is recognized as one of the greatest Baroque composers, and his influence during his lifetime was widespread across Europe. He composed many instrumental concertos, for the violin and a variety of other instruments, as well as sacred choral works and more than forty operas. His best-known work is a series of violin concertos known as \"The Four Seasons\".", "source_id": "Antonio Vivaldi" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-1", "doc-7" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-1", "text": "Six Violin Sonatas, Op. 5 (Vivaldi)\nAntonio Vivaldi wrote a set of sonatas, Op. 5, in 1716.", "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)" }, { "id": "doc-7", "text": "Antonio Vivaldi\nAntonio Lucio Vivaldi (] ; 4 March 1678 \u2013 28 July 1741) was an Italian Baroque composer, virtuoso violinist, teacher and cleric. Born in Venice, he is recognized as one of the greatest Baroque composers, and his influence during his lifetime was widespread across Europe. He composed many instrumental concertos, for the violin and a variety of other instruments, as well as sacred choral works and more than forty operas. His best-known work is a series of violin concertos known as \"The Four Seasons\".", "source_id": "Antonio Vivaldi" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5adccdd55542992c1e3a2408", "question": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?", "target": "Waimea Bay", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-94e3ad1fd876e00389d5", "text": "Butch Van Artsdalen was known for surfing 25-foot waves at Sunset Beach .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.099187850952148, "probability": 0.44628089666366577, "verifications": [ { "source_id": "Waimea Bay, Hawaii", "distribution": { "contradiction": 0.0286415908485651, "support": 0.011072869412600994, "unknown": 0.9602855443954468 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" }, { "source_id": "Butch Van Artsdalen", "distribution": { "contradiction": 0.05794946849346161, "support": 0.02507428079843521, "unknown": 0.9169761538505554 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-bd619c3d7f1a9681d174", "text": "Butch Van Artsdalen was known for surfing 25-foot waves at Waimea Bay .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.476259231567383, "probability": 0.23937225341796875, "verifications": [ { "source_id": "Waimea Bay, Hawaii", "distribution": { "contradiction": 0.021749958395957947, "support": 0.010007117874920368, "unknown": 0.9682429432868958 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" }, { "source_id": "Butch Van Artsdalen", "distribution": { "contradiction": 0.035855360329151154, "support": 0.017075106501579285, "unknown": 0.9470694661140442 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-0e4f9993d6dd6fe5a920", "text": "Butch Van Artsdalen was known for surfing 25-foot waves at Waimea Bay and Sunset Beach .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.748736381530762, "probability": 0.31434687972068787, "verifications": [ { "source_id": "Waimea Bay, Hawaii", "distribution": { "contradiction": 0.024171432480216026, "support": 0.010524339973926544, "unknown": 0.9653041362762451 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" }, { "source_id": "Butch Van Artsdalen", "distribution": { "contradiction": 0.030038075521588326, "support": 0.01801122911274433, "unknown": 0.95195072889328 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-5", "text": "Waimea Bay, Hawaii\nWaimea Bay is located in Haleiwa on the North Shore of O'ahu in the Hawaiian Islands at the mouth of the Waimea River. Waimea Valley extends behind Waimea Bay. \"Waimea\" means \"reddish water\" in Hawaiian.", "source_id": "Waimea Bay, Hawaii" }, { "id": "doc-7", "text": "Butch Van Artsdalen\nCharles M. Van Artsdalen (January 31, 1941 \u2013 July 18, 1979) was a legendary surfer. He moved to La Jolla, California, from his birthplace of Norfolk, Virginia, at age 14. Van Artsdalen is best known as a pioneer of surfing 25-foot waves at such North Shore locations as Waimea Bay and Sunset Beach; and tube riding in Hawaii during the early 1960-1970s. A member of the Duke Kahanamoku Surf Team, he appeared in several surf movies and remained a fixture of the international surfing community until his death from alcohol-related illness in 1979.", "source_id": "Butch Van Artsdalen" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-5", "doc-7" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-5", "text": "Waimea Bay, Hawaii\nWaimea Bay is located in Haleiwa on the North Shore of O'ahu in the Hawaiian Islands at the mouth of the Waimea River. Waimea Valley extends behind Waimea Bay. \"Waimea\" means \"reddish water\" in Hawaiian.", "source_id": "Waimea Bay, Hawaii" }, { "id": "doc-7", "text": "Butch Van Artsdalen\nCharles M. Van Artsdalen (January 31, 1941 \u2013 July 18, 1979) was a legendary surfer. He moved to La Jolla, California, from his birthplace of Norfolk, Virginia, at age 14. Van Artsdalen is best known as a pioneer of surfing 25-foot waves at such North Shore locations as Waimea Bay and Sunset Beach; and tube riding in Hawaii during the early 1960-1970s. A member of the Duke Kahanamoku Surf Team, he appeared in several surf movies and remained a fixture of the international surfing community until his death from alcohol-related illness in 1979.", "source_id": "Butch Van Artsdalen" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5abd1529554299114383a1e3", "question": "What type of magazine is Eugene Habecker the chairman of?", "target": "evangelical Christian periodical", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What type of magazine is Eugene Habecker the chairman of?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-063f1d8ff076e0dd0f53", "text": "Eugene Habecker is the chairman of Christianity Today .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.196399688720703, "probability": 0.609771728515625, "verifications": [ { "source_id": "Eugene Habecker", "distribution": { "contradiction": 0.015894543379545212, "support": 0.016972847282886505, "unknown": 0.9671326875686646 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Christianity Today", "distribution": { "contradiction": 0.0265209823846817, "support": 0.007062033750116825, "unknown": 0.9664170145988464 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-de2f83ae9df7ec1252ff", "text": "Eugene Habecker the chairman of Christianity Today .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.750046730041504, "probability": 0.3902282416820526, "verifications": [ { "source_id": "Eugene Habecker", "distribution": { "contradiction": 0.014907455071806908, "support": 0.01663331687450409, "unknown": 0.9684591293334961 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Christianity Today", "distribution": { "contradiction": 0.032860610634088516, "support": 0.009577625431120396, "unknown": 0.9575616717338562 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "Eugene Habecker\nDr. Eugene B. Habecker is the 30th president of Taylor University and chairman of Christianity Today. Previously, Habecker was president of the American Bible Society. He is also the author of several books, two coauthored with his wife, Marylou Habecker. He was inaugurated in a ceremony on Friday, April 28, 2006, in Odle Arena on the Upland campus, only two days after a tragic fatal car accident involving several students and employees.", "source_id": "Eugene Habecker" }, { "id": "doc-5", "text": "Christianity Today\nChristianity Today magazine is an evangelical Christian periodical that was founded in 1956 and is based in Carol Stream, Illinois. \" The Washington Post\" calls \"Christianity Today,\" \"evangelicalism\u2019s flagship magazine\"; \"The New York Times\" describes it as a \"mainstream evangelical magazine.\"", "source_id": "Christianity Today" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-5" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Eugene Habecker\nDr. Eugene B. Habecker is the 30th president of Taylor University and chairman of Christianity Today. Previously, Habecker was president of the American Bible Society. He is also the author of several books, two coauthored with his wife, Marylou Habecker. He was inaugurated in a ceremony on Friday, April 28, 2006, in Odle Arena on the Upland campus, only two days after a tragic fatal car accident involving several students and employees.", "source_id": "Eugene Habecker" }, { "id": "doc-5", "text": "Christianity Today\nChristianity Today magazine is an evangelical Christian periodical that was founded in 1956 and is based in Carol Stream, Illinois. \" The Washington Post\" calls \"Christianity Today,\" \"evangelicalism\u2019s flagship magazine\"; \"The New York Times\" describes it as a \"mainstream evangelical magazine.\"", "source_id": "Christianity Today" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a78db8055429970f5fffdb2", "question": "Flyboys stars which actor who was nominated for an Academy Award for \"127 Hours\"?", "target": "James Franco", "answer": "James Franco was nominated for an Academy Award for 127 Hours '' for his role in 127 Hours '' .", "answer_exact_match": false, "answer_wholeword_containment": true, "candidate_answer_substring_coverage": true, "selected_text_preserved": true, "abstained": false, "abstention_enforced": false, "default_abstention_text": false, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "James Franco was nominated for an Academy Award for 127 Hours '' for his role in 127 Hours '' .", "cognition": { "question": "Flyboys stars which actor who was nominated for an Academy Award for \"127 Hours\"?", "selected_id": "hypothesis-7b5092602135b71d1970", "abstained": false, "candidates": [ { "id": "hypothesis-9a12a934a2b0fa0dae79", "text": "James Franco was nominated for an Academy Award for 127 Hours '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.66514778137207, "probability": 0.24741891026496887, "verifications": [ { "source_id": "James Franco", "distribution": { "contradiction": 0.08240272104740143, "support": 0.6895154118537903, "unknown": 0.22808188199996948 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Flyboys (film)", "distribution": { "contradiction": 0.03295936435461044, "support": 0.007199402432888746, "unknown": 0.9598411917686462 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-8146938b02d5afe9266c", "text": "Flyboys stars James Franco .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.643052101135254, "probability": 0.6578567624092102, "verifications": [ { "source_id": "James Franco", "distribution": { "contradiction": 0.2980218529701233, "support": 0.042609941214323044, "unknown": 0.6593681573867798 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Flyboys (film)", "distribution": { "contradiction": 0.049388304352760315, "support": 0.022691849619150162, "unknown": 0.9279197454452515 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-7b5092602135b71d1970", "text": "James Franco was nominated for an Academy Award for 127 Hours '' for his role in 127 Hours '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.705035209655762, "probability": 0.09472427517175674, "verifications": [ { "source_id": "James Franco", "distribution": { "contradiction": 0.0664210170507431, "support": 0.7765237092971802, "unknown": 0.15705527365207672 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Flyboys (film)", "distribution": { "contradiction": 0.0298065934330225, "support": 0.0075899334624409676, "unknown": 0.9626034498214722 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "James Franco\nJames Edward Franco (born April 19, 1978) is an American actor, director, screenwriter, and producer. For his role in \"127 Hours\" (2010), Franco was nominated for an Academy Award for Best Actor. He is known for his roles in live-action films such as \"Milk\" (2008), \"Pineapple Express\" (2008), \"Rise of the Planet of the Apes\" (2011), \"Spring Breakers\" (2012), \"Oz the Great and Powerful\" (2013), \"This Is the End\" (2013), \" The Disaster Artist\" (2017), and Sam Raimi's \"Spider-Man\" trilogy, while also voicing characters in the animated films \"The Little Prince\" (2015) and \"Sausage Party\" (2016).", "source_id": "James Franco" }, { "id": "doc-4", "text": "Flyboys (film)\nFlyboys is a 2006 war drama film set during World War I, starring James Franco, Martin Henderson, Jean Reno, Jennifer Decker, David Ellison, Abdul Salis, Philip Winchester, and Tyler Labine. It was directed by Tony Bill, a pilot and aviation enthusiast. The screenplay about men in aerial combat was written by Phil Sears, Blake T. Evans and David S. Ward with the story by Blake T. Evans. Themes of friendship, racial prejudice, revenge and love are also explored in the film.", "source_id": "Flyboys (film)" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "James Franco was nominated for an Academy Award for 127 Hours '' for his role in 127 Hours '' .", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-4" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": false, "realization_verifications": [ { "source_id": "doc-0", "distribution": { "contradiction": 0.0664210170507431, "support": 0.7765237092971802, "unknown": 0.15705527365207672 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false }, { "source_id": "doc-4", "distribution": { "contradiction": 0.0298065934330225, "support": 0.0075899334624409676, "unknown": 0.9626034498214722 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "James Franco\nJames Edward Franco (born April 19, 1978) is an American actor, director, screenwriter, and producer. For his role in \"127 Hours\" (2010), Franco was nominated for an Academy Award for Best Actor. He is known for his roles in live-action films such as \"Milk\" (2008), \"Pineapple Express\" (2008), \"Rise of the Planet of the Apes\" (2011), \"Spring Breakers\" (2012), \"Oz the Great and Powerful\" (2013), \"This Is the End\" (2013), \" The Disaster Artist\" (2017), and Sam Raimi's \"Spider-Man\" trilogy, while also voicing characters in the animated films \"The Little Prince\" (2015) and \"Sausage Party\" (2016).", "source_id": "James Franco" }, { "id": "doc-4", "text": "Flyboys (film)\nFlyboys is a 2006 war drama film set during World War I, starring James Franco, Martin Henderson, Jean Reno, Jennifer Decker, David Ellison, Abdul Salis, Philip Winchester, and Tyler Labine. It was directed by Tony Bill, a pilot and aviation enthusiast. The screenplay about men in aerial combat was written by Phil Sears, Blake T. Evans and David S. Ward with the story by Blake T. Evans. Themes of friendship, racial prejudice, revenge and love are also explored in the film.", "source_id": "Flyboys (film)" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false }, "realization_nli": [ { "source_id": "hypothesis-7b5092602135b71d1970", "distribution": { "contradiction": 0.03601401671767235, "support": 0.8815348148345947, "unknown": 0.08245111256837845 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ] }, { "id": "5ab9debb55429955dce3ed97", "question": "The English actor Kris Marshall played which character in the highly rated comedy-drama television series \"Death in Paradise\"?", "target": "DI Humphrey Goodman", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "The English actor Kris Marshall played which character in the highly rated comedy-drama television series \"Death in Paradise\"?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-56b454585b04010b3baf", "text": "Kris Marshall played Nick Harper in the highly rated comedy-drama television series Death in Paradise .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.393385887145996, "probability": 0.22319844365119934, "verifications": [ { "source_id": "Kris Marshall", "distribution": { "contradiction": 0.019464176148176193, "support": 0.011412149295210838, "unknown": 0.9691237211227417 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Death in Paradise (TV series)", "distribution": { "contradiction": 0.1850120723247528, "support": 0.007372277323156595, "unknown": 0.8076156973838806 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-dff235e330ef04cb19c2", "text": "Kris Marshall played Nick Harper in the highly rated comedy-drama television series Death in Paradise '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.213777542114258, "probability": 0.5069705247879028, "verifications": [ { "source_id": "Kris Marshall", "distribution": { "contradiction": 0.0233817920088768, "support": 0.011309584602713585, "unknown": 0.9653086066246033 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Death in Paradise (TV series)", "distribution": { "contradiction": 0.20030590891838074, "support": 0.0070434152148664, "unknown": 0.7926506996154785 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-44c30a1ff0ce91bafbc1", "text": "The English actor Kris Marshall played Nick Harper in the highly rated comedy-drama television series Death in Paradise .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.583120346069336, "probability": 0.26983100175857544, "verifications": [ { "source_id": "Kris Marshall", "distribution": { "contradiction": 0.012393330223858356, "support": 0.014515819028019905, "unknown": 0.9730907678604126 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Death in Paradise (TV series)", "distribution": { "contradiction": 0.12414231151342392, "support": 0.00923158973455429, "unknown": 0.8666261434555054 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-4", "text": "Kris Marshall\nKristopher \"Kris\" Marshall (born 11 April 1973) is an English actor. He has played Nick Harper in \"My Family\", Colin Frissell in the 2003 film \"Love Actually\", Adam in BT Retail adverts from 2005 until 2011, and Dave in the first series of \"Citizen Khan\" (2012). He played DI Humphrey Goodman in \"Death in Paradise\", taking over the role April 2013 for the show's third series and leaving it in February 2017 in the sixth series.", "source_id": "Kris Marshall" }, { "id": "doc-7", "text": "Death in Paradise (TV series)\nDeath in Paradise is a British-French crime comedy-drama television series created by Robert Thorogood, starring Ben Miller (series 1\u20133), Kris Marshall (series 3\u20136) & Ardal O'Hanlon (series 6\u2013present). The programme is a joint UK and French production filmed on the French Caribbean island of Guadeloupe and broadcast on BBC One in the United Kingdom and France 2 in France. \"Death in Paradise\" has enjoyed high ratings, leading to repeated renewals. A sixth series began broadcasting on 5 January 2017 and aired its finale on 23 February, with the subsequent DVD release occurring three days later on 27 February 2017. The show will return in 2018 for a seventh series.", "source_id": "Death in Paradise (TV series)" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-4", "doc-7" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-4", "text": "Kris Marshall\nKristopher \"Kris\" Marshall (born 11 April 1973) is an English actor. He has played Nick Harper in \"My Family\", Colin Frissell in the 2003 film \"Love Actually\", Adam in BT Retail adverts from 2005 until 2011, and Dave in the first series of \"Citizen Khan\" (2012). He played DI Humphrey Goodman in \"Death in Paradise\", taking over the role April 2013 for the show's third series and leaving it in February 2017 in the sixth series.", "source_id": "Kris Marshall" }, { "id": "doc-7", "text": "Death in Paradise (TV series)\nDeath in Paradise is a British-French crime comedy-drama television series created by Robert Thorogood, starring Ben Miller (series 1\u20133), Kris Marshall (series 3\u20136) & Ardal O'Hanlon (series 6\u2013present). The programme is a joint UK and French production filmed on the French Caribbean island of Guadeloupe and broadcast on BBC One in the United Kingdom and France 2 in France. \"Death in Paradise\" has enjoyed high ratings, leading to repeated renewals. A sixth series began broadcasting on 5 January 2017 and aired its finale on 23 February, with the subsequent DVD release occurring three days later on 27 February 2017. The show will return in 2018 for a seventh series.", "source_id": "Death in Paradise (TV series)" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ae791ef55429952e35ea979", "question": "In what month was the relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born?", "target": "February", "answer": "The relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born was in October , 1999 .", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": true, "abstained": false, "abstention_enforced": false, "default_abstention_text": false, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "The relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born was in October , 1999 .", "cognition": { "question": "In what month was the relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born?", "selected_id": "hypothesis-6599c3fae481609ff2ee", "abstained": false, "candidates": [ { "id": "hypothesis-30ebaaec64c1e5e506bb", "text": "The relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born was in October .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.1623640060424805, "probability": 0.32325583696365356, "verifications": [ { "source_id": "Gameel Al-Batouti", "distribution": { "contradiction": 0.02628043293952942, "support": 0.008986782282590866, "unknown": 0.9647328853607178 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "EgyptAir Flight 990", "distribution": { "contradiction": 0.0373227559030056, "support": 0.6852281093597412, "unknown": 0.2774491310119629 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-ff22f8f3387486f6d083", "text": "The relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born in October , 1999 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.2941999435424805, "probability": 0.36880943179130554, "verifications": [ { "source_id": "Gameel Al-Batouti", "distribution": { "contradiction": 0.03001588210463524, "support": 0.0076003363355994225, "unknown": 0.9623838067054749 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "EgyptAir Flight 990", "distribution": { "contradiction": 0.05611244961619377, "support": 0.5660147070884705, "unknown": 0.37787285447120667 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-6599c3fae481609ff2ee", "text": "The relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born was in October , 1999 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.113807678222656, "probability": 0.3079347014427185, "verifications": [ { "source_id": "Gameel Al-Batouti", "distribution": { "contradiction": 0.021178152412176132, "support": 0.009489379823207855, "unknown": 0.9693324565887451 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "EgyptAir Flight 990", "distribution": { "contradiction": 0.017701882869005203, "support": 0.8357990980148315, "unknown": 0.14649896323680878 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Gameel Al-Batouti\nGameel Al-Batouti (Arabic: \u062c\u0645\u064a\u0644 \u0627\u0644\u0628\u0637\u0648\u0637\u064a\u200e \u200e ; also rendered \"Gamil El Batouti\" or \"El Batouty\" in U.S. official reports; 2 February 1940 \u2013 31 October 1999) was a pilot for EgyptAir and a former officer for the Egyptian Air Force. On 31 October 1999, all 217 people aboard EgyptAir Flight 990 were killed when it crashed into the Atlantic Ocean about 60 mi southeast of Nantucket Island, Massachusetts. The US National Transportation Safety Board (NTSB) concluded that the official probable cause of the crash was a series of control inputs made by Al-Batouti, who was in the position of relief first officer in command at the time of the crash.", "source_id": "Gameel Al-Batouti" }, { "id": "doc-3", "text": "EgyptAir Flight 990\nEgyptAir Flight 990 (MS990/MSR990) was a regularly scheduled flight from Los Angeles International Airport, United States, to Cairo International Airport, Egypt, with a stop at John F. Kennedy International Airport, New York City. On October 31, 1999, the Boeing 767 operating the route crashed into the Atlantic Ocean about 60 mi south of Nantucket Island, Massachusetts, killing all 217 people on board. The official probable cause of the crash was deliberate action by the relief first officer.", "source_id": "EgyptAir Flight 990" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "The relief first officer of the flight that crashed on its way to Cairo with a stop in New York Born was in October , 1999 .", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-3" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": false, "realization_verifications": [ { "source_id": "doc-3", "distribution": { "contradiction": 0.017701882869005203, "support": 0.8357990980148315, "unknown": 0.14649896323680878 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false }, { "source_id": "doc-2", "distribution": { "contradiction": 0.021178152412176132, "support": 0.009489379823207855, "unknown": 0.9693324565887451 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-3", "text": "EgyptAir Flight 990\nEgyptAir Flight 990 (MS990/MSR990) was a regularly scheduled flight from Los Angeles International Airport, United States, to Cairo International Airport, Egypt, with a stop at John F. Kennedy International Airport, New York City. On October 31, 1999, the Boeing 767 operating the route crashed into the Atlantic Ocean about 60 mi south of Nantucket Island, Massachusetts, killing all 217 people on board. The official probable cause of the crash was deliberate action by the relief first officer.", "source_id": "EgyptAir Flight 990" }, { "id": "doc-2", "text": "Gameel Al-Batouti\nGameel Al-Batouti (Arabic: \u062c\u0645\u064a\u0644 \u0627\u0644\u0628\u0637\u0648\u0637\u064a\u200e \u200e ; also rendered \"Gamil El Batouti\" or \"El Batouty\" in U.S. official reports; 2 February 1940 \u2013 31 October 1999) was a pilot for EgyptAir and a former officer for the Egyptian Air Force. On 31 October 1999, all 217 people aboard EgyptAir Flight 990 were killed when it crashed into the Atlantic Ocean about 60 mi southeast of Nantucket Island, Massachusetts. The US National Transportation Safety Board (NTSB) concluded that the official probable cause of the crash was a series of control inputs made by Al-Batouti, who was in the position of relief first officer in command at the time of the crash.", "source_id": "Gameel Al-Batouti" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false }, "realization_nli": [ { "source_id": "hypothesis-6599c3fae481609ff2ee", "distribution": { "contradiction": 0.058757856488227844, "support": 0.8214112520217896, "unknown": 0.11983086913824081 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ] }, { "id": "5ab8f7535542991b5579f0a7", "question": "Which film was released first: Sacred Planet or Oz the Great and Powerful?", "target": "Sacred Planet", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which film was released first: Sacred Planet or Oz the Great and Powerful?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-a7ed499b7a2370a1deda", "text": "Oz the Great and Powerful was released first .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.192813873291016, "probability": 0.4548712372779846, "verifications": [ { "source_id": "Oz the Great and Powerful", "distribution": { "contradiction": 0.0992186889052391, "support": 0.01659807562828064, "unknown": 0.8841832280158997 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-0" }, { "source_id": "Sacred Planet", "distribution": { "contradiction": 0.9532524347305298, "support": 0.0026915837079286575, "unknown": 0.044056035578250885 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-56b22b701506eeefcdc7", "text": "Oz the Great and Powerful was released first , Sacred Planet .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.808558464050293, "probability": 0.30974796414375305, "verifications": [ { "source_id": "Oz the Great and Powerful", "distribution": { "contradiction": 0.4263613224029541, "support": 0.01076545286923647, "unknown": 0.5628732442855835 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-0" }, { "source_id": "Sacred Planet", "distribution": { "contradiction": 0.821264386177063, "support": 0.003846194129437208, "unknown": 0.17488941550254822 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-2e16561a07f8b54284cb", "text": "Sacred Planet was released first , Oz the Great and Powerful .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.534004211425781, "probability": 0.23538081347942352, "verifications": [ { "source_id": "Oz the Great and Powerful", "distribution": { "contradiction": 0.291410893201828, "support": 0.01314097736030817, "unknown": 0.695448100566864 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-0" }, { "source_id": "Sacred Planet", "distribution": { "contradiction": 0.12322711944580078, "support": 0.006453621666878462, "unknown": 0.8703192472457886 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" } ], "evidence": [ { "id": "doc-0", "text": "Oz the Great and Powerful\nOz the Great and Powerful is a 2013 American fantasy adventure film directed by Sam Raimi and produced by Joe Roth, from a screenplay written by David Lindsay-Abaire and Mitchell Kapner. The film stars James Franco, Michelle Williams, Rachel Weisz and Mila Kunis, with Zach Braff, Bill Cobbs, Joey King, and Tony Cox in supporting roles. Based on L. Frank Baum's \"Oz\" novels and set 20 years before the events of the original novel, \"Oz the Great and Powerful\" is a spiritual prequel to the 1939 Metro-Goldwyn-Mayer film, \"The Wizard of Oz\". The film tells the story of Oscar Diggs, a deceptive magician who arrives in the Land of Oz and encounters three witches: Theodora, Evanora, and Glinda. Oscar is then enlisted to restore order in Oz, while struggling to resolve conflicts with the witches and himself.", "source_id": "Oz the Great and Powerful" }, { "id": "doc-9", "text": "Sacred Planet\nSacred Planet is a 2004 documentary directed by Jon Long and Hairul Salleh Askor. Robert Redford provided narration for the film. The film was released by Walt Disney Pictures on April 22, 2004, and grossed $1,108,356.", "source_id": "Sacred Planet" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-9" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Oz the Great and Powerful\nOz the Great and Powerful is a 2013 American fantasy adventure film directed by Sam Raimi and produced by Joe Roth, from a screenplay written by David Lindsay-Abaire and Mitchell Kapner. The film stars James Franco, Michelle Williams, Rachel Weisz and Mila Kunis, with Zach Braff, Bill Cobbs, Joey King, and Tony Cox in supporting roles. Based on L. Frank Baum's \"Oz\" novels and set 20 years before the events of the original novel, \"Oz the Great and Powerful\" is a spiritual prequel to the 1939 Metro-Goldwyn-Mayer film, \"The Wizard of Oz\". The film tells the story of Oscar Diggs, a deceptive magician who arrives in the Land of Oz and encounters three witches: Theodora, Evanora, and Glinda. Oscar is then enlisted to restore order in Oz, while struggling to resolve conflicts with the witches and himself.", "source_id": "Oz the Great and Powerful" }, { "id": "doc-9", "text": "Sacred Planet\nSacred Planet is a 2004 documentary directed by Jon Long and Hairul Salleh Askor. Robert Redford provided narration for the film. The film was released by Walt Disney Pictures on April 22, 2004, and grossed $1,108,356.", "source_id": "Sacred Planet" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ac119335542992a796dede4", "question": "I'd Come for You is a single by the band formed in what Alberta city?", "target": "Hanna", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "I'd Come for You is a single by the band formed in what Alberta city?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-bbaca9bf360a975bcb3b", "text": "I'd Come for You is a single by the band formed in Alberta .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.389909744262695, "probability": 0.2984141409397125, "verifications": [ { "source_id": "I'd Come for You", "distribution": { "contradiction": 0.0406794548034668, "support": 0.005187211558222771, "unknown": 0.9541333913803101 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Nickelback", "distribution": { "contradiction": 0.02782057784497738, "support": 0.0082534896209836, "unknown": 0.9639259576797485 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-0e56c2070de015482a23", "text": "I'd Come for You is a single by the band formed in Hanna , Alberta .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.690790176391602, "probability": 0.40317174792289734, "verifications": [ { "source_id": "I'd Come for You", "distribution": { "contradiction": 0.07658380270004272, "support": 0.006229919847100973, "unknown": 0.9171863198280334 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Nickelback", "distribution": { "contradiction": 0.029721030965447426, "support": 0.008212526328861713, "unknown": 0.9620664715766907 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-dfb25949d0a1a3617f58", "text": "I 'd Come for You is a single by the band formed in Alberta .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.389909744262695, "probability": 0.2984141409397125, "verifications": [ { "source_id": "I'd Come for You", "distribution": { "contradiction": 0.04132503271102905, "support": 0.00520398048684001, "unknown": 0.9534710645675659 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Nickelback", "distribution": { "contradiction": 0.02805384434759617, "support": 0.008242697454988956, "unknown": 0.963703453540802 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "I'd Come for You\n\"I'd Come for You\" is the second European single (fourth in Canada) from Nickelback's 2008 album \"Dark Horse\". The song was written by the band and Mutt Lange. It was released March 23, 2009. \" If Today Was Your Last Day\" served as the third single instead in North America (after \"Gotta Be Somebody\" and the rock radio-only single \"Something in Your Mouth\").", "source_id": "I'd Come for You" }, { "id": "doc-3", "text": "Nickelback\nNickelback is a Canadian rock band formed in 1995 in Hanna, Alberta, Canada. The band is composed of guitarist and lead vocalist Chad Kroeger, guitarist, keyboardist and backing vocalist Ryan Peake, bassist Mike Kroeger, and drummer Daniel Adair. The band went through a few drummer changes between 1995 and 2005, achieving its current lineup when Adair replaced drummer Ryan Vikedal.", "source_id": "Nickelback" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-3" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "I'd Come for You\n\"I'd Come for You\" is the second European single (fourth in Canada) from Nickelback's 2008 album \"Dark Horse\". The song was written by the band and Mutt Lange. It was released March 23, 2009. \" If Today Was Your Last Day\" served as the third single instead in North America (after \"Gotta Be Somebody\" and the rock radio-only single \"Something in Your Mouth\").", "source_id": "I'd Come for You" }, { "id": "doc-3", "text": "Nickelback\nNickelback is a Canadian rock band formed in 1995 in Hanna, Alberta, Canada. The band is composed of guitarist and lead vocalist Chad Kroeger, guitarist, keyboardist and backing vocalist Ryan Peake, bassist Mike Kroeger, and drummer Daniel Adair. The band went through a few drummer changes between 1995 and 2005, achieving its current lineup when Adair replaced drummer Ryan Vikedal.", "source_id": "Nickelback" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a7d2afd5542995f4f402221", "question": "Who is older, Ferdi Taygan or Mahesh Bhupathi?", "target": "Ferdi Taygan", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Who is older, Ferdi Taygan or Mahesh Bhupathi?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-31eaf33c8bcc21f84d25", "text": "Ferdi Taygan is older , Mahesh Bhupathi .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.580793380737305, "probability": 0.28601446747779846, "verifications": [ { "source_id": "Ferdi Taygan", "distribution": { "contradiction": 0.22723527252674103, "support": 0.012632536701858044, "unknown": 0.7601321339607239 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Mahesh Bhupathi", "distribution": { "contradiction": 0.7454468011856079, "support": 0.028695283457636833, "unknown": 0.22585788369178772 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-284f95a9961b9cf8e9d7", "text": "Ferdi Taygan is older , Ferdi Taygan or Mahesh Bhupathi .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.670782089233398, "probability": 0.3129461109638214, "verifications": [ { "source_id": "Ferdi Taygan", "distribution": { "contradiction": 0.19368351995944977, "support": 0.0376962386071682, "unknown": 0.7686201930046082 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Mahesh Bhupathi", "distribution": { "contradiction": 0.11393293738365173, "support": 0.03517303243279457, "unknown": 0.850894033908844 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-fc65c5e28851de5d6649", "text": "Ferdi Taygan is older , Ferdi Taygan .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.918810844421387, "probability": 0.4010394513607025, "verifications": [ { "source_id": "Ferdi Taygan", "distribution": { "contradiction": 0.04267928749322891, "support": 0.0497889518737793, "unknown": 0.9075317978858948 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Mahesh Bhupathi", "distribution": { "contradiction": 0.053081754595041275, "support": 0.021608687937259674, "unknown": 0.9253094792366028 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Ferdi Taygan\nFerdi Taygan (born December 5, 1956), is a former professional tennis player from the United States. He is of Turkish descent.", "source_id": "Ferdi Taygan" }, { "id": "doc-3", "text": "Mahesh Bhupathi\nMahesh Shrinivas Bhupathi (born 7 June 1974) is a retired Indian professional tennis player. In 1997, he became the first Indian to win a Grand Slam tournament (with Rika Hiraki). With his win at the Australian Open mixed doubles in 2006, he joined the elite group of eight tennis players who have achieved a career Grand Slam in mixed doubles. He is also the founder of International Premier Tennis League. In December 2016, Bhupathi was appointed as India's next non-playing Davis Cup captain and took over the reins from Anand Amritraj in February 2017.", "source_id": "Mahesh Bhupathi" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-3" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Ferdi Taygan\nFerdi Taygan (born December 5, 1956), is a former professional tennis player from the United States. He is of Turkish descent.", "source_id": "Ferdi Taygan" }, { "id": "doc-3", "text": "Mahesh Bhupathi\nMahesh Shrinivas Bhupathi (born 7 June 1974) is a retired Indian professional tennis player. In 1997, he became the first Indian to win a Grand Slam tournament (with Rika Hiraki). With his win at the Australian Open mixed doubles in 2006, he joined the elite group of eight tennis players who have achieved a career Grand Slam in mixed doubles. He is also the founder of International Premier Tennis League. In December 2016, Bhupathi was appointed as India's next non-playing Davis Cup captain and took over the reins from Anand Amritraj in February 2017.", "source_id": "Mahesh Bhupathi" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a8dd5f955429917b4a5bcc7", "question": "Other than racing, what sport does the 1998 champion of the Toyota GRand Prix practice?", "target": "paracyclist", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Other than racing, what sport does the 1998 champion of the Toyota GRand Prix practice?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-bfedb7d968cf7792be03", "text": "The 1998 champion of the Toyota GRand Prix practices racing .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.063251495361328, "probability": 0.5659299492835999, "verifications": [ { "source_id": "1998 Toyota Grand Prix of Long Beach", "distribution": { "contradiction": 0.04297966882586479, "support": 0.01562805287539959, "unknown": 0.9413922429084778 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Alex Zanardi", "distribution": { "contradiction": 0.04473697766661644, "support": 0.00799970980733633, "unknown": 0.9472633600234985 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-818d34084139dfce9eb5", "text": "The 1998 champion of the Toyota GRand Prix practiced racing .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.79798698425293, "probability": 0.43407002091407776, "verifications": [ { "source_id": "1998 Toyota Grand Prix of Long Beach", "distribution": { "contradiction": 0.028499923646450043, "support": 0.019115513190627098, "unknown": 0.9523845911026001 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Alex Zanardi", "distribution": { "contradiction": 0.04415164515376091, "support": 0.011729319579899311, "unknown": 0.9441190361976624 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "1998 Toyota Grand Prix of Long Beach\nThe 1998 Toyota Grand Prix of Long Beach was the third round of the 1998 CART FedEx Champ Car World Series season, held on April 5, 1998, on the streets of Long Beach, California. Alex Zanardi won the race, even though he was a lap down at one point.", "source_id": "1998 Toyota Grand Prix of Long Beach" }, { "id": "doc-8", "text": "Alex Zanardi\nAlessandro \"Alex\" Zanardi (] ; born 23 October 1966) is an Italian professional racing driver and paracyclist.", "source_id": "Alex Zanardi" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-8" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "1998 Toyota Grand Prix of Long Beach\nThe 1998 Toyota Grand Prix of Long Beach was the third round of the 1998 CART FedEx Champ Car World Series season, held on April 5, 1998, on the streets of Long Beach, California. Alex Zanardi won the race, even though he was a lap down at one point.", "source_id": "1998 Toyota Grand Prix of Long Beach" }, { "id": "doc-8", "text": "Alex Zanardi\nAlessandro \"Alex\" Zanardi (] ; born 23 October 1966) is an Italian professional racing driver and paracyclist.", "source_id": "Alex Zanardi" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a83a532554299334474606f", "question": "Where did Ian Harland study prior to studying at the oldest college at the University of Cambridge?", "target": "The Dragon School in Oxford", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Where did Ian Harland study prior to studying at the oldest college at the University of Cambridge?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-d6ee548d0b02f8c01796", "text": "Ian Harland studied at The Dragon School in Oxford and Haileybury prior to studying at the oldest college at the University of Cambridge .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-7", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.661622047424316, "probability": 0.22175882756710052, "verifications": [ { "source_id": "Peterhouse, Cambridge", "distribution": { "contradiction": 0.904731035232544, "support": 0.009750592522323132, "unknown": 0.08551842719316483 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" }, { "source_id": "Ian Harland", "distribution": { "contradiction": 0.020935237407684326, "support": 0.019472818821668625, "unknown": 0.9595919847488403 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-aeb127e324939a5a95b0", "text": "Ian Harland studied at The Dragon School in Oxford and Haileybury before studying at the oldest college at the University of Cambridge .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-7", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.778013229370117, "probability": 0.24913166463375092, "verifications": [ { "source_id": "Peterhouse, Cambridge", "distribution": { "contradiction": 0.9226866960525513, "support": 0.009396927431225777, "unknown": 0.0679164007306099 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" }, { "source_id": "Ian Harland", "distribution": { "contradiction": 0.01886509172618389, "support": 0.011316285468637943, "unknown": 0.9698185920715332 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-7e5b8815c494ef2ce3ce", "text": "Ian Harland studied at The Dragon School in Oxford and Haileybury .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-7", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.531227111816406, "probability": 0.529109537601471, "verifications": [ { "source_id": "Peterhouse, Cambridge", "distribution": { "contradiction": 0.9006561636924744, "support": 0.009685935452580452, "unknown": 0.08965790271759033 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" }, { "source_id": "Ian Harland", "distribution": { "contradiction": 0.07725904136896133, "support": 0.450405091047287, "unknown": 0.4723358452320099 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-7", "text": "Peterhouse, Cambridge\nPeterhouse is a constituent college of the University of Cambridge, England. It is the oldest college of the university, having been founded in 1284 by Hugo de Balsham, Bishop of Ely, and granted its charter by King Edward I. Today, Peterhouse has 226 undergraduates, 86 full-time graduate students and 45 fellows. The modern name of Peterhouse does not include the word \"college\".", "source_id": "Peterhouse, Cambridge" }, { "id": "doc-8", "text": "Ian Harland\nFrom a clerical family (Samuel Harland, general secretary of the Commonwealth and Continental Church Society, was his father), Harland was educated at The Dragon School in Oxford and Haileybury. He then went to university at Peterhouse, Cambridge, taking a law degree. After two years as a schoolmaster at Sunningdale School he studied for the priesthood at Wycliffe Hall, Oxford and began his ministry as a curate in Melton Mowbray in 1960.", "source_id": "Ian Harland" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-7", "doc-8" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-7", "text": "Peterhouse, Cambridge\nPeterhouse is a constituent college of the University of Cambridge, England. It is the oldest college of the university, having been founded in 1284 by Hugo de Balsham, Bishop of Ely, and granted its charter by King Edward I. Today, Peterhouse has 226 undergraduates, 86 full-time graduate students and 45 fellows. The modern name of Peterhouse does not include the word \"college\".", "source_id": "Peterhouse, Cambridge" }, { "id": "doc-8", "text": "Ian Harland\nFrom a clerical family (Samuel Harland, general secretary of the Commonwealth and Continental Church Society, was his father), Harland was educated at The Dragon School in Oxford and Haileybury. He then went to university at Peterhouse, Cambridge, taking a law degree. After two years as a schoolmaster at Sunningdale School he studied for the priesthood at Wycliffe Hall, Oxford and began his ministry as a curate in Melton Mowbray in 1960.", "source_id": "Ian Harland" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a90692f5542990a98493625", "question": "In what 1997 American mystery crime film did Johnny Galecki start in?", "target": "Suicide Kings", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "In what 1997 American mystery crime film did Johnny Galecki start in?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-c66ba935236800f0daba", "text": "Johnny Galecki started in The Hostage .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.959005355834961, "probability": 0.36313962936401367, "verifications": [ { "source_id": "Johnny Galecki", "distribution": { "contradiction": 0.09628074616193771, "support": 0.01563471555709839, "unknown": 0.8880845904350281 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Suicide Kings", "distribution": { "contradiction": 0.025215497240424156, "support": 0.010233335196971893, "unknown": 0.9645511507987976 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-ed25c86eeaf8ead7e696", "text": "Johnny Galecki started in the 1997 American mystery crime film The Hostage .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.91036319732666, "probability": 0.34589841961860657, "verifications": [ { "source_id": "Johnny Galecki", "distribution": { "contradiction": 0.0435635931789875, "support": 0.011176716536283493, "unknown": 0.9452595710754395 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Suicide Kings", "distribution": { "contradiction": 0.036331821233034134, "support": 0.015369752421975136, "unknown": 0.9482983946800232 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-343ac0fdb3bf4e2f5ae0", "text": "Johnny Galecki started in the 1997 American mystery crime film .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.737410545349121, "probability": 0.29096198081970215, "verifications": [ { "source_id": "Johnny Galecki", "distribution": { "contradiction": 0.030312582850456238, "support": 0.010501077398657799, "unknown": 0.959186315536499 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Suicide Kings", "distribution": { "contradiction": 0.03155641257762909, "support": 0.01243013795465231, "unknown": 0.9560134410858154 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Johnny Galecki\nJohn Mark Galecki (born April 30, 1975) is an American actor. He is known for playing David Healy in the ABC sitcom \"Roseanne\" from 1992 to 1997 and Dr. Leonard Hofstadter in the CBS sitcom \"The Big Bang Theory\" since 2007. Galecki also appeared in the films \"National Lampoon's Christmas Vacation\" (1989), \"Prancer\" (1989), \"Suicide Kings\" (1997), \"I Know What You Did Last Summer\" (1997), \"Bookies\" (2003), and \"In Time\" (2011).", "source_id": "Johnny Galecki" }, { "id": "doc-3", "text": "Suicide Kings\nSuicide Kings is a 1997 American mystery crime film based on Don Stanford's short story \"The Hostage\" and directed by Peter O'Fallon. It stars Christopher Walken, Denis Leary, Sean Patrick Flanery, Johnny Galecki, Jay Mohr, Jeremy Sisto and Henry Thomas. The film follows the group of criminals who kidnap a respected Mafia figure. It has a 34% approval rating on Rotten Tomatoes and grossed $1.7 million in the US.", "source_id": "Suicide Kings" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-3" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Johnny Galecki\nJohn Mark Galecki (born April 30, 1975) is an American actor. He is known for playing David Healy in the ABC sitcom \"Roseanne\" from 1992 to 1997 and Dr. Leonard Hofstadter in the CBS sitcom \"The Big Bang Theory\" since 2007. Galecki also appeared in the films \"National Lampoon's Christmas Vacation\" (1989), \"Prancer\" (1989), \"Suicide Kings\" (1997), \"I Know What You Did Last Summer\" (1997), \"Bookies\" (2003), and \"In Time\" (2011).", "source_id": "Johnny Galecki" }, { "id": "doc-3", "text": "Suicide Kings\nSuicide Kings is a 1997 American mystery crime film based on Don Stanford's short story \"The Hostage\" and directed by Peter O'Fallon. It stars Christopher Walken, Denis Leary, Sean Patrick Flanery, Johnny Galecki, Jay Mohr, Jeremy Sisto and Henry Thomas. The film follows the group of criminals who kidnap a respected Mafia figure. It has a 34% approval rating on Rotten Tomatoes and grossed $1.7 million in the US.", "source_id": "Suicide Kings" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a7555215542996c70cfaee1", "question": "Which came to market first, \"Hey Pa! There's a Goat on the Roof\" or \"Poleconomy\"", "target": "Hey Pa! There's a Goat on the Roof", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which came to market first, \"Hey Pa! There's a Goat on the Roof\" or \"Poleconomy\"", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-83ed9e444ca446a81bf1", "text": "Hey Pa! There's a Goat on the Roof came to market first .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 3.9375033378601074, "probability": 0.36114299297332764, "verifications": [ { "source_id": "Hey Pa! There's a Goat on the Roof", "distribution": { "contradiction": 0.15151548385620117, "support": 0.007934889756143093, "unknown": 0.840549647808075 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Poleconomy", "distribution": { "contradiction": 0.30143240094184875, "support": 0.009525144472718239, "unknown": 0.6890424489974976 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-2" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-9367a76e28ae57d783a6", "text": "Hey Pa! There's a Goat on the Roof came to market first , Hey Pa! There's a Goat on the Roof '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 2.676607370376587, "probability": 0.10234791785478592, "verifications": [ { "source_id": "Hey Pa! There's a Goat on the Roof", "distribution": { "contradiction": 0.0799437016248703, "support": 0.017558841034770012, "unknown": 0.9024975299835205 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Poleconomy", "distribution": { "contradiction": 0.0246999803930521, "support": 0.007976620458066463, "unknown": 0.967323362827301 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-2" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-6996d0933d1967ea1e49", "text": "Hey Pa! There's a Goat on the Roof was a children 's board game issued by Parker Brothers in 1966 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.33331298828125, "probability": 0.536509096622467, "verifications": [ { "source_id": "Hey Pa! There's a Goat on the Roof", "distribution": { "contradiction": 0.17290528118610382, "support": 0.4308496415615082, "unknown": 0.3962450623512268 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Poleconomy", "distribution": { "contradiction": 0.03899545967578888, "support": 0.008101169019937515, "unknown": 0.9529034495353699 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-2" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" } ], "evidence": [ { "id": "doc-1", "text": "Hey Pa! There's a Goat on the Roof\nHey Pa! There's a Goat on the Roof was a children's board game issued by Parker Brothers in 1966.", "source_id": "Hey Pa! There's a Goat on the Roof" }, { "id": "doc-2", "text": "Poleconomy\nPoleconomy is a board game invented by New Zealander Bruce E. Hatherley and first published in 1980. It is similar to \"Monopoly\" but the board is divided into companies rather than properties. Players compete to acquire properties and investments through stylized economic and political activity. The playing time is set by agreement from all players, e.g. 2 hours, and the winner is determined by the monetary sum of all player cash, companies, insurance or other assets. The game can also end when there are no more companies, advertisements in the bank or bonds in the treasury. Another objective of the game is \"to teach players some of the fundamentals of economics, and the ways in which the economy and the government interact.\" Different editions were published for several different countries. The name \"Poleconomy\" is a portmanteau of \"politics\" and \"economy\".", "source_id": "Poleconomy" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-1", "doc-2" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-1", "text": "Hey Pa! There's a Goat on the Roof\nHey Pa! There's a Goat on the Roof was a children's board game issued by Parker Brothers in 1966.", "source_id": "Hey Pa! There's a Goat on the Roof" }, { "id": "doc-2", "text": "Poleconomy\nPoleconomy is a board game invented by New Zealander Bruce E. Hatherley and first published in 1980. It is similar to \"Monopoly\" but the board is divided into companies rather than properties. Players compete to acquire properties and investments through stylized economic and political activity. The playing time is set by agreement from all players, e.g. 2 hours, and the winner is determined by the monetary sum of all player cash, companies, insurance or other assets. The game can also end when there are no more companies, advertisements in the bank or bonds in the treasury. Another objective of the game is \"to teach players some of the fundamentals of economics, and the ways in which the economy and the government interact.\" Different editions were published for several different countries. The name \"Poleconomy\" is a portmanteau of \"politics\" and \"economy\".", "source_id": "Poleconomy" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a90580655429933b8a204ef", "question": "What record did the Super Bowl XXXIV champions finish the regular-season with in 1999?", "target": "a record of 13\u20133", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What record did the Super Bowl XXXIV champions finish the regular-season with in 1999?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-76486e1100d274449317", "text": "The Super Bowl XXXIV champions finished the regular-season with 13\u20133, and the NFC West Championship .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-6" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.945463180541992, "probability": 0.3375437557697296, "verifications": [ { "source_id": "1999 St. Louis Rams season", "distribution": { "contradiction": 0.016670437529683113, "support": 0.013860741630196571, "unknown": 0.9694687724113464 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "2000 Tennessee Titans season", "distribution": { "contradiction": 0.7189968824386597, "support": 0.015149225480854511, "unknown": 0.2658538520336151 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-851313ce15747424af9c", "text": "The Super Bowl XXXIV champions finished the regular-season with 13\u20133 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-6" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.021794319152832, "probability": 0.3643176853656769, "verifications": [ { "source_id": "1999 St. Louis Rams season", "distribution": { "contradiction": 0.019746337085962296, "support": 0.013004982843995094, "unknown": 0.9672486186027527 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "2000 Tennessee Titans season", "distribution": { "contradiction": 0.4679832458496094, "support": 0.02157123014330864, "unknown": 0.5104455351829529 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-71b680ee3fda55ba0815", "text": "The Super Bowl XXXIV champions finished the regular-season with 13\u20133 and the NFC West Championship .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-6" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.82132625579834, "probability": 0.2981385290622711, "verifications": [ { "source_id": "1999 St. Louis Rams season", "distribution": { "contradiction": 0.015114299952983856, "support": 0.014739803969860077, "unknown": 0.9701458811759949 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "2000 Tennessee Titans season", "distribution": { "contradiction": 0.8475109934806824, "support": 0.012417738325893879, "unknown": 0.14007125794887543 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "1999 St. Louis Rams season\nThe 1999 St. Louis Rams season was the team's 62nd year with the National Football League and the fifth season in St. Louis, Missouri. The Rams finished the regular-season with a record of 13\u20133, and the NFC West Championship.", "source_id": "1999 St. Louis Rams season" }, { "id": "doc-6", "text": "2000 Tennessee Titans season\nThe 2000 Tennessee Titans season was the franchise's 41st season and their 31st in the National Football League. It was the team's second being known as the \"Titans.\" The team entered the season as the defending AFC Champions, having narrowly lost Super Bowl XXXIV to the St. Louis Rams.", "source_id": "2000 Tennessee Titans season" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-6" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "1999 St. Louis Rams season\nThe 1999 St. Louis Rams season was the team's 62nd year with the National Football League and the fifth season in St. Louis, Missouri. The Rams finished the regular-season with a record of 13\u20133, and the NFC West Championship.", "source_id": "1999 St. Louis Rams season" }, { "id": "doc-6", "text": "2000 Tennessee Titans season\nThe 2000 Tennessee Titans season was the franchise's 41st season and their 31st in the National Football League. It was the team's second being known as the \"Titans.\" The team entered the season as the defending AFC Champions, having narrowly lost Super Bowl XXXIV to the St. Louis Rams.", "source_id": "2000 Tennessee Titans season" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a88e605554299206df2b39c", "question": "What is the population of the town at the intersection of Interstate Highways 77 and the longest Interstate highway within the borders of Virginia?", "target": "8,211 at the 2010 census", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What is the population of the town at the intersection of Interstate Highways 77 and the longest Interstate highway within the borders of Virginia?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-844f7107ae07fb8d753a", "text": "The population of the town at the intersection of Interstate Highways 77 and the longest Interstate highway within the borders of Virginia is 8,211 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.634575843811035, "probability": 0.2960655093193054, "verifications": [ { "source_id": "Wytheville, Virginia", "distribution": { "contradiction": 0.030589260160923004, "support": 0.00828075036406517, "unknown": 0.9611299633979797 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Interstate 81 in Virginia", "distribution": { "contradiction": 0.061826758086681366, "support": 0.008416498079895973, "unknown": 0.9297567009925842 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-87fed04a43b199487388", "text": "The population of the town at the intersection of Interstate Highways 77 and the longest Interstate highway within the borders of Virginia was 8,211 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.649720191955566, "probability": 0.3005833327770233, "verifications": [ { "source_id": "Wytheville, Virginia", "distribution": { "contradiction": 0.031719010323286057, "support": 0.00795915350317955, "unknown": 0.960321843624115 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Interstate 81 in Virginia", "distribution": { "contradiction": 0.06432180106639862, "support": 0.007784183137118816, "unknown": 0.9278939962387085 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-460fcdbe1e92df99a825", "text": "The population of the town at the intersection of Interstate Highways 77 and the longest Interstate highway within the borders of Virginia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.943802833557129, "probability": 0.40335121750831604, "verifications": [ { "source_id": "Wytheville, Virginia", "distribution": { "contradiction": 0.038669705390930176, "support": 0.007768440991640091, "unknown": 0.9535619020462036 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Interstate 81 in Virginia", "distribution": { "contradiction": 0.14872783422470093, "support": 0.013277231715619564, "unknown": 0.8379949927330017 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Wytheville, Virginia\nWytheville ( ) is a town in, and the county seat of, Wythe County, Virginia, United States. It is named after George Wythe, a signer of the United States Declaration of Independence, and mentor to Thomas Jefferson. Wytheville's population was 8,211 at the 2010 census. Located at the intersection of Interstate Highways 77 and 81, the town has long been a crossroads for travelers. During the American Civil War, Wytheville had a strategic importance, and was attacked in 1863 (Toland's Raid) and 1865 (Stoneman's 1865 Raid). The town is also the birthplace of Edith Bolling Wilson, wife of President Woodrow Wilson.", "source_id": "Wytheville, Virginia" }, { "id": "doc-7", "text": "Interstate 81 in Virginia\nInterstate 81 (I-81) is an 855 mi highway. In the U.S. state of Virginia, I-81 runs for 324.92 mi , making the portion in Virginia longer than any other state's portion. It is also the longest Interstate highway within the borders of Virginia. It stretches from the Tennessee state line near Bristol to the West Virginia state line near Winchester.", "source_id": "Interstate 81 in Virginia" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-7" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Wytheville, Virginia\nWytheville ( ) is a town in, and the county seat of, Wythe County, Virginia, United States. It is named after George Wythe, a signer of the United States Declaration of Independence, and mentor to Thomas Jefferson. Wytheville's population was 8,211 at the 2010 census. Located at the intersection of Interstate Highways 77 and 81, the town has long been a crossroads for travelers. During the American Civil War, Wytheville had a strategic importance, and was attacked in 1863 (Toland's Raid) and 1865 (Stoneman's 1865 Raid). The town is also the birthplace of Edith Bolling Wilson, wife of President Woodrow Wilson.", "source_id": "Wytheville, Virginia" }, { "id": "doc-7", "text": "Interstate 81 in Virginia\nInterstate 81 (I-81) is an 855 mi highway. In the U.S. state of Virginia, I-81 runs for 324.92 mi , making the portion in Virginia longer than any other state's portion. It is also the longest Interstate highway within the borders of Virginia. It stretches from the Tennessee state line near Bristol to the West Virginia state line near Winchester.", "source_id": "Interstate 81 in Virginia" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5add67915542992200553af8", "question": "What was the nickname of the hitman hired by an Italian American Criminal Organization?", "target": "The Crowned Prince of the Philadelphia Mob", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What was the nickname of the hitman hired by an Italian American Criminal Organization?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-9a2974ff15b0f539a753", "text": "The nickname of the hitman hired by an Italian American Criminal Organization was The Crowned Prince of the Philadelphia Mob .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.748218536376953, "probability": 0.30799779295921326, "verifications": [ { "source_id": "Salvatore Testa", "distribution": { "contradiction": 0.025270670652389526, "support": 0.13108612596988678, "unknown": 0.8436432480812073 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Philadelphia crime family", "distribution": { "contradiction": 0.021512331441044807, "support": 0.012054924853146076, "unknown": 0.9664328098297119 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-4f331fd0122bc094cd48", "text": "The Crowned Prince of the Philadelphia Mob was the nickname of the hitman hired by an Italian American Criminal Organization .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.855047225952148, "probability": 0.3427225947380066, "verifications": [ { "source_id": "Salvatore Testa", "distribution": { "contradiction": 0.020722830668091774, "support": 0.025531742721796036, "unknown": 0.953745424747467 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Philadelphia crime family", "distribution": { "contradiction": 0.01849929615855217, "support": 0.01303701102733612, "unknown": 0.9684635996818542 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-b89d581a9c17ad00ec94", "text": "The nickname of the hitman hired by an Italian American Criminal Organization was the Philadelphia Mafia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-3" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.873998641967773, "probability": 0.34927961230278015, "verifications": [ { "source_id": "Salvatore Testa", "distribution": { "contradiction": 0.021863048896193504, "support": 0.09989193081855774, "unknown": 0.8782449960708618 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Philadelphia crime family", "distribution": { "contradiction": 0.031060855835676193, "support": 0.03612345829606056, "unknown": 0.932815670967102 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "Salvatore Testa\nSalvatore \"Salvie\" Testa (March 31, 1956 - September 14, 1984), nicknamed The Crowned Prince of the Philadelphia Mob, was a Philadelphia gangster who served as a hitman for the Philadelphia crime family during a period of internal gang conflict. The son of former Philadelphia boss Philip, Testa was a rising star in the mob until he was killed on orders from Nicky Scarfo Sr.", "source_id": "Salvatore Testa" }, { "id": "doc-3", "text": "Philadelphia crime family\nThe Philadelphia crime family, (pronounced ] ) also known as the Philadelphia Mafia, the Philly Mob/Mafia, the Bruno-Scarfo family, the South Philly Mob/Mafia, or the Philadelphia-South Jersey Mob/Mafia is an Italian American criminal organization based in Philadelphia, Pennsylvania. Formed and based in South Philadelphia, the family primarily operates in various areas and neighborhoods throughout the Philadelphia metropolitan area and Delaware Valley, Southeastern Pennsylvania, and New Jersey, especially South Jersey. The Philadelphia family is notorious for its violence, due in particular to its succession of violent bosses and multiple mob wars.", "source_id": "Philadelphia crime family" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-3" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Salvatore Testa\nSalvatore \"Salvie\" Testa (March 31, 1956 - September 14, 1984), nicknamed The Crowned Prince of the Philadelphia Mob, was a Philadelphia gangster who served as a hitman for the Philadelphia crime family during a period of internal gang conflict. The son of former Philadelphia boss Philip, Testa was a rising star in the mob until he was killed on orders from Nicky Scarfo Sr.", "source_id": "Salvatore Testa" }, { "id": "doc-3", "text": "Philadelphia crime family\nThe Philadelphia crime family, (pronounced ] ) also known as the Philadelphia Mafia, the Philly Mob/Mafia, the Bruno-Scarfo family, the South Philly Mob/Mafia, or the Philadelphia-South Jersey Mob/Mafia is an Italian American criminal organization based in Philadelphia, Pennsylvania. Formed and based in South Philadelphia, the family primarily operates in various areas and neighborhoods throughout the Philadelphia metropolitan area and Delaware Valley, Southeastern Pennsylvania, and New Jersey, especially South Jersey. The Philadelphia family is notorious for its violence, due in particular to its succession of violent bosses and multiple mob wars.", "source_id": "Philadelphia crime family" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a8463545542990548d0b2a3", "question": "What was the position of Jean Baptiste Androuet du Cerceau under the man who was King of the Polish-Lithuanian Commonwealth from 1573 to 1575?", "target": "supervisor of the royal works", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What was the position of Jean Baptiste Androuet du Cerceau under the man who was King of the Polish-Lithuanian Commonwealth from 1573 to 1575?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-520860bf23937e510afe", "text": "Jean Baptiste Androuet du Cerceau was the position of Jean Baptiste Androuet du Cerceau under the man who was King of the Polish-Lithuanian Commonwealth from 1573 to 1575 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.28257417678833, "probability": 0.1547876000404358, "verifications": [ { "source_id": "Jean Baptiste Androuet du Cerceau", "distribution": { "contradiction": 0.12634646892547607, "support": 0.021425988525152206, "unknown": 0.8522275686264038 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-5" }, { "source_id": "Henry III of France", "distribution": { "contradiction": 0.14701123535633087, "support": 0.7402291297912598, "unknown": 0.11275970190763474 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-3836b95f397468c41ece", "text": "The position of Jean Baptiste Androuet du Cerceau was under the man who was King of the Polish-Lithuanian Commonwealth from 1573 to 1575 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.425115585327148, "probability": 0.48521655797958374, "verifications": [ { "source_id": "Jean Baptiste Androuet du Cerceau", "distribution": { "contradiction": 0.9709450602531433, "support": 0.004510772880166769, "unknown": 0.0245442446321249 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-5" }, { "source_id": "Henry III of France", "distribution": { "contradiction": 0.1738957315683365, "support": 0.7350271940231323, "unknown": 0.09107709676027298 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-469cb9777664c92ee59a", "text": "The position of Jean Baptiste Androuet du Cerceau under the man who was King of the Polish-Lithuanian Commonwealth from 1573 to 1575 was the position of Henri III and Henri IV .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-8" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.126612663269043, "probability": 0.3599958121776581, "verifications": [ { "source_id": "Jean Baptiste Androuet du Cerceau", "distribution": { "contradiction": 0.04646584391593933, "support": 0.015645047649741173, "unknown": 0.9378891587257385 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-5" }, { "source_id": "Henry III of France", "distribution": { "contradiction": 0.593043863773346, "support": 0.1514010727405548, "unknown": 0.25555509328842163 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" } ], "evidence": [ { "id": "doc-5", "text": "Jean Baptiste Androuet du Cerceau\nJean Baptiste Androuet du Cerceau (1544/47\u20131590) was a French architect who designed the Pont Neuf (1579), spanning the Seine, Paris, and became supervisor of the royal works under Henri III and Henri IV, including the Louvre. Several \"h\u00f4tels particuliers\" are ascribed to him. The H\u00f4tel d'Angoul\u00eame, the H\u00f4tel de Lamoignon (1584), which houses the Historical Library of the City of Paris, and the H\u00f4tel de Mayenne (rue St-Antoine in the Marais). The H\u00f4tel de Mayenne, with rhythmically varied dormer windows set in a high slate roof, has the pediments of its piano nobile windows superposed on the frieze above.", "source_id": "Jean Baptiste Androuet du Cerceau" }, { "id": "doc-8", "text": "Henry III of France\nHenry III (19 September 1551 \u2013 2 August 1589; born \"Alexandre \u00c9douard de France\", Polish: \"Henryk Walezy\" , Lithuanian: \"Henrikas Valua\" ) was King of the Polish-Lithuanian Commonwealth from 1573 to 1575 and King of France from 1574 until his death. He was the last French monarch of the House of Valois.", "source_id": "Henry III of France" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-5", "doc-8" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-5", "text": "Jean Baptiste Androuet du Cerceau\nJean Baptiste Androuet du Cerceau (1544/47\u20131590) was a French architect who designed the Pont Neuf (1579), spanning the Seine, Paris, and became supervisor of the royal works under Henri III and Henri IV, including the Louvre. Several \"h\u00f4tels particuliers\" are ascribed to him. The H\u00f4tel d'Angoul\u00eame, the H\u00f4tel de Lamoignon (1584), which houses the Historical Library of the City of Paris, and the H\u00f4tel de Mayenne (rue St-Antoine in the Marais). The H\u00f4tel de Mayenne, with rhythmically varied dormer windows set in a high slate roof, has the pediments of its piano nobile windows superposed on the frieze above.", "source_id": "Jean Baptiste Androuet du Cerceau" }, { "id": "doc-8", "text": "Henry III of France\nHenry III (19 September 1551 \u2013 2 August 1589; born \"Alexandre \u00c9douard de France\", Polish: \"Henryk Walezy\" , Lithuanian: \"Henrikas Valua\" ) was King of the Polish-Lithuanian Commonwealth from 1573 to 1575 and King of France from 1574 until his death. He was the last French monarch of the House of Valois.", "source_id": "Henry III of France" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a74684655429929fddd8410", "question": "Which American music star appeared as a guest in a Will & Grace episode called \"Back Up, Dancer\"?", "target": "Janet Damita Jo Jackson", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which American music star appeared as a guest in a Will & Grace episode called \"Back Up, Dancer\"?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-498dbee0ea3af4b53047", "text": "Janet Jackson appeared as a guest in a Will & Grace episode called Back Up , Dancer '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.225672721862793, "probability": 0.47221407294273376, "verifications": [ { "source_id": "Janet Jackson", "distribution": { "contradiction": 0.02429276891052723, "support": 0.006046165246516466, "unknown": 0.9696609973907471 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Back Up, Dancer", "distribution": { "contradiction": 0.21656407415866852, "support": 0.43828535079956055, "unknown": 0.3451506197452545 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-ffc00bb01432ccfa975f", "text": "Janet Jackson appeared as a guest in a Will & Grace episode titled Back Up , Dancer '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.336931228637695, "probability": 0.5277859568595886, "verifications": [ { "source_id": "Janet Jackson", "distribution": { "contradiction": 0.0255202054977417, "support": 0.005850854329764843, "unknown": 0.968629002571106 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Back Up, Dancer", "distribution": { "contradiction": 0.19364139437675476, "support": 0.6021021604537964, "unknown": 0.20425647497177124 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-4", "text": "Janet Jackson\nJanet Damita Jo Jackson (born May 16, 1966) is an American singer, songwriter, dancer and actress. Known for sonically innovative, socially conscious, and sexually provocative records, elaborate stage shows, and high-profile television and film roles, she has been a prominent figure in popular culture for over 30 years.", "source_id": "Janet Jackson" }, { "id": "doc-9", "text": "Back Up, Dancer\n\"Back Up, Dancer\" is the second episode of the seventh season of the American television series \"Will & Grace\". It was written by Tracy Poust and Jon Kinnally and directed by series producer James Burrows. The episode originally aired on the National Broadcasting Company (NBC) in the United States on September 23, 2004. Guest stars in \"Back Up, Dancer\" include Will Arnett, Bobby Cannavale, and Janet Jackson.", "source_id": "Back Up, Dancer" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-4", "doc-9" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-4", "text": "Janet Jackson\nJanet Damita Jo Jackson (born May 16, 1966) is an American singer, songwriter, dancer and actress. Known for sonically innovative, socially conscious, and sexually provocative records, elaborate stage shows, and high-profile television and film roles, she has been a prominent figure in popular culture for over 30 years.", "source_id": "Janet Jackson" }, { "id": "doc-9", "text": "Back Up, Dancer\n\"Back Up, Dancer\" is the second episode of the seventh season of the American television series \"Will & Grace\". It was written by Tracy Poust and Jon Kinnally and directed by series producer James Burrows. The episode originally aired on the National Broadcasting Company (NBC) in the United States on September 23, 2004. Guest stars in \"Back Up, Dancer\" include Will Arnett, Bobby Cannavale, and Janet Jackson.", "source_id": "Back Up, Dancer" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a835c9f5542992ef85e228a", "question": "What army did the namesake of the ship launched as the M\u00fcnchen in 1930 fight in during the American Revolutionary War?", "target": "Continental Army", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What army did the namesake of the ship launched as the M\u00fcnchen in 1930 fight in during the American Revolutionary War?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-08840dba879429f88d03", "text": "The namesake of the ship launched as the M\u00fcnchen in 1930 fighting in the Kriegsmarine during the American Revolutionary War .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.000082969665527, "probability": 0.31784558296203613, "verifications": [ { "source_id": "Friedrich Wilhelm von Steuben", "distribution": { "contradiction": 0.9680171012878418, "support": 0.0033403115812689066, "unknown": 0.02864254079759121 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "SS General von Steuben", "distribution": { "contradiction": 0.033412862569093704, "support": 0.904312014579773, "unknown": 0.06227511912584305 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-c64cf5bbf94aac9dd07e", "text": "The namesake of the ship launched as the M\u00fcnchen in 1930 fought in the Kriegsmarine of Nazi Germany .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.073198318481445, "probability": 0.34195563197135925, "verifications": [ { "source_id": "Friedrich Wilhelm von Steuben", "distribution": { "contradiction": 0.9772113561630249, "support": 0.0025687094312161207, "unknown": 0.02021997980773449 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "SS General von Steuben", "distribution": { "contradiction": 0.03138459473848343, "support": 0.9176158308982849, "unknown": 0.05099960416555405 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-ad4c73d5e47e32013770", "text": "The namesake of the ship launched as the M\u00fcnchen in 1930 fight in the Kriegsmarine of Nazi Germany .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.068047523498535, "probability": 0.340198814868927, "verifications": [ { "source_id": "Friedrich Wilhelm von Steuben", "distribution": { "contradiction": 0.9649524092674255, "support": 0.0030103155877441168, "unknown": 0.03203726187348366 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "SS General von Steuben", "distribution": { "contradiction": 0.028050372377038002, "support": 0.9230844974517822, "unknown": 0.04886516556143761 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "Friedrich Wilhelm von Steuben\nFriedrich Wilhelm August Heinrich Ferdinand Steuben (born Friedrich Wilhelm Ludolf Gerhard Augustin von Steuben; September 17, 1730 \u2013 November 28, 1794), also referred to as the Baron von Steuben, was a Prussian and later an American military officer. He served as inspector general and major general of the Continental Army during the American Revolutionary War. He is credited with being one of the fathers of the Continental Army in teaching them the essentials of military drills, tactics, and disciplines. He wrote \"Regulations for the Order and Discipline of the Troops of the United States\", the book that served as standard United States drill manual until the American Civil War. He served as General George Washington's chief of staff in the final years of the war.", "source_id": "Friedrich Wilhelm von Steuben" }, { "id": "doc-9", "text": "SS General von Steuben\nSS \"General von Steuben was a German passenger liner and later an armed transport in the Kriegsmarine of Nazi Germany that was sunk during World War II. She was launched as M\u00fcnchen (sometimes spelled Muenchen), renamed in 1930 as General von Steuben (after the famous German officer of the American Revolutionary War), and renamed again in 1938 as Steuben\".", "source_id": "SS General von Steuben" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-9" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Friedrich Wilhelm von Steuben\nFriedrich Wilhelm August Heinrich Ferdinand Steuben (born Friedrich Wilhelm Ludolf Gerhard Augustin von Steuben; September 17, 1730 \u2013 November 28, 1794), also referred to as the Baron von Steuben, was a Prussian and later an American military officer. He served as inspector general and major general of the Continental Army during the American Revolutionary War. He is credited with being one of the fathers of the Continental Army in teaching them the essentials of military drills, tactics, and disciplines. He wrote \"Regulations for the Order and Discipline of the Troops of the United States\", the book that served as standard United States drill manual until the American Civil War. He served as General George Washington's chief of staff in the final years of the war.", "source_id": "Friedrich Wilhelm von Steuben" }, { "id": "doc-9", "text": "SS General von Steuben\nSS \"General von Steuben was a German passenger liner and later an armed transport in the Kriegsmarine of Nazi Germany that was sunk during World War II. She was launched as M\u00fcnchen (sometimes spelled Muenchen), renamed in 1930 as General von Steuben (after the famous German officer of the American Revolutionary War), and renamed again in 1938 as Steuben\".", "source_id": "SS General von Steuben" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ae7edee554299540e5a56ad", "question": "The Church of the Guanche People was founded in the city that is on the most populated island of what larger area?", "target": "Macaronesia", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "The Church of the Guanche People was founded in the city that is on the most populated island of what larger area?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-47f8a73aa495de6a03b1", "text": "The Church of the Guanche People was founded in the city that is on the most populated island of the seven Canary Islands .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-3", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.182321548461914, "probability": 0.3881720006465912, "verifications": [ { "source_id": "Tenerife", "distribution": { "contradiction": 0.3574800491333008, "support": 0.009178894571959972, "unknown": 0.6333410739898682 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" }, { "source_id": "Church of the Guanche People", "distribution": { "contradiction": 0.018673796206712723, "support": 0.008179545402526855, "unknown": 0.9731466770172119 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-e76a87cc6b4cc6035c11", "text": "The Church of the Guanche People was founded in the city that is on the most populated island of Macaronesia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-3", "doc-7" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.637324333190918, "probability": 0.6118280291557312, "verifications": [ { "source_id": "Tenerife", "distribution": { "contradiction": 0.03464386612176895, "support": 0.011779768392443657, "unknown": 0.9535763263702393 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-3" }, { "source_id": "Church of the Guanche People", "distribution": { "contradiction": 0.15401211380958557, "support": 0.007609628606587648, "unknown": 0.8383782505989075 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-3", "text": "Tenerife\nTenerife ( ; ] ) is the largest and most populated island of the seven Canary Islands. It is also the most populated island of Spain, with a land area of 2034.38 km2 and 898,680 inhabitants, 43 percent of the total population of the Canary Islands. Tenerife is the largest and most populous island of Macaronesia.", "source_id": "Tenerife" }, { "id": "doc-7", "text": "Church of the Guanche People\nThe Church of the Guanche People (Spanish: \"Iglesia del Pueblo Guanche\" ) is a religious organisation, founded in 2001 in the city of San Crist\u00f3bal de La Laguna (Tenerife, Canary Islands, Spain). It aims to perpetuate and spread the pagan religion of the ancient Guanche people.", "source_id": "Church of the Guanche People" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-3", "doc-7" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-3", "text": "Tenerife\nTenerife ( ; ] ) is the largest and most populated island of the seven Canary Islands. It is also the most populated island of Spain, with a land area of 2034.38 km2 and 898,680 inhabitants, 43 percent of the total population of the Canary Islands. Tenerife is the largest and most populous island of Macaronesia.", "source_id": "Tenerife" }, { "id": "doc-7", "text": "Church of the Guanche People\nThe Church of the Guanche People (Spanish: \"Iglesia del Pueblo Guanche\" ) is a religious organisation, founded in 2001 in the city of San Crist\u00f3bal de La Laguna (Tenerife, Canary Islands, Spain). It aims to perpetuate and spread the pagan religion of the ancient Guanche people.", "source_id": "Church of the Guanche People" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a8ee4315542990e94052ba7", "question": "What officially ended the first phase of the military conflict between the British Raj and the Emirate of Afghanistan from 1878 to 1880?", "target": "Treaty of Gandamak", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What officially ended the first phase of the military conflict between the British Raj and the Emirate of Afghanistan from 1878 to 1880?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-d59d8e5a85dec85eb2b9", "text": "The Treaty of Gandamak officially ended the first phase of the military conflict between the British Raj and the Emirate of Afghanistan from 1878 to 1880 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.868804931640625, "probability": 0.4975591003894806, "verifications": [ { "source_id": "Treaty of Gandamak", "distribution": { "contradiction": 0.05630002170801163, "support": 0.006578035186976194, "unknown": 0.9371219873428345 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Second Anglo-Afghan War", "distribution": { "contradiction": 0.12613356113433838, "support": 0.21684010326862335, "unknown": 0.6570263504981995 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" }, { "id": "hypothesis-f0c12a800bc4244a01a6", "text": "Treaty of Gandamak officially ended the first phase of the military conflict between the British Raj and the Emirate of Afghanistan from 1878 to 1880 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.878568649291992, "probability": 0.5024409294128418, "verifications": [ { "source_id": "Treaty of Gandamak", "distribution": { "contradiction": 0.052902787923812866, "support": 0.006346053909510374, "unknown": 0.9407511949539185 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1" }, { "source_id": "Second Anglo-Afghan War", "distribution": { "contradiction": 0.10524678230285645, "support": 0.18207021057605743, "unknown": 0.7126829624176025 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": true, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "truncated; abstention required" } ], "evidence": [ { "id": "doc-1", "text": "Treaty of Gandamak\nThe Treaty of Gandamak officially ended the first phase of the Second Anglo-Afghan War. Mohammad Yaqub Khan ceded various frontier areas to Britain while retaining full sovereignty over Afghanistan.", "source_id": "Treaty of Gandamak" }, { "id": "doc-4", "text": "Second Anglo-Afghan War\nThe Second Anglo\u2013Afghan War (Pashto: \u062f \u0627\u0641\u063a\u0627\u0646-\u0627\u0646\u06ab\u0631\u06d0\u0632 \u062f\u0648\u064a\u0645\u0647 \u062c\u06ab\u0693\u0647) was a military conflict fought between the British Raj and the Emirate of Afghanistan from 1878 to 1880, when the latter was ruled by Sher Ali Khan of the Barakzai dynasty, the son of former Emir Dost Mohammad Khan. This was the second time British India invaded Afghanistan. The war ended after the British emerged victorious against the Afghan rebels and the Afghans agreed to let the British attain all of their geopolitical objectives from the Treaty of Gandamak. Most of the British and Indian soldiers withdrew from Afghanistan. The Afghan tribes were permitted to maintain internal rule and local customs but they had to cede control of the area's foreign relations to the British, who, in turn, guaranteed the area's freedom from foreign military domination as a buffer between the British Raj and the Russian Empire.", "source_id": "Second Anglo-Afghan War" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-1", "doc-4" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-1", "text": "Treaty of Gandamak\nThe Treaty of Gandamak officially ended the first phase of the Second Anglo-Afghan War. Mohammad Yaqub Khan ceded various frontier areas to Britain while retaining full sovereignty over Afghanistan.", "source_id": "Treaty of Gandamak" }, { "id": "doc-4", "text": "Second Anglo-Afghan War\nThe Second Anglo\u2013Afghan War (Pashto: \u062f \u0627\u0641\u063a\u0627\u0646-\u0627\u0646\u06ab\u0631\u06d0\u0632 \u062f\u0648\u064a\u0645\u0647 \u062c\u06ab\u0693\u0647) was a military conflict fought between the British Raj and the Emirate of Afghanistan from 1878 to 1880, when the latter was ruled by Sher Ali Khan of the Barakzai dynasty, the son of former Emir Dost Mohammad Khan. This was the second time British India invaded Afghanistan. The war ended after the British emerged victorious against the Afghan rebels and the Afghans agreed to let the British attain all of their geopolitical objectives from the Treaty of Gandamak. Most of the British and Indian soldiers withdrew from Afghanistan. The Afghan tribes were permitted to maintain internal rule and local customs but they had to cede control of the area's foreign relations to the British, who, in turn, guaranteed the area's freedom from foreign military domination as a buffer between the British Raj and the Russian Empire.", "source_id": "Second Anglo-Afghan War" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a8cb288554299585d9e3726", "question": "The mass killing that took place at Oakland, California on April 2, 2012 was less deadly than the one that took place on October 1, 2015 in which state?", "target": "Oregon", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "The mass killing that took place at Oakland, California on April 2, 2012 was less deadly than the one that took place on October 1, 2015 in which state?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-970d4228f16349b0d01a", "text": "The mass killing that took place at Oakland , California on April 2, 2012 was less deadly than the one that took place on October 1, 2015 in Oregon .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 3.8414347171783447, "probability": 0.3199048936367035, "verifications": [ { "source_id": "Oikos University shooting", "distribution": { "contradiction": 0.14475315809249878, "support": 0.017235951498150826, "unknown": 0.8380109071731567 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Umpqua Community College shooting", "distribution": { "contradiction": 0.060531776398420334, "support": 0.007257749792188406, "unknown": 0.9322105050086975 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-ade31e3633f3b528b8e5", "text": "The mass killing that took place at Oakland , California on April 2 , 2012 was less deadly than the one that took place on October 1, 2015 in Oregon .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 3.8414347171783447, "probability": 0.3199048936367035, "verifications": [ { "source_id": "Oikos University shooting", "distribution": { "contradiction": 0.15181118249893188, "support": 0.017173172906041145, "unknown": 0.8310157060623169 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Umpqua Community College shooting", "distribution": { "contradiction": 0.061914727091789246, "support": 0.0073425960727036, "unknown": 0.9307426810264587 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-c0930e75369fd29c178a", "text": "The mass killing that took place at Oakland , California on April 2, 2012 was less deadly than the one that took place on October 1, 2015 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 3.960043430328369, "probability": 0.3601902723312378, "verifications": [ { "source_id": "Oikos University shooting", "distribution": { "contradiction": 0.23489221930503845, "support": 0.018192525953054428, "unknown": 0.7469152808189392 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "Umpqua Community College shooting", "distribution": { "contradiction": 0.0744226723909378, "support": 0.007238669786602259, "unknown": 0.9183385968208313 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-4", "text": "Oikos University shooting\nThe Oikos University shooting occurred on April 2, 2012, when a gunman shot at people inside Oikos University, a Korean Christian college in Oakland, California, United States. Within a few hours, the number of reported fatalities reached seven. 43-year-old One L. Goh, a former student at the school, was taken into custody and identified as the suspect in the shootings. Along with the California State University, Fullerton massacre, this was the fourth-deadliest university shooting in United States history, after the Virginia Tech massacre, the University of Texas Clock Tower shooting, and the Umpqua Community College shooting, and the eighth-deadliest U.S. school massacre overall. It is also considered the deadliest mass killing in the city's history.", "source_id": "Oikos University shooting" }, { "id": "doc-9", "text": "Umpqua Community College shooting\nThe Umpqua Community College shooting occurred on October 1, 2015, at the UCC campus near Roseburg, Oregon, United States. Chris Harper-Mercer, a 26-year-old enrolled at the school, fatally shot an assistant professor and eight students in a classroom. Seven to nine others were injured. Roseburg police detectives responding to the incident engaged Harper-Mercer in a brief shootout. After being wounded, he killed himself by shooting himself in the head. The mass shooting was the deadliest in Oregon's modern history.", "source_id": "Umpqua Community College shooting" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-4", "doc-9" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-4", "text": "Oikos University shooting\nThe Oikos University shooting occurred on April 2, 2012, when a gunman shot at people inside Oikos University, a Korean Christian college in Oakland, California, United States. Within a few hours, the number of reported fatalities reached seven. 43-year-old One L. Goh, a former student at the school, was taken into custody and identified as the suspect in the shootings. Along with the California State University, Fullerton massacre, this was the fourth-deadliest university shooting in United States history, after the Virginia Tech massacre, the University of Texas Clock Tower shooting, and the Umpqua Community College shooting, and the eighth-deadliest U.S. school massacre overall. It is also considered the deadliest mass killing in the city's history.", "source_id": "Oikos University shooting" }, { "id": "doc-9", "text": "Umpqua Community College shooting\nThe Umpqua Community College shooting occurred on October 1, 2015, at the UCC campus near Roseburg, Oregon, United States. Chris Harper-Mercer, a 26-year-old enrolled at the school, fatally shot an assistant professor and eight students in a classroom. Seven to nine others were injured. Roseburg police detectives responding to the incident engaged Harper-Mercer in a brief shootout. After being wounded, he killed himself by shooting himself in the head. The mass shooting was the deadliest in Oregon's modern history.", "source_id": "Umpqua Community College shooting" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5adfbca255429942ec259b2c", "question": "When did the rock band that sang \"All Join Hands\" rise to prominence? ", "target": "early 1970s", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "When did the rock band that sang \"All Join Hands\" rise to prominence? ", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-4e730573ea095a880adc", "text": "The rock band that sang All Join Hands '' rose to prominence in the early 1970s .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.1633148193359375, "probability": 0.3357376456260681, "verifications": [ { "source_id": "All Join Hands", "distribution": { "contradiction": 0.3967834711074829, "support": 0.012328086420893669, "unknown": 0.590888500213623 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" }, { "source_id": "Slade", "distribution": { "contradiction": 0.13708434998989105, "support": 0.02151566930115223, "unknown": 0.8414000868797302 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-a39aed2cf584abc2696e", "text": "The rock band that sang All Join Hands '' rose to prominence during the early 1970s .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.097614288330078, "probability": 0.3143885135650635, "verifications": [ { "source_id": "All Join Hands", "distribution": { "contradiction": 0.29438528418540955, "support": 0.012553245760500431, "unknown": 0.6930614113807678 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" }, { "source_id": "Slade", "distribution": { "contradiction": 0.1404750943183899, "support": 0.018613498657941818, "unknown": 0.8409114480018616 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-4a13c27cf6f7ad7b4116", "text": "The rock band that sang All Join Hands '' rose to prominence .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5", "doc-9" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.204557418823242, "probability": 0.3498738706111908, "verifications": [ { "source_id": "All Join Hands", "distribution": { "contradiction": 0.04140690714120865, "support": 0.018426958471536636, "unknown": 0.940166175365448 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" }, { "source_id": "Slade", "distribution": { "contradiction": 0.4952187240123749, "support": 0.02018982172012329, "unknown": 0.48459145426750183 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-9" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-5", "text": "All Join Hands\n\"All Join Hands\" is a song by the British rock band Slade, released in 1984 as the lead single from the band's twelfth studio album \"Rogues Gallery\". The song was written by lead vocalist Noddy Holder and bassist Jim Lea, and produced by John Punter. It reached No. 15 in the UK, remaining in the chart for ten weeks.", "source_id": "All Join Hands" }, { "id": "doc-9", "text": "Slade\nSlade are an English glam rock band from Wolverhampton. They rose to prominence during the early 1970s with 17 consecutive top 20 hits and six number ones on the UK Singles Chart. The \"British Hit Singles & Albums\" names them as the most successful British group of the 1970s based on sales of singles. They were the first act to have three singles enter the charts at number one; all six of the band's chart-toppers were penned by Noddy Holder and Jim Lea. As of 2006, total UK sales stand at 6,520,171, and their best-selling single, \"Merry Xmas Everybody\", has sold in excess of one million copies.", "source_id": "Slade" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-5", "doc-9" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-5", "text": "All Join Hands\n\"All Join Hands\" is a song by the British rock band Slade, released in 1984 as the lead single from the band's twelfth studio album \"Rogues Gallery\". The song was written by lead vocalist Noddy Holder and bassist Jim Lea, and produced by John Punter. It reached No. 15 in the UK, remaining in the chart for ten weeks.", "source_id": "All Join Hands" }, { "id": "doc-9", "text": "Slade\nSlade are an English glam rock band from Wolverhampton. They rose to prominence during the early 1970s with 17 consecutive top 20 hits and six number ones on the UK Singles Chart. The \"British Hit Singles & Albums\" names them as the most successful British group of the 1970s based on sales of singles. They were the first act to have three singles enter the charts at number one; all six of the band's chart-toppers were penned by Noddy Holder and Jim Lea. As of 2006, total UK sales stand at 6,520,171, and their best-selling single, \"Merry Xmas Everybody\", has sold in excess of one million copies.", "source_id": "Slade" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ae24abc5542994d89d5b3c3", "question": "Where is the company that owns thanKing Horse pub restaurant chain based ?", "target": "Bury St Edmunds,", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Where is the company that owns thanKing Horse pub restaurant chain based ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-c02333ef2d735c9e5428", "text": "The company that owns thanKing Horse pub restaurant chain is based in Bury St Edmunds , Suffolk .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.998575210571289, "probability": 0.2958565354347229, "verifications": [ { "source_id": "Hungry Horse", "distribution": { "contradiction": 0.443890243768692, "support": 0.004474181681871414, "unknown": 0.5516356229782104 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Greene King", "distribution": { "contradiction": 0.031680263578891754, "support": 0.07474913448095322, "unknown": 0.8935706615447998 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-ace4c980ba4746235b0f", "text": "The company that owns thanKing Horse pub restaurant chain is based in Bury St Edmunds , Suffolk , England .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.198752403259277, "probability": 0.3614240288734436, "verifications": [ { "source_id": "Hungry Horse", "distribution": { "contradiction": 0.3383963704109192, "support": 0.00513276644051075, "unknown": 0.6564708948135376 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Greene King", "distribution": { "contradiction": 0.045817095786333084, "support": 0.07577192038297653, "unknown": 0.8784109950065613 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-dffb656f80287c3f776b", "text": "The company that owns thanKing Horse pub restaurant chain based in Bury St Edmunds , Suffolk , England .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.145612716674805, "probability": 0.3427194058895111, "verifications": [ { "source_id": "Hungry Horse", "distribution": { "contradiction": 0.1469450443983078, "support": 0.006232664454728365, "unknown": 0.8468222618103027 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Greene King", "distribution": { "contradiction": 0.0724518746137619, "support": 0.105087049305439, "unknown": 0.8224610090255737 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "Hungry Horse\nHungry Horse is a chain of 225 pub-restaurants in England, Wales and Scotland, owned by Greene King Brewery. It was founded in 1995, and promotes itself as offering low cost meals for families and groups.", "source_id": "Hungry Horse" }, { "id": "doc-4", "text": "Greene King\nGreene King is the UK's largest pub retailer and brewer. It is based in Bury St Edmunds, Suffolk, England. The company owns pubs, restaurants and hotels. It is listed on the London Stock Exchange and is a constituent of the FTSE 250 Index.", "source_id": "Greene King" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-4" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Hungry Horse\nHungry Horse is a chain of 225 pub-restaurants in England, Wales and Scotland, owned by Greene King Brewery. It was founded in 1995, and promotes itself as offering low cost meals for families and groups.", "source_id": "Hungry Horse" }, { "id": "doc-4", "text": "Greene King\nGreene King is the UK's largest pub retailer and brewer. It is based in Bury St Edmunds, Suffolk, England. The company owns pubs, restaurants and hotels. It is listed on the London Stock Exchange and is a constituent of the FTSE 250 Index.", "source_id": "Greene King" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5ab802085542990e739ec7ca", "question": "What is the uppermost age range for the sort of fiction written by Alexander Gordon Smith?", "target": "early 20s", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What is the uppermost age range for the sort of fiction written by Alexander Gordon Smith?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-82e21392d4362611461a", "text": "The uppermost age range for the sort of fiction written by Alexander Gordon Smith is 12\u201318 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.727494239807129, "probability": 0.3695588707923889, "verifications": [ { "source_id": "Alexander Gordon Smith", "distribution": { "contradiction": 0.02183365263044834, "support": 0.06736060231924057, "unknown": 0.9108057022094727 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Young adult fiction", "distribution": { "contradiction": 0.020204029977321625, "support": 0.01138698123395443, "unknown": 0.9684090614318848 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-06b709a5a1d2dbf87e2b", "text": "The uppermost age range for the sort of fiction written by Alexander Gordon Smith is 12-18 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.568456649780273, "probability": 0.31522053480148315, "verifications": [ { "source_id": "Alexander Gordon Smith", "distribution": { "contradiction": 0.0322323776781559, "support": 0.08772686123847961, "unknown": 0.8800408244132996 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Young adult fiction", "distribution": { "contradiction": 0.020550798624753952, "support": 0.01258501410484314, "unknown": 0.9668641090393066 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-35180d9376e676fde2be", "text": "The uppermost age range for the sort of fiction written by Alexander Gordon Smith is 12 - 18 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0", "doc-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.568456649780273, "probability": 0.31522053480148315, "verifications": [ { "source_id": "Alexander Gordon Smith", "distribution": { "contradiction": 0.01744992285966873, "support": 0.09398932009935379, "unknown": 0.8885607719421387 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0" }, { "source_id": "Young adult fiction", "distribution": { "contradiction": 0.019477548077702522, "support": 0.014643505215644836, "unknown": 0.9658790230751038 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0", "text": "Alexander Gordon Smith\nAlexander Gordon Smith (27 February 1979, Norwich, England) is an author of children and Young Adult fiction.", "source_id": "Alexander Gordon Smith" }, { "id": "doc-2", "text": "Young adult fiction\nYoung adult fiction or young adult literature (YA) is fiction published for readers in their youth. The age range for young adult fiction is subjective. Some sources claim it ranges from ages 12\u201318, while authors and readers of \"young teen novels\" often define it as written for those aged 15 to the early 20s. The terms young adult novel, juvenile novel, teenage fiction, young adult book, etc., refer to the works in this category.", "source_id": "Young adult fiction" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-0", "doc-2" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0", "text": "Alexander Gordon Smith\nAlexander Gordon Smith (27 February 1979, Norwich, England) is an author of children and Young Adult fiction.", "source_id": "Alexander Gordon Smith" }, { "id": "doc-2", "text": "Young adult fiction\nYoung adult fiction or young adult literature (YA) is fiction published for readers in their youth. The age range for young adult fiction is subjective. Some sources claim it ranges from ages 12\u201318, while authors and readers of \"young teen novels\" often define it as written for those aged 15 to the early 20s. The terms young adult novel, juvenile novel, teenage fiction, young adult book, etc., refer to the works in this category.", "source_id": "Young adult fiction" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5adfa3da554299025d62a317", "question": "What is the birthday of the actress who was the Duchess in \"The Revengers Tragedy\"?", "target": "23 November 1946", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What is the birthday of the actress who was the Duchess in \"The Revengers Tragedy\"?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-0f28ff932ff029362954", "text": "The birthday of the actress who was the Duchess in The Revengers Tragedy '' is 23 November 1946 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.384853363037109, "probability": 0.2819131314754486, "verifications": [ { "source_id": "Revengers Tragedy", "distribution": { "contradiction": 0.8365234136581421, "support": 0.013609480112791061, "unknown": 0.14986717700958252 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Diana Quick", "distribution": { "contradiction": 0.013174738734960556, "support": 0.01781589351594448, "unknown": 0.9690093994140625 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-1d55985821f0c5a8d1ed", "text": "The birthday of the actress who was the Duchess in The Revengers Tragedy '' was 23 November 1946 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.339719772338867, "probability": 0.26947224140167236, "verifications": [ { "source_id": "Revengers Tragedy", "distribution": { "contradiction": 0.8042590618133545, "support": 0.013907553628087044, "unknown": 0.18183334171772003 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Diana Quick", "distribution": { "contradiction": 0.01245725154876709, "support": 0.017647603526711464, "unknown": 0.96989506483078 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-3d48f7a79c7758c27173", "text": "Diana Quick is the birthday of the actress who was the Duchess in The Revengers Tragedy '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2", "doc-4" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.849418640136719, "probability": 0.4486146569252014, "verifications": [ { "source_id": "Revengers Tragedy", "distribution": { "contradiction": 0.81223464012146, "support": 0.013635052368044853, "unknown": 0.1741304099559784 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2" }, { "source_id": "Diana Quick", "distribution": { "contradiction": 0.02610793150961399, "support": 0.010164105333387852, "unknown": 0.9637279510498047 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2", "text": "Revengers Tragedy\nRevengers Tragedy is a film adaptation of the 1606 play \"The Revenger's Tragedy\" (attributed to Thomas Middleton in the credits, following the current scholarly consensus). It was directed by Alex Cox and adapted for the screen by Cox's fellow Liverpudlian, Frank Cottrell Boyce. The film stars Christopher Eccleston as the revenge-obsessed Vindice, with Derek Jacobi as the evil Duke, Eddie Izzard as his lecherous son Lussurioso, Diana Quick as the Duchess, Andrew Schofield as Vindice's brother Carlo (a version of the play's Hippolito), Carla Henry as his virtuous sister Castiza, and Marc Warren and Justin Salinger as the Duchess's sons Supervacuo and Ambitioso.", "source_id": "Revengers Tragedy" }, { "id": "doc-4", "text": "Diana Quick\nDiana Marilyn Quick (born 23 November 1946) is an English actress.", "source_id": "Diana Quick" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-2", "doc-4" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2", "text": "Revengers Tragedy\nRevengers Tragedy is a film adaptation of the 1606 play \"The Revenger's Tragedy\" (attributed to Thomas Middleton in the credits, following the current scholarly consensus). It was directed by Alex Cox and adapted for the screen by Cox's fellow Liverpudlian, Frank Cottrell Boyce. The film stars Christopher Eccleston as the revenge-obsessed Vindice, with Derek Jacobi as the evil Duke, Eddie Izzard as his lecherous son Lussurioso, Diana Quick as the Duchess, Andrew Schofield as Vindice's brother Carlo (a version of the play's Hippolito), Carla Henry as his virtuous sister Castiza, and Marc Warren and Justin Salinger as the Duchess's sons Supervacuo and Ambitioso.", "source_id": "Revengers Tragedy" }, { "id": "doc-4", "text": "Diana Quick\nDiana Marilyn Quick (born 23 November 1946) is an English actress.", "source_id": "Diana Quick" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, { "id": "5a76e88555429972597f13f4", "question": "Which film starring Patrick Tatten is based on a book written by Steve Lopez?", "target": "The Soloist", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which film starring Patrick Tatten is based on a book written by Steve Lopez?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-10cedcc951896d614016", "text": "The Soloist starring Patrick Tatten is based on a book written by Steve Lopez .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.680432319641113, "probability": 0.3643563389778137, "verifications": [ { "source_id": "Patrick Tatten", "distribution": { "contradiction": 0.07069768756628036, "support": 0.006206480786204338, "unknown": 0.9230958819389343 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "The Soloist", "distribution": { "contradiction": 0.9158467650413513, "support": 0.01724245958030224, "unknown": 0.06691071391105652 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-b2ff2b370d592da711cb", "text": "The film starring Patrick Tatten is based on a book written by Steve Lopez .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.549880027770996, "probability": 0.3197629749774933, "verifications": [ { "source_id": "Patrick Tatten", "distribution": { "contradiction": 0.05091123655438423, "support": 0.005719228181988001, "unknown": 0.9433695673942566 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "The Soloist", "distribution": { "contradiction": 0.9475535154342651, "support": 0.0161969605833292, "unknown": 0.03624958544969559 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-0484334b25f097f87d17", "text": "The Soloist starred Patrick Tatten based on a book written by Steve Lopez .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-4", "doc-5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.537664413452148, "probability": 0.315880686044693, "verifications": [ { "source_id": "Patrick Tatten", "distribution": { "contradiction": 0.0502929650247097, "support": 0.005627445410937071, "unknown": 0.9440795183181763 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-4" }, { "source_id": "The Soloist", "distribution": { "contradiction": 0.9546684622764587, "support": 0.013449957594275475, "unknown": 0.03188156336545944 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-4", "text": "Patrick Tatten\nPatrick James Bolger Tatten (born January 21, 1981) is an American actor and producer. He is best known for roles in films such as: \"Dirty Deeds\", \"The Good German\", \"The Soloist\" , \"Lucky Bastard\" & the t.v. series Boystown on \"OutTV\".", "source_id": "Patrick Tatten" }, { "id": "doc-5", "text": "The Soloist\nThe Soloist is a 2009 British-American drama film directed by Joe Wright, and starring Jamie Foxx and Robert Downey, Jr.. It is based on the true story of Nathaniel Ayers, a musician who developed schizophrenia and became homeless. The screenplay by Susannah Grant is based on the book, \"The Soloist\" by Steve Lopez.", "source_id": "The Soloist" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "doc-4", "doc-5" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-4", "text": "Patrick Tatten\nPatrick James Bolger Tatten (born January 21, 1981) is an American actor and producer. He is best known for roles in films such as: \"Dirty Deeds\", \"The Good German\", \"The Soloist\" , \"Lucky Bastard\" & the t.v. series Boystown on \"OutTV\".", "source_id": "Patrick Tatten" }, { "id": "doc-5", "text": "The Soloist\nThe Soloist is a 2009 British-American drama film directed by Joe Wright, and starring Jamie Foxx and Robert Downey, Jr.. It is based on the true story of Nathaniel Ayers, a musician who developed schizophrenia and became homeless. The screenplay by Susannah Grant is based on the book, \"The Soloist\" by Steve Lopez.", "source_id": "The Soloist" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } } ] }, "controls": { "records": [ { "id": "5ae048a255429924de1b708e", "omission": { "id": "5ae048a255429924de1b708e", "question": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?", "target": "Catwoman", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-ef8771f1a1bd113a262b", "text": "Pitof directs the action-adventure tie-in video game based off of it in 2004 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.591056823730469, "probability": 0.3223026692867279, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-fa69b357702924e14eee", "text": "Pitof directed the action-adventure tie-in video game based off of it in 2004 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.625574111938477, "probability": 0.33362191915512085, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-0a886f65c84a561ff0ac", "text": "Pitof directs the action-adventure tie-in video game based on it in 2004 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.656426429748535, "probability": 0.34407535195350647, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5ae048a255429924de1b708e", "question": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?", "target": "Catwoman", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-a9b5417435eb1272bfb0", "text": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.700798988342285, "probability": 0.146567240357399, "verifications": [ { "source_id": "Catwoman (film)", "distribution": { "contradiction": 0.629417896270752, "support": 0.02211170084774494, "unknown": 0.3484703600406647 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Catwoman (video game)", "distribution": { "contradiction": 0.629417896270752, "support": 0.02211170084774494, "unknown": 0.3484703600406647 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-742cc9da437a06a19626", "text": "Pitof directs Catwoman .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.227077484130859, "probability": 0.674359142780304, "verifications": [ { "source_id": "Catwoman (film)", "distribution": { "contradiction": 0.9763675332069397, "support": 0.0057112001813948154, "unknown": 0.017921283841133118 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Catwoman (video game)", "distribution": { "contradiction": 0.9763675332069397, "support": 0.0057112001813948154, "unknown": 0.017921283841133118 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-47e2af92ff56b6cde4dd", "text": "Catwoman , which had an action-adventure tie-in video game based off of it in 2004 , was directed by Pitof .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.901111602783203, "probability": 0.17907360196113586, "verifications": [ { "source_id": "Catwoman (film)", "distribution": { "contradiction": 0.7424694299697876, "support": 0.013194307684898376, "unknown": 0.24433627724647522 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Catwoman (video game)", "distribution": { "contradiction": 0.7424694299697876, "support": 0.013194307684898376, "unknown": 0.24433627724647522 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Catwoman (film)" }, { "id": "doc-5@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Catwoman (video game)" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?" }, { "role": "assistant", "text": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 ." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?\"}, {\"role\": \"assistant\", \"text\": \"Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 .\"}]\nCurrent question: What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?" }, "conversation_context": [ { "role": "user", "text": "What movie did Pitof direct which had an action-adventure tie-in video game based off of it in 2004?" }, { "role": "assistant", "text": "Pitof directs Catwoman , which had an action-adventure tie-in video game based off of it in 2004 ." } ], "conversation_context_truncated": false, "response_proposal": { "text": "[Catwoman (video game)]", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Catwoman (film)" }, { "id": "doc-5@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Catwoman (video game)" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5a9068a95542990a98493621", "omission": { "id": "5a9068a95542990a98493621", "question": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?", "target": "7 miles", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-c862127d2b964a3774f0", "text": "On 16 January 1995 , First Knight was being filmed at a studio located how far from Windsor .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.420130729675293, "probability": 0.3287990093231201, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-77344090c17f5ff78672", "text": "On 16 January 1995 First Knight '' was being filmed at a studio located how far from Windsor .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.5525665283203125, "probability": 0.37535881996154785, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-0687ca7836a88c90003e", "text": "On 16 January 1995 , First Knight was being filmed at a studio located .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.314510345458984, "probability": 0.2958422005176544, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5a9068a95542990a98493621", "question": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?", "target": "7 miles", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-cc5c383f4b6f0ac39ee4", "text": "On 16 January 1995 , First Knight was being filmed at a studio located how far from Windsor .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-6@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.5701494216918945, "probability": 0.28629884123802185, "verifications": [ { "source_id": "Filming of James Bond in the 1990s", "distribution": { "contradiction": 0.8930366635322571, "support": 0.007859524339437485, "unknown": 0.09910384565591812 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Pinewood Studios", "distribution": { "contradiction": 0.8930366635322571, "support": 0.007859524339437485, "unknown": 0.09910384565591812 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-8f1c058762b75f03cebf", "text": "On 16 January 1995 First Knight '' was being filmed at a studio located how far from Windsor .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-6@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.702256202697754, "probability": 0.326732873916626, "verifications": [ { "source_id": "Filming of James Bond in the 1990s", "distribution": { "contradiction": 0.9221093058586121, "support": 0.008567828685045242, "unknown": 0.06932295858860016 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Pinewood Studios", "distribution": { "contradiction": 0.9221093058586121, "support": 0.008567828685045242, "unknown": 0.06932295858860016 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-54c3d872ae379ca1c7d5", "text": "The first knight was being filmed at a studio located how far from Windsor .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-6@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.871456146240234, "probability": 0.38696831464767456, "verifications": [ { "source_id": "Filming of James Bond in the 1990s", "distribution": { "contradiction": 0.7919839024543762, "support": 0.009286276064813137, "unknown": 0.19872984290122986 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Pinewood Studios", "distribution": { "contradiction": 0.7919839024543762, "support": 0.009286276064813137, "unknown": 0.19872984290122986 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-6@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Filming of James Bond in the 1990s" }, { "id": "doc-6@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Pinewood Studios" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"on 16 January 1995 \\\"First Knight\\\" was being filmed at a studio located how far from Windsor ?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?" }, "conversation_context": [ { "role": "user", "text": "on 16 January 1995 \"First Knight\" was being filmed at a studio located how far from Windsor ?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Filming of James Bond in the 1990s" }, { "id": "doc-6@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Pinewood Studios" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5adf9dc3554299025d62a2e6", "omission": { "id": "5adf9dc3554299025d62a2e6", "question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "target": "Laura Ann Osnes", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-e74891ab30a0a89bf9e0", "text": "The name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical Griese '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.681377410888672, "probability": 0.38779234886169434, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-5a666190b8d3f12cdcc9", "text": "The name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Griese .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.275866508483887, "probability": 0.258516401052475, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-cd7f5498d450bc7f32b6", "text": "The name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Griese '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.58933162689209, "probability": 0.3536912202835083, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5adf9dc3554299025d62a2e6", "question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "target": "Laura Ann Osnes", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-a0fe66f2a2281c8cefa9", "text": "The name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical Griese '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-8@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.777976989746094, "probability": 0.38375699520111084, "verifications": [ { "source_id": "Laura Osnes", "distribution": { "contradiction": 0.9160857200622559, "support": 0.004222359508275986, "unknown": 0.07969187945127487 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Grease: The New Broadway Cast Recording", "distribution": { "contradiction": 0.9160857200622559, "support": 0.004222359508275986, "unknown": 0.07969187945127487 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-6cb34c38ddfb27db0bba", "text": "The name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Grrease .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-8@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.412642478942871, "probability": 0.26631370186805725, "verifications": [ { "source_id": "Laura Osnes", "distribution": { "contradiction": 0.8733256459236145, "support": 0.004924276378005743, "unknown": 0.1217501237988472 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Grease: The New Broadway Cast Recording", "distribution": { "contradiction": 0.8733256459236145, "support": 0.004924276378005743, "unknown": 0.1217501237988472 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-715db09e94652a0a05a1", "text": "The name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical , Griese '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-8@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.685698509216309, "probability": 0.3499293029308319, "verifications": [ { "source_id": "Laura Osnes", "distribution": { "contradiction": 0.9138984680175781, "support": 0.0045785061083734035, "unknown": 0.08152303099632263 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Grease: The New Broadway Cast Recording", "distribution": { "contradiction": 0.9138984680175781, "support": 0.0045785061083734035, "unknown": 0.08152303099632263 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-8@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Laura Osnes" }, { "id": "doc-8@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Grease: The New Broadway Cast Recording" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \\\"Grease\\\"?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?" }, "conversation_context": [ { "role": "user", "text": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Laura Osnes" }, { "id": "doc-8@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Grease: The New Broadway Cast Recording" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5ae406245542995dadf242e2", "omission": { "id": "5ae406245542995dadf242e2", "question": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?", "target": "20", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-b9c4109d04f91fd2acf9", "text": "Marion Greene was a health policy analyst for St. Judt Medical company .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.152797698974609, "probability": 0.5423980355262756, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-2558e898a3ad5522cf10", "text": "Marion Greene was a health policy analyst for St. Judt Medical company , which had a total of five principal operations worldwide .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 3.982797145843506, "probability": 0.457601934671402, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5ae406245542995dadf242e2", "question": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?", "target": "20", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-e46df970b9b77dfa43bd", "text": "Marion Greene was a health policy analyst for St. Judt Medical company .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.261619567871094, "probability": 0.5154074430465698, "verifications": [ { "source_id": "St. Jude Medical", "distribution": { "contradiction": 0.8359507918357849, "support": 0.014763128012418747, "unknown": 0.14928612112998962 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Marion Greene", "distribution": { "contradiction": 0.8359507918357849, "support": 0.014763128012418747, "unknown": 0.14928612112998962 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-c1f8c0d079506eecc65f", "text": "Marion Greene was a health policy analyst for St. Judt Medical company , which had five principal operations worldwide .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-2@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.199970245361328, "probability": 0.4845925569534302, "verifications": [ { "source_id": "St. Jude Medical", "distribution": { "contradiction": 0.14782339334487915, "support": 0.016868695616722107, "unknown": 0.8353079557418823 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-2@4" }, { "source_id": "Marion Greene", "distribution": { "contradiction": 0.14782339334487915, "support": 0.016868695616722107, "unknown": 0.8353079557418823 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "St. Jude Medical" }, { "id": "doc-5@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Marion Greene" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?" }, "conversation_context": [ { "role": "user", "text": "Marion Greene was a health policy analyst for St. Judt Medical company, which had how many principal operations worldwide?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-2@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "St. Jude Medical" }, { "id": "doc-5@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Marion Greene" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5ab6369655429953192ad2a1", "omission": { "id": "5ab6369655429953192ad2a1", "question": "Which Genus has more species Eucryphia or Lepidozamia ?", "target": "Eucryphia", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which Genus has more species Eucryphia or Lepidozamia ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-f103ae6ba67219126f76", "text": "Eucryphia has more species Eucryphia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.917658805847168, "probability": 0.42701467871665955, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-5aed2298d1d64eff8810", "text": "Eucryphia has more species Eucryphia or Lepidozamia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.704158782958984, "probability": 0.3449220359325409, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-d7a934e5faa88afc7938", "text": "Eucryphia has more species .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.290463447570801, "probability": 0.22806325554847717, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5ab6369655429953192ad2a1", "question": "Which Genus has more species Eucryphia or Lepidozamia ?", "target": "Eucryphia", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": true, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Which Genus has more species Eucryphia or Lepidozamia ?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-1e9b97fa7a9cada984e9", "text": "Lepidozamia has more species Eucryphia or Lepidozamia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0@4", "doc-1@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.1977219581604, "probability": 0.23025985062122345, "verifications": [ { "source_id": "Eucryphia", "distribution": { "contradiction": 0.0329010970890522, "support": 0.014731304720044136, "unknown": 0.9523676037788391 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0@4" }, { "source_id": "Lepidozamia", "distribution": { "contradiction": 0.0329010970890522, "support": 0.014731304720044136, "unknown": 0.9523676037788391 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-078a1242fd06c29e0d44", "text": "Eucryphia has more species Eucryphia or Lepidozamia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0@4", "doc-1@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.863576889038086, "probability": 0.448121041059494, "verifications": [ { "source_id": "Eucryphia", "distribution": { "contradiction": 0.0312345027923584, "support": 0.01226382702589035, "unknown": 0.9565016031265259 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0@4" }, { "source_id": "Lepidozamia", "distribution": { "contradiction": 0.0312345027923584, "support": 0.01226382702589035, "unknown": 0.9565016031265259 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-be8a9231e3b6e015c8a2", "text": "Lepidozamia has more species Eucryphia .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0@4", "doc-1@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.531881332397461, "probability": 0.32161909341812134, "verifications": [ { "source_id": "Eucryphia", "distribution": { "contradiction": 0.06557577848434448, "support": 0.018554022535681725, "unknown": 0.915870189666748 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0@4" }, { "source_id": "Lepidozamia", "distribution": { "contradiction": 0.06557577848434448, "support": 0.018554022535681725, "unknown": 0.915870189666748 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Eucryphia" }, { "id": "doc-1@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Lepidozamia" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "Which Genus has more species Eucryphia or Lepidozamia ?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"Which Genus has more species Eucryphia or Lepidozamia ?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: Which Genus has more species Eucryphia or Lepidozamia ?" }, "conversation_context": [ { "role": "user", "text": "Which Genus has more species Eucryphia or Lepidozamia ?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "[Eucryphia] This source is unavailable and supplies no evidence about the question.", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Eucryphia" }, { "id": "doc-1@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Lepidozamia" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5ae628f55542995703ce8b32", "omission": { "id": "5ae628f55542995703ce8b32", "question": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?", "target": "Venice", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-51a2669bb92da134c17c", "text": "The Italian Baroque composer who composed a set of sonatas , titled Op. 5, was born in Italy .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.683119773864746, "probability": 0.33587130904197693, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-497733583217ed8c7e5c", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5 was born in Italy .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.481593132019043, "probability": 0.27456867694854736, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-cf7f90ed49b4007f8d45", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5, was born in Italy .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.831409454345703, "probability": 0.38955995440483093, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5ae628f55542995703ce8b32", "question": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?", "target": "Venice", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-5a13fa47650dd5d5e9fa", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5, born in Italy .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1@4", "doc-7@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.66159725189209, "probability": 0.3142234981060028, "verifications": [ { "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)", "distribution": { "contradiction": 0.07607351243495941, "support": 0.011697004549205303, "unknown": 0.9122294783592224 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1@4" }, { "source_id": "Antonio Vivaldi", "distribution": { "contradiction": 0.07607351243495941, "support": 0.011697004549205303, "unknown": 0.9122294783592224 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-b9faf03e36723cc62439", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5 was born in Italy .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1@4", "doc-7@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.532426834106445, "probability": 0.27614718675613403, "verifications": [ { "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)", "distribution": { "contradiction": 0.031187303364276886, "support": 0.011486772447824478, "unknown": 0.9573259353637695 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1@4" }, { "source_id": "Antonio Vivaldi", "distribution": { "contradiction": 0.031187303364276886, "support": 0.011486772447824478, "unknown": 0.9573259353637695 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-62b38115bcd6ccade26c", "text": "The Italian Baroque composer who composed a set of sonatas titled Op. 5, born in the city of Rome .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-1@4", "doc-7@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.926745414733887, "probability": 0.40962928533554077, "verifications": [ { "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)", "distribution": { "contradiction": 0.042475610971450806, "support": 0.009643088094890118, "unknown": 0.947881281375885 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-1@4" }, { "source_id": "Antonio Vivaldi", "distribution": { "contradiction": 0.042475610971450806, "support": 0.009643088094890118, "unknown": 0.947881281375885 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-1@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)" }, { "id": "doc-7@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Antonio Vivaldi" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?" }, "conversation_context": [ { "role": "user", "text": "In what city was the Italian Baroque composer who composed a set of sonatas, titled Op. 5, born?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-1@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Six Violin Sonatas, Op. 5 (Vivaldi)" }, { "id": "doc-7@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Antonio Vivaldi" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5adccdd55542992c1e3a2408", "omission": { "id": "5adccdd55542992c1e3a2408", "question": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?", "target": "Waimea Bay", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-49960eb3536c579c7408", "text": "Butch Van Artsdalen was known for surfing 25-foot waves at Hawaiian .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.491645812988281, "probability": 0.3022330105304718, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-9313ef57dd905e696571", "text": "Butch Van Artsdalen was known for surfing 25-foot waves in Hawaiian .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.443598747253418, "probability": 0.2880549132823944, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-1c69794c4cc099e977d2", "text": "Butch Van Artsdalen was known for surfing 25-foot waves in Hawaii .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.795902252197266, "probability": 0.4097120761871338, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5adccdd55542992c1e3a2408", "question": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?", "target": "Waimea Bay", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-d1043bbd53ca1948eb59", "text": "Butch Van Artsdalen was known for surfing 25-foot waves at Hawaiian .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5@4", "doc-7@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.606036186218262, "probability": 0.2893631160259247, "verifications": [ { "source_id": "Waimea Bay, Hawaii", "distribution": { "contradiction": 0.408378928899765, "support": 0.0243392251431942, "unknown": 0.5672818422317505 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@4" }, { "source_id": "Butch Van Artsdalen", "distribution": { "contradiction": 0.408378928899765, "support": 0.0243392251431942, "unknown": 0.5672818422317505 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-9864cbb04a53b8f9e522", "text": "Butch Van Artsdalen was known for surfing 25-foot waves in Hawaiian .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5@4", "doc-7@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.55405330657959, "probability": 0.2747054696083069, "verifications": [ { "source_id": "Waimea Bay, Hawaii", "distribution": { "contradiction": 0.7514291405677795, "support": 0.020856209099292755, "unknown": 0.22771461308002472 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@4" }, { "source_id": "Butch Van Artsdalen", "distribution": { "contradiction": 0.7514291405677795, "support": 0.020856209099292755, "unknown": 0.22771461308002472 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-5b37c0247432b52ee8fa", "text": "Butch Van Artsdalen was known for surfing 25-foot waves at Kauai .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-5@4", "doc-7@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.015838623046875, "probability": 0.43593135476112366, "verifications": [ { "source_id": "Waimea Bay, Hawaii", "distribution": { "contradiction": 0.5794227719306946, "support": 0.024596882984042168, "unknown": 0.3959803283214569 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@4" }, { "source_id": "Butch Van Artsdalen", "distribution": { "contradiction": 0.5794227719306946, "support": 0.024596882984042168, "unknown": 0.3959803283214569 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-7@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-5@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Waimea Bay, Hawaii" }, { "id": "doc-7@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Butch Van Artsdalen" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"Meaning \\\"reddish water\\\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?" }, "conversation_context": [ { "role": "user", "text": "Meaning \"reddish water\" in Hawaiian, which location was Butch Van Artsdalen known for surfing 25-foot waves at?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "[Butch Van Artsdalen]", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-5@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Waimea Bay, Hawaii" }, { "id": "doc-7@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Butch Van Artsdalen" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 }, { "id": "5abd1529554299114383a1e3", "omission": { "id": "5abd1529554299114383a1e3", "question": "What type of magazine is Eugene Habecker the chairman of?", "target": "evangelical Christian periodical", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What type of magazine is Eugene Habecker the chairman of?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-7e4986e2604fd81fd6f0", "text": "Eugene Habecker is the chairman of the magazine .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.842167854309082, "probability": 0.16127796471118927, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-5453e4bb5b45d930c9d4", "text": "Eugene Habecker is the chairman of a magazine .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.915125846862793, "probability": 0.17348434031009674, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-de2b17ec5c74cc7bcd5a", "text": "Eugene Habecker is the chairman of the magazine magazine '' .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 6.259182929992676, "probability": 0.6652376651763916, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "5abd1529554299114383a1e3", "question": "What type of magazine is Eugene Habecker the chairman of?", "target": "evangelical Christian periodical", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "What type of magazine is Eugene Habecker the chairman of?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-c920905e96c27a8fa2be", "text": "Eugene Habecker is the chairman of a type of magazine .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.50040340423584, "probability": 0.25950029492378235, "verifications": [ { "source_id": "Eugene Habecker", "distribution": { "contradiction": 0.9071362018585205, "support": 0.012067893519997597, "unknown": 0.08079583942890167 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0@4" }, { "source_id": "Christianity Today", "distribution": { "contradiction": 0.9071362018585205, "support": 0.012067893519997597, "unknown": 0.08079583942890167 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-3f2e27fa4a8e5ef37242", "text": "Eugene Habecker is the chairman of the type of magazine .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.653697967529297, "probability": 0.3024912476539612, "verifications": [ { "source_id": "Eugene Habecker", "distribution": { "contradiction": 0.8550026416778564, "support": 0.010368334129452705, "unknown": 0.134629026055336 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0@4" }, { "source_id": "Christianity Today", "distribution": { "contradiction": 0.8550026416778564, "support": 0.010368334129452705, "unknown": 0.134629026055336 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-bdff56f6f4d04aba90fb", "text": "Eugene Habecker is the chairman of a magazine .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "doc-0@4", "doc-5@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.023883819580078, "probability": 0.43800848722457886, "verifications": [ { "source_id": "Eugene Habecker", "distribution": { "contradiction": 0.9409489631652832, "support": 0.009001334197819233, "unknown": 0.05004969611763954 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-0@4" }, { "source_id": "Christianity Today", "distribution": { "contradiction": 0.9409489631652832, "support": 0.009001334197819233, "unknown": 0.05004969611763954 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "doc-5@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "doc-0@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Eugene Habecker" }, { "id": "doc-5@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Christianity Today" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "What type of magazine is Eugene Habecker the chairman of?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"What type of magazine is Eugene Habecker the chairman of?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: What type of magazine is Eugene Habecker the chairman of?" }, "conversation_context": [ { "role": "user", "text": "What type of magazine is Eugene Habecker the chairman of?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "doc-0@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Eugene Habecker" }, { "id": "doc-5@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "Christianity Today" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 } ], "omission": { "count": 8, "failed_calls": 0, "short_answer_exact_match_format_sensitive": 0.0, "answer_wholeword_containment_not_accuracy": 0.0, "candidate_answer_substring_coverage": 0.125, "cognitive_abstention_rate": 1.0, "final_enforced_abstention_rate": 1.0, "default_abstention_text_rate": 1.0, "selected_text_preserved_rate": 0.0, "selected_contradiction_veto_violations": 0, "generation_truncations": 0, "realization_nli_support_mean_on_selected": null, "uncalibrated_verifier_records": 0 }, "replacement": { "count": 8, "failed_calls": 0, "short_answer_exact_match_format_sensitive": 0.0, "answer_wholeword_containment_not_accuracy": 0.0, "candidate_answer_substring_coverage": 0.25, "cognitive_abstention_rate": 1.0, "final_enforced_abstention_rate": 1.0, "default_abstention_text_rate": 1.0, "selected_text_preserved_rate": 0.0, "selected_contradiction_veto_violations": 0, "generation_truncations": 0, "realization_nli_support_mean_on_selected": null, "uncalibrated_verifier_records": 0 } }, "authored_fixtures": { "real_data": { "metrics": { "count": 1, "failed_calls": 0, "short_answer_exact_match_format_sensitive": 0.0, "answer_wholeword_containment_not_accuracy": 0.0, "candidate_answer_substring_coverage": 0.0, "cognitive_abstention_rate": 1.0, "final_enforced_abstention_rate": 1.0, "default_abstention_text_rate": 1.0, "selected_text_preserved_rate": 0.0, "selected_contradiction_veto_violations": 0, "generation_truncations": 0, "realization_nli_support_mean_on_selected": null, "uncalibrated_verifier_records": 0 }, "records": [ { "id": "authored-conflict", "question": "Is the door open?", "target": "insufficient evidence", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Is the door open?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-722ddf074b3ad348647f", "text": "The door is open .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "report-1", "report-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.503460884094238, "probability": 0.469615638256073, "verifications": [ { "source_id": "authored-observer-A", "distribution": { "contradiction": 0.02036396600306034, "support": 0.7655325531959534, "unknown": 0.21410346031188965 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-1" }, { "source_id": "authored-observer-B", "distribution": { "contradiction": 0.9675791263580322, "support": 0.0167155172675848, "unknown": 0.0157054103910923 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-2" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-230f6dd55b6d1555a5f8", "text": "The door is closed .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "report-1", "report-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.359971046447754, "probability": 0.4068419635295868, "verifications": [ { "source_id": "authored-observer-A", "distribution": { "contradiction": 0.968310534954071, "support": 0.01515771821141243, "unknown": 0.01653173565864563 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-1" }, { "source_id": "authored-observer-B", "distribution": { "contradiction": 0.013815037906169891, "support": 0.9282950758934021, "unknown": 0.0578899160027504 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-2" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-35b808aa5e1c94ebd06c", "text": "The door is closed and is not open .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "report-1", "report-2" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.168130874633789, "probability": 0.1235424131155014, "verifications": [ { "source_id": "authored-observer-A", "distribution": { "contradiction": 0.9848495721817017, "support": 0.007875997573137283, "unknown": 0.007274429779499769 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-1" }, { "source_id": "authored-observer-B", "distribution": { "contradiction": 0.016712747514247894, "support": 0.832832396030426, "unknown": 0.15045486390590668 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-2" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "report-1", "text": "The door is open.", "source_id": "authored-observer-A" }, { "id": "report-2", "text": "The door is closed and is not open.", "source_id": "authored-observer-B" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "report-1", "report-2" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "report-1", "text": "The door is open.", "source_id": "authored-observer-A" }, { "id": "report-2", "text": "The door is closed and is not open.", "source_id": "authored-observer-B" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } } ] }, "controls": { "records": [ { "id": "authored-conflict", "omission": { "id": "authored-conflict", "question": "Is the door open?", "target": "insufficient evidence", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": null, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Is the door open?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-c9d2c4ec99392e5f911d", "text": "The door is open .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.377785682678223, "probability": 0.40225130319595337, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-378d3e58dcef8e8ac838", "text": "The door is open to the public .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.138113975524902, "probability": 0.31652596592903137, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-528c4d308d3cf0d4f593", "text": "The door is open to the people .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.019855499267578, "probability": 0.2812226712703705, "verifications": [], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 2, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement": { "id": "authored-conflict", "question": "Is the door open?", "target": "insufficient evidence", "answer": "I do not have enough supported evidence to answer.", "answer_exact_match": false, "answer_wholeword_containment": false, "candidate_answer_substring_coverage": false, "selected_text_preserved": false, "abstained": true, "abstention_enforced": true, "default_abstention_text": true, "contradicted_selection": false, "input_truncated": false, "verifier_calibrated": true, "receipt": { "text": "I do not have enough supported evidence to answer.", "cognition": { "question": "Is the door open?", "selected_id": null, "abstained": true, "candidates": [ { "id": "hypothesis-f7d621250e95e4e6e0bd", "text": "The door is open to the user .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "report-1@4", "report-2@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.62752628326416, "probability": 0.33239808678627014, "verifications": [ { "source_id": "authored-observer-A", "distribution": { "contradiction": 0.8959447741508484, "support": 0.013552166521549225, "unknown": 0.09050313383340836 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-1@4" }, { "source_id": "authored-observer-B", "distribution": { "contradiction": 0.8959447741508484, "support": 0.013552166521549225, "unknown": 0.09050313383340836 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-2@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-076801b00f94778d49cc", "text": "The door is open .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "report-1@4", "report-2@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.913629531860352, "probability": 0.44249826669692993, "verifications": [ { "source_id": "authored-observer-A", "distribution": { "contradiction": 0.7398958802223206, "support": 0.014104507863521576, "unknown": 0.2459995597600937 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-1@4" }, { "source_id": "authored-observer-B", "distribution": { "contradiction": 0.7398958802223206, "support": 0.014104507863521576, "unknown": 0.2459995597600937 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-2@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-b42119643115e4496933", "text": "The door is open by user .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "report-1@4", "report-2@5" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 5.237753868103027, "probability": 0.22510363161563873, "verifications": [ { "source_id": "authored-observer-A", "distribution": { "contradiction": 0.8755722045898438, "support": 0.012348653748631477, "unknown": 0.11207913607358932 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-1@4" }, { "source_id": "authored-observer-B", "distribution": { "contradiction": 0.8755722045898438, "support": 0.012348653748631477, "unknown": 0.11207913607358932 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "report-2@5" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "report-1@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "authored-observer-A" }, { "id": "report-2@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "authored-observer-B" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 7, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth.", "conversation_context": [ { "role": "user", "text": "Is the door open?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "retrieval_query": "Prior dialogue (context only; not source evidence; assistant statements are unverified):\n[{\"role\": \"user\", \"text\": \"Is the door open?\"}, {\"role\": \"assistant\", \"text\": \"I do not have enough supported evidence to answer.\"}]\nCurrent question: Is the door open?" }, "conversation_context": [ { "role": "user", "text": "Is the door open?" }, { "role": "assistant", "text": "I do not have enough supported evidence to answer." } ], "conversation_context_truncated": false, "response_proposal": { "text": "No supported selection", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": true, "realization_verifications": [], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "report-1@4", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "authored-observer-A" }, { "id": "report-2@5", "text": "This source is unavailable and supplies no evidence about the question.", "source_id": "authored-observer-B" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "replacement_kind": "authored source-withdrawal notice, not real-world evidence", "state_revision_before": 3, "state_revision_after": 7 } ], "omission": { "count": 1, "failed_calls": 0, "short_answer_exact_match_format_sensitive": 0.0, "answer_wholeword_containment_not_accuracy": 0.0, "candidate_answer_substring_coverage": 0.0, "cognitive_abstention_rate": 1.0, "final_enforced_abstention_rate": 1.0, "default_abstention_text_rate": 1.0, "selected_text_preserved_rate": 0.0, "selected_contradiction_veto_violations": 0, "generation_truncations": 0, "realization_nli_support_mean_on_selected": null, "uncalibrated_verifier_records": 0 }, "replacement": { "count": 1, "failed_calls": 0, "short_answer_exact_match_format_sensitive": 0.0, "answer_wholeword_containment_not_accuracy": 0.0, "candidate_answer_substring_coverage": 0.0, "cognitive_abstention_rate": 1.0, "final_enforced_abstention_rate": 1.0, "default_abstention_text_rate": 1.0, "selected_text_preserved_rate": 0.0, "selected_contradiction_veto_violations": 0, "generation_truncations": 0, "realization_nli_support_mean_on_selected": null, "uncalibrated_verifier_records": 0 } }, "conflict_cases_with_selected_candidate": 0, "expected_behavior": "Authored mutually conflicting sources should prevent an unqualified selection; this is a fixture, not a public benchmark." }, "episodic_retrieval": { "metrics": { "learned_hit_at_1": 1.0, "learned_hit_at_5": 1.0, "lexical_hit_at_1": 0.96875, "lexical_hit_at_5": 1.0 }, "records": [ { "id": "5ae048a255429924de1b708e", "relevant_ids": [ "5ae048a255429924de1b708e:doc-2", "5ae048a255429924de1b708e:doc-5" ], "learned_ids": [ "5ae048a255429924de1b708e:doc-5", "5adfa3da554299025d62a317:doc-2", "5a90692f5542990a98493625:doc-3", "5ae048a255429924de1b708e:doc-2", "5a78db8055429970f5fffdb2:doc-0" ], "lexical_ids": [ "5ae048a255429924de1b708e:doc-5", "5ae048a255429924de1b708e:doc-2", "5a9068a95542990a98493621:doc-2", "5a7555215542996c70cfaee1:doc-2", "5adf9dc3554299025d62a2e6:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a9068a95542990a98493621", "relevant_ids": [ "5a9068a95542990a98493621:doc-2", "5a9068a95542990a98493621:doc-6" ], "learned_ids": [ "5a9068a95542990a98493621:doc-2", "5a9068a95542990a98493621:doc-6", "5ab8f7535542991b5579f0a7:doc-9", "5a8463545542990548d0b2a3:doc-5", "5ab9debb55429955dce3ed97:doc-7" ], "lexical_ids": [ "5a9068a95542990a98493621:doc-2", "5ae791ef55429952e35ea979:doc-3", "5a9068a95542990a98493621:doc-6", "5adccdd55542992c1e3a2408:doc-7", "5ae791ef55429952e35ea979:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5adf9dc3554299025d62a2e6", "relevant_ids": [ "5adf9dc3554299025d62a2e6:doc-2", "5adf9dc3554299025d62a2e6:doc-8" ], "learned_ids": [ "5adf9dc3554299025d62a2e6:doc-8", "5adf9dc3554299025d62a2e6:doc-2", "5a74684655429929fddd8410:doc-4", "5a90692f5542990a98493625:doc-2", "5adfa3da554299025d62a317:doc-4" ], "lexical_ids": [ "5adf9dc3554299025d62a2e6:doc-2", "5adf9dc3554299025d62a2e6:doc-8", "5ab9debb55429955dce3ed97:doc-7", "5ae406245542995dadf242e2:doc-5", "5a88e605554299206df2b39c:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ae406245542995dadf242e2", "relevant_ids": [ "5ae406245542995dadf242e2:doc-2", "5ae406245542995dadf242e2:doc-5" ], "learned_ids": [ "5ae406245542995dadf242e2:doc-5", "5ae406245542995dadf242e2:doc-2", "5ae24abc5542994d89d5b3c3:doc-4", "5a83a532554299334474606f:doc-8", "5a835c9f5542992ef85e228a:doc-0" ], "lexical_ids": [ "5ae406245542995dadf242e2:doc-5", "5ae406245542995dadf242e2:doc-2", "5a9068a95542990a98493621:doc-2", "5ae628f55542995703ce8b32:doc-7", "5a88e605554299206df2b39c:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ab6369655429953192ad2a1", "relevant_ids": [ "5ab6369655429953192ad2a1:doc-0", "5ab6369655429953192ad2a1:doc-1" ], "learned_ids": [ "5ab6369655429953192ad2a1:doc-0", "5ab6369655429953192ad2a1:doc-1", "5ae7edee554299540e5a56ad:doc-3", "5a8dd5f955429917b4a5bcc7:doc-8", "5a83a532554299334474606f:doc-7" ], "lexical_ids": [ "5ab6369655429953192ad2a1:doc-0", "5ab6369655429953192ad2a1:doc-1", "5a7555215542996c70cfaee1:doc-2", "5adf9dc3554299025d62a2e6:doc-2", "5a8463545542990548d0b2a3:doc-5" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ae628f55542995703ce8b32", "relevant_ids": [ "5ae628f55542995703ce8b32:doc-1", "5ae628f55542995703ce8b32:doc-7" ], "learned_ids": [ "5ae628f55542995703ce8b32:doc-1", "5ae628f55542995703ce8b32:doc-7", "5a8dd5f955429917b4a5bcc7:doc-8", "5adf9dc3554299025d62a2e6:doc-8", "5a76e88555429972597f13f4:doc-5" ], "lexical_ids": [ "5ae628f55542995703ce8b32:doc-7", "5ae406245542995dadf242e2:doc-5", "5a8463545542990548d0b2a3:doc-5", "5ae628f55542995703ce8b32:doc-1", "5a7d2afd5542995f4f402221:doc-3" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5adccdd55542992c1e3a2408", "relevant_ids": [ "5adccdd55542992c1e3a2408:doc-5", "5adccdd55542992c1e3a2408:doc-7" ], "learned_ids": [ "5adccdd55542992c1e3a2408:doc-7", "5adccdd55542992c1e3a2408:doc-5", "5ae7edee554299540e5a56ad:doc-3", "5a7d2afd5542995f4f402221:doc-2", "5a8dd5f955429917b4a5bcc7:doc-8" ], "lexical_ids": [ "5adccdd55542992c1e3a2408:doc-7", "5a9068a95542990a98493621:doc-2", "5adccdd55542992c1e3a2408:doc-5", "5adf9dc3554299025d62a2e6:doc-2", "5ae406245542995dadf242e2:doc-5" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5abd1529554299114383a1e3", "relevant_ids": [ "5abd1529554299114383a1e3:doc-0", "5abd1529554299114383a1e3:doc-5" ], "learned_ids": [ "5abd1529554299114383a1e3:doc-0", "5abd1529554299114383a1e3:doc-5", "5ae406245542995dadf242e2:doc-5", "5ae24abc5542994d89d5b3c3:doc-4", "5ae406245542995dadf242e2:doc-2" ], "lexical_ids": [ "5abd1529554299114383a1e3:doc-0", "5ae048a255429924de1b708e:doc-2", "5ae048a255429924de1b708e:doc-5", "5adf9dc3554299025d62a2e6:doc-2", "5adf9dc3554299025d62a2e6:doc-8" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a78db8055429970f5fffdb2", "relevant_ids": [ "5a78db8055429970f5fffdb2:doc-0", "5a78db8055429970f5fffdb2:doc-4" ], "learned_ids": [ "5a78db8055429970f5fffdb2:doc-4", "5a78db8055429970f5fffdb2:doc-0", "5a76e88555429972597f13f4:doc-4", "5a90692f5542990a98493625:doc-2", "5adf9dc3554299025d62a2e6:doc-8" ], "lexical_ids": [ "5a78db8055429970f5fffdb2:doc-0", "5adf9dc3554299025d62a2e6:doc-2", "5ae406245542995dadf242e2:doc-5", "5a9068a95542990a98493621:doc-2", "5ae628f55542995703ce8b32:doc-7" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ab9debb55429955dce3ed97", "relevant_ids": [ "5ab9debb55429955dce3ed97:doc-4", "5ab9debb55429955dce3ed97:doc-7" ], "learned_ids": [ "5ab9debb55429955dce3ed97:doc-7", "5ab9debb55429955dce3ed97:doc-4", "5a90692f5542990a98493625:doc-2", "5a90692f5542990a98493625:doc-3", "5adfa3da554299025d62a317:doc-2" ], "lexical_ids": [ "5ab9debb55429955dce3ed97:doc-4", "5ab9debb55429955dce3ed97:doc-7", "5adf9dc3554299025d62a2e6:doc-2", "5adf9dc3554299025d62a2e6:doc-8", "5a74684655429929fddd8410:doc-9" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ae791ef55429952e35ea979", "relevant_ids": [ "5ae791ef55429952e35ea979:doc-2", "5ae791ef55429952e35ea979:doc-3" ], "learned_ids": [ "5ae791ef55429952e35ea979:doc-3", "5ae791ef55429952e35ea979:doc-2", "5add67915542992200553af8:doc-0", "5a8dd5f955429917b4a5bcc7:doc-8", "5a78db8055429970f5fffdb2:doc-4" ], "lexical_ids": [ "5ae791ef55429952e35ea979:doc-3", "5ae791ef55429952e35ea979:doc-2", "5a835c9f5542992ef85e228a:doc-0", "5a9068a95542990a98493621:doc-2", "5a7d2afd5542995f4f402221:doc-3" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ab8f7535542991b5579f0a7", "relevant_ids": [ "5ab8f7535542991b5579f0a7:doc-0", "5ab8f7535542991b5579f0a7:doc-9" ], "learned_ids": [ "5ab8f7535542991b5579f0a7:doc-9", "5ab8f7535542991b5579f0a7:doc-0", "5a78db8055429970f5fffdb2:doc-0", "5a9068a95542990a98493621:doc-2", "5ab9debb55429955dce3ed97:doc-7" ], "lexical_ids": [ "5a78db8055429970f5fffdb2:doc-0", "5ab8f7535542991b5579f0a7:doc-9", "5ab8f7535542991b5579f0a7:doc-0", "5a9068a95542990a98493621:doc-2", "5ae791ef55429952e35ea979:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": false, "lexical_hit_at_5": true }, { "id": "5ac119335542992a796dede4", "relevant_ids": [ "5ac119335542992a796dede4:doc-2", "5ac119335542992a796dede4:doc-3" ], "learned_ids": [ "5ac119335542992a796dede4:doc-2", "5adfbca255429942ec259b2c:doc-5", "5ac119335542992a796dede4:doc-3", "5adfbca255429942ec259b2c:doc-9", "5a9068a95542990a98493621:doc-6" ], "lexical_ids": [ "5ac119335542992a796dede4:doc-2", "5adfbca255429942ec259b2c:doc-5", "5ac119335542992a796dede4:doc-3", "5a90692f5542990a98493625:doc-2", "5adf9dc3554299025d62a2e6:doc-8" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a7d2afd5542995f4f402221", "relevant_ids": [ "5a7d2afd5542995f4f402221:doc-2", "5a7d2afd5542995f4f402221:doc-3" ], "learned_ids": [ "5a7d2afd5542995f4f402221:doc-2", "5a7d2afd5542995f4f402221:doc-3", "5a8dd5f955429917b4a5bcc7:doc-8", "5a74684655429929fddd8410:doc-4", "5adccdd55542992c1e3a2408:doc-7" ], "lexical_ids": [ "5a7d2afd5542995f4f402221:doc-3", "5a7d2afd5542995f4f402221:doc-2", "5ae406245542995dadf242e2:doc-5", "5ae791ef55429952e35ea979:doc-2", "5ab8f7535542991b5579f0a7:doc-0" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a8dd5f955429917b4a5bcc7", "relevant_ids": [ "5a8dd5f955429917b4a5bcc7:doc-0", "5a8dd5f955429917b4a5bcc7:doc-8" ], "learned_ids": [ "5a8dd5f955429917b4a5bcc7:doc-0", "5a8dd5f955429917b4a5bcc7:doc-8", "5a7d2afd5542995f4f402221:doc-2", "5a90580655429933b8a204ef:doc-6", "5a90580655429933b8a204ef:doc-2" ], "lexical_ids": [ "5a8dd5f955429917b4a5bcc7:doc-0", "5ae628f55542995703ce8b32:doc-7", "5a7555215542996c70cfaee1:doc-2", "5a88e605554299206df2b39c:doc-7", "5ae406245542995dadf242e2:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a83a532554299334474606f", "relevant_ids": [ "5a83a532554299334474606f:doc-7", "5a83a532554299334474606f:doc-8" ], "learned_ids": [ "5a83a532554299334474606f:doc-8", "5a83a532554299334474606f:doc-7", "5a8cb288554299585d9e3726:doc-9", "5abd1529554299114383a1e3:doc-0", "5a8cb288554299585d9e3726:doc-4" ], "lexical_ids": [ "5a83a532554299334474606f:doc-8", "5a83a532554299334474606f:doc-7", "5a8cb288554299585d9e3726:doc-4", "5a9068a95542990a98493621:doc-2", "5adccdd55542992c1e3a2408:doc-7" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a90692f5542990a98493625", "relevant_ids": [ "5a90692f5542990a98493625:doc-2", "5a90692f5542990a98493625:doc-3" ], "learned_ids": [ "5a90692f5542990a98493625:doc-2", "5a90692f5542990a98493625:doc-3", "5a78db8055429970f5fffdb2:doc-0", "5adf9dc3554299025d62a2e6:doc-2", "5a9068a95542990a98493621:doc-2" ], "lexical_ids": [ "5a90692f5542990a98493625:doc-3", "5a90692f5542990a98493625:doc-2", "5ab8f7535542991b5579f0a7:doc-0", "5add67915542992200553af8:doc-3", "5a74684655429929fddd8410:doc-4" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a7555215542996c70cfaee1", "relevant_ids": [ "5a7555215542996c70cfaee1:doc-1", "5a7555215542996c70cfaee1:doc-2" ], "learned_ids": [ "5a7555215542996c70cfaee1:doc-1", "5a7555215542996c70cfaee1:doc-2", "5add67915542992200553af8:doc-3", "5ab8f7535542991b5579f0a7:doc-9", "5add67915542992200553af8:doc-0" ], "lexical_ids": [ "5a7555215542996c70cfaee1:doc-1", "5a7555215542996c70cfaee1:doc-2", "5ae791ef55429952e35ea979:doc-2", "5a8463545542990548d0b2a3:doc-5", "5a9068a95542990a98493621:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a90580655429933b8a204ef", "relevant_ids": [ "5a90580655429933b8a204ef:doc-2", "5a90580655429933b8a204ef:doc-6" ], "learned_ids": [ "5a90580655429933b8a204ef:doc-2", "5a90580655429933b8a204ef:doc-6", "5a8dd5f955429917b4a5bcc7:doc-0", "5adfbca255429942ec259b2c:doc-9", "5ab6369655429953192ad2a1:doc-1" ], "lexical_ids": [ "5a90580655429933b8a204ef:doc-2", "5a90580655429933b8a204ef:doc-6", "5a90692f5542990a98493625:doc-2", "5ae406245542995dadf242e2:doc-2", "5abd1529554299114383a1e3:doc-0" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a88e605554299206df2b39c", "relevant_ids": [ "5a88e605554299206df2b39c:doc-2", "5a88e605554299206df2b39c:doc-7" ], "learned_ids": [ "5a88e605554299206df2b39c:doc-7", "5a88e605554299206df2b39c:doc-2", "5ae406245542995dadf242e2:doc-5", "5ae7edee554299540e5a56ad:doc-3", "5adccdd55542992c1e3a2408:doc-7" ], "lexical_ids": [ "5a88e605554299206df2b39c:doc-2", "5a88e605554299206df2b39c:doc-7", "5a8cb288554299585d9e3726:doc-4", "5adccdd55542992c1e3a2408:doc-7", "5adf9dc3554299025d62a2e6:doc-8" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5add67915542992200553af8", "relevant_ids": [ "5add67915542992200553af8:doc-0", "5add67915542992200553af8:doc-3" ], "learned_ids": [ "5add67915542992200553af8:doc-3", "5add67915542992200553af8:doc-0", "5a8dd5f955429917b4a5bcc7:doc-8", "5a8cb288554299585d9e3726:doc-4", "5a76e88555429972597f13f4:doc-4" ], "lexical_ids": [ "5add67915542992200553af8:doc-3", "5ae406245542995dadf242e2:doc-2", "5ae628f55542995703ce8b32:doc-7", "5a78db8055429970f5fffdb2:doc-0", "5a74684655429929fddd8410:doc-9" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a8463545542990548d0b2a3", "relevant_ids": [ "5a8463545542990548d0b2a3:doc-5", "5a8463545542990548d0b2a3:doc-8" ], "learned_ids": [ "5a8463545542990548d0b2a3:doc-5", "5a8463545542990548d0b2a3:doc-8", "5a83a532554299334474606f:doc-8", "5add67915542992200553af8:doc-0", "5a835c9f5542992ef85e228a:doc-9" ], "lexical_ids": [ "5a8463545542990548d0b2a3:doc-5", "5a8463545542990548d0b2a3:doc-8", "5ab8f7535542991b5579f0a7:doc-0", "5a7d2afd5542995f4f402221:doc-3", "5a83a532554299334474606f:doc-8" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a74684655429929fddd8410", "relevant_ids": [ "5a74684655429929fddd8410:doc-4", "5a74684655429929fddd8410:doc-9" ], "learned_ids": [ "5a74684655429929fddd8410:doc-9", "5adf9dc3554299025d62a2e6:doc-2", "5a74684655429929fddd8410:doc-4", "5a76e88555429972597f13f4:doc-5", "5adf9dc3554299025d62a2e6:doc-8" ], "lexical_ids": [ "5a74684655429929fddd8410:doc-9", "5adf9dc3554299025d62a2e6:doc-2", "5adccdd55542992c1e3a2408:doc-7", "5add67915542992200553af8:doc-0", "5a74684655429929fddd8410:doc-4" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a835c9f5542992ef85e228a", "relevant_ids": [ "5a835c9f5542992ef85e228a:doc-0", "5a835c9f5542992ef85e228a:doc-9" ], "learned_ids": [ "5a835c9f5542992ef85e228a:doc-9", "5a835c9f5542992ef85e228a:doc-0", "5ae791ef55429952e35ea979:doc-2", "5ae406245542995dadf242e2:doc-2", "5ae791ef55429952e35ea979:doc-3" ], "lexical_ids": [ "5a835c9f5542992ef85e228a:doc-9", "5a835c9f5542992ef85e228a:doc-0", "5a88e605554299206df2b39c:doc-2", "5adf9dc3554299025d62a2e6:doc-2", "5ae628f55542995703ce8b32:doc-7" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ae7edee554299540e5a56ad", "relevant_ids": [ "5ae7edee554299540e5a56ad:doc-3", "5ae7edee554299540e5a56ad:doc-7" ], "learned_ids": [ "5ae7edee554299540e5a56ad:doc-7", "5ae7edee554299540e5a56ad:doc-3", "5a8463545542990548d0b2a3:doc-5", "5ae24abc5542994d89d5b3c3:doc-0", "5a83a532554299334474606f:doc-8" ], "lexical_ids": [ "5ae7edee554299540e5a56ad:doc-7", "5ae791ef55429952e35ea979:doc-2", "5a8cb288554299585d9e3726:doc-4", "5ae406245542995dadf242e2:doc-5", "5ae791ef55429952e35ea979:doc-3" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a8ee4315542990e94052ba7", "relevant_ids": [ "5a8ee4315542990e94052ba7:doc-1", "5a8ee4315542990e94052ba7:doc-4" ], "learned_ids": [ "5a8ee4315542990e94052ba7:doc-4", "5a8ee4315542990e94052ba7:doc-1", "5a7d2afd5542995f4f402221:doc-3", "5a90580655429933b8a204ef:doc-6", "5ab802085542990e739ec7ca:doc-0" ], "lexical_ids": [ "5a8ee4315542990e94052ba7:doc-4", "5a8ee4315542990e94052ba7:doc-1", "5adfbca255429942ec259b2c:doc-9", "5a7d2afd5542995f4f402221:doc-3", "5a7555215542996c70cfaee1:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a8cb288554299585d9e3726", "relevant_ids": [ "5a8cb288554299585d9e3726:doc-4", "5a8cb288554299585d9e3726:doc-9" ], "learned_ids": [ "5a8cb288554299585d9e3726:doc-4", "5a8cb288554299585d9e3726:doc-9", "5a90692f5542990a98493625:doc-3", "5ae791ef55429952e35ea979:doc-2", "5ae791ef55429952e35ea979:doc-3" ], "lexical_ids": [ "5a8cb288554299585d9e3726:doc-4", "5a8cb288554299585d9e3726:doc-9", "5ae791ef55429952e35ea979:doc-2", "5a8dd5f955429917b4a5bcc7:doc-0", "5a78db8055429970f5fffdb2:doc-0" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5adfbca255429942ec259b2c", "relevant_ids": [ "5adfbca255429942ec259b2c:doc-5", "5adfbca255429942ec259b2c:doc-9" ], "learned_ids": [ "5adfbca255429942ec259b2c:doc-5", "5adfbca255429942ec259b2c:doc-9", "5ac119335542992a796dede4:doc-3", "5a7555215542996c70cfaee1:doc-1", "5a76e88555429972597f13f4:doc-5" ], "lexical_ids": [ "5adfbca255429942ec259b2c:doc-5", "5adfbca255429942ec259b2c:doc-9", "5ae791ef55429952e35ea979:doc-2", "5ac119335542992a796dede4:doc-3", "5a7555215542996c70cfaee1:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ae24abc5542994d89d5b3c3", "relevant_ids": [ "5ae24abc5542994d89d5b3c3:doc-0", "5ae24abc5542994d89d5b3c3:doc-4" ], "learned_ids": [ "5ae24abc5542994d89d5b3c3:doc-0", "5ae24abc5542994d89d5b3c3:doc-4", "5a7555215542996c70cfaee1:doc-1", "5ae406245542995dadf242e2:doc-2", "5a9068a95542990a98493621:doc-6" ], "lexical_ids": [ "5ae24abc5542994d89d5b3c3:doc-4", "5abd1529554299114383a1e3:doc-5", "5ae24abc5542994d89d5b3c3:doc-0", "5ae048a255429924de1b708e:doc-2", "5ae048a255429924de1b708e:doc-5" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5ab802085542990e739ec7ca", "relevant_ids": [ "5ab802085542990e739ec7ca:doc-0", "5ab802085542990e739ec7ca:doc-2" ], "learned_ids": [ "5ab802085542990e739ec7ca:doc-0", "5ab802085542990e739ec7ca:doc-2", "5ae628f55542995703ce8b32:doc-1", "5adfa3da554299025d62a317:doc-2", "5a83a532554299334474606f:doc-8" ], "lexical_ids": [ "5ab802085542990e739ec7ca:doc-2", "5ab802085542990e739ec7ca:doc-0", "5ae048a255429924de1b708e:doc-2", "5adf9dc3554299025d62a2e6:doc-8", "5a78db8055429970f5fffdb2:doc-4" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5adfa3da554299025d62a317", "relevant_ids": [ "5adfa3da554299025d62a317:doc-2", "5adfa3da554299025d62a317:doc-4" ], "learned_ids": [ "5adfa3da554299025d62a317:doc-2", "5adfa3da554299025d62a317:doc-4", "5a74684655429929fddd8410:doc-4", "5ae048a255429924de1b708e:doc-5", "5ae048a255429924de1b708e:doc-2" ], "lexical_ids": [ "5adfa3da554299025d62a317:doc-2", "5ae406245542995dadf242e2:doc-5", "5a7d2afd5542995f4f402221:doc-3", "5ae048a255429924de1b708e:doc-5", "5adf9dc3554299025d62a2e6:doc-2" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true }, { "id": "5a76e88555429972597f13f4", "relevant_ids": [ "5a76e88555429972597f13f4:doc-4", "5a76e88555429972597f13f4:doc-5" ], "learned_ids": [ "5a76e88555429972597f13f4:doc-4", "5a78db8055429970f5fffdb2:doc-0", "5a90692f5542990a98493625:doc-3", "5a90692f5542990a98493625:doc-2", "5ab802085542990e739ec7ca:doc-0" ], "lexical_ids": [ "5a76e88555429972597f13f4:doc-5", "5ae048a255429924de1b708e:doc-2", "5ab8f7535542991b5579f0a7:doc-0", "5a90692f5542990a98493625:doc-3", "5a78db8055429970f5fffdb2:doc-4" ], "learned_hit_at_1": true, "learned_hit_at_5": true, "lexical_hit_at_1": true, "lexical_hit_at_5": true } ], "corpus_size": 64, "limitations": "Oracle supporting-passage corpus built from these diagnostic questions, not open-corpus retrieval. Ranker encoder was trained on a separate HotpotQA subset; no fitting here." }, "repository_smoke": { "kind": "real repository document smoke, separate from the statistical benchmark", "document": "docs/pretrained.md", "sha256": "a345be80d1baeff4c75e07def99af2a66f83cbec885f29021e21d67b8ed0e2bb", "excerpt": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "question": "Which service is the preferred model host for TensorCode?", "first": { "text": "The preferred model host for TensorCode is nugging Face .", "receipt": { "text": "The preferred model host for TensorCode is nugging Face .", "cognition": { "question": "Which service is the preferred model host for TensorCode?", "selected_id": "hypothesis-38e2ce675ff3591d7c68", "abstained": false, "candidates": [ { "id": "hypothesis-6e4793c7544ee2a77d80", "text": "The preferred model host for TensorCode is nHugging Face .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.708287239074707, "probability": 0.3669797480106354, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.20080351829528809, "support": 0.13878197968006134, "unknown": 0.6604145169258118 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-815fb6c0f41f8392d24b", "text": "TensorCode is the preferred model host for TensorCode .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.482316017150879, "probability": 0.29275479912757874, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.8401834964752197, "support": 0.03588845580816269, "unknown": 0.123928003013134 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-38e2ce675ff3591d7c68", "text": "The preferred model host for TensorCode is nugging Face .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.632706642150879, "probability": 0.3402654528617859, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.05651784688234329, "support": 0.8642567992210388, "unknown": 0.07922527939081192 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" } ], "retrieval": [], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 3, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "The preferred model host for TensorCode is nugging Face .", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [ "developer-guide" ], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": false, "realization_verifications": [ { "source_id": "developer-guide", "distribution": { "contradiction": 0.05651784688234329, "support": 0.8642567992210388, "unknown": 0.07922527939081192 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "new_episode": { "text": "The preferred model host for TensorCode is nugging Face .", "receipt": { "text": "The preferred model host for TensorCode is nugging Face .", "cognition": { "question": "Which service is the preferred model host for TensorCode?", "selected_id": "hypothesis-38e2ce675ff3591d7c68", "abstained": false, "candidates": [ { "id": "hypothesis-6e4793c7544ee2a77d80", "text": "The preferred model host for TensorCode is nHugging Face .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.708287239074707, "probability": 0.3669797480106354, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.20080351829528809, "support": 0.13878197968006134, "unknown": 0.6604145169258118 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-815fb6c0f41f8392d24b", "text": "TensorCode is the preferred model host for TensorCode .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.482316017150879, "probability": 0.29275479912757874, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.8401834964752197, "support": 0.03588845580816269, "unknown": 0.123928003013134 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-38e2ce675ff3591d7c68", "text": "The preferred model host for TensorCode is nugging Face .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.632706642150879, "probability": 0.3402654528617859, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.05651784688234329, "support": 0.8642567992210388, "unknown": 0.07922527939081192 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" } ], "retrieval": [ { "evidence": { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" }, "score": 0.6029560441511032, "episode_id": "episode-0", "question": "Which service is the preferred model host for TensorCode?", "outcome": "" } ], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 6, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "The preferred model host for TensorCode is nugging Face .", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": false, "realization_verifications": [ { "source_id": "developer-guide", "distribution": { "contradiction": 0.05651784688234329, "support": 0.8642567992210388, "unknown": 0.07922527939081192 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "save_load_snapshot_exact": true, "reloaded_new_episode": { "text": "The preferred model host for TensorCode is nugging Face .", "receipt": { "text": "The preferred model host for TensorCode is nugging Face .", "cognition": { "question": "Which service is the preferred model host for TensorCode?", "selected_id": "hypothesis-38e2ce675ff3591d7c68", "abstained": false, "candidates": [ { "id": "hypothesis-6e4793c7544ee2a77d80", "text": "The preferred model host for TensorCode is nHugging Face .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.708287239074707, "probability": 0.3669797480106354, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.20080351829528809, "support": 0.13878197968006134, "unknown": 0.6604145169258118 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-815fb6c0f41f8392d24b", "text": "TensorCode is the preferred model host for TensorCode .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.482316017150879, "probability": 0.29275479912757874, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.8401834964752197, "support": 0.03588845580816269, "unknown": 0.123928003013134 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": false, "verification_coverage": "complete supplied pairs" }, { "id": "hypothesis-38e2ce675ff3591d7c68", "text": "The preferred model host for TensorCode is nugging Face .", "origin": "generated", "generated_by": "4e95373d89416a65c0240a32475d433f210e22125ef941552ac70cfde26a6385", "proposal_template_version": 1, "generator_identity_kind": "configuration_and_prompt_template_fingerprint", "generator_configuration_fingerprint": "e5340d020cc490fea88df2a3a68ae408e3fcd77a964fc818dbd89245fc931084", "generator_foundation": { "repository": "../artifacts/evaluate-cognition/flan-small-foundation", "revision": "0fc9ddf78a1e988dac52e2dac162b0ede4fd74ab", "workspace_initialization": "random" }, "epistemic_status": "hypothesis", "source_ids": [ "developer-guide" ], "source_reference_kind": "generation_context", "input_truncated": false, "predicted_score": 4.632706642150879, "probability": 0.3402654528617859, "verifications": [ { "source_id": "docs/pretrained.md", "distribution": { "contradiction": 0.05651784688234329, "support": 0.8642567992210388, "unknown": 0.07922527939081192 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false, "evidence_id": "developer-guide" } ], "accepted_by_policy": true, "verification_coverage": "complete supplied pairs" } ], "evidence": [ { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" } ], "retrieval": [ { "evidence": { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" }, "score": 0.6029560441511032, "episode_id": "episode-0", "question": "Which service is the preferred model host for TensorCode?", "outcome": "" } ], "retrieval_encoder": { "encoder": "owned_retrieval_encoder", "pooling": "masked_mean", "normalized": true, "max_tokens": 256, "dimensions": 384, "foundation": { "pooling_contract": "caller_declared_masked_mean_l2", "repository": "../artifacts/minilm-foundation", "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "weights": "loaded_foundation" }, "compatibility": "requires a foundation trained/documented for masked-mean pooling followed by L2 normalization", "semantics": "embedding proximity, not truth or calibrated evidence support" }, "state_revision": 9, "policy": { "min_support": 0.7, "max_contradiction": 0.2, "max_unknown": 0.3, "origin": "authored_policy", "aggregation": "strongest-source support/unknown; maximum contradiction across current sources", "semantics": "model-score screening, not established truth; source trust supplied by caller" }, "verification_scope": "source", "model_provenance": "14692c02f2ca820f3332587a355117c9aa64f76079de3a2aab41c6a1cad8b2e6", "semantics": "Generated and supplied candidates remain hypotheses; selection is authored screening of model scores, not truth." }, "conversation_context": [], "conversation_context_truncated": false, "response_proposal": { "text": "The preferred model host for TensorCode is nugging Face .", "origin": "model_generation", "epistemic_status": "unverified_proposal" }, "retained_evidence_ids": [], "retention_policy": "authored: retain explicitly supplied active sources after successful turn", "abstention_enforced": false, "realization_verifications": [ { "source_id": "developer-guide", "distribution": { "contradiction": 0.05651784688234329, "support": 0.8642567992210388, "unknown": 0.07922527939081192 }, "origin": "model_inference", "calibrated": true, "calibration_sample_count": 256, "model": { "repository": "cross-encoder/nli-deberta-v3-small", "revision": "fa2804872c3b4bd748f38c0185cc85775361e735" }, "input_truncated": false } ], "verification_scope": "source", "realization_joint_verification": null, "full_realization_joint_verification": null, "realization_sources": [ { "id": "developer-guide", "text": "Install `tensorcode[tools]` as shown in the [quickstart](quickstart.md).\nHugging Face is the preferred host. Tools construct their own encoders, workspace\nand prediction/decoding components; callers do not supply a separate model.", "source_id": "docs/pretrained.md" } ], "source_truncation": [], "realization_semantics": "Authored screening of model NLI scores; not a factual guarantee", "capabilities": { "language_realization": true, "persistent_cognitive_state": true, "hypothesis_generation": true, "source_verification": true, "abstention_policy": "authored" }, "input_truncated": false } }, "limitations": "One actual document; source retrieval and persistence do not establish answer correctness." }, "data_manifest": { "dataset": "hotpotqa/hotpot_qa", "revision": "1908d6afbbead072334abe2965f91bd2709910ab", "file": "distractor/validation-00000-of-00001.parquet", "sha256": "c20b638ca82b21d04fe12e14ff417ad05153d4d215a65de54497fca4e972f7c6", "offset": 272, "count": 32, "ids": [ "5ae048a255429924de1b708e", "5a9068a95542990a98493621", "5adf9dc3554299025d62a2e6", "5ae406245542995dadf242e2", "5ab6369655429953192ad2a1", "5ae628f55542995703ce8b32", "5adccdd55542992c1e3a2408", "5abd1529554299114383a1e3", "5a78db8055429970f5fffdb2", "5ab9debb55429955dce3ed97", "5ae791ef55429952e35ea979", "5ab8f7535542991b5579f0a7", "5ac119335542992a796dede4", "5a7d2afd5542995f4f402221", "5a8dd5f955429917b4a5bcc7", "5a83a532554299334474606f", "5a90692f5542990a98493625", "5a7555215542996c70cfaee1", "5a90580655429933b8a204ef", "5a88e605554299206df2b39c", "5add67915542992200553af8", "5a8463545542990548d0b2a3", "5a74684655429929fddd8410", "5a835c9f5542992ef85e228a", "5ae7edee554299540e5a56ad", "5a8ee4315542990e94052ba7", "5a8cb288554299585d9e3726", "5adfbca255429942ec259b2c", "5ae24abc5542994d89d5b3c3", "5ab802085542990e739ec7ca", "5adfa3da554299025d62a317", "5a76e88555429972597f13f4" ], "retrieval": "oracle supporting passages", "role": "final, fixed before this run; disjoint this adaptation and diagnostic subsets, not a claim of unknown foundation pretraining exposure" }, "protocol": { "primary_count": 32, "control_count": 8, "control_selection": "first fixed cases in input order" }, "model_fingerprint": "591c06bed994fd6c69e0679beae24480c93b85058066c627a285d0791379717a", "foundation": { "repository": "../artifacts/evaluate-cognition/flan-base-foundation", "revision": "7bcac572ce56db69c1ea7c8af255c5d7c9672fc2", "workspace_initialization": "random" }, "evaluation_role": "fixed-configuration final evaluation; do not tune on these results", "cases_sha256": "380aebdc9ec1a17134ec3e83b2b0f6b40e97ee3902e68b1b31593dc3786a35db", "manual_factual_review": { "status": "pending", "scope": "Review every non-abstained primary response against the complete source passages and gold answer; containment is not correctness." }, "limitations": [ "Exact short-answer EM is format-mismatched for declarative responses and must not be presented as factual accuracy.", "Oracle supporting passages, not learned retrieval.", "Lexical answer coverage and fidelity diagnostics do not establish semantic correctness; realization NLI uses the same separately calibrated SNLI verifier and is an additional model judgment.", "NLI calibration is inherited from a separate SNLI calibration split, not evidence-QA calibration.", "Authored threshold policy screens model scores; acceptance is not proof of truth.", "Foundation generation workspace is unadapted unless separately documented." ] }