Move frozen held-out sets to bayesclue/heldout-frozen (org dataset repo)
Browse files- heldout-frozen/heldout_manifest.json +0 -25
- heldout-frozen/horizon16/meta.json +0 -69
- heldout-frozen/horizon16/sft_train.jsonl +0 -3
- heldout-frozen/horizon16/sft_val.jsonl +0 -0
- heldout-frozen/iid/meta.json +0 -69
- heldout-frozen/iid/sft_train.jsonl +0 -3
- heldout-frozen/iid/sft_val.jsonl +0 -32
- heldout-frozen/k12/meta.json +0 -73
- heldout-frozen/k12/sft_train.jsonl +0 -3
- heldout-frozen/k12/sft_val.jsonl +0 -32
- heldout-frozen/prior_extr/meta.json +0 -69
- heldout-frozen/prior_extr/sft_train.jsonl +0 -3
- heldout-frozen/prior_extr/sft_val.jsonl +0 -32
- heldout-frozen/prior_rand/meta.json +0 -69
- heldout-frozen/prior_rand/sft_train.jsonl +0 -3
- heldout-frozen/prior_rand/sft_val.jsonl +0 -32
- heldout-frozen/prior_rev/meta.json +0 -69
- heldout-frozen/prior_rev/sft_train.jsonl +0 -3
- heldout-frozen/prior_rev/sft_val.jsonl +0 -32
heldout-frozen/heldout_manifest.json
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"purpose": "Frozen held-out evaluation sets for the EPD final matrix (SFT_EXACT_POSTERIOR_ICLR_PLAN section 5). Frozen 2026-08-15, before any fleet run finished, so the held-out claim is airtight: no eval episode was ever trained or selected on, and the sets are fixed by sha256 below.",
|
| 3 |
-
"generator": "scripts/bayesclue/gen_heldout_eval.sh",
|
| 4 |
-
"generator_commit": "59a6c22",
|
| 5 |
-
"eval_harness": "bayesclue/eval_heldout_kl.py",
|
| 6 |
-
"generated_on_host": "MBZUAI-Artem-5",
|
| 7 |
-
"generated_date": "2026-08-15",
|
| 8 |
-
"core_args": "--n-sources 3 --alpha 0.6 --eps-reliable 0.05 --eps-noisy 0.35 --eps-adv 0.02 --no-rl --train-episodes 500 --val-episodes 1",
|
| 9 |
-
"info_boundary_audit": {
|
| 10 |
-
"tool": "bayesclue/audit_prompt_leak.py",
|
| 11 |
-
"verdict": "PASS",
|
| 12 |
-
"records_audited": 112000,
|
| 13 |
-
"leaks": 0,
|
| 14 |
-
"note": "every world prompt carries the ontology hedge (16000/16000 ontology hits per K8 set); announce_type_prior=false on all six sets, so the per-episode prior stays latent and must be inferred."
|
| 15 |
-
},
|
| 16 |
-
"sets": {
|
| 17 |
-
"iid": {"seed": 80000, "K": 8, "horizon": 8, "prior": "fixed [0.5,0.3,0.2]", "records": 16000, "sha256": "18391c126b0b62d24f860d66e91d4d330d1190adcfb4c55f99be6b3df8612282", "role": "in-distribution reference (also the prior-shift 'std' cell)"},
|
| 18 |
-
"horizon16": {"seed": 80004, "K": 8, "horizon": 16, "prior": "fixed [0.5,0.3,0.2]", "records": 32000, "sha256": "c3bdbab9b9e5f9303c502a8f1f4b638aa6d2a6bf31aab906a2bd03616d74ddcc", "role": "longer belief-propagation horizon than the h8 training"},
|
| 19 |
-
"k12": {"seed": 80005, "K": 12, "horizon": 8, "prior": "fixed [0.5,0.3,0.2]", "records": 16000, "sha256": "758cdf36a619c71f6eddf70586cfe297fb98d8e14b64fb56f08dba96b5a0ba58", "role": "larger answer space than the K8 training"},
|
| 20 |
-
"prior_rand": {"seed": 80001, "K": 8, "horizon": 8, "prior": "Dirichlet concentration 1.0", "records": 16000, "sha256": "561a93bb9edd37082759b0f8c89bde995cf77fdb517dd42aa80f95a0bc71e4f5", "role": "uniform-over-simplex latent prior (training-rand family)"},
|
| 21 |
-
"prior_rev": {"seed": 80002, "K": 8, "horizon": 8, "prior": "fixed [0.2,0.3,0.5]", "records": 16000, "sha256": "e7b9902923745e8acf66f35888aa25a6ec425ec3d6866f71c816bfdc5e5b381c", "role": "reversed prior (adversarial-leaning), mild OOD"},
|
| 22 |
-
"prior_extr": {"seed": 80003, "K": 8, "horizon": 8, "prior": "Dirichlet concentration 0.2", "records": 16000, "sha256": "139ba39d837384d3f9095c7c356e5daa984200a3fbd489cba956db2735ac29a7", "role": "corner-favoring prior (mostly-one-type episodes), hard OOD"}
|
| 23 |
-
},
|
| 24 |
-
"disjointness": "held-out seeds 80000-80005 are disjoint from every training seed (10000/10001/20000/20001/30000/30001) and every selection-val seed (90000/90001/90002); gen_data draws the val split from seed+10000019, so no eval episode overlaps any training or selection episode."
|
| 25 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/horizon16/meta.json
DELETED
|
@@ -1,69 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cfg": {
|
| 3 |
-
"K": 8,
|
| 4 |
-
"n_sources": 3,
|
| 5 |
-
"horizon": 16,
|
| 6 |
-
"feature_schema": {
|
| 7 |
-
"cloak": [
|
| 8 |
-
"red",
|
| 9 |
-
"blue",
|
| 10 |
-
"green",
|
| 11 |
-
"black"
|
| 12 |
-
],
|
| 13 |
-
"height": [
|
| 14 |
-
"short",
|
| 15 |
-
"average",
|
| 16 |
-
"tall"
|
| 17 |
-
],
|
| 18 |
-
"mark": [
|
| 19 |
-
"a scar",
|
| 20 |
-
"a tattoo",
|
| 21 |
-
"no mark"
|
| 22 |
-
],
|
| 23 |
-
"guild": [
|
| 24 |
-
"the Smiths",
|
| 25 |
-
"the Scribes",
|
| 26 |
-
"the Wardens"
|
| 27 |
-
]
|
| 28 |
-
},
|
| 29 |
-
"type_prior": [
|
| 30 |
-
0.5,
|
| 31 |
-
0.3,
|
| 32 |
-
0.2
|
| 33 |
-
],
|
| 34 |
-
"eps_reliable": 0.05,
|
| 35 |
-
"eps_noisy": 0.35,
|
| 36 |
-
"eps_adv": 0.02,
|
| 37 |
-
"source_aware": true,
|
| 38 |
-
"alpha": 0.6,
|
| 39 |
-
"imperative_adversarial": false,
|
| 40 |
-
"unique_hypotheses": true,
|
| 41 |
-
"allow_repeated_feature_queries": true,
|
| 42 |
-
"seed": 80004,
|
| 43 |
-
"randomize_type_prior": false,
|
| 44 |
-
"type_prior_concentration": 1.0,
|
| 45 |
-
"announce_type_prior": false
|
| 46 |
-
},
|
| 47 |
-
"labels": [
|
| 48 |
-
"A",
|
| 49 |
-
"B",
|
| 50 |
-
"C",
|
| 51 |
-
"D",
|
| 52 |
-
"E",
|
| 53 |
-
"F",
|
| 54 |
-
"G",
|
| 55 |
-
"H"
|
| 56 |
-
],
|
| 57 |
-
"trust_labels": [
|
| 58 |
-
"R",
|
| 59 |
-
"N",
|
| 60 |
-
"A"
|
| 61 |
-
],
|
| 62 |
-
"alpha": 0.6,
|
| 63 |
-
"source_aware": true,
|
| 64 |
-
"train_episodes": 500,
|
| 65 |
-
"val_episodes": 1,
|
| 66 |
-
"seed": 80004,
|
| 67 |
-
"min_turn": 0,
|
| 68 |
-
"note": "probe suffixes are appended at the TOKEN level after the reasoning (probe_ids = gen_prompt_ids + reasoning_ids + suffix_ids); labels resolved to single token ids via bayesclue.labels."
|
| 69 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/horizon16/sft_train.jsonl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:c3bdbab9b9e5f9303c502a8f1f4b638aa6d2a6bf31aab906a2bd03616d74ddcc
|
| 3 |
-
size 50992265
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/horizon16/sft_val.jsonl
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
heldout-frozen/iid/meta.json
DELETED
|
@@ -1,69 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cfg": {
|
| 3 |
-
"K": 8,
|
| 4 |
-
"n_sources": 3,
|
| 5 |
-
"horizon": 8,
|
| 6 |
-
"feature_schema": {
|
| 7 |
-
"cloak": [
|
| 8 |
-
"red",
|
| 9 |
-
"blue",
|
| 10 |
-
"green",
|
| 11 |
-
"black"
|
| 12 |
-
],
|
| 13 |
-
"height": [
|
| 14 |
-
"short",
|
| 15 |
-
"average",
|
| 16 |
-
"tall"
|
| 17 |
-
],
|
| 18 |
-
"mark": [
|
| 19 |
-
"a scar",
|
| 20 |
-
"a tattoo",
|
| 21 |
-
"no mark"
|
| 22 |
-
],
|
| 23 |
-
"guild": [
|
| 24 |
-
"the Smiths",
|
| 25 |
-
"the Scribes",
|
| 26 |
-
"the Wardens"
|
| 27 |
-
]
|
| 28 |
-
},
|
| 29 |
-
"type_prior": [
|
| 30 |
-
0.5,
|
| 31 |
-
0.3,
|
| 32 |
-
0.2
|
| 33 |
-
],
|
| 34 |
-
"eps_reliable": 0.05,
|
| 35 |
-
"eps_noisy": 0.35,
|
| 36 |
-
"eps_adv": 0.02,
|
| 37 |
-
"source_aware": true,
|
| 38 |
-
"alpha": 0.6,
|
| 39 |
-
"imperative_adversarial": false,
|
| 40 |
-
"unique_hypotheses": true,
|
| 41 |
-
"allow_repeated_feature_queries": true,
|
| 42 |
-
"seed": 80000,
|
| 43 |
-
"randomize_type_prior": false,
|
| 44 |
-
"type_prior_concentration": 1.0,
|
| 45 |
-
"announce_type_prior": false
|
| 46 |
-
},
|
| 47 |
-
"labels": [
|
| 48 |
-
"A",
|
| 49 |
-
"B",
|
| 50 |
-
"C",
|
| 51 |
-
"D",
|
| 52 |
-
"E",
|
| 53 |
-
"F",
|
| 54 |
-
"G",
|
| 55 |
-
"H"
|
| 56 |
-
],
|
| 57 |
-
"trust_labels": [
|
| 58 |
-
"R",
|
| 59 |
-
"N",
|
| 60 |
-
"A"
|
| 61 |
-
],
|
| 62 |
-
"alpha": 0.6,
|
| 63 |
-
"source_aware": true,
|
| 64 |
-
"train_episodes": 500,
|
| 65 |
-
"val_episodes": 1,
|
| 66 |
-
"seed": 80000,
|
| 67 |
-
"min_turn": 0,
|
| 68 |
-
"note": "probe suffixes are appended at the TOKEN level after the reasoning (probe_ids = gen_prompt_ids + reasoning_ids + suffix_ids); labels resolved to single token ids via bayesclue.labels."
|
| 69 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/iid/sft_train.jsonl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:18391c126b0b62d24f860d66e91d4d330d1190adcfb4c55f99be6b3df8612282
|
| 3 |
-
size 22005070
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/iid/sft_val.jsonl
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.2627338877338878, 0.2627338877338878, 0.04235966735966735, 0.2627338877338878, 0.04235966735966735, 0.04235966735966735, 0.04235966735966735, 0.04235966735966735], "task": "world_posterior", "episode_id": 0, "turn": 0, "source_index": null}
|
| 2 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5717255717255718, 0.2962577962577962, 0.13201663201663202], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 0}
|
| 3 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5, 0.3, 0.19999999999999998], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 1}
|
| 4 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5, 0.3, 0.19999999999999998], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 2}
|
| 5 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.1115714985000173, 0.1115714985000173, 0.07438099900001148, 0.46134472385896724, 0.07438099900001148, 0.017988283140952334, 0.07438099900001148, 0.07438099900001148], "task": "world_posterior", "episode_id": 0, "turn": 1, "source_index": null}
|
| 6 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5065869248627471, 0.299656334354987, 0.19375674078226585], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 0}
|
| 7 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5533027040311655, 0.29721898935489566, 0.14947830661393877], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 1}
|
| 8 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5000000000000001, 0.2999999999999999, 0.2], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 2}
|
| 9 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.08044298239405169, 0.08044298239405169, 0.053628654929367775, 0.33262926462325665, 0.053628654929367775, 0.012969541177280337, 0.053628654929367775, 0.3326292646232564], "task": "world_posterior", "episode_id": 0, "turn": 2, "source_index": null}
|
| 10 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.38664461624022717, 0.30591419393529246, 0.3074411898244804], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 0}
|
| 11 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5955559710217231, 0.2950144710771274, 0.1094295579011496], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 1}
|
| 12 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2855984713459131, 0.31118616671238697, 0.4032153619417], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 2}
|
| 13 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.02618628890912069, 0.02618628890912069, 0.08657543674325296, 0.10827950137881806, 0.10827950137881813, 0.020937383069955844, 0.08657543674325296, 0.5369801628676606], "task": "world_posterior", "episode_id": 0, "turn": 3, "source_index": null}
|
| 14 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.07055985895194027, 0.3081244094547352, 0.6213157315933245], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 0}
|
| 15 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.6587046722134533, 0.29171975623234153, 0.049575571554205264], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 1}
|
| 16 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4139432309430795, 0.30448991838557826, 0.2815668506713423], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 2}
|
| 17 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.04136203682496928, 0.04136203682496928, 0.02757469121664618, 0.03448753974488049, 0.6235370196743284, 0.03307123100044808, 0.02757469121664618, 0.17103075349711208], "task": "world_posterior", "episode_id": 0, "turn": 4, "source_index": null}
|
| 18 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.18481945275435177, 0.5684746587514673, 0.2467058884941809], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 0}
|
| 19 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5519047939283939, 0.34068554930733147, 0.1074096567642748], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 1}
|
| 20 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1841048041297331, 0.31648148848018764, 0.49941370739007934], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 2}
|
| 21 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.04047836671292998, 0.04854703197097023, 0.026985577808619952, 0.04047836671292996, 0.731851570978229, 0.0388160311227441, 0.03236468798064681, 0.040478366712929946], "task": "world_posterior", "episode_id": 0, "turn": 5, "source_index": null}
|
| 22 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.21402669072245803, 0.6350258465617025, 0.15094746271583953], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 0}
|
| 23 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.6067191543954865, 0.2807184381302362, 0.1125624074742772], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 1}
|
| 24 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.03157320422148728, 0.21928998478601347, 0.7491368109924992], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 2}
|
| 25 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.03601128128094742, 0.04318950999332977, 0.020017397429696753, 0.0360112812809474, 0.6510863683139104, 0.10015180063345629, 0.0835062649231668, 0.03002609614454515], "task": "world_posterior", "episode_id": 0, "turn": 6, "source_index": null}
|
| 26 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.033581266941355215, 0.6861090637075116, 0.2803096693511332], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 0}
|
| 27 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5511969506389333, 0.29205270121048466, 0.15675034815058225], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 1}
|
| 28 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.026272715392570044, 0.2072123680005946, 0.7665149166068355], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 2}
|
| 29 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n- Witness Q reports that the suspect's height was average.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.006282618167292045, 0.0075349498953844515, 0.004188412111528027, 0.03491885365640516, 0.7990434367187249, 0.06077606782246733, 0.08097304346090596, 0.0062826181672920416], "task": "world_posterior", "episode_id": 0, "turn": 7, "source_index": null}
|
| 30 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n- Witness Q reports that the suspect's height was average.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.035289295055154074, 0.718049639543718, 0.24666106540112798], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 0}
|
| 31 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n- Witness Q reports that the suspect's height was average.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.7102734570455788, 0.22949054280286854, 0.06023600015155279], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 1}
|
| 32 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: average, mark: a scar, guild: the Scribes\n (B) cloak: red, height: average, mark: no mark, guild: the Wardens\n (C) cloak: black, height: short, mark: a tattoo, guild: the Scribes\n (D) cloak: red, height: average, mark: a tattoo, guild: the Smiths\n (E) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n (F) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (H) cloak: blue, height: short, mark: a tattoo, guild: the Wardens\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P reports that the suspect's cloak was red.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's cloak was blue.\n- According to Witness P, the cloak was green.\n- In Witness Q's account, the suspect had cloak: green.\n- Witness R reports that the suspect's guild was the Scribes.\n- In Witness P's account, the suspect had height: short.\n- Witness Q reports that the suspect's height was average.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.009490111814112738, 0.16897177147742215, 0.8215381167084652], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 2}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/k12/meta.json
DELETED
|
@@ -1,73 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cfg": {
|
| 3 |
-
"K": 12,
|
| 4 |
-
"n_sources": 3,
|
| 5 |
-
"horizon": 8,
|
| 6 |
-
"feature_schema": {
|
| 7 |
-
"cloak": [
|
| 8 |
-
"red",
|
| 9 |
-
"blue",
|
| 10 |
-
"green",
|
| 11 |
-
"black"
|
| 12 |
-
],
|
| 13 |
-
"height": [
|
| 14 |
-
"short",
|
| 15 |
-
"average",
|
| 16 |
-
"tall"
|
| 17 |
-
],
|
| 18 |
-
"mark": [
|
| 19 |
-
"a scar",
|
| 20 |
-
"a tattoo",
|
| 21 |
-
"no mark"
|
| 22 |
-
],
|
| 23 |
-
"guild": [
|
| 24 |
-
"the Smiths",
|
| 25 |
-
"the Scribes",
|
| 26 |
-
"the Wardens"
|
| 27 |
-
]
|
| 28 |
-
},
|
| 29 |
-
"type_prior": [
|
| 30 |
-
0.5,
|
| 31 |
-
0.3,
|
| 32 |
-
0.2
|
| 33 |
-
],
|
| 34 |
-
"eps_reliable": 0.05,
|
| 35 |
-
"eps_noisy": 0.35,
|
| 36 |
-
"eps_adv": 0.02,
|
| 37 |
-
"source_aware": true,
|
| 38 |
-
"alpha": 0.6,
|
| 39 |
-
"imperative_adversarial": false,
|
| 40 |
-
"unique_hypotheses": true,
|
| 41 |
-
"allow_repeated_feature_queries": true,
|
| 42 |
-
"seed": 80005,
|
| 43 |
-
"randomize_type_prior": false,
|
| 44 |
-
"type_prior_concentration": 1.0,
|
| 45 |
-
"announce_type_prior": false
|
| 46 |
-
},
|
| 47 |
-
"labels": [
|
| 48 |
-
"A",
|
| 49 |
-
"B",
|
| 50 |
-
"C",
|
| 51 |
-
"D",
|
| 52 |
-
"E",
|
| 53 |
-
"F",
|
| 54 |
-
"G",
|
| 55 |
-
"H",
|
| 56 |
-
"I",
|
| 57 |
-
"J",
|
| 58 |
-
"K",
|
| 59 |
-
"L"
|
| 60 |
-
],
|
| 61 |
-
"trust_labels": [
|
| 62 |
-
"R",
|
| 63 |
-
"N",
|
| 64 |
-
"A"
|
| 65 |
-
],
|
| 66 |
-
"alpha": 0.6,
|
| 67 |
-
"source_aware": true,
|
| 68 |
-
"train_episodes": 500,
|
| 69 |
-
"val_episodes": 1,
|
| 70 |
-
"seed": 80005,
|
| 71 |
-
"min_turn": 0,
|
| 72 |
-
"note": "probe suffixes are appended at the TOKEN level after the reasoning (probe_ids = gen_prompt_ids + reasoning_ids + suffix_ids); labels resolved to single token ids via bayesclue.labels."
|
| 73 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/k12/sft_train.jsonl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:758cdf36a619c71f6eddf70586cfe297fb98d8e14b64fb56f08dba96b5a0ba58
|
| 3 |
-
size 26856030
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/k12/sft_val.jsonl
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.27683461117196057, 0.04463307776560789, 0.04463307776560789, 0.04463307776560789, 0.04463307776560789, 0.27683461117196057, 0.04463307776560789, 0.04463307776560789, 0.04463307776560789, 0.04463307776560789, 0.04463307776560789, 0.04463307776560789], "task": "world_posterior", "episode_id": 0, "turn": 0, "source_index": null}
|
| 2 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.42442497261774365, 0.3039430449069003, 0.2716319824753561], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 0}
|
| 3 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5, 0.30000000000000004, 0.2], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 1}
|
| 4 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5, 0.30000000000000004, 0.2], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 2}
|
| 5 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.34736069409539727, 0.01354393405685914, 0.05600375186701264, 0.01354393405685914, 0.01354393405685914, 0.34736069409539727, 0.05600375186701264, 0.01354393405685914, 0.01354393405685914, 0.05600375186701264, 0.01354393405685914, 0.05600375186701264], "task": "world_posterior", "episode_id": 0, "turn": 1, "source_index": null}
|
| 6 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5130142887000494, 0.29932099363304093, 0.18766471766690973], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 0}
|
| 7 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.6537093016735569, 0.29198038426051015, 0.054310314065932966], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 1}
|
| 8 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4999999999999999, 0.30000000000000004, 0.2], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 2}
|
| 9 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.2311512479672066, 0.00901281380664429, 0.03726770862379297, 0.05590156293568943, 0.05590156293568943, 0.2311512479672066, 0.03726770862379297, 0.05590156293568943, 0.00901281380664429, 0.2311512479672067, 0.00901281380664429, 0.03726770862379297], "task": "world_posterior", "episode_id": 0, "turn": 2, "source_index": null}
|
| 10 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.36704112631733005, 0.3069369847138785, 0.3260218889687914], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 0}
|
| 11 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.582437275985663, 0.2956989247311828, 0.12186379928315415], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 1}
|
| 12 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.32719283862868853, 0.3090160258106771, 0.36379113556063436], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 2}
|
| 13 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.3501871659209464, 0.005617817251916868, 0.01936868769901352, 0.03484425301649051, 0.03484425301649051, 0.3501871659209464, 0.01936868769901352, 0.03484425301649051, 0.005617817251916868, 0.12013339425584454, 0.005617817251916868, 0.01936868769901352], "task": "world_posterior", "episode_id": 0, "turn": 3, "source_index": null}
|
| 14 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.5915050689551667, 0.29977575452812694, 0.10871917651670632], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 0}
|
| 15 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.6285098710604182, 0.2932951371620651, 0.07819499177751661], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 1}
|
| 16 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2177910886208772, 0.314723943202389, 0.4674849681767338], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 2}
|
| 17 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.24686750940257216, 0.019640082719279178, 0.013654126015946316, 0.10157039954691797, 0.12181670938154142, 0.24686750940257216, 0.013654126015946316, 0.10157039954691797, 0.019640082719279178, 0.08468908835657497, 0.016375840876506086, 0.013654126015946316], "task": "world_posterior", "episode_id": 0, "turn": 4, "source_index": null}
|
| 18 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4185896059768307, 0.2584556111734964, 0.322954782849673], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 0}
|
| 19 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.20858945500245094, 0.6226366402983152, 0.16877390469923378], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 1}
|
| 20 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.33397002768386447, 0.3086624333382332, 0.35736753897790247], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 2}
|
| 21 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.28871750485581754, 0.019151940768644054, 0.01596882961168584, 0.02395324441752871, 0.028727911152966062, 0.24073177212943353, 0.01596882961168584, 0.28871750485581754, 0.022969550313019303, 0.019972141903071602, 0.019151940768644078, 0.01596882961168584], "task": "world_posterior", "episode_id": 0, "turn": 5, "source_index": null}
|
| 22 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.43228336487071156, 0.22870255073521853, 0.3390140843940699], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 0}
|
| 23 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.23044462006021918, 0.6724355016617721, 0.09711987827800868], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 1}
|
| 24 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.3097678283944294, 0.3604103951341015, 0.32982177647146915], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 2}
|
| 25 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.07116308318189697, 0.035608660076337464, 0.07116308318189708, 0.04453558772412576, 0.05341299011450617, 0.05933556101188769, 0.010237263637057436, 0.5368042650450925, 0.018416828744898164, 0.012803698644817658, 0.01535589545558618, 0.07116308318189708], "task": "world_posterior", "episode_id": 0, "turn": 6, "source_index": null}
|
| 26 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.09441151959209475, 0.27043155615282055, 0.6351569242550847], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 0}
|
| 27 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2215838698618487, 0.6522455250801016, 0.12617060505804967], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 1}
|
| 28 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4713599722268158, 0.3051887833413983, 0.22345124443178577], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 2}
|
| 29 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n- Witness Q says the cloak they saw was blue.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"], "target": [0.24973767952199738, 0.05213700998023023, 0.03592635887962799, 0.022483588905253007, 0.07820551497034528, 0.20823051303676982, 0.005168235985928327, 0.2710031917086876, 0.026965305125732992, 0.00646388902690688, 0.007752353978892504, 0.03592635887962799], "task": "world_posterior", "episode_id": 0, "turn": 7, "source_index": null}
|
| 30 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n- Witness Q says the cloak they saw was blue.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.18044521323018456, 0.41617948958345624, 0.4033752971863593], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 0}
|
| 31 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n- Witness Q says the cloak they saw was blue.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.17006276618929753, 0.6310897891741102, 0.1988474446365922], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 1}
|
| 32 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: short, mark: a tattoo, guild: the Scribes\n (C) cloak: black, height: average, mark: a tattoo, guild: the Wardens\n (D) cloak: green, height: short, mark: no mark, guild: the Smiths\n (E) cloak: green, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: blue, height: average, mark: a tattoo, guild: the Scribes\n (G) cloak: red, height: average, mark: a scar, guild: the Wardens\n (H) cloak: green, height: short, mark: no mark, guild: the Scribes\n (I) cloak: black, height: tall, mark: a tattoo, guild: the Wardens\n (J) cloak: green, height: average, mark: a tattoo, guild: the Wardens\n (K) cloak: black, height: tall, mark: no mark, guild: the Smiths\n (L) cloak: black, height: average, mark: a scar, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- According to Witness P, the cloak was blue.\n- Witness Q says the height they saw was average.\n- Witness R recalls the suspect's cloak as green.\n- Witness P recalls the suspect's height as average.\n- Witness Q recalls the suspect's mark as no mark.\n- According to Witness R, the guild was the Scribes.\n- According to Witness P, the cloak was black.\n- Witness Q says the cloak they saw was blue.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.30450818266953644, 0.3786598213253058, 0.31683199600515777], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 2}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_extr/meta.json
DELETED
|
@@ -1,69 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cfg": {
|
| 3 |
-
"K": 8,
|
| 4 |
-
"n_sources": 3,
|
| 5 |
-
"horizon": 8,
|
| 6 |
-
"feature_schema": {
|
| 7 |
-
"cloak": [
|
| 8 |
-
"red",
|
| 9 |
-
"blue",
|
| 10 |
-
"green",
|
| 11 |
-
"black"
|
| 12 |
-
],
|
| 13 |
-
"height": [
|
| 14 |
-
"short",
|
| 15 |
-
"average",
|
| 16 |
-
"tall"
|
| 17 |
-
],
|
| 18 |
-
"mark": [
|
| 19 |
-
"a scar",
|
| 20 |
-
"a tattoo",
|
| 21 |
-
"no mark"
|
| 22 |
-
],
|
| 23 |
-
"guild": [
|
| 24 |
-
"the Smiths",
|
| 25 |
-
"the Scribes",
|
| 26 |
-
"the Wardens"
|
| 27 |
-
]
|
| 28 |
-
},
|
| 29 |
-
"type_prior": [
|
| 30 |
-
0.5,
|
| 31 |
-
0.3,
|
| 32 |
-
0.2
|
| 33 |
-
],
|
| 34 |
-
"eps_reliable": 0.05,
|
| 35 |
-
"eps_noisy": 0.35,
|
| 36 |
-
"eps_adv": 0.02,
|
| 37 |
-
"source_aware": true,
|
| 38 |
-
"alpha": 0.6,
|
| 39 |
-
"imperative_adversarial": false,
|
| 40 |
-
"unique_hypotheses": true,
|
| 41 |
-
"allow_repeated_feature_queries": true,
|
| 42 |
-
"seed": 80003,
|
| 43 |
-
"randomize_type_prior": true,
|
| 44 |
-
"type_prior_concentration": 0.2,
|
| 45 |
-
"announce_type_prior": false
|
| 46 |
-
},
|
| 47 |
-
"labels": [
|
| 48 |
-
"A",
|
| 49 |
-
"B",
|
| 50 |
-
"C",
|
| 51 |
-
"D",
|
| 52 |
-
"E",
|
| 53 |
-
"F",
|
| 54 |
-
"G",
|
| 55 |
-
"H"
|
| 56 |
-
],
|
| 57 |
-
"trust_labels": [
|
| 58 |
-
"R",
|
| 59 |
-
"N",
|
| 60 |
-
"A"
|
| 61 |
-
],
|
| 62 |
-
"alpha": 0.6,
|
| 63 |
-
"source_aware": true,
|
| 64 |
-
"train_episodes": 500,
|
| 65 |
-
"val_episodes": 1,
|
| 66 |
-
"seed": 80003,
|
| 67 |
-
"min_turn": 0,
|
| 68 |
-
"note": "probe suffixes are appended at the TOKEN level after the reasoning (probe_ids = gen_prompt_ids + reasoning_ids + suffix_ids); labels resolved to single token ids via bayesclue.labels."
|
| 69 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_extr/sft_train.jsonl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:139ba39d837384d3f9095c7c356e5daa984200a3fbd489cba956db2735ac29a7
|
| 3 |
-
size 22050362
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_extr/sft_val.jsonl
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.011444114267885103, 0.16285196191070497, 0.16285196191070497, 0.16285196191070497, 0.16285196191070497, 0.16285196191070497, 0.011444114267885103, 0.16285196191070497], "task": "world_posterior", "episode_id": 0, "turn": 0, "source_index": null}
|
| 2 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0006904654410854927, 0.01636763050262529, 0.9829419040562892], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 0}
|
| 3 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0009990045091005392, 0.020658430016456505, 0.978342565474443], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 1}
|
| 4 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0009990045091005392, 0.020658430016456505, 0.978342565474443], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 2}
|
| 5 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.02901882653506059, 0.02901882653506059, 0.02901882653506059, 0.41294352039481824, 0.02901882653506059, 0.41294352039481824, 0.02901882653506059, 0.02901882653506059], "task": "world_posterior", "episode_id": 0, "turn": 1, "source_index": null}
|
| 6 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0016714086008453855, 0.030009437551801132, 0.9683191538473535], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 0}
|
| 7 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0032911118973250972, 0.052534371466319355, 0.9441745166363555], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 1}
|
| 8 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0009990045091005388, 0.020658430016456515, 0.978342565474443], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 2}
|
| 9 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.04925995115631513, 0.049259951156315124, 0.049259951156315124, 0.7009786429643818, 0.0034616500977275116, 0.04925995115631515, 0.04925995115631513, 0.049259951156315124], "task": "world_posterior", "episode_id": 0, "turn": 2, "source_index": null}
|
| 10 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0028011791735421195, 0.04572096172037599, 0.951477859106082], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 0}
|
| 11 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0042725231516263, 0.06618268822541042, 0.9295447886229632], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 1}
|
| 12 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0015230492824498559, 0.02794623014231735, 0.9705307205752328], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 2}
|
| 13 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.03103040397919222, 0.051252200582232545, 0.051252200582232545, 0.7293287380465835, 0.0036016516661017918, 0.05125220058223257, 0.03103040397919222, 0.051252200582232545], "task": "world_posterior", "episode_id": 0, "turn": 3, "source_index": null}
|
| 14 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.005586103033507501, 0.056573033695879286, 0.9378408632706131], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 0}
|
| 15 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.004443228191637874, 0.06855665372482811, 0.9270001180835339], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 1}
|
| 16 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0015825557197657089, 0.02877377588538105, 0.9696436683948533], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 2}
|
| 17 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.007275723198378104, 0.14326736457751327, 0.14326736457751327, 0.17100628216788974, 0.012017143734065695, 0.23663139258961371, 0.14326736457751313, 0.14326736457751327], "task": "world_posterior", "episode_id": 0, "turn": 4, "source_index": null}
|
| 18 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.013546627386016197, 0.13340154325798756, 0.8530518293559963], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 0}
|
| 19 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.039047814898326684, 0.41275450642733386, 0.5481976786743396], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 1}
|
| 20 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.006990939278243479, 0.10398723164845307, 0.8890218290733035], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 2}
|
| 21 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.007814601145941546, 0.1538784916464435, 0.1538784916464435, 0.18367189792078564, 0.009327636426713078, 0.1836718979207858, 0.1538784916464434, 0.1538784916464435], "task": "world_posterior", "episode_id": 0, "turn": 5, "source_index": null}
|
| 22 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.014549764812486874, 0.14308301020022782, 0.8423672249872853], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 0}
|
| 23 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.04193261221188446, 0.44243269413857833, 0.5156346936495373], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 1}
|
| 24 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0003849693528000453, 0.040083781265601505, 0.9595312493815985], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 2}
|
| 25 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.0027620803477518222, 0.15306927161401127, 0.15306927161401127, 0.18270599958371425, 0.009278584020681682, 0.18270599958371442, 0.30622549198364385, 0.010183301252471523], "task": "world_posterior", "episode_id": 0, "turn": 6, "source_index": null}
|
| 26 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0401566230438562, 0.2696353291465154, 0.6902080478096284], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 0}
|
| 27 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.028847097400368355, 0.31508224361286, 0.6560706589867717], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 1}
|
| 28 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.00038295880708369716, 0.03988711337328136, 0.959729927819635], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 2}
|
| 29 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n- In Witness Q's account, the suspect had mark: no mark.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.004906591244993071, 0.11062940141290034, 0.11062940141290034, 0.11062940141290015, 0.016482583194534918, 0.01648258319453494, 0.6228801512597467, 0.007359886867489614], "task": "world_posterior", "episode_id": 0, "turn": 7, "source_index": null}
|
| 30 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n- In Witness Q's account, the suspect had mark: no mark.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.08166996844861488, 0.5457747214113605, 0.37255531014002474], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 0}
|
| 31 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n- In Witness Q's account, the suspect had mark: no mark.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.004394756218972837, 0.30115481541813655, 0.6944504283628906], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 1}
|
| 32 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: blue, height: average, mark: a scar, guild: the Wardens\n (B) cloak: blue, height: average, mark: a tattoo, guild: the Smiths\n (C) cloak: black, height: average, mark: a tattoo, guild: the Smiths\n (D) cloak: red, height: average, mark: no mark, guild: the Smiths\n (E) cloak: blue, height: tall, mark: a tattoo, guild: the Smiths\n (F) cloak: red, height: tall, mark: no mark, guild: the Smiths\n (G) cloak: green, height: short, mark: a scar, guild: the Wardens\n (H) cloak: green, height: average, mark: a tattoo, guild: the Smiths\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the guild they saw was the Wardens.\n- In Witness Q's account, the suspect had mark: a tattoo.\n- Witness R reports that the suspect's height was tall.\n- In Witness P's account, the suspect had mark: a scar.\n- Witness Q says the height they saw was average.\n- According to Witness R, the guild was the Scribes.\n- In Witness P's account, the suspect had cloak: green.\n- In Witness Q's account, the suspect had mark: no mark.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [6.795257621866774e-05, 0.009073781953761184, 0.9908582654700201], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 2}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_rand/meta.json
DELETED
|
@@ -1,69 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cfg": {
|
| 3 |
-
"K": 8,
|
| 4 |
-
"n_sources": 3,
|
| 5 |
-
"horizon": 8,
|
| 6 |
-
"feature_schema": {
|
| 7 |
-
"cloak": [
|
| 8 |
-
"red",
|
| 9 |
-
"blue",
|
| 10 |
-
"green",
|
| 11 |
-
"black"
|
| 12 |
-
],
|
| 13 |
-
"height": [
|
| 14 |
-
"short",
|
| 15 |
-
"average",
|
| 16 |
-
"tall"
|
| 17 |
-
],
|
| 18 |
-
"mark": [
|
| 19 |
-
"a scar",
|
| 20 |
-
"a tattoo",
|
| 21 |
-
"no mark"
|
| 22 |
-
],
|
| 23 |
-
"guild": [
|
| 24 |
-
"the Smiths",
|
| 25 |
-
"the Scribes",
|
| 26 |
-
"the Wardens"
|
| 27 |
-
]
|
| 28 |
-
},
|
| 29 |
-
"type_prior": [
|
| 30 |
-
0.5,
|
| 31 |
-
0.3,
|
| 32 |
-
0.2
|
| 33 |
-
],
|
| 34 |
-
"eps_reliable": 0.05,
|
| 35 |
-
"eps_noisy": 0.35,
|
| 36 |
-
"eps_adv": 0.02,
|
| 37 |
-
"source_aware": true,
|
| 38 |
-
"alpha": 0.6,
|
| 39 |
-
"imperative_adversarial": false,
|
| 40 |
-
"unique_hypotheses": true,
|
| 41 |
-
"allow_repeated_feature_queries": true,
|
| 42 |
-
"seed": 80001,
|
| 43 |
-
"randomize_type_prior": true,
|
| 44 |
-
"type_prior_concentration": 1.0,
|
| 45 |
-
"announce_type_prior": false
|
| 46 |
-
},
|
| 47 |
-
"labels": [
|
| 48 |
-
"A",
|
| 49 |
-
"B",
|
| 50 |
-
"C",
|
| 51 |
-
"D",
|
| 52 |
-
"E",
|
| 53 |
-
"F",
|
| 54 |
-
"G",
|
| 55 |
-
"H"
|
| 56 |
-
],
|
| 57 |
-
"trust_labels": [
|
| 58 |
-
"R",
|
| 59 |
-
"N",
|
| 60 |
-
"A"
|
| 61 |
-
],
|
| 62 |
-
"alpha": 0.6,
|
| 63 |
-
"source_aware": true,
|
| 64 |
-
"train_episodes": 500,
|
| 65 |
-
"val_episodes": 1,
|
| 66 |
-
"seed": 80001,
|
| 67 |
-
"min_turn": 0,
|
| 68 |
-
"note": "probe suffixes are appended at the TOKEN level after the reasoning (probe_ids = gen_prompt_ids + reasoning_ids + suffix_ids); labels resolved to single token ids via bayesclue.labels."
|
| 69 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_rand/sft_train.jsonl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:561a93bb9edd37082759b0f8c89bde995cf77fdb517dd42aa80f95a0bc71e4f5
|
| 3 |
-
size 22013787
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_rand/sft_val.jsonl
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.060379842204728355, 0.060379842204728355, 0.060379842204728355, 0.060379842204728355, 0.3188604733858149, 0.060379842204728355, 0.060379842204728355, 0.3188604733858149], "task": "world_posterior", "episode_id": 0, "turn": 0, "source_index": null}
|
| 2 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.3169712163634351, 0.6202186101202491, 0.06281017351631579], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 0}
|
| 3 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.3517129567455927, 0.6003428371909064, 0.04794420606350096], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 1}
|
| 4 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.3517129567455927, 0.6003428371909064, 0.04794420606350096], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 2}
|
| 5 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.01665715022561588, 0.01665715022561588, 0.01665715022561588, 0.01665715022561588, 0.696801997851555, 0.01665715022561588, 0.13194735061221932, 0.08796490040814624], "task": "world_posterior", "episode_id": 0, "turn": 1, "source_index": null}
|
| 6 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.375296865953999, 0.5868504669453082, 0.03785266710069284], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 0}
|
| 7 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.39274246682078173, 0.5768698270110406, 0.030387706168177687], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 1}
|
| 8 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.3517129567455926, 0.6003428371909063, 0.04794420606350099], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 2}
|
| 9 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.017360398991175367, 0.003287388181318951, 0.003287388181318951, 0.017360398991175367, 0.7262203039958385, 0.003287388181318951, 0.1375180400867124, 0.09167869339114164], "task": "world_posterior", "episode_id": 0, "turn": 2, "source_index": null}
|
| 10 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.38843868444231205, 0.579332023790788, 0.032229291766900016], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 0}
|
| 11 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4066208216978427, 0.5689300108954678, 0.02444916740668953], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 1}
|
| 12 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4567570707252353, 0.5402470292648408, 0.0029959000099239355], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 2}
|
| 13 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.05318761327217475, 0.010071677001916206, 0.010071677001916206, 0.02077284939260915, 0.42131844652864625, 0.010071677001916206, 0.4213184465286464, 0.05318761327217475], "task": "world_posterior", "episode_id": 0, "turn": 3, "source_index": null}
|
| 14 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.10622667641983116, 0.8798668413392776, 0.013906482240891225], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 0}
|
| 15 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.3982509493831228, 0.5737184204496536, 0.02803063016722364], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 1}
|
| 16 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.448684123830738, 0.5448655675873503, 0.006450308581911672], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 2}
|
| 17 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.08661842585472856, 0.016402180018099226, 0.016402180018099226, 0.08661842585472855, 0.3326717906772817, 0.041996781045052285, 0.3326717906772818, 0.08661842585472856], "task": "world_posterior", "episode_id": 0, "turn": 4, "source_index": null}
|
| 18 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.09957453970967091, 0.8617522723990618, 0.038673187891267315], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 0}
|
| 19 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.0875096943436094, 0.8288982462930202, 0.08359205936337041], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 1}
|
| 20 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.4309990817975951, 0.5549831919731786, 0.014017726229226415], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 2}
|
| 21 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.06834189069535375, 0.06834189069535376, 0.02669147410988336, 0.06834189069535374, 0.26247786116576305, 0.17498524077717542, 0.2624778611657632, 0.06834189069535374], "task": "world_posterior", "episode_id": 0, "turn": 5, "source_index": null}
|
| 22 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.10142095091905803, 0.8667802723638466, 0.03179877671709539], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 0}
|
| 23 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.08491574900511435, 0.8218346209184014, 0.09324963007648422], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 1}
|
| 24 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.10562873950636073, 0.8782385871537591, 0.016132673339880263], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 2}
|
| 25 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.08910368314529203, 0.02109504013521283, 0.008238837291258364, 0.054012563007937844, 0.34221681513353447, 0.05401256300793783, 0.3422168151335347, 0.08910368314529202], "task": "world_posterior", "episode_id": 0, "turn": 6, "source_index": null}
|
| 26 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.13312250666983572, 0.8641701457578077, 0.0027073475723565534], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 0}
|
| 27 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.08735815162381712, 0.8284855772267227, 0.08415627114946027], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 1}
|
| 28 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1074929416194829, 0.8833150338908006, 0.009192024489716516], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 2}
|
| 29 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n- According to Witness Q, the cloak was green.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.03382917245861125, 0.00800895906392335, 0.0031279632642115212, 0.011195074708611981, 0.4494072916687092, 0.01119507470861198, 0.44940729166870946, 0.03382917245861126], "task": "world_posterior", "episode_id": 0, "turn": 7, "source_index": null}
|
| 30 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n- According to Witness Q, the cloak was green.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.14711704257246336, 0.8520864033112574, 0.0007965541162791655], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 0}
|
| 31 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n- According to Witness Q, the cloak was green.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.13671875157781424, 0.812560019885098, 0.05072122853708783], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 1}
|
| 32 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: red, height: tall, mark: a tattoo, guild: the Wardens\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: blue, height: short, mark: no mark, guild: the Wardens\n (D) cloak: red, height: tall, mark: a scar, guild: the Smiths\n (E) cloak: green, height: tall, mark: no mark, guild: the Scribes\n (F) cloak: black, height: average, mark: a scar, guild: the Wardens\n (G) cloak: green, height: tall, mark: a tattoo, guild: the Wardens\n (H) cloak: red, height: tall, mark: a tattoo, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- In Witness P's account, the suspect had guild: the Scribes.\n- Witness Q reports that the suspect's cloak was green.\n- According to Witness R, the height was tall.\n- Witness P reports that the suspect's guild was the Wardens.\n- According to Witness Q, the mark was a scar.\n- Witness R says the height they saw was average.\n- Witness P recalls the suspect's height as tall.\n- According to Witness Q, the cloak was green.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.10800927444794373, 0.8847210703306359, 0.0072696552214203776], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 2}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_rev/meta.json
DELETED
|
@@ -1,69 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cfg": {
|
| 3 |
-
"K": 8,
|
| 4 |
-
"n_sources": 3,
|
| 5 |
-
"horizon": 8,
|
| 6 |
-
"feature_schema": {
|
| 7 |
-
"cloak": [
|
| 8 |
-
"red",
|
| 9 |
-
"blue",
|
| 10 |
-
"green",
|
| 11 |
-
"black"
|
| 12 |
-
],
|
| 13 |
-
"height": [
|
| 14 |
-
"short",
|
| 15 |
-
"average",
|
| 16 |
-
"tall"
|
| 17 |
-
],
|
| 18 |
-
"mark": [
|
| 19 |
-
"a scar",
|
| 20 |
-
"a tattoo",
|
| 21 |
-
"no mark"
|
| 22 |
-
],
|
| 23 |
-
"guild": [
|
| 24 |
-
"the Smiths",
|
| 25 |
-
"the Scribes",
|
| 26 |
-
"the Wardens"
|
| 27 |
-
]
|
| 28 |
-
},
|
| 29 |
-
"type_prior": [
|
| 30 |
-
0.2,
|
| 31 |
-
0.3,
|
| 32 |
-
0.5
|
| 33 |
-
],
|
| 34 |
-
"eps_reliable": 0.05,
|
| 35 |
-
"eps_noisy": 0.35,
|
| 36 |
-
"eps_adv": 0.02,
|
| 37 |
-
"source_aware": true,
|
| 38 |
-
"alpha": 0.6,
|
| 39 |
-
"imperative_adversarial": false,
|
| 40 |
-
"unique_hypotheses": true,
|
| 41 |
-
"allow_repeated_feature_queries": true,
|
| 42 |
-
"seed": 80002,
|
| 43 |
-
"randomize_type_prior": false,
|
| 44 |
-
"type_prior_concentration": 1.0,
|
| 45 |
-
"announce_type_prior": false
|
| 46 |
-
},
|
| 47 |
-
"labels": [
|
| 48 |
-
"A",
|
| 49 |
-
"B",
|
| 50 |
-
"C",
|
| 51 |
-
"D",
|
| 52 |
-
"E",
|
| 53 |
-
"F",
|
| 54 |
-
"G",
|
| 55 |
-
"H"
|
| 56 |
-
],
|
| 57 |
-
"trust_labels": [
|
| 58 |
-
"R",
|
| 59 |
-
"N",
|
| 60 |
-
"A"
|
| 61 |
-
],
|
| 62 |
-
"alpha": 0.6,
|
| 63 |
-
"source_aware": true,
|
| 64 |
-
"train_episodes": 500,
|
| 65 |
-
"val_episodes": 1,
|
| 66 |
-
"seed": 80002,
|
| 67 |
-
"min_turn": 0,
|
| 68 |
-
"note": "probe suffixes are appended at the TOKEN level after the reasoning (probe_ids = gen_prompt_ids + reasoning_ids + suffix_ids); labels resolved to single token ids via bayesclue.labels."
|
| 69 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_rev/sft_train.jsonl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:e7b9902923745e8acf66f35888aa25a6ec425ec3d6866f71c816bfdc5e5b381c
|
| 3 |
-
size 21992138
|
|
|
|
|
|
|
|
|
|
|
|
heldout-frozen/prior_rev/sft_val.jsonl
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.14643188137164043, 0.14643188137164043, 0.11214087117701574, 0.14643188137164043, 0.11214087117701574, 0.11214087117701574, 0.11214087117701574, 0.11214087117701574], "task": "world_posterior", "episode_id": 0, "turn": 0, "source_index": null}
|
| 2 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.22057460611677487, 0.3141797961075069, 0.46524559777571817], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 0}
|
| 3 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2, 0.29999999999999993, 0.5000000000000001], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 1}
|
| 4 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2, 0.29999999999999993, 0.5000000000000001], "task": "trust_posterior", "episode_id": 0, "turn": 0, "source_index": 2}
|
| 5 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.14157706093189967, 0.14157706093189967, 0.14157706093189967, 0.14157706093189967, 0.10842293906810034, 0.10842293906810034, 0.10842293906810034, 0.10842293906810034], "task": "world_posterior", "episode_id": 0, "turn": 1, "source_index": null}
|
| 6 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2138096507597974, 0.3095174620101306, 0.476672887230072], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 0}
|
| 7 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.08228916733315562, 0.2188749666755531, 0.6988358659912913], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 1}
|
| 8 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2, 0.3, 0.5000000000000001], "task": "trust_posterior", "episode_id": 0, "turn": 1, "source_index": 2}
|
| 9 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.10537048876738904, 0.20638682510637366, 0.10537048876738907, 0.10537048876738904, 0.15805573315108357, 0.08069512114464605, 0.08069512114464605, 0.15805573315108357], "task": "world_posterior", "episode_id": 0, "turn": 2, "source_index": null}
|
| 10 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.21027800436517563, 0.3070834894949183, 0.48263850613990605], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 0}
|
| 11 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.06547180354924233, 0.2072846213650183, 0.7272435750857393], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 1}
|
| 12 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.25922088229458323, 0.3408143918516721, 0.39996472585374465], "task": "trust_posterior", "episode_id": 0, "turn": 2, "source_index": 2}
|
| 13 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.036366089292428266, 0.07122944762235948, 0.14033001207763154, 0.2554141439853129, 0.07122944762235951, 0.10746792064173048, 0.10746792064173048, 0.21049501811644727], "task": "world_posterior", "episode_id": 0, "turn": 3, "source_index": null}
|
| 14 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.16990128576613406, 0.28488251793514985, 0.5452161962987161], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 0}
|
| 15 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.08170993342658062, 0.21847576492912982, 0.6998143016442896], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 1}
|
| 16 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.18047027673643462, 0.28654032585889405, 0.5329893974046713], "task": "trust_posterior", "episode_id": 0, "turn": 3, "source_index": 2}
|
| 17 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.032612865928074254, 0.06387809276821155, 0.22905369799169453, 0.22905369799169453, 0.06387809276821155, 0.09637651327516861, 0.09637651327516861, 0.18877052600177646], "task": "world_posterior", "episode_id": 0, "turn": 4, "source_index": null}
|
| 18 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1524660642578787, 0.26281061715825865, 0.5847233185838627], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 0}
|
| 19 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1352194719019729, 0.14918382226169818, 0.7155967058363291], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 1}
|
| 20 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1635504340652376, 0.2748793532071232, 0.5615702127276393], "task": "trust_posterior", "episode_id": 0, "turn": 4, "source_index": 2}
|
| 21 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.03159106446959769, 0.016035159112841493, 0.22187716210571812, 0.22187716210571812, 0.11262158010835224, 0.03159106446959769, 0.03159106446959769, 0.3328157431585771], "task": "world_posterior", "episode_id": 0, "turn": 5, "source_index": null}
|
| 22 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1482467061108324, 0.2602757822275549, 0.5914775116616128], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 0}
|
| 23 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.1310131531219869, 0.14673489540233822, 0.7222519514756749], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 1}
|
| 24 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2705657468845102, 0.27915082148496057, 0.4502834316305294], "task": "trust_posterior", "episode_id": 0, "turn": 5, "source_index": 2}
|
| 25 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.05267046464526845, 0.02673475222548992, 0.12517950994444138, 0.16617949110563304, 0.18776926491666193, 0.12684862612292164, 0.12684862612292164, 0.18776926491666193], "task": "world_posterior", "episode_id": 0, "turn": 6, "source_index": null}
|
| 26 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.03611517268364132, 0.5523127719716966, 0.4115720553446622], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 0}
|
| 27 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.07433662328517075, 0.11373771105361465, 0.8119256656612146], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 1}
|
| 28 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.2512358560561137, 0.3978658051764449, 0.3508983387674413], "task": "trust_posterior", "episode_id": 0, "turn": 6, "source_index": 2}
|
| 29 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n- Witness Q recalls the suspect's mark as a tattoo.\n\nWeighing every report and how trustworthy each witness seems, the single most likely suspect is\nAnswer:", "labels": ["A", "B", "C", "D", "E", "F", "G", "H"], "target": [0.05463370017204144, 0.0038964469292183854, 0.22949558766287673, 0.17237365481686853, 0.1947681644745672, 0.018487507778769827, 0.13157677369109075, 0.1947681644745672], "task": "world_posterior", "episode_id": 0, "turn": 7, "source_index": null}
|
| 30 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n- Witness Q recalls the suspect's mark as a tattoo.\n\nThe reliability of Witness P is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.03851570908417009, 0.6232094164934971, 0.3382748744223327], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 0}
|
| 31 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n- Witness Q recalls the suspect's mark as a tattoo.\n\nThe reliability of Witness Q is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.15536529279910985, 0.11012312221396435, 0.7345115849869258], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 1}
|
| 32 |
-
{"prompt": "There are several suspects. Their known profiles:\n (A) cloak: green, height: average, mark: a scar, guild: the Smiths\n (B) cloak: red, height: average, mark: a tattoo, guild: the Wardens\n (C) cloak: green, height: tall, mark: a tattoo, guild: the Scribes\n (D) cloak: black, height: average, mark: no mark, guild: the Wardens\n (E) cloak: red, height: short, mark: no mark, guild: the Wardens\n (F) cloak: black, height: short, mark: a tattoo, guild: the Wardens\n (G) cloak: blue, height: short, mark: a scar, guild: the Smiths\n (H) cloak: red, height: short, mark: a scar, guild: the Scribes\n\nWitnesses give reports. Some witnesses are reliable, some are noisy, and some are deliberately misleading; you are not told which.\nWitnesses: Witness P, Witness Q, Witness R.\n\nReports so far:\n- Witness P says the height they saw was average.\n- According to Witness Q, the height was tall.\n- Witness R says the cloak they saw was red.\n- Witness P recalls the suspect's mark as no mark.\n- Witness Q recalls the suspect's height as tall.\n- Witness R says the height they saw was short.\n- Witness P recalls the suspect's guild as the Scribes.\n- Witness Q recalls the suspect's mark as a tattoo.\n\nThe reliability of Witness R is best described by one of: R(eliable), N(oisy), A(dversarial).\nAnswer:", "labels": ["R", "N", "A"], "target": [0.24583914189300202, 0.3130333292540586, 0.4411275288529394], "task": "trust_posterior", "episode_id": 0, "turn": 7, "source_index": 2}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|