| { |
| "exp": "exp22", |
| "cell": "s1_string", |
| "variant": "cot", |
| "arm": "correct", |
| "seed": 0, |
| "composable": [ |
| "interleave_halves", |
| "reverse_blocks3", |
| "reverse_first_half", |
| "swap_pairs" |
| ], |
| "distractors": [ |
| "charset", |
| "nice_chars" |
| ], |
| "atoms": [ |
| "charset", |
| "interleave_halves", |
| "nice_chars", |
| "reverse_blocks3", |
| "reverse_first_half", |
| "swap_pairs" |
| ], |
| "fourth_atom": "reverse_blocks3", |
| "chain_lengths": [ |
| 5, |
| 7, |
| 8 |
| ], |
| "excluded_pairs": [ |
| [ |
| "reverse_first_half", |
| "swap_pairs" |
| ] |
| ], |
| "train_comps": [ |
| "reverse_first_half|interleave_halves", |
| "interleave_halves|reverse_first_half", |
| "reverse_blocks3|swap_pairs", |
| "swap_pairs|reverse_blocks3", |
| "reverse_blocks3|interleave_halves", |
| "interleave_halves|reverse_blocks3" |
| ], |
| "heldout_comps": [ |
| "reverse_blocks3|reverse_first_half", |
| "reverse_first_half|reverse_blocks3", |
| "swap_pairs|interleave_halves", |
| "interleave_halves|swap_pairs" |
| ], |
| "heldout_labels": { |
| "reverse_blocks3|reverse_first_half": "never_paired", |
| "reverse_first_half|reverse_blocks3": "never_paired", |
| "swap_pairs|interleave_halves": "never_paired", |
| "interleave_halves|swap_pairs": "never_paired" |
| }, |
| "n_comp_train_per_pairing": 200, |
| "total_comp_train": 1200, |
| "compositions": { |
| "reverse_first_half|interleave_halves": { |
| "role": "train", |
| "n_train": 200, |
| "n_seen_eval": 200 |
| }, |
| "interleave_halves|reverse_first_half": { |
| "role": "train", |
| "n_train": 200, |
| "n_seen_eval": 200 |
| }, |
| "reverse_blocks3|swap_pairs": { |
| "role": "train", |
| "n_train": 200, |
| "n_seen_eval": 200 |
| }, |
| "swap_pairs|reverse_blocks3": { |
| "role": "train", |
| "n_train": 200, |
| "n_seen_eval": 200 |
| }, |
| "reverse_blocks3|interleave_halves": { |
| "role": "train", |
| "n_train": 200, |
| "n_seen_eval": 200 |
| }, |
| "interleave_halves|reverse_blocks3": { |
| "role": "train", |
| "n_train": 200, |
| "n_seen_eval": 200 |
| }, |
| "reverse_blocks3|reverse_first_half": { |
| "role": "heldout", |
| "label": "never_paired", |
| "n_eval": 200, |
| "distinct_answers": 200, |
| "top_answer_share": 0.005 |
| }, |
| "reverse_first_half|reverse_blocks3": { |
| "role": "heldout", |
| "label": "never_paired", |
| "n_eval": 200, |
| "distinct_answers": 200, |
| "top_answer_share": 0.005 |
| }, |
| "swap_pairs|interleave_halves": { |
| "role": "heldout", |
| "label": "never_paired", |
| "n_eval": 200, |
| "distinct_answers": 200, |
| "top_answer_share": 0.005 |
| }, |
| "interleave_halves|swap_pairs": { |
| "role": "heldout", |
| "label": "never_paired", |
| "n_eval": 200, |
| "distinct_answers": 200, |
| "top_answer_share": 0.005 |
| } |
| }, |
| "train_heldout_chain_overlap": [], |
| "heldout_instance_collisions": 0, |
| "coverage": { |
| "train_inner_atoms": [ |
| "interleave_halves", |
| "reverse_blocks3", |
| "reverse_first_half", |
| "swap_pairs" |
| ], |
| "train_outer_atoms": [ |
| "interleave_halves", |
| "reverse_blocks3", |
| "reverse_first_half", |
| "swap_pairs" |
| ], |
| "heldout_inner_atoms": [ |
| "interleave_halves", |
| "reverse_blocks3", |
| "reverse_first_half", |
| "swap_pairs" |
| ], |
| "heldout_outer_atoms": [ |
| "interleave_halves", |
| "reverse_blocks3", |
| "reverse_first_half", |
| "swap_pairs" |
| ], |
| "heldout_inner_count": 4, |
| "heldout_outer_count": 4, |
| "per_slot_coverage": { |
| "interleave_halves": { |
| "train_as_inner": 2, |
| "train_as_outer": 2 |
| }, |
| "reverse_blocks3": { |
| "train_as_inner": 2, |
| "train_as_outer": 2 |
| }, |
| "reverse_first_half": { |
| "train_as_inner": 1, |
| "train_as_outer": 1 |
| }, |
| "swap_pairs": { |
| "train_as_inner": 1, |
| "train_as_outer": 1 |
| } |
| }, |
| "train_covers_both_slots": true, |
| "pairing_collisions": [] |
| }, |
| "collisions_total": 0, |
| "chain_lengths_out_of_range": [] |
| } |
