| """Gold-preserving typed-choice augmentation; never applied to evaluation. |
| |
| The none-option wording matters more than it looks. The pinned Apache-2.0 Kev recipe records that its first |
| training run learned "this wording means pick it", so its generator varies both the key and the description |
| and uses the same set for correct and incorrect none options. Our earlier version used one fixed key, |
| `flymyjev_none`, with four descriptions, which is exactly the shortcut that failure describes. |
| """ |
| from copy import deepcopy |
|
|
| |
| |
| NONE_KEYS = ['other', 'none', 'not_listed', 'none_of_the_above', 'unknown', 'no_match', 'something_else'] |
| NONE_DESCRIPTIONS = ['None of the above', 'None of these', 'A reason that fits none of the above', |
| 'Something else', 'Not listed here', 'None of the listed options apply', |
| 'An answer not covered by the other options', 'No option matches', |
| 'Cannot be determined from the options given', 'Other', None] |
| DISTRACTORS = ['Bad weather caused it', 'The colour purple', 'A recipe for pancakes', |
| 'Unrelated: quarterly tax filing', 'The shipping label printer ran out of paper'] |
|
|
|
|
| def none_option(question, rng): |
| """A fresh (key, description) for a none option that does not collide with the existing options.""" |
| key = rng.choice(NONE_KEYS) |
| while key in question['criteria']: |
| key = key + '_' + str(rng.randint(2, 99)) |
| return key, rng.choice(NONE_DESCRIPTIONS) |
|
|
|
|
| def none_pair(record, rng): |
| """Same input with gold present/absent; a none option is wrong in one member and right in the other. |
| |
| Pair members share the none wording and the surviving option order. Score and Noul semantics never change. |
| """ |
| eligible = [(qid, q) for qid, q in record['questions'].items() |
| if q['type'] == 'choice' and 3 <= len(q['criteria']) < 255] |
| if not eligible: |
| return [] |
| qid, original = rng.choice(eligible) |
| key, description = none_option(original, rng) |
| present = deepcopy(original) |
| |
| present.pop('probabilities', None) |
| items = list(present['criteria'].items()) + [(key, description)] |
| rng.shuffle(items) |
| present['criteria'] = dict(items) |
| absent = deepcopy(present) |
| absent['criteria'].pop(original['label']) |
| absent['label'] = key |
| return [dict(deepcopy(record), questions={qid: q}) for q in [present, absent]] |
|
|
|
|
| def add_none_distractor(record, rng): |
| """Add a none option that is present and wrong; the gold never moves. |
| |
| The measured reason. On the replay's 96 none_present questions the gold is never the none option, and v20 |
| answers "none of these" 45 times - 46.9%, which is almost exactly the 50% rate at which our none pair |
| makes that option correct. The model is obeying the distribution we trained it on. Kev carries three |
| arms where we carry one: p_none 0.1, p_none_distract 0.12 and p_none_pair 0.25, so for them a present |
| none option is usually wrong. This is the missing middle arm. |
| """ |
| eligible = [(qid, q) for qid, q in record['questions'].items() |
| if q['type'] == 'choice' and 2 <= len(q['criteria']) < 255] |
| if not eligible: |
| return None |
| qid, original = rng.choice(eligible) |
| key, description = none_option(original, rng) |
| question = deepcopy(original) |
| question.pop('probabilities', None) |
| items = list(question['criteria'].items()) + [(key, description)] |
| rng.shuffle(items) |
| question['criteria'] = dict(items) |
| if question['label'] not in question['criteria'] or question['label'] == key: |
| return None |
| return dict(deepcopy(record), questions={qid: question}) |
|
|
|
|
| def add_distractor(record, rng): |
| """Insert one clearly irrelevant option into a choice question; the gold never changes. |
| |
| Without this the model only ever sees plausible options and can learn to spread probability over all of |
| them. The distractor is unrelated to every domain we train on, so leaving mass on it is always an error. |
| """ |
| eligible = [(qid, q) for qid, q in record['questions'].items() |
| if q['type'] == 'choice' and 2 <= len(q['criteria']) < 255] |
| if not eligible: |
| return None |
| qid, original = rng.choice(eligible) |
| key = 'unrelated' |
| while key in original['criteria']: |
| key = key + '_' + str(rng.randint(2, 99)) |
| changed = deepcopy(original) |
| changed.pop('probabilities', None) |
| items = list(changed['criteria'].items()) + [(key, rng.choice(DISTRACTORS))] |
| rng.shuffle(items) |
| changed['criteria'] = dict(items) |
| if changed['label'] not in changed['criteria']: |
| raise ValueError('Distractor insertion lost the gold option') |
| return dict(deepcopy(record), questions={qid: changed}) |
|
|
