File size: 5,041 Bytes
df57b75 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """Gold-preserving typed-choice augmentation; never applied to evaluation.
The none-option wording matters more than it looks. The pinned Apache-2.0 Kev recipe records that its first
training run learned "this wording means pick it", so its generator varies both the key and the description
and uses the same set for correct and incorrect none options. Our earlier version used one fixed key,
`flymyjev_none`, with four descriptions, which is exactly the shortcut that failure describes.
"""
from copy import deepcopy
# Keys and descriptions are drawn independently, and a null description is allowed, so neither the key nor
# the wording identifies the correct answer. Same pool for the present and the absent member of a pair.
NONE_KEYS = ['other', 'none', 'not_listed', 'none_of_the_above', 'unknown', 'no_match', 'something_else']
NONE_DESCRIPTIONS = ['None of the above', 'None of these', 'A reason that fits none of the above',
'Something else', 'Not listed here', 'None of the listed options apply',
'An answer not covered by the other options', 'No option matches',
'Cannot be determined from the options given', 'Other', None]
DISTRACTORS = ['Bad weather caused it', 'The colour purple', 'A recipe for pancakes',
'Unrelated: quarterly tax filing', 'The shipping label printer ran out of paper']
def none_option(question, rng):
"""A fresh (key, description) for a none option that does not collide with the existing options."""
key = rng.choice(NONE_KEYS)
while key in question['criteria']:
key = key + '_' + str(rng.randint(2, 99))
return key, rng.choice(NONE_DESCRIPTIONS)
def none_pair(record, rng):
"""Same input with gold present/absent; a none option is wrong in one member and right in the other.
Pair members share the none wording and the surviving option order. Score and Noul semantics never change.
"""
eligible = [(qid, q) for qid, q in record['questions'].items()
if q['type'] == 'choice' and 3 <= len(q['criteria']) < 255]
if not eligible:
return []
qid, original = rng.choice(eligible)
key, description = none_option(original, rng)
present = deepcopy(original)
# The option set changes, so an exact distribution no longer applies to either member.
present.pop('probabilities', None)
items = list(present['criteria'].items()) + [(key, description)]
rng.shuffle(items)
present['criteria'] = dict(items)
absent = deepcopy(present)
absent['criteria'].pop(original['label'])
absent['label'] = key
return [dict(deepcopy(record), questions={qid: q}) for q in [present, absent]]
def add_none_distractor(record, rng):
"""Add a none option that is present and wrong; the gold never moves.
The measured reason. On the replay's 96 none_present questions the gold is never the none option, and v20
answers "none of these" 45 times - 46.9%, which is almost exactly the 50% rate at which our none pair
makes that option correct. The model is obeying the distribution we trained it on. Kev carries three
arms where we carry one: p_none 0.1, p_none_distract 0.12 and p_none_pair 0.25, so for them a present
none option is usually wrong. This is the missing middle arm.
"""
eligible = [(qid, q) for qid, q in record['questions'].items()
if q['type'] == 'choice' and 2 <= len(q['criteria']) < 255]
if not eligible:
return None
qid, original = rng.choice(eligible)
key, description = none_option(original, rng)
question = deepcopy(original)
question.pop('probabilities', None)
items = list(question['criteria'].items()) + [(key, description)]
rng.shuffle(items)
question['criteria'] = dict(items)
if question['label'] not in question['criteria'] or question['label'] == key:
return None
return dict(deepcopy(record), questions={qid: question})
def add_distractor(record, rng):
"""Insert one clearly irrelevant option into a choice question; the gold never changes.
Without this the model only ever sees plausible options and can learn to spread probability over all of
them. The distractor is unrelated to every domain we train on, so leaving mass on it is always an error.
"""
eligible = [(qid, q) for qid, q in record['questions'].items()
if q['type'] == 'choice' and 2 <= len(q['criteria']) < 255]
if not eligible:
return None
qid, original = rng.choice(eligible)
key = 'unrelated'
while key in original['criteria']:
key = key + '_' + str(rng.randint(2, 99))
changed = deepcopy(original)
changed.pop('probabilities', None)
items = list(changed['criteria'].items()) + [(key, rng.choice(DISTRACTORS))]
rng.shuffle(items)
changed['criteria'] = dict(items)
if changed['label'] not in changed['criteria']:
raise ValueError('Distractor insertion lost the gold option')
return dict(deepcopy(record), questions={qid: changed})
|