File size: 10,137 Bytes
f91d419
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
"""Rule-checked synthetic contrasts; never presented as human annotations.

Template identities are disjoint across splits. Correct/incorrect editing cases
share a context within a split, so evaluation uncertainty must group by context.
"""
import hashlib
import json
import random
from pathlib import Path

NAMES = ['Mia', 'Tom', 'Lily', 'Ben', 'Ana', 'Sam', 'Lucy', 'Max', 'Ella', 'Leo',
         'Nora', 'Jack', 'Zoe', 'Alex', 'Rose', 'Finn']
COLORS = ['blue', 'red', 'green', 'yellow', 'pink', 'white', 'black', 'purple']
OBJECTS = ['hat', 'cup', 'ball', 'bag', 'box', 'coat', 'shirt', 'kite']
FILLERS = ['', ' It was a quiet day.', ' A little bird sang nearby.',
           ' The sun was shining.', ' A friend was waiting.', ' It was early in the morning.']
CATEGORIES = ['color', 'location', 'emotion', 'state', 'identity', 'agreement']


def render(category, template, rng):
    name = rng.choice(NAMES)
    if category == 'color':
        good = rng.choice(COLORS); obj = rng.choice(OBJECTS)
        wrong = rng.sample([c for c in COLORS if c != good], 3)
        facts = [f'{name} owned a {good} {obj}.', f'The {obj} was painted {good}.',
                 f'{name} chose the {good} {obj}.']
        middles = [f'{name} picked up the [X] {obj}.', f'The [X] {obj} belonged to {name}.',
                   f'{name} held the same [X] {obj}.', f'Later, {name} carried the [X] {obj}.']
        text = facts[template % 3] + ' ' + middles[template // 3] + f' Its color had not changed.'
        accepted = [good]
    elif category == 'location':
        good = rng.choice(['under', 'behind', 'beside', 'near'])
        wrong = rng.sample([c for c in ['under', 'behind', 'beside', 'near', 'above'] if c != good], 3)
        obj = rng.choice(['bed', 'chair', 'table', 'tree', 'bench', 'desk'])
        item = rng.choice(['toy', 'ball', 'box', 'bag', 'shoe'])
        facts = [f'{name} left the {item} {good} the {obj}.',
                 f'The {item} lay {good} the {obj}.', f'{name} placed a {item} {good} the {obj}.']
        middles = [f'Later, {name} found it [X] the {obj}.', f'The {item} was still [X] the {obj}.',
                   f'{name} looked [X] the {obj} for the {item}.', f'That {item} remained [X] the {obj}.']
        text = facts[template % 3] + ' ' + middles[template // 3] + ' Nobody had moved it.'
        accepted = [good]
    elif category in ('emotion', 'state'):
        options = [
            ('lost a favorite toy', ['sad', 'upset'], ['happy', 'excited', 'proud'], 'Tears ran down the little face.'),
            ('won a fun game', ['happy', 'excited'], ['sad', 'angry', 'afraid'], 'There was a big smile.'),
            ('heard a scary noise', ['scared', 'afraid'], ['calm', 'happy', 'brave'], 'Both hands shook with fear.'),
            ('had not eaten all day', ['hungry'], ['full', 'sleepy', 'cold'], 'The empty stomach growled.'),
            ('played all day without resting', ['tired', 'sleepy'], ['awake', 'hungry', 'excited'], 'It was time to rest.'),
        ] if category == 'emotion' else [
            ('got caught in the rain', ['wet'], ['dry', 'hot', 'clean'], 'Water dripped from the coat.'),
            ('sat by a warm fire', ['warm'], ['cold', 'wet', 'frozen'], 'The fire gave off plenty of heat.'),
            ('turned off the only lamp', ['dark'], ['bright', 'warm', 'wet'], 'There was no light in the room.'),
            ('emptied a box completely', ['empty'], ['full', 'heavy', 'wet'], 'There was nothing left inside.'),
            ('filled a box to the top', ['full'], ['empty', 'wet', 'broken'], 'There was no room for anything else.'),
        ]
        event, accepted, wrong, evidence = rng.choice(options)
        beginnings = [f'{name} {event}.', f'One day, {name} {event}.', f'After lunch, {name} {event}.']
        subject = name if category == 'emotion' or accepted[0] in ('wet', 'warm') else ('The room' if accepted[0] == 'dark' else 'The box')
        middles = [f'{subject} felt [X].', f'Now {subject.lower() if subject.startswith("The") else subject} was [X].',
                   f'{subject} was very [X].', f'At that moment, {subject.lower() if subject.startswith("The") else subject} seemed [X].']
        text = beginnings[template % 3] + ' ' + middles[template // 3] + ' ' + evidence
        # "box felt full" is unnatural: use a grammatical state predicate.
        if category == 'state':
            text = text.replace('felt [X]', 'was [X]')
    elif category == 'identity':
        gender = rng.choice(['girl', 'boy', 'children'])
        good = {'girl': 'She', 'boy': 'He', 'children': 'They'}[gender]
        wrong = [p for p in ['She', 'He', 'They', 'It'] if p != good]
        facts = [f'{name} was a little {gender}.', f'A {gender} named {name} came by.', f'{name}, a {gender}, went out.'] if gender != 'children' else [
            f'{name} and a friend were children.', 'Two children went out together.', f'{name} and two friends came by.']
        tails = ['[X] smiled and waved.', '[X] wanted to play a game.', '[X] walked to the park.', '[X] found a little ball.']
        text = facts[template % 3] + ' ' + tails[template // 3]
        accepted = [good]
    else:
        kind = rng.randrange(4)
        if kind == 0:
            good, bad = rng.choice([('walked','walk'), ('played','play'), ('went','go'), ('ate','eat'), ('ran','run'), ('saw','see')])
            starts = ['Yesterday,', 'Last week,', 'Last night,']
            tails = ['in the garden.', 'near the house.', 'with a friend.', 'before going home.']
            text = f'{starts[template % 3]} {name} [X] {tails[template // 3]}'
            wrong = [bad, 'is', 'are']; accepted = [good]
        elif kind == 1:
            good, bad = rng.choice([('walks','walk'), ('plays','play'), ('eats','eat'), ('runs','run'), ('goes','go')])
            starts = ['Every morning,', 'Each day,', 'Every afternoon,']
            tails = ['before school.', 'near the house.', 'with a friend.', 'in the park.']
            text = f'{starts[template % 3]} {name} [X] {tails[template // 3]}'
            accepted = [good]; wrong = [bad, 'were', 'are']
        elif kind == 2:
            plural = rng.choice(['dogs', 'cats', 'children', 'birds', 'puppies', 'boys'])
            starts = [f'The two {plural}', f'All three {plural}', f'Those {plural}']
            tails = ['playing together.', 'happy to be there.', 'waiting outside.', 'very tired.']
            text = f'{starts[template % 3]} [X] {tails[template // 3]}'
            accepted = ['were']; wrong = ['was', 'is', 'am']
        else:
            vowel = rng.choice(['apple', 'egg', 'orange', 'umbrella', 'elephant'])
            starts = [f'{name} saw', f'{name} found', f'{name} drew']
            tails = ['yesterday.', 'in a picture.', 'this morning.', 'for a friend.']
            text = f'{starts[template % 3]} [X] {vowel} {tails[template // 3]}'
            accepted = ['an']; wrong = ['a', 'two', 'many']
    # Same distractors across split identities; no category token is fed to ranker.
    text += rng.choice(FILLERS)
    prefix, suffix = text.split('[X]')
    return prefix, suffix, accepted, wrong


def alternatives(prefix,suffix,category,words):
    """Bounded explicit equivalences; NOT an open-ended semantic verifier."""
    values=list(words)
    if category in ('emotion','state') and not prefix.rstrip().endswith('very'):
        values += [modifier+word for word in words for modifier in ('very ','so ')]
    if category=='location':
        for word in words:
            values += {'under':['underneath','beneath'],'behind':['just behind','close behind'],
                       'beside':['next to'],'near':['close to']}[word]
    return list(dict.fromkeys(values))


def build(output, train=12000, validation=1200, test_contexts=300, seed=173, version=1):
    output = Path(output)
    if output.exists():
        raise ValueError('Choose a fresh dataset directory; frozen data must not be overwritten')
    output.mkdir(parents=True)
    rng = random.Random(seed); seen = set(); manifest = {'version': version, 'seed': seed,
        'provenance': 'Procedural rule-checked synthetic contrasts; no model teacher; no human labels.',
        'license': 'Apache-2.0', 'splits': {}}
    for split, count, templates in [('train', train, list(range(8))), ('validation', validation, [8,9]), ('test', test_contexts, [10,11])]:
        contexts=[]; tries=0
        while len(contexts)<count:
            tries+=1
            if tries>count*100: raise ValueError('Requested too many unique contexts')
            category=CATEGORIES[len(contexts)%len(CATEGORIES)]
            template=rng.choice(templates)
            prefix,suffix,accepted,wrong=render(category,template,rng)
            if version>=2:accepted=alternatives(prefix,suffix,category,accepted)
            key=hashlib.sha256((prefix+'[X]'+suffix).encode()).hexdigest()
            if key in seen: continue
            seen.add(key)
            contexts.append({'id':f'{split}-{len(contexts):05}', 'context_sha256':key,
                'category':category,'template':template,'prefix':prefix,'suffix':suffix,
                'accepted':accepted,'negative':wrong})
        path=output/f'{split}.jsonl'
        path.write_text(''.join(json.dumps(c,ensure_ascii=False)+'\n' for c in contexts),encoding='utf-8')
        manifest['splits'][split]={'contexts':len(contexts),'templates':templates,
            'sha256':hashlib.sha256(path.read_bytes()).hexdigest()}
    (output/'manifest.json').write_text(json.dumps(manifest,indent=2),encoding='utf-8')
    return manifest


def read(path):
    return [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines()]


def editing_cases(contexts):
    """Balanced matched clean/error inputs; accepted list is deliberately bounded."""
    for c in contexts:
        for clean in (False, True):
            fragment = c['accepted'][0] if clean else c['negative'][0]
            text=c['prefix']+fragment+c['suffix']; start=len(c['prefix'])
            yield {**c,'id':c['id']+('-clean' if clean else '-error'), 'clean':clean,
                   'text':text,'start':start,'end':start+len(fragment)}