File size: 12,138 Bytes
7bcd331 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | """Authored English chatbot diagnostic v1, frozen before seeing model outputs.
30 cases per objective category, plus 12 qualitative conversations. Variants
within a family are correlated; this is not a representative population sample.
"""
def build():
rows = []
def add(category, family, prompt, answers, *, messages=None, case_sensitive=False,
answerable=True, qualitative=False):
number = sum(r['category'] == category for r in rows)
rows.append(dict(id=f'{category}-{number:02d}', category=category, family=family,
messages=messages or [dict(role='user', content=prompt)],
answers=answers, case_sensitive=case_sensitive,
answerable=answerable, qualitative=qualitative,
max_tokens=160 if qualitative else 128))
facts = [
('What is the capital of Japan?', ['Tokyo']),
('Which planet is closest to the Sun?', ['Mercury']),
('What is the chemical symbol for gold?', ['Au']),
('How many sides does a hexagon have?', ['six', '6']),
('Which ocean is the largest on Earth?', ['Pacific', 'Pacific Ocean', 'the Pacific Ocean']),
('Who wrote Pride and Prejudice?', ['Jane Austen']),
('On which continent is Egypt?', ['Africa']),
('What force pulls objects toward Earth?', ['gravity', 'gravitational force']),
('What gas do humans need to breathe to survive?', ['oxygen']),
('What is the process by which plants use sunlight to make food called?', ['photosynthesis']),
('How many minutes are in one hour?', ['60', 'sixty']),
('At standard atmospheric pressure, at what Celsius temperature does pure water freeze?', ['0', 'zero', '0 degrees Celsius', '0°C', 'zero degrees Celsius']),
('Which instrument is used to measure temperature?', ['thermometer', 'a thermometer']),
('Which organ pumps blood around the human body?', ['heart', 'the heart']),
('What is the largest living land animal?', ['African elephant', 'African bush elephant', 'African savanna elephant', 'elephant', 'the African elephant']),
('What is the opposite of north?', ['south']),
('What is the name of the natural satellite that orbits Earth?', ['Moon', 'the Moon']),
('Who painted the Mona Lisa?', ['Leonardo da Vinci', 'da Vinci']),
('In which country is the city of Barcelona?', ['Spain']),
('How many days does a leap year have?', ['366', 'three hundred sixty-six', 'three hundred and sixty-six']),
('What is the freezing of a liquid into a solid called?', ['freezing', 'solidification']),
('What is the square root of 81?', ['9', 'nine']),
('Which language is primarily spoken in Brazil?', ['Portuguese', 'Brazilian Portuguese']),
('Which animal is known for building dams?', ['beaver', 'beavers', 'a beaver']),
('What part of a plant typically absorbs water from the soil?', ['roots', 'the roots', 'root']),
('What is the main gas in Earth’s atmosphere?', ['nitrogen']),
('How many degrees are in a right angle?', ['90', 'ninety', '90 degrees']),
('Which metal is liquid at ordinary room temperature, around 20 degrees Celsius?', ['mercury']),
('What is the name of the galaxy that contains our solar system?', ['Milky Way', 'the Milky Way']),
('Which sense uses the ears?', ['hearing']),
]
for q, answers in facts:
add('knowledge', 'stable_fact', q + ' Give a short answer.', answers)
names = ['Anika', 'Bram', 'Celia', 'Dante', 'Esme', 'Farid']
other = ['Greta', 'Hector', 'Ines', 'Jules', 'Kira', 'Lucio']
colors = ['teal', 'maroon', 'ivory', 'indigo', 'ochre', 'turquoise']
items = ['notebook', 'umbrella', 'camera', 'scarf', 'compass', 'lantern']
cities = ['Kyoto', 'Porto', 'Bergen', 'Ghent', 'Bari', 'Lyon']
days = ['Tuesday', 'Friday', 'Sunday', 'Monday', 'Thursday', 'Wednesday']
foods = ['rice', 'pasta', 'lentils', 'potatoes', 'noodles', 'beans']
for i, name in enumerate(names):
lead = 'Use only the information below. Give a short answer.\n\n'
add('reading', 'contrast', lead + f'{name} brought a {colors[i]} {items[i]}. {other[i]} brought a black {items[i]}.\nWhat color was {name}’s {items[i]}?', [colors[i]])
add('reading', 'schedule_exception', lead + f'The club normally meets on Saturday. This week its meeting has been moved to {days[i]}.\nOn which day is this week’s meeting?', [days[i]])
add('reading', 'relation', lead + f'{name} visited {cities[i]}. {other[i]} stayed in Naples.\nWhich city did {name} visit?', [cities[i]])
add('reading', 'negation', lead + f'For dinner, {name} chose {foods[i]}, not bread.\nWhat did {name} choose for dinner?', [foods[i]])
add('reading', 'quantity', lead + f'The green shelf has {11+i} books. The orange shelf has {23+i} books.\nHow many books are on the orange shelf?', [str(23+i)])
a, b = 7+i, 3+i
add('reasoning', 'addition', f'{name} has {a} postcards and receives {b} more. How many postcards does {name} have now? Give only the number.', [str(a+b)])
add('reasoning', 'subtraction', f'A tray contains {20+2*i} cups. Someone removes {b} cups. How many remain? Give only the number.', [str(20+2*i-b)])
add('reasoning', 'multiplication', f'There are {b} packets, each containing 4 stickers. How many stickers are there altogether? Give only the number.', [str(b*4)])
add('reasoning', 'ordering', f'{name} is taller than {other[i]}. {other[i]} is taller than Zola. Who is tallest? Give only the name.', [name])
add('reasoning', 'state_change', f'{name} puts a coin in a drawer, then moves it into a cup. The drawer is closed. Where is the coin now? Give only the container.', ['cup', 'the cup'])
add('instructions', 'case', f'Convert this word to uppercase and return only the result: {items[i]}', [items[i].upper()], case_sensitive=True)
words = [colors[i], items[i], cities[i].lower()]
add('instructions', 'reverse_list', 'Reverse the order of these three items. Return the items separated by commas, with no extra words: ' + ', '.join(words), [', '.join(reversed(words))])
values = [10+i*2, 3+i*2, 8+i*2]
add('instructions', 'sort_numbers', 'Put these numbers in ascending order. Return only the comma-separated numbers: ' + ', '.join(map(str, values)), [', '.join(map(str, sorted(values)))])
add('instructions', 'replacement', f'Replace "small" with "large" in this sentence and return only the revised sentence: The small {items[i]} is here.', [f'The large {items[i]} is here.'], case_sensitive=True)
add('instructions', 'copy', f'Repeat this label exactly, without a prefix or explanation: QZ-{417+i}-R', [f'QZ-{417+i}-R'], case_sensitive=True)
def dialogue(first, last, middle=None):
turns = [dict(role='user', content=first), dict(role='assistant', content='Understood.')]
if middle:
turns += [dict(role='user', content=middle), dict(role='assistant', content='Understood.')]
return turns + [dict(role='user', content=last)]
add('conversation', 'correction', '', [str(317+i)], messages=dialogue(f'My reservation number is {217+i}.', 'What is my current reservation number? Give only the number.', f'Correction: my reservation number is {317+i}.'))
add('conversation', 'preference', '', [foods[i]], messages=dialogue(f'My favorite dinner is {foods[i]}.', 'What did I say my favorite dinner was? Give only the food.'))
add('conversation', 'entity_tracking', '', [colors[i]], messages=dialogue(f'My name is {name}. My bag is {colors[i]}. {other[i]} has a black bag.', 'What color is my bag? Give only the color.'))
add('conversation', 'distraction', '', [cities[i]], messages=dialogue(f'My next trip is to {cities[i]}.', 'Which city did I say I am visiting? Give only the city.', 'I am packing a coat and two books.'))
add('conversation', 'updated_plan', '', [days[i]], messages=dialogue('I planned to visit the museum on Saturday.', 'On which day do I now plan to visit the museum? Give only the day.', f'I changed my plans. I will visit the museum on {days[i]} instead.'))
available = i % 2 == 0
suffix = ' Use only these facts. If the answer is missing, reply "Not provided." Otherwise give a short answer.'
info = [
('missing_time', f'{name} booked a train ticket.' + (f' Departure is at {8+i}:30.' if available else ''), 'What time does the train leave?', [f'{8+i}:30']),
('missing_quantity', f'{name} bought apples.' + (f' The bag contains {8+i} apples.' if available else ''), 'How many apples did they buy?', [str(8+i)]),
('missing_identity', f'A {items[i]} was found in the park.' + (f' It belongs to {name}.' if available else ''), 'Who owns it?', [name]),
('missing_location', f'{name} mailed a parcel.' + (f' The destination is {cities[i]}.' if available else ''), 'To which city was the parcel sent?', [cities[i]]),
('missing_attribute', f'{name} has a bicycle.' + (f' The bicycle is {colors[i]}.' if available else ''), 'What color is the bicycle?', [colors[i]]),
]
for family, passage, question, answer in info:
add('answerability', family, passage + '\n' + question + suffix,
answer if available else ['Not provided.', 'Not stated.', 'Not specified.', 'Unknown.', 'I don’t know.', 'I do not know.', 'Not enough information.', 'The information is not provided.'],
answerable=available)
qualitative = [
('greeting', 'Hi! How are you?', 'A brief friendly greeting; no invented personal history.'),
('thanks', 'Thanks, that helped.', 'A brief relevant acknowledgement.'),
('explanation', 'Why does an ice cube melt in a warm room? Explain simply in one or two sentences.', 'Heat from the surroundings makes solid ice turn into liquid water.'),
('explanation', 'Why is a bicycle easier to pedal in a lower gear when going uphill?', 'Lower gearing reduces the pedal force needed at the cost of more pedal turns or lower speed.'),
('practical', 'Give me two simple ways to keep my desk organized.', 'Two distinct useful organization suggestions, without irrelevant filler.'),
('practical', 'I keep forgetting to bring my reusable bag to the store. Suggest one practical habit.', 'One relevant concrete reminder or preparation habit.'),
('writing', 'Make this request more polite without changing its meaning: Send me the notes today.', 'A polite request for the notes today; preserve the deadline.'),
('comparison', 'What is the difference between borrowing and buying a book?', 'Borrowing is temporary access with return; buying confers ownership.'),
('explanation', 'What does the word "fragile" mean? Give a simple example.', 'Easily broken or damaged, with a suitable example.'),
('planning', 'I have ten minutes to tidy my room. What should I do first?', 'A feasible brief starting action, not an elaborate or unrelated plan.'),
('uncertainty', 'What color is the mug on my desk right now?', 'Acknowledge lack of visual/contextual information; do not invent a color.'),
('followup', None, 'Use the previously stated no-spicy-food preference and rice/vegetables ingredients.'),
]
for family, prompt, rubric in qualitative:
messages = None
if prompt is None:
messages = [dict(role='user',content='I do not like spicy food. I have rice and vegetables.'),
dict(role='assistant',content='Understood.'),
dict(role='user',content='Suggest a simple dinner using those ingredients.')]
add('open_chat', family, prompt, [], messages=messages, qualitative=True)
rows[-1]['review_rubric'] = rubric
assert len(rows) == 192
assert len({r['id'] for r in rows}) == len(rows)
assert len({str(r['messages']) for r in rows}) == len(rows)
return rows
|