File size: 7,748 Bytes
c7893fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | """Zero-shot evaluation sources. These datasets never appear in train/val/test, and their
tasks (question-type classification, formality rating, spam filtering) have no
counterpart in the training mix, so heldout.jsonl measures generalization to unseen tasks.
"""
from __future__ import annotations
from jevlike.data.common import (
Example, Label, Scale, balanced_order, clean, cuts, dedupe_rows, derived_noul, make_choice, make_score, noul,
pick_instr, reverse, truncate, weighted,
)
from jevlike.data.hub import read_many, read_rows
from jevlike.data.label_names import TREC_FINE
from jevlike.data.sources_choice import GENERIC_CHOICE, Ctx
TREC_COARSE = [
Label("abbreviation", ["asks what an abbreviation stands for, or for an abbreviation"], "an abbreviation", key="ABBR"),
Label("entity", ["asks for a thing: an animal, food, product, color, event, sport, etc."], "an entity or thing",
key="ENTY"),
Label("description", ["asks for a definition, description, manner or reason"], "a description or explanation",
key="DESC"),
Label("human", ["asks about a person or a group of people"], "a person or group of people", key="HUM"),
Label("location", ["asks for a place: city, country, mountain, etc."], "a location", key="LOC"),
Label("numeric value", ["asks for a number, date, amount, distance, duration, etc."], "a number or quantity",
key="NUM"),
]
_FINE_DESC = {
"abb": "abbreviation", "exp": "expansion of an abbreviation", "animal": "animal", "body": "body part or organ",
"color": "color", "cremat": "creative work (book, film, song, invention)", "currency": "currency",
"dismed": "disease or medicine", "event": "event", "food": "food", "instru": "musical instrument",
"lang": "language", "letter": "letter or character", "plant": "plant", "product": "product",
"religion": "religion", "sport": "sport", "substance": "substance, element or material",
"symbol": "symbol or sign", "techmeth": "technique or method", "termeq": "equivalent term",
"veh": "vehicle", "word": "word with a special property", "def": "definition of something",
"manner": "manner of an action (how)", "reason": "reason (why)", "gr": "group or organization",
"ind": "individual person", "title": "title or position of a person", "city": "city", "country": "country",
"mount": "mountain", "state": "state or province", "code": "postal code, phone number or other code",
"count": "count (how many)", "date": "date or time", "dist": "distance or length", "money": "price or amount of money",
"ord": "rank or order", "period": "duration", "perc": "percentage or fraction", "speed": "speed",
"temp": "temperature", "volsize": "size, area or volume", "weight": "weight",
}
def _fine_label(n: str) -> Label:
coarse, fine = n.split(":")
if fine == "other":
d = {"ENTY": "other entity", "LOC": "other location", "NUM": "other number"}[coarse]
elif fine == "desc":
d = "description of a person" if coarse == "HUM" else "description of something"
else:
d = _FINE_DESC[fine]
return Label(d, [f"the answer is a {d}", d], f"a {d}", key=n.replace(":", "_").lower())
def trec(ctx: Ctx):
rows = read_many("CogComp/trec", ["default/train/0000.parquet", "default/test/0000.parquet"],
revision="refs/convert/parquet")
rng = ctx.rng
fine = [_fine_label(n) for n in TREC_FINE]
rows = dedupe_rows(rows, lambda r: r["text"], lambda r: r["fine_label"])
c_instr = ["What type of answer does this question expect?", "Classify the question by answer type.",
"What is the question asking for?", "Which kind of information is being requested?"]
pos = ["The question asks for {x}.", "The expected answer is {x}.", "Is this question asking for {x}?"]
neg = ["The question does not ask for {x}."]
for r in balanced_order(rng, rows, lambda r: int(r["coarse_label"]), 0.5):
text = clean(r["text"])
c, f = int(r["coarse_label"]), int(r["fine_label"])
kind = weighted(rng, ["coarse", "fine", "noul"], [0.5, 0.2, 0.3])
if kind == "coarse":
q = make_choice(rng, TREC_COARSE, c, pick_instr(rng, c_instr, GENERIC_CHOICE), n=6 if rng.random() < 0.7 else None)
elif kind == "fine":
q = make_choice(rng, fine, f, pick_instr(rng, c_instr, GENERIC_CHOICE), full_prob=0.15)
else:
q = derived_noul(rng, TREC_COARSE, c, pos, neg, p_other_group=0.0)
yield Example(text, [q])
def formality(ctx: Ctx):
rows = read_many("osyvokon/pavlick-formality-scores", ["train.csv", "test.csv"])
rng = ctx.rng
rows = dedupe_rows(rows, lambda r: str(r["sentence"]), lambda r: round(float(r["avg_score"]), 1))
ins = ["How formal is this sentence?", "Rate the formality of the text.", "What register is this written in?",
"Formality level?"]
scales = [
Scale(["very informal", "informal", "neutral", "formal", "very formal"], cuts(-1.8, -0.6, 0.6, 1.8), ins, 1.3),
Scale(["casual", "neutral", "formal"], cuts(-0.7, 0.7), ins, 1.0),
Scale(["slang or chatty", "everyday conversational", "plain neutral prose", "professional", "academic or legal"],
cuts(-1.8, -0.6, 0.6, 1.8), ins, 0.6),
]
scales.append(reverse(scales[1], ["How casual is this sentence?", "Rate how informal the text sounds."], weight=0.6))
stm = [("The sentence is written in a formal style.", lambda v: True if v >= 1.0 else (False if v <= -1.0 else None)),
("This sounds casual and conversational.", lambda v: True if v <= -1.0 else (False if v >= 1.0 else None)),
("This text would fit in an official document.", lambda v: True if v >= 1.5 else (False if v <= -0.5 else None))]
for r in balanced_order(rng, rows, lambda r: cuts(-1.8, -0.6, 0.6, 1.8)(float(r["avg_score"])), 0.5):
v = float(r["avg_score"])
text = clean(r["sentence"])
if rng.random() < 0.7:
q = make_score(rng, scales, v)
else:
q = None
for _ in range(4):
s, fn = rng.choice(stm)
t = fn(v)
if t is not None:
q = noul(s, t)
break
if q:
yield Example(text, [q])
def enron(ctx: Ctx):
rows = read_rows("SetFit/enron_spam", "test.jsonl")
rng = ctx.rng
def text(r):
subj, msg = clean(r.get("subject") or ""), clean(r.get("message") or "")
return truncate(f"Subject: {subj}\n\n{msg}" if subj else msg)
rows = [dict(r, _t=text(r)) for r in rows]
rows = dedupe_rows(rows, lambda r: r["_t"], lambda r: r["label"])
labels = [Label("inbox", ["legitimate email (ham)", "normal work or personal email"], key="ham"),
Label("spam", ["unsolicited bulk or advertising email", "junk / spam"])]
pos = ["This email is spam.", "This is an unsolicited bulk or advertising email.", "Is this email spam?",
"The message is junk mail."]
neg = ["This is a legitimate email.", "This email is not spam.", "The message is a normal work email."]
instr = ["Which folder should this email go to?", "Spam or ham?", "Filter this email.",
"How should this message be filed?"]
for r in balanced_order(rng, rows, lambda r: int(r["label"]), 0.0):
y = int(r["label"])
if rng.random() < 0.3:
q = make_choice(rng, labels, y, pick_instr(rng, instr), n=2)
elif rng.random() < 0.3:
q = noul(rng.choice(neg), y == 0)
else:
q = noul(rng.choice(pos), y == 1)
yield Example(r["_t"], [q])
SOURCES = {"trec": trec, "pavlick_formality": formality, "enron_spam": enron}
|