File size: 3,397 Bytes
788274d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | import pandas as pd
from uvw.data import leakage
from uvw.data.sources import MINOR_PATTERN, _artwork_key, _named_subject, church_rows, vangogh_rows
from uvw.data.splits import assign
def _fake_ner(entities):
return lambda text: entities(text)
def test_minor_pattern_catches_minors_but_not_adult_terms():
for p in ["a teen at the beach", "17 year old", "16yo", "little girl", "schoolgirl outfit",
"young beautiful girl posing", "young girls"]:
assert MINOR_PATTERN.search(p), p
for p in ["a woman at the beach", "girlfriend portrait", "grown man"]:
assert not MINOR_PATTERN.search(p), p
def test_named_subject_distinguishes_credit_from_subject():
def ner(text):
i = text.lower().find("jane doe")
return [{"entity_group": "PER", "score": 0.99, "start": i, "end": i + 8}] if i >= 0 else []
assert _named_subject("portrait of Jane Doe on a beach", ner)
assert _named_subject("jane doe lying in a bathtub", ner) # lowercase subject
assert not _named_subject("a beach scene, art by Jane Doe", ner)
assert not _named_subject("by jane doe, a collage", ner) # credit at the start
assert not _named_subject("a beach scene by greg rutkowski and Jane Doe", ner)
assert not _named_subject("in the style of artgerm, greg rutkowski, jane doe", ner)
def test_artwork_aliases_group_alternative_titles():
assert _artwork_key("Starry Night by Vincent van Gogh") == _artwork_key("The Starry Night by Vincent van Gogh")
assert _artwork_key("Bedroom in Arles by Vincent van Gogh") == _artwork_key("The Bedroom by Vincent van Gogh")
assert _artwork_key("original replica of The Starry Night") == _artwork_key("Starry Night by X")
def _frame():
rows = [r.__dict__ for r in vangogh_rows() + church_rows()]
return pd.DataFrame(rows)
def test_splits_deterministic_and_template_disjoint():
a, b = assign(_frame()), assign(_frame())
pd.testing.assert_frame_equal(a, b)
assert leakage.template_overlap(a).empty
# Paired designs: every artist / class for one subject or context lands in the same split.
assert a.groupby("template_id").split.nunique().max() == 1
assert set(a.split) == {"dev", "val", "test"}
def test_leakage_detects_injected_leak():
df = assign(_frame())
leaked = df.iloc[[0]].copy()
leaked["split"] = "test" if df.split.iloc[0] != "test" else "dev"
bad = pd.concat([df, leaked.assign(prompt_id="leak")])
assert not leakage.template_overlap(bad).empty
assert not leakage.exact_duplicates(bad).empty
def test_summarise_clusters_and_agreement():
from uvw.evaluate import summarise
rows = [{"role": "target", "category": "a", "group_id": f"g{i // 3}", "target_hit": i % 2 == 0,
"second_hit": i % 2 == 0, "clip_score": 30.0} for i in range(12)]
s = summarise(rows)
t = s["role:target"]
assert t["n_images"] == 12 and t["n_clusters"] == 4
assert t["target_hit"]["ci95"][0] <= t["target_hit"]["mean"] <= t["target_hit"]["ci95"][1]
assert s["detector_agreement"] == 1.0
def test_expand_seeds_keeps_source_seed_first():
from uvw.evaluate import expand_seeds
df = pd.DataFrame([{"prompt_id": "p", "text": "x", "seed": 42}])
seeds = [it.seed for _, it in expand_seeds(df, 3)]
assert seeds[0] == 42 and len(set(seeds)) == 3
assert seeds == [it.seed for _, it in expand_seeds(df, 3)]
|