Spaces:

viv
/

men4dem

Running

men4dem / prep.py

Upload 10 files

010acb3 verified about 1 month ago

15.9 kB

"""Build the compact data file the Researchers' Night page reads."""
import json
import sys
from pathlib import Path

import pandas as pd

sys.path.insert(0, "../space")
import extract  # noqa: E402

ISO = {
    "Greece": "gr", "Germany": "de", "Italy": "it", "Netherlands": "nl",
    "Poland": "pl", "Sweden": "se", "Austria": "at", "Belgium": "be",
    "Bulgaria": "bg", "Croatia": "hr", "Cyprus": "cy", "Czech Republic": "cz",
    "Denmark": "dk", "Estonia": "ee", "Finland": "fi", "France": "fr",
    "Hungary": "hu", "Ireland": "ie", "Latvia": "lv", "Lithuania": "lt",
    "Luxembourg": "lu", "Malta": "mt", "Portugal": "pt", "Romania": "ro",
    "Slovakia": "sk", "Slovenia": "si", "Spain": "es", "United Kingdom": "gb",
}

EL_COUNTRY = {
    "Greece": "Ελλάδα", "Germany": "Γερμανία", "Italy": "Ιταλία",
    "Netherlands": "Ολλανδία", "Poland": "Πολωνία", "Sweden": "Σουηδία",
    "EU": "Ευρωπαϊκό Κοινοβούλιο", "Austria": "Αυστρία", "Belgium": "Βέλγιο",
    "Bulgaria": "Βουλγαρία", "Croatia": "Κροατία", "Cyprus": "Κύπρος",
    "Czech Republic": "Τσεχία", "Denmark": "Δανία", "Estonia": "Εσθονία",
    "Finland": "Φινλανδία", "France": "Γαλλία", "Hungary": "Ουγγαρία",
    "Ireland": "Ιρλανδία", "Latvia": "Λετονία", "Lithuania": "Λιθουανία",
    "Luxembourg": "Λουξεμβούργο", "Malta": "Μάλτα", "Portugal": "Πορτογαλία",
    "Romania": "Ρουμανία", "Slovakia": "Σλοβακία", "Slovenia": "Σλοβενία",
    "Spain": "Ισπανία", "United Kingdom": "Ηνωμένο Βασίλειο",
}

EL_TOPIC = {
    "Health": "Υγεία", "Environment": "Περιβάλλον", "Economy": "Οικονομία",
    "Conflicts": "Συγκρούσεις", "Migration": "Μετανάστευση",
    "Agriculture": "Γεωργία", "Gender": "Φύλο", "Religion": "Θρησκεία",
    "Cybersecurity": "Κυβερνοασφάλεια", "Hate Speech": "Ρητορική μίσους",
    "Debt crises": "Κρίση χρέους", "Terrorism - Extremism": "Τρομοκρατία",
    "Border Control": "Έλεγχος συνόρων", "Refugee crises": "Προσφυγικό",
}

EL_BLOC = {
    "Far-right": "Άκρα δεξιά", "Right": "Δεξιά", "Centre-right": "Κεντροδεξιά",
    "Centre": "Κέντρο", "Centre-left": "Κεντροαριστερά", "Left": "Αριστερά",
    "Far-left": "Άκρα αριστερά", "Other": "Άλλο", "Unknown": "Άγνωστο",
}

BLOC_ORDER = ["Far-left", "Left", "Centre-left", "Centre", "Centre-right",
              "Right", "Far-right"]


def main():
    D = extract.load("../space/topics.xlsx")
    out = {}

    # ---------------------------------------------------------- headline
    seg = D["country_volume"].query("unit == 'Segments'")["value"].sum()
    ses = D["country_volume"].query("unit == 'Sessions'")["value"].sum()
    speakers = D["stats"].query("metric == 'speakers'")["value"].sum()
    years = D["temporal"]["year"]
    out["headline"] = {
        "segments": int(seg), "sessions": int(ses), "speakers": int(speakers),
        "topics": int(D["orientation"].topic.nunique()),
        "year_min": int(years.min()), "year_max": int(years.max()),
        "parties": int(D["parties"].party.nunique()),
        "countries": len(ISO),
    }

    # ------------------------------------------------- who gets to speak
    s = D["sex"].pivot(index="topic", columns="sex", values="value").dropna()
    s["share"] = 100 * s["Female"] / (s["Female"] + s["Male"])
    s = s.sort_values("share", ascending=False)
    out["gender_topics"] = [
        {"topic": t, "el": EL_TOPIC.get(t, t), "share": round(r["share"], 1),
         "female": int(r["Female"]), "male": int(r["Male"])}
        for t, r in s.iterrows()]
    out["gender_overall"] = round(
        100 * s["Female"].sum() / (s["Female"].sum() + s["Male"].sum()), 1)

    g = D["gender_orientation"].pivot_table(
        index="orientation", columns="sex", values="value", aggfunc="sum").dropna()
    g["share"] = 100 * g["Female"] / (g["Female"] + g["Male"])
    out["gender_blocs"] = [
        {"bloc": b, "el": EL_BLOC.get(b, b), "share": round(g.loc[b, "share"], 1)}
        for b in BLOC_ORDER if b in g.index]

    # --------------------------------------------------- map: parliaments
    p = D["prominence"].copy()
    p = p[p.value > 0]
    topic_mean = p.groupby("topic")["value"].mean()
    p["ratio"] = p.apply(lambda r: r["value"] / topic_mean[r["topic"]], axis=1)

    vol = D["country_volume"].query("unit == 'Segments'")
    parliaments = {}
    for c in D["prominence"].country.unique():
        sub = p[p.country == c].sort_values("ratio", ascending=False)
        v = vol[vol.country == c].sort_values("value", ascending=False)
        parliaments[c] = {
            "name_en": c, "name_el": EL_COUNTRY.get(c, c),
            "iso": ISO.get(c), "tier": "parliament",
            "segments": int(v["value"].sum()),
            "distinct": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic),
                          "x": round(r.ratio, 1)}
                         for r in sub.head(3).itertuples() if r.ratio >= 1.15],
            "biggest": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic),
                         "n": int(r.value)} for r in v.head(3).itertuples()],
            "topics_all": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic),
                            "n": int(r.value)}
                           for r in v.sort_values("value", ascending=False).itertuples()
                           if r.value > 0],
        }
    out["parliaments"] = parliaments

    # ------------------------------------------------------- map: parties
    pa = D["parties"]
    countries = {}
    for name, iso in ISO.items():
        sub = pa[pa.party_country == name]
        if sub.empty:
            continue
        top_topic = (sub.groupby("topic")["segments"].sum()
                     .sort_values(ascending=False))
        big = sub.sort_values("party_total", ascending=False).drop_duplicates("party")
        countries[name] = {
            "name_en": name, "name_el": EL_COUNTRY.get(name, name), "iso": iso,
            "tier": "parliament" if name in parliaments else "party",
            "parties": int(sub.party.nunique()),
            "top_topics": [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)}
                           for t, n in top_topic.head(3).items()],
            "topics_all": [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)}
                           for t, n in top_topic.items() if n > 0],
            "big_parties": [str(x).rsplit(" (", 1)[0] for x in big.party.head(3)],
        }
        if name in parliaments:
            countries[name].update({
                k: parliaments[name][k]
                for k in ("distinct", "biggest", "segments", "topics_all")})
    for name, pl in parliaments.items():
        if name in countries or name == "EU":
            continue
        countries[name] = pl
    out["countries"] = countries
    out["eu"] = parliaments.get("EU")

    # --------------------------------------------------------- timeline
    t = D["temporal"].query("unit == 'Sessions'")
    lo, hi = t.year.min(), t.year.max()
    t = t[(t.year > lo) & (t.year < hi)]
    piv = t.pivot_table(index="year", columns="topic", values="value", aggfunc="sum")
    order = piv.sum().sort_values(ascending=False).index.tolist()
    out["timeline"] = {
        "years": [int(y) for y in piv.index],
        "series": [{"topic": tp, "el": EL_TOPIC.get(tp, tp),
                    "values": [int(v) for v in piv[tp].fillna(0)]}
                   for tp in order],
    }

    # ------------------------------------------------ topic totals (bars)
    totals = (D["country_volume"].query("unit == 'Segments'")
              .groupby("topic")["value"].sum().sort_values(ascending=False))
    out["topic_totals"] = [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)}
                           for t, n in totals.items()]


    # ------------------------------------- optional: bloc x year x country
    # Dropped in as far_right.csv (see DATA_REQUEST.md). Absent -> the exhibit
    # simply has no such chapter; nothing else changes.
    fr_path = Path("far_right.csv")
    if fr_path.exists():
        fr = pd.read_csv(fr_path)
        need = {"country", "year", "bloc", "segments", "total_segments"}
        missing = need - set(fr.columns)
        if missing:
            raise SystemExit(f"far_right.csv is missing columns: {sorted(missing)}")
        fr["year"] = fr["year"].astype(int)
        fr["share"] = 100 * fr["segments"] / fr["total_segments"].replace(0, pd.NA)
        fr = fr.dropna(subset=["share"])
        out["far_right"] = {
            "blocs": sorted(fr.bloc.unique().tolist()),
            "years": sorted(int(y) for y in fr.year.unique()),
            "rows": [{"country": r.country, "iso": ISO.get(r.country),
                      "el": EL_COUNTRY.get(r.country, r.country),
                      "year": int(r.year), "bloc": r.bloc,
                      "segments": int(r.segments),
                      "share": round(float(r.share), 2)}
                     for r in fr.itertuples()],
        }
        n = len(out["far_right"]["rows"])
        print(f"  far_right.csv: {n} rows, "
              f"{fr.country.nunique()} countries, "
              f"{fr.year.min()}-{fr.year.max()}, {fr.bloc.nunique()} blocs")
    else:
        print("  far_right.csv not present - bloc/year/country chapter skipped")


    # --------------------------------------------- topic co-focus network
    # Two topics are linked when the same parties give both a large share of
    # their own speech. Spearman across parties, so party size does not drive it.
    # This is co-focus, not co-occurrence inside a speech, and not causation.
    import numpy as np
    pa2 = D["parties"].dropna(subset=["focus_pct"])
    mat = pa2.pivot_table(index="party", columns="topic", values="focus_pct",
                          aggfunc="sum")
    mat = mat[mat.notna().sum(axis=1) >= 6].fillna(0.0)
    mat.columns.name = None
    corr = mat.corr(method="spearman")
    names = [t for t in extract.TOPICS if t in corr.columns]
    names += [t for t in corr.columns if t not in names]

    R_MIN = 0.30
    edges = []
    for i, a in enumerate(names):
        for b in names[i + 1:]:
            r = float(corr.loc[a, b])
            if r >= R_MIN:
                edges.append({"a": a, "b": b, "r": round(r, 3)})
    edges.sort(key=lambda e: -e["r"])

    # deterministic spring layout so the drawing is stable between builds
    rng = np.random.default_rng(7)
    idx = {t: i for i, t in enumerate(names)}
    n = len(names)
    pos = rng.normal(0, 0.35, size=(n, 2))
    adj = np.zeros((n, n))
    for e in edges:
        w = (e["r"] - R_MIN) / (1 - R_MIN) + 0.25
        adj[idx[e["a"]], idx[e["b"]]] = adj[idx[e["b"]], idx[e["a"]]] = w
    for step in range(600):
        delta = pos[:, None, :] - pos[None, :, :]
        dist = np.sqrt((delta ** 2).sum(-1)) + 1e-6
        rep = (delta / dist[..., None] ** 3) * 0.010          # everything pushes apart
        att = -(delta * (adj * dist)[..., None]) * 0.012      # edges pull together
        force = (rep + att).sum(axis=1) - pos * 0.010         # mild pull to centre
        pos += force * (1.0 - step / 700)
    pos -= pos.mean(axis=0)
    span = np.abs(pos).max() or 1.0
    pos = pos / span

    seg_by_topic = (D["country_volume"].query("unit == 'Segments'")
                    .groupby("topic")["value"].sum())
    out["network"] = {
        "r_min": R_MIN,
        "parties": int(mat.shape[0]),
        "nodes": [{"topic": t, "el": EL_TOPIC.get(t, t),
                   "x": round(float(pos[idx[t], 0]), 4),
                   "y": round(float(pos[idx[t], 1]), 4),
                   "n": int(seg_by_topic.get(t, 0))} for t in names],
        "edges": edges,
    }
    print(f"  network: {len(names)} topics, {len(edges)} links "
          f"(r>={R_MIN}) from {mat.shape[0]} parties; "
          f"strongest {edges[0]['a']}-{edges[0]['b']} r={edges[0]['r']}")


    # ------------------------------- blocs and topics on one map (CA)
    # Correspondence analysis of the bloc x topic table: the standard method for
    # a contingency table, and it places blocs and topics in the same space, so
    # a bloc sitting in the direction of a topic means that bloc gives it more
    # attention than the corpus as a whole does.
    tab = (D["orientation"].pivot(index="orientation", columns="topic", values="value")
           .fillna(0.0))
    # "Other" and "Unknown" are residual buckets, not political positions:
    # including them distorts the axes and confuses a lay reader.
    rows_order = [b for b in BLOC_ORDER if b in tab.index]
    dropped_rows = [b for b in tab.index if b not in rows_order]
    tab = tab.loc[rows_order]
    cols_order = [t for t in extract.TOPICS if t in tab.columns]
    cols_order += [t for t in tab.columns if t not in cols_order]
    tab = tab[cols_order]

    N = tab.values.astype(float)
    total = N.sum()
    P = N / total
    rm, cm = P.sum(1), P.sum(0)
    Sres = (P - np.outer(rm, cm)) / np.sqrt(np.outer(rm, cm))
    U, sv, Vt = np.linalg.svd(Sres, full_matrices=False)
    inertia = sv ** 2
    pct = (inertia / inertia.sum() * 100)

    Fr = (U * sv) / np.sqrt(rm)[:, None]          # blocs
    Fc = (Vt.T * sv) / np.sqrt(cm)[:, None]       # topics
    scale = max(np.abs(Fr[:, :2]).max(), np.abs(Fc[:, :2]).max()) or 1.0

    over = (P / np.outer(rm, cm))                 # 1.0 = exactly as expected
    over = pd.DataFrame(over, index=tab.index, columns=tab.columns)

    blocs_out = []
    for i, b in enumerate(tab.index):
        top = over.loc[b].sort_values(ascending=False).head(3)
        blocs_out.append({
            "key": b, "el": EL_BLOC.get(b, b),
            "x": round(float(Fr[i, 0] / scale), 4),
            "y": round(float(Fr[i, 1] / scale), 4),
            "n": int(N[i].sum()),
            "top": [{"topic": t, "el": EL_TOPIC.get(t, t), "x": round(float(v), 2)}
                    for t, v in top.items()],
        })
    topics_out = []
    for j, t in enumerate(tab.columns):
        lead = over[t].sort_values(ascending=False).head(2)
        topics_out.append({
            "topic": t, "el": EL_TOPIC.get(t, t),
            "x": round(float(Fc[j, 0] / scale), 4),
            "y": round(float(Fc[j, 1] / scale), 4),
            "n": int(N[:, j].sum()),
            "lead": [{"bloc": b, "el": EL_BLOC.get(b, b), "x": round(float(v), 2)}
                     for b, v in lead.items()],
        })

    out["camap"] = {
        "blocs": blocs_out, "topics": topics_out,
        "d1": round(float(pct[0]), 1), "d2": round(float(pct[1]), 1),
        "explained": round(float(pct[:2].sum()), 1),
        "segments": int(total),
        "excluded": dropped_rows,
    }
    print(f"  bloc-topic map: {len(blocs_out)} blocs, {len(topics_out)} topics, "
          f"D1 {pct[0]:.1f}% + D2 {pct[1]:.1f}% = {pct[:2].sum():.1f}% of the structure"
          + (f"; excluded {dropped_rows}" if dropped_rows else ""))

    json.dump(out, open("data.json", "w", encoding="utf-8"),
              ensure_ascii=False, separators=(",", ":"))
    print("wrote data.json")
    print("  headline:", out["headline"])
    print("  parliaments:", list(parliaments))
    print("  countries on map:", len(out["countries"]))
    print("\n  most distinctive per parliament:")
    for c, v in parliaments.items():
        d = ", ".join(f"{x['topic']} {x['x']}x" for x in v["distinct"]) or "—"
        print(f"    {c:12s} {d}")
    print("\n  female share ladder:", [(b["bloc"], b["share"]) for b in out["gender_blocs"]])


if __name__ == "__main__":
    main()

	"""Build the compact data file the Researchers' Night page reads."""
	import json
	import sys
	from pathlib import Path

	import pandas as pd

	sys.path.insert(0, "../space")
	import extract # noqa: E402

	ISO = {
	"Greece": "gr", "Germany": "de", "Italy": "it", "Netherlands": "nl",
	"Poland": "pl", "Sweden": "se", "Austria": "at", "Belgium": "be",
	"Bulgaria": "bg", "Croatia": "hr", "Cyprus": "cy", "Czech Republic": "cz",
	"Denmark": "dk", "Estonia": "ee", "Finland": "fi", "France": "fr",
	"Hungary": "hu", "Ireland": "ie", "Latvia": "lv", "Lithuania": "lt",
	"Luxembourg": "lu", "Malta": "mt", "Portugal": "pt", "Romania": "ro",
	"Slovakia": "sk", "Slovenia": "si", "Spain": "es", "United Kingdom": "gb",
	}

	EL_COUNTRY = {
	"Greece": "Ελλάδα", "Germany": "Γερμανία", "Italy": "Ιταλία",
	"Netherlands": "Ολλανδία", "Poland": "Πολωνία", "Sweden": "Σουηδία",
	"EU": "Ευρωπαϊκό Κοινοβούλιο", "Austria": "Αυστρία", "Belgium": "Βέλγιο",
	"Bulgaria": "Βουλγαρία", "Croatia": "Κροατία", "Cyprus": "Κύπρος",
	"Czech Republic": "Τσεχία", "Denmark": "Δανία", "Estonia": "Εσθονία",
	"Finland": "Φινλανδία", "France": "Γαλλία", "Hungary": "Ουγγαρία",
	"Ireland": "Ιρλανδία", "Latvia": "Λετονία", "Lithuania": "Λιθουανία",
	"Luxembourg": "Λουξεμβούργο", "Malta": "Μάλτα", "Portugal": "Πορτογαλία",
	"Romania": "Ρουμανία", "Slovakia": "Σλοβακία", "Slovenia": "Σλοβενία",
	"Spain": "Ισπανία", "United Kingdom": "Ηνωμένο Βασίλειο",
	}

	EL_TOPIC = {
	"Health": "Υγεία", "Environment": "Περιβάλλον", "Economy": "Οικονομία",
	"Conflicts": "Συγκρούσεις", "Migration": "Μετανάστευση",
	"Agriculture": "Γεωργία", "Gender": "Φύλο", "Religion": "Θρησκεία",
	"Cybersecurity": "Κυβερνοασφάλεια", "Hate Speech": "Ρητορική μίσους",
	"Debt crises": "Κρίση χρέους", "Terrorism - Extremism": "Τρομοκρατία",
	"Border Control": "Έλεγχος συνόρων", "Refugee crises": "Προσφυγικό",
	}

	EL_BLOC = {
	"Far-right": "Άκρα δεξιά", "Right": "Δεξιά", "Centre-right": "Κεντροδεξιά",
	"Centre": "Κέντρο", "Centre-left": "Κεντροαριστερά", "Left": "Αριστερά",
	"Far-left": "Άκρα αριστερά", "Other": "Άλλο", "Unknown": "Άγνωστο",
	}

	BLOC_ORDER = ["Far-left", "Left", "Centre-left", "Centre", "Centre-right",
	"Right", "Far-right"]


	def main():
	D = extract.load("../space/topics.xlsx")
	out = {}

	# ---------------------------------------------------------- headline
	seg = D["country_volume"].query("unit == 'Segments'")["value"].sum()
	ses = D["country_volume"].query("unit == 'Sessions'")["value"].sum()
	speakers = D["stats"].query("metric == 'speakers'")["value"].sum()
	years = D["temporal"]["year"]
	out["headline"] = {
	"segments": int(seg), "sessions": int(ses), "speakers": int(speakers),
	"topics": int(D["orientation"].topic.nunique()),
	"year_min": int(years.min()), "year_max": int(years.max()),
	"parties": int(D["parties"].party.nunique()),
	"countries": len(ISO),
	}

	# ------------------------------------------------- who gets to speak
	s = D["sex"].pivot(index="topic", columns="sex", values="value").dropna()
	s["share"] = 100 * s["Female"] / (s["Female"] + s["Male"])
	s = s.sort_values("share", ascending=False)
	out["gender_topics"] = [
	{"topic": t, "el": EL_TOPIC.get(t, t), "share": round(r["share"], 1),
	"female": int(r["Female"]), "male": int(r["Male"])}
	for t, r in s.iterrows()]
	out["gender_overall"] = round(
	100 * s["Female"].sum() / (s["Female"].sum() + s["Male"].sum()), 1)

	g = D["gender_orientation"].pivot_table(
	index="orientation", columns="sex", values="value", aggfunc="sum").dropna()
	g["share"] = 100 * g["Female"] / (g["Female"] + g["Male"])
	out["gender_blocs"] = [
	{"bloc": b, "el": EL_BLOC.get(b, b), "share": round(g.loc[b, "share"], 1)}
	for b in BLOC_ORDER if b in g.index]

	# --------------------------------------------------- map: parliaments
	p = D["prominence"].copy()
	p = p[p.value > 0]
	topic_mean = p.groupby("topic")["value"].mean()
	p["ratio"] = p.apply(lambda r: r["value"] / topic_mean[r["topic"]], axis=1)

	vol = D["country_volume"].query("unit == 'Segments'")
	parliaments = {}
	for c in D["prominence"].country.unique():
	sub = p[p.country == c].sort_values("ratio", ascending=False)
	v = vol[vol.country == c].sort_values("value", ascending=False)
	parliaments[c] = {
	"name_en": c, "name_el": EL_COUNTRY.get(c, c),
	"iso": ISO.get(c), "tier": "parliament",
	"segments": int(v["value"].sum()),
	"distinct": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic),
	"x": round(r.ratio, 1)}
	for r in sub.head(3).itertuples() if r.ratio >= 1.15],
	"biggest": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic),
	"n": int(r.value)} for r in v.head(3).itertuples()],
	"topics_all": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic),
	"n": int(r.value)}
	for r in v.sort_values("value", ascending=False).itertuples()
	if r.value > 0],
	}
	out["parliaments"] = parliaments

	# ------------------------------------------------------- map: parties
	pa = D["parties"]
	countries = {}
	for name, iso in ISO.items():
	sub = pa[pa.party_country == name]
	if sub.empty:
	continue
	top_topic = (sub.groupby("topic")["segments"].sum()
	.sort_values(ascending=False))
	big = sub.sort_values("party_total", ascending=False).drop_duplicates("party")
	countries[name] = {
	"name_en": name, "name_el": EL_COUNTRY.get(name, name), "iso": iso,
	"tier": "parliament" if name in parliaments else "party",
	"parties": int(sub.party.nunique()),
	"top_topics": [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)}
	for t, n in top_topic.head(3).items()],
	"topics_all": [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)}
	for t, n in top_topic.items() if n > 0],
	"big_parties": [str(x).rsplit(" (", 1)[0] for x in big.party.head(3)],
	}
	if name in parliaments:
	countries[name].update({
	k: parliaments[name][k]
	for k in ("distinct", "biggest", "segments", "topics_all")})
	for name, pl in parliaments.items():
	if name in countries or name == "EU":
	continue
	countries[name] = pl
	out["countries"] = countries
	out["eu"] = parliaments.get("EU")

	# --------------------------------------------------------- timeline
	t = D["temporal"].query("unit == 'Sessions'")
	lo, hi = t.year.min(), t.year.max()
	t = t[(t.year > lo) & (t.year < hi)]
	piv = t.pivot_table(index="year", columns="topic", values="value", aggfunc="sum")
	order = piv.sum().sort_values(ascending=False).index.tolist()
	out["timeline"] = {
	"years": [int(y) for y in piv.index],
	"series": [{"topic": tp, "el": EL_TOPIC.get(tp, tp),
	"values": [int(v) for v in piv[tp].fillna(0)]}
	for tp in order],
	}

	# ------------------------------------------------ topic totals (bars)
	totals = (D["country_volume"].query("unit == 'Segments'")
	.groupby("topic")["value"].sum().sort_values(ascending=False))
	out["topic_totals"] = [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)}
	for t, n in totals.items()]


	# ------------------------------------- optional: bloc x year x country
	# Dropped in as far_right.csv (see DATA_REQUEST.md). Absent -> the exhibit
	# simply has no such chapter; nothing else changes.
	fr_path = Path("far_right.csv")
	if fr_path.exists():
	fr = pd.read_csv(fr_path)
	need = {"country", "year", "bloc", "segments", "total_segments"}
	missing = need - set(fr.columns)
	if missing:
	raise SystemExit(f"far_right.csv is missing columns: {sorted(missing)}")
	fr["year"] = fr["year"].astype(int)
	fr["share"] = 100 * fr["segments"] / fr["total_segments"].replace(0, pd.NA)
	fr = fr.dropna(subset=["share"])
	out["far_right"] = {
	"blocs": sorted(fr.bloc.unique().tolist()),
	"years": sorted(int(y) for y in fr.year.unique()),
	"rows": [{"country": r.country, "iso": ISO.get(r.country),
	"el": EL_COUNTRY.get(r.country, r.country),
	"year": int(r.year), "bloc": r.bloc,
	"segments": int(r.segments),
	"share": round(float(r.share), 2)}
	for r in fr.itertuples()],
	}
	n = len(out["far_right"]["rows"])
	print(f" far_right.csv: {n} rows, "
	f"{fr.country.nunique()} countries, "
	f"{fr.year.min()}-{fr.year.max()}, {fr.bloc.nunique()} blocs")
	else:
	print(" far_right.csv not present - bloc/year/country chapter skipped")


	# --------------------------------------------- topic co-focus network
	# Two topics are linked when the same parties give both a large share of
	# their own speech. Spearman across parties, so party size does not drive it.
	# This is co-focus, not co-occurrence inside a speech, and not causation.
	import numpy as np
	pa2 = D["parties"].dropna(subset=["focus_pct"])
	mat = pa2.pivot_table(index="party", columns="topic", values="focus_pct",
	aggfunc="sum")
	mat = mat[mat.notna().sum(axis=1) >= 6].fillna(0.0)
	mat.columns.name = None
	corr = mat.corr(method="spearman")
	names = [t for t in extract.TOPICS if t in corr.columns]
	names += [t for t in corr.columns if t not in names]

	R_MIN = 0.30
	edges = []
	for i, a in enumerate(names):
	for b in names[i + 1:]:
	r = float(corr.loc[a, b])
	if r >= R_MIN:
	edges.append({"a": a, "b": b, "r": round(r, 3)})
	edges.sort(key=lambda e: -e["r"])

	# deterministic spring layout so the drawing is stable between builds
	rng = np.random.default_rng(7)
	idx = {t: i for i, t in enumerate(names)}
	n = len(names)
	pos = rng.normal(0, 0.35, size=(n, 2))
	adj = np.zeros((n, n))
	for e in edges:
	w = (e["r"] - R_MIN) / (1 - R_MIN) + 0.25
	adj[idx[e["a"]], idx[e["b"]]] = adj[idx[e["b"]], idx[e["a"]]] = w
	for step in range(600):
	delta = pos[:, None, :] - pos[None, :, :]
	dist = np.sqrt((delta ** 2).sum(-1)) + 1e-6
	rep = (delta / dist[..., None] ** 3) * 0.010 # everything pushes apart
	att = -(delta * (adj * dist)[..., None]) * 0.012 # edges pull together
	force = (rep + att).sum(axis=1) - pos * 0.010 # mild pull to centre
	pos += force * (1.0 - step / 700)
	pos -= pos.mean(axis=0)
	span = np.abs(pos).max() or 1.0
	pos = pos / span

	seg_by_topic = (D["country_volume"].query("unit == 'Segments'")
	.groupby("topic")["value"].sum())
	out["network"] = {
	"r_min": R_MIN,
	"parties": int(mat.shape[0]),
	"nodes": [{"topic": t, "el": EL_TOPIC.get(t, t),
	"x": round(float(pos[idx[t], 0]), 4),
	"y": round(float(pos[idx[t], 1]), 4),
	"n": int(seg_by_topic.get(t, 0))} for t in names],
	"edges": edges,
	}
	print(f" network: {len(names)} topics, {len(edges)} links "
	f"(r>={R_MIN}) from {mat.shape[0]} parties; "
	f"strongest {edges[0]['a']}-{edges[0]['b']} r={edges[0]['r']}")


	# ------------------------------- blocs and topics on one map (CA)
	# Correspondence analysis of the bloc x topic table: the standard method for
	# a contingency table, and it places blocs and topics in the same space, so
	# a bloc sitting in the direction of a topic means that bloc gives it more
	# attention than the corpus as a whole does.
	tab = (D["orientation"].pivot(index="orientation", columns="topic", values="value")
	.fillna(0.0))
	# "Other" and "Unknown" are residual buckets, not political positions:
	# including them distorts the axes and confuses a lay reader.
	rows_order = [b for b in BLOC_ORDER if b in tab.index]
	dropped_rows = [b for b in tab.index if b not in rows_order]
	tab = tab.loc[rows_order]
	cols_order = [t for t in extract.TOPICS if t in tab.columns]
	cols_order += [t for t in tab.columns if t not in cols_order]
	tab = tab[cols_order]

	N = tab.values.astype(float)
	total = N.sum()
	P = N / total
	rm, cm = P.sum(1), P.sum(0)
	Sres = (P - np.outer(rm, cm)) / np.sqrt(np.outer(rm, cm))
	U, sv, Vt = np.linalg.svd(Sres, full_matrices=False)
	inertia = sv ** 2
	pct = (inertia / inertia.sum() * 100)

	Fr = (U * sv) / np.sqrt(rm)[:, None] # blocs
	Fc = (Vt.T * sv) / np.sqrt(cm)[:, None] # topics
	scale = max(np.abs(Fr[:, :2]).max(), np.abs(Fc[:, :2]).max()) or 1.0

	over = (P / np.outer(rm, cm)) # 1.0 = exactly as expected
	over = pd.DataFrame(over, index=tab.index, columns=tab.columns)

	blocs_out = []
	for i, b in enumerate(tab.index):
	top = over.loc[b].sort_values(ascending=False).head(3)
	blocs_out.append({
	"key": b, "el": EL_BLOC.get(b, b),
	"x": round(float(Fr[i, 0] / scale), 4),
	"y": round(float(Fr[i, 1] / scale), 4),
	"n": int(N[i].sum()),
	"top": [{"topic": t, "el": EL_TOPIC.get(t, t), "x": round(float(v), 2)}
	for t, v in top.items()],
	})
	topics_out = []
	for j, t in enumerate(tab.columns):
	lead = over[t].sort_values(ascending=False).head(2)
	topics_out.append({
	"topic": t, "el": EL_TOPIC.get(t, t),
	"x": round(float(Fc[j, 0] / scale), 4),
	"y": round(float(Fc[j, 1] / scale), 4),
	"n": int(N[:, j].sum()),
	"lead": [{"bloc": b, "el": EL_BLOC.get(b, b), "x": round(float(v), 2)}
	for b, v in lead.items()],
	})

	out["camap"] = {
	"blocs": blocs_out, "topics": topics_out,
	"d1": round(float(pct[0]), 1), "d2": round(float(pct[1]), 1),
	"explained": round(float(pct[:2].sum()), 1),
	"segments": int(total),
	"excluded": dropped_rows,
	}
	print(f" bloc-topic map: {len(blocs_out)} blocs, {len(topics_out)} topics, "
	f"D1 {pct[0]:.1f}% + D2 {pct[1]:.1f}% = {pct[:2].sum():.1f}% of the structure"
	+ (f"; excluded {dropped_rows}" if dropped_rows else ""))

	json.dump(out, open("data.json", "w", encoding="utf-8"),
	ensure_ascii=False, separators=(",", ":"))
	print("wrote data.json")
	print(" headline:", out["headline"])
	print(" parliaments:", list(parliaments))
	print(" countries on map:", len(out["countries"]))
	print("\n most distinctive per parliament:")
	for c, v in parliaments.items():
	d = ", ".join(f"{x['topic']} {x['x']}x" for x in v["distinct"]) or "—"
	print(f" {c:12s} {d}")
	print("\n female share ladder:", [(b["bloc"], b["share"]) for b in out["gender_blocs"]])


	if __name__ == "__main__":
	main()