"""Build the compact data file the Researchers' Night page reads.""" import json import sys from pathlib import Path import pandas as pd sys.path.insert(0, "../space") import extract # noqa: E402 ISO = { "Greece": "gr", "Germany": "de", "Italy": "it", "Netherlands": "nl", "Poland": "pl", "Sweden": "se", "Austria": "at", "Belgium": "be", "Bulgaria": "bg", "Croatia": "hr", "Cyprus": "cy", "Czech Republic": "cz", "Denmark": "dk", "Estonia": "ee", "Finland": "fi", "France": "fr", "Hungary": "hu", "Ireland": "ie", "Latvia": "lv", "Lithuania": "lt", "Luxembourg": "lu", "Malta": "mt", "Portugal": "pt", "Romania": "ro", "Slovakia": "sk", "Slovenia": "si", "Spain": "es", "United Kingdom": "gb", } EL_COUNTRY = { "Greece": "Ελλάδα", "Germany": "Γερμανία", "Italy": "Ιταλία", "Netherlands": "Ολλανδία", "Poland": "Πολωνία", "Sweden": "Σουηδία", "EU": "Ευρωπαϊκό Κοινοβούλιο", "Austria": "Αυστρία", "Belgium": "Βέλγιο", "Bulgaria": "Βουλγαρία", "Croatia": "Κροατία", "Cyprus": "Κύπρος", "Czech Republic": "Τσεχία", "Denmark": "Δανία", "Estonia": "Εσθονία", "Finland": "Φινλανδία", "France": "Γαλλία", "Hungary": "Ουγγαρία", "Ireland": "Ιρλανδία", "Latvia": "Λετονία", "Lithuania": "Λιθουανία", "Luxembourg": "Λουξεμβούργο", "Malta": "Μάλτα", "Portugal": "Πορτογαλία", "Romania": "Ρουμανία", "Slovakia": "Σλοβακία", "Slovenia": "Σλοβενία", "Spain": "Ισπανία", "United Kingdom": "Ηνωμένο Βασίλειο", } EL_TOPIC = { "Health": "Υγεία", "Environment": "Περιβάλλον", "Economy": "Οικονομία", "Conflicts": "Συγκρούσεις", "Migration": "Μετανάστευση", "Agriculture": "Γεωργία", "Gender": "Φύλο", "Religion": "Θρησκεία", "Cybersecurity": "Κυβερνοασφάλεια", "Hate Speech": "Ρητορική μίσους", "Debt crises": "Κρίση χρέους", "Terrorism - Extremism": "Τρομοκρατία", "Border Control": "Έλεγχος συνόρων", "Refugee crises": "Προσφυγικό", } EL_BLOC = { "Far-right": "Άκρα δεξιά", "Right": "Δεξιά", "Centre-right": "Κεντροδεξιά", "Centre": "Κέντρο", "Centre-left": "Κεντροαριστερά", "Left": "Αριστερά", "Far-left": "Άκρα αριστερά", "Other": "Άλλο", "Unknown": "Άγνωστο", } BLOC_ORDER = ["Far-left", "Left", "Centre-left", "Centre", "Centre-right", "Right", "Far-right"] def main(): D = extract.load("../space/topics.xlsx") out = {} # ---------------------------------------------------------- headline seg = D["country_volume"].query("unit == 'Segments'")["value"].sum() ses = D["country_volume"].query("unit == 'Sessions'")["value"].sum() speakers = D["stats"].query("metric == 'speakers'")["value"].sum() years = D["temporal"]["year"] out["headline"] = { "segments": int(seg), "sessions": int(ses), "speakers": int(speakers), "topics": int(D["orientation"].topic.nunique()), "year_min": int(years.min()), "year_max": int(years.max()), "parties": int(D["parties"].party.nunique()), "countries": len(ISO), } # ------------------------------------------------- who gets to speak s = D["sex"].pivot(index="topic", columns="sex", values="value").dropna() s["share"] = 100 * s["Female"] / (s["Female"] + s["Male"]) s = s.sort_values("share", ascending=False) out["gender_topics"] = [ {"topic": t, "el": EL_TOPIC.get(t, t), "share": round(r["share"], 1), "female": int(r["Female"]), "male": int(r["Male"])} for t, r in s.iterrows()] out["gender_overall"] = round( 100 * s["Female"].sum() / (s["Female"].sum() + s["Male"].sum()), 1) g = D["gender_orientation"].pivot_table( index="orientation", columns="sex", values="value", aggfunc="sum").dropna() g["share"] = 100 * g["Female"] / (g["Female"] + g["Male"]) out["gender_blocs"] = [ {"bloc": b, "el": EL_BLOC.get(b, b), "share": round(g.loc[b, "share"], 1)} for b in BLOC_ORDER if b in g.index] # --------------------------------------------------- map: parliaments p = D["prominence"].copy() p = p[p.value > 0] topic_mean = p.groupby("topic")["value"].mean() p["ratio"] = p.apply(lambda r: r["value"] / topic_mean[r["topic"]], axis=1) vol = D["country_volume"].query("unit == 'Segments'") parliaments = {} for c in D["prominence"].country.unique(): sub = p[p.country == c].sort_values("ratio", ascending=False) v = vol[vol.country == c].sort_values("value", ascending=False) parliaments[c] = { "name_en": c, "name_el": EL_COUNTRY.get(c, c), "iso": ISO.get(c), "tier": "parliament", "segments": int(v["value"].sum()), "distinct": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic), "x": round(r.ratio, 1)} for r in sub.head(3).itertuples() if r.ratio >= 1.15], "biggest": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic), "n": int(r.value)} for r in v.head(3).itertuples()], "topics_all": [{"topic": r.topic, "el": EL_TOPIC.get(r.topic, r.topic), "n": int(r.value)} for r in v.sort_values("value", ascending=False).itertuples() if r.value > 0], } out["parliaments"] = parliaments # ------------------------------------------------------- map: parties pa = D["parties"] countries = {} for name, iso in ISO.items(): sub = pa[pa.party_country == name] if sub.empty: continue top_topic = (sub.groupby("topic")["segments"].sum() .sort_values(ascending=False)) big = sub.sort_values("party_total", ascending=False).drop_duplicates("party") countries[name] = { "name_en": name, "name_el": EL_COUNTRY.get(name, name), "iso": iso, "tier": "parliament" if name in parliaments else "party", "parties": int(sub.party.nunique()), "top_topics": [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)} for t, n in top_topic.head(3).items()], "topics_all": [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)} for t, n in top_topic.items() if n > 0], "big_parties": [str(x).rsplit(" (", 1)[0] for x in big.party.head(3)], } if name in parliaments: countries[name].update({ k: parliaments[name][k] for k in ("distinct", "biggest", "segments", "topics_all")}) for name, pl in parliaments.items(): if name in countries or name == "EU": continue countries[name] = pl out["countries"] = countries out["eu"] = parliaments.get("EU") # --------------------------------------------------------- timeline t = D["temporal"].query("unit == 'Sessions'") lo, hi = t.year.min(), t.year.max() t = t[(t.year > lo) & (t.year < hi)] piv = t.pivot_table(index="year", columns="topic", values="value", aggfunc="sum") order = piv.sum().sort_values(ascending=False).index.tolist() out["timeline"] = { "years": [int(y) for y in piv.index], "series": [{"topic": tp, "el": EL_TOPIC.get(tp, tp), "values": [int(v) for v in piv[tp].fillna(0)]} for tp in order], } # ------------------------------------------------ topic totals (bars) totals = (D["country_volume"].query("unit == 'Segments'") .groupby("topic")["value"].sum().sort_values(ascending=False)) out["topic_totals"] = [{"topic": t, "el": EL_TOPIC.get(t, t), "n": int(n)} for t, n in totals.items()] # ------------------------------------- optional: bloc x year x country # Dropped in as far_right.csv (see DATA_REQUEST.md). Absent -> the exhibit # simply has no such chapter; nothing else changes. fr_path = Path("far_right.csv") if fr_path.exists(): fr = pd.read_csv(fr_path) need = {"country", "year", "bloc", "segments", "total_segments"} missing = need - set(fr.columns) if missing: raise SystemExit(f"far_right.csv is missing columns: {sorted(missing)}") fr["year"] = fr["year"].astype(int) fr["share"] = 100 * fr["segments"] / fr["total_segments"].replace(0, pd.NA) fr = fr.dropna(subset=["share"]) out["far_right"] = { "blocs": sorted(fr.bloc.unique().tolist()), "years": sorted(int(y) for y in fr.year.unique()), "rows": [{"country": r.country, "iso": ISO.get(r.country), "el": EL_COUNTRY.get(r.country, r.country), "year": int(r.year), "bloc": r.bloc, "segments": int(r.segments), "share": round(float(r.share), 2)} for r in fr.itertuples()], } n = len(out["far_right"]["rows"]) print(f" far_right.csv: {n} rows, " f"{fr.country.nunique()} countries, " f"{fr.year.min()}-{fr.year.max()}, {fr.bloc.nunique()} blocs") else: print(" far_right.csv not present - bloc/year/country chapter skipped") # --------------------------------------------- topic co-focus network # Two topics are linked when the same parties give both a large share of # their own speech. Spearman across parties, so party size does not drive it. # This is co-focus, not co-occurrence inside a speech, and not causation. import numpy as np pa2 = D["parties"].dropna(subset=["focus_pct"]) mat = pa2.pivot_table(index="party", columns="topic", values="focus_pct", aggfunc="sum") mat = mat[mat.notna().sum(axis=1) >= 6].fillna(0.0) mat.columns.name = None corr = mat.corr(method="spearman") names = [t for t in extract.TOPICS if t in corr.columns] names += [t for t in corr.columns if t not in names] R_MIN = 0.30 edges = [] for i, a in enumerate(names): for b in names[i + 1:]: r = float(corr.loc[a, b]) if r >= R_MIN: edges.append({"a": a, "b": b, "r": round(r, 3)}) edges.sort(key=lambda e: -e["r"]) # deterministic spring layout so the drawing is stable between builds rng = np.random.default_rng(7) idx = {t: i for i, t in enumerate(names)} n = len(names) pos = rng.normal(0, 0.35, size=(n, 2)) adj = np.zeros((n, n)) for e in edges: w = (e["r"] - R_MIN) / (1 - R_MIN) + 0.25 adj[idx[e["a"]], idx[e["b"]]] = adj[idx[e["b"]], idx[e["a"]]] = w for step in range(600): delta = pos[:, None, :] - pos[None, :, :] dist = np.sqrt((delta ** 2).sum(-1)) + 1e-6 rep = (delta / dist[..., None] ** 3) * 0.010 # everything pushes apart att = -(delta * (adj * dist)[..., None]) * 0.012 # edges pull together force = (rep + att).sum(axis=1) - pos * 0.010 # mild pull to centre pos += force * (1.0 - step / 700) pos -= pos.mean(axis=0) span = np.abs(pos).max() or 1.0 pos = pos / span seg_by_topic = (D["country_volume"].query("unit == 'Segments'") .groupby("topic")["value"].sum()) out["network"] = { "r_min": R_MIN, "parties": int(mat.shape[0]), "nodes": [{"topic": t, "el": EL_TOPIC.get(t, t), "x": round(float(pos[idx[t], 0]), 4), "y": round(float(pos[idx[t], 1]), 4), "n": int(seg_by_topic.get(t, 0))} for t in names], "edges": edges, } print(f" network: {len(names)} topics, {len(edges)} links " f"(r>={R_MIN}) from {mat.shape[0]} parties; " f"strongest {edges[0]['a']}-{edges[0]['b']} r={edges[0]['r']}") # ------------------------------- blocs and topics on one map (CA) # Correspondence analysis of the bloc x topic table: the standard method for # a contingency table, and it places blocs and topics in the same space, so # a bloc sitting in the direction of a topic means that bloc gives it more # attention than the corpus as a whole does. tab = (D["orientation"].pivot(index="orientation", columns="topic", values="value") .fillna(0.0)) # "Other" and "Unknown" are residual buckets, not political positions: # including them distorts the axes and confuses a lay reader. rows_order = [b for b in BLOC_ORDER if b in tab.index] dropped_rows = [b for b in tab.index if b not in rows_order] tab = tab.loc[rows_order] cols_order = [t for t in extract.TOPICS if t in tab.columns] cols_order += [t for t in tab.columns if t not in cols_order] tab = tab[cols_order] N = tab.values.astype(float) total = N.sum() P = N / total rm, cm = P.sum(1), P.sum(0) Sres = (P - np.outer(rm, cm)) / np.sqrt(np.outer(rm, cm)) U, sv, Vt = np.linalg.svd(Sres, full_matrices=False) inertia = sv ** 2 pct = (inertia / inertia.sum() * 100) Fr = (U * sv) / np.sqrt(rm)[:, None] # blocs Fc = (Vt.T * sv) / np.sqrt(cm)[:, None] # topics scale = max(np.abs(Fr[:, :2]).max(), np.abs(Fc[:, :2]).max()) or 1.0 over = (P / np.outer(rm, cm)) # 1.0 = exactly as expected over = pd.DataFrame(over, index=tab.index, columns=tab.columns) blocs_out = [] for i, b in enumerate(tab.index): top = over.loc[b].sort_values(ascending=False).head(3) blocs_out.append({ "key": b, "el": EL_BLOC.get(b, b), "x": round(float(Fr[i, 0] / scale), 4), "y": round(float(Fr[i, 1] / scale), 4), "n": int(N[i].sum()), "top": [{"topic": t, "el": EL_TOPIC.get(t, t), "x": round(float(v), 2)} for t, v in top.items()], }) topics_out = [] for j, t in enumerate(tab.columns): lead = over[t].sort_values(ascending=False).head(2) topics_out.append({ "topic": t, "el": EL_TOPIC.get(t, t), "x": round(float(Fc[j, 0] / scale), 4), "y": round(float(Fc[j, 1] / scale), 4), "n": int(N[:, j].sum()), "lead": [{"bloc": b, "el": EL_BLOC.get(b, b), "x": round(float(v), 2)} for b, v in lead.items()], }) out["camap"] = { "blocs": blocs_out, "topics": topics_out, "d1": round(float(pct[0]), 1), "d2": round(float(pct[1]), 1), "explained": round(float(pct[:2].sum()), 1), "segments": int(total), "excluded": dropped_rows, } print(f" bloc-topic map: {len(blocs_out)} blocs, {len(topics_out)} topics, " f"D1 {pct[0]:.1f}% + D2 {pct[1]:.1f}% = {pct[:2].sum():.1f}% of the structure" + (f"; excluded {dropped_rows}" if dropped_rows else "")) json.dump(out, open("data.json", "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":")) print("wrote data.json") print(" headline:", out["headline"]) print(" parliaments:", list(parliaments)) print(" countries on map:", len(out["countries"])) print("\n most distinctive per parliament:") for c, v in parliaments.items(): d = ", ".join(f"{x['topic']} {x['x']}x" for x in v["distinct"]) or "—" print(f" {c:12s} {d}") print("\n female share ladder:", [(b["bloc"], b["share"]) for b in out["gender_blocs"]]) if __name__ == "__main__": main()