""" build_data.py - African Books Recovery Project ETL: normalise the hand-researched source JSON (*_presses.json, *_titles.json, rights.json, digitization.json, holdings.json) into the CSV tables and the data.js file that the static dashboard (index.html) reads. Everything lives side by side in one folder. Governance rules (mirrored in METHODS.md and enforced here): 1. Every record keeps its source URLs. Records with no source are dropped. 2. No value is invented. Unknown year stays null; it is never estimated. 3. Every record keeps the researcher confidence flag (high/medium/low) and the dashboard always exposes it. 4. Derived fields (region, decade, recovery stage, pilot cluster) are computed here, deterministically, and documented - never hand-entered. 5. "Out of print" is never rendered as "out of copyright". Rights status is a separate, explicitly unresolved field. Run: python build_data.py (needs pandas; the Space itself needs nothing) """ from __future__ import annotations import json import pathlib import re import sys import pandas as pd # Flat layout: source JSON, fixtures and code all live in this one folder. ROOT = pathlib.Path(__file__).parent RESEARCH = ROOT DATA = ROOT BATCHES = [ ("nigeria", "Nigeria"), ("ghana", "Ghana & Anglophone West Africa"), ("eastafrica", "East Africa & Horn"), ("franco", "Francophone, Lusophone & North Africa"), ("southern", "Southern Africa"), ] # -------------------------------------------------------------------------- # Country -> region / normalised country name # -------------------------------------------------------------------------- COUNTRY_FIX = { "eSwatini": "Eswatini", "Republic of the Congo": "Congo", "United Republic of Tanzania": "Tanzania", "Ivory Coast": "Côte d'Ivoire", "São Tomé and Principe": "São Tomé and Príncipe", "Zaire (now Democratic Republic of the Congo)": "DR Congo", "Zimbabwe (Rhodesia)": "Zimbabwe", "Namibia (South West Africa)": "Namibia", "Senegal / Côte d'Ivoire / Togo": "Senegal", "The Gambia": "Gambia", } REGION = { # Les 54 pays, tous assignes : quand le catalogue ne couvrait que cinq # regions, une table partielle suffisait ; des que le moissonnage rapporte # du Mali ou du Rwanda, un pays absent d'ici tombe dans « Unrecorded » et # disparait des cartes comme des graphiques. La table doit etre complete. "Nigeria": "West Africa", "Ghana": "West Africa", "Sierra Leone": "West Africa", "Liberia": "West Africa", "Gambia": "West Africa", "Senegal": "West Africa", "Côte d'Ivoire": "West Africa", "Togo": "West Africa", "Benin": "West Africa", "Mali": "West Africa", "Burkina Faso": "West Africa", "Niger": "West Africa", "Guinea": "West Africa", "Guinea-Bissau": "West Africa", "Mauritania": "West Africa", "Cabo Verde": "West Africa", "Cameroon": "Central Africa", "DR Congo": "Central Africa", "Congo": "Central Africa", "Gabon": "Central Africa", "Chad": "Central Africa", "Central African Republic": "Central Africa", "Equatorial Guinea": "Central Africa", "São Tomé and Príncipe": "Central Africa", "Kenya": "East Africa", "Tanzania": "East Africa", "Uganda": "East Africa", "Rwanda": "East Africa", "Burundi": "East Africa", "Ethiopia": "Horn of Africa", "Somalia": "Horn of Africa", "Somaliland": "Horn of Africa", "Sudan": "Horn of Africa", "South Sudan": "Horn of Africa", "Eritrea": "Horn of Africa", "Djibouti": "Horn of Africa", "Egypt": "North Africa", "Algeria": "North Africa", "Tunisia": "North Africa", "Morocco": "North Africa", "Libya": "North Africa", "Western Sahara": "North Africa", "South Africa": "Southern Africa", "Zimbabwe": "Southern Africa", "Zambia": "Southern Africa", "Malawi": "Southern Africa", "Lesotho": "Southern Africa", "Botswana": "Southern Africa", "Namibia": "Southern Africa", "Mozambique": "Southern Africa", "Angola": "Southern Africa", "Eswatini": "Southern Africa", "Madagascar": "Indian Ocean", "Mauritius": "Indian Ocean", "Seychelles": "Indian Ocean", "Comoros": "Indian Ocean", "France": "Diaspora (Europe)", "Portugal": "Diaspora (Europe)", "United Kingdom": "Diaspora (Europe)", } # Country name as it appears in contours_afrique.csv (Natural Earth 1:50m). # Only the three that differ are listed; everything else matches on its own name. # A country with no entry and no match simply has no polygon, and the map says so # rather than silently dropping it. MAP_COUNTRY = { "DR Congo": "Democratic Republic of the Congo", "Tanzania": "United Republic of Tanzania", "Côte d'Ivoire": "Ivory Coast", "Congo": "Republic of the Congo", "Eswatini": "eSwatini", "São Tomé and Príncipe": "São Tomé and Principe", } # Cities: canonical name, coordinates and the spelling variants that occur in the # research files. Loaded once here so that normalisation is a reviewable table # rather than a regex buried in the app. _CITIES = json.loads((RESEARCH / "villes.json").read_text(encoding="utf-8"))["cities"] CITY_COORD = {c["city"]: (c["lat"], c["lon"], c["country"]) for c in _CITIES} CITY_CANON = {} for _c in _CITIES: CITY_CANON[_c["city"].lower()] = _c["city"] for _v in _c["variants"]: CITY_CANON[_v.lower()] = _c["city"] def canon_city(raw): """Canonical city name. An unmapped spelling is returned unchanged and shows up in selftest.py as a city with no coordinates - visible, not silently lost.""" r = (raw or "").strip() return CITY_CANON.get(r.lower(), r) def map_name(country): c = norm_country(country) return MAP_COUNTRY.get(c, c) # Presses whose imprint sits outside the continent. Kept in the register because # they were African-founded and African-directed, but never counted as # "published in Africa" without the flag being visible. DIASPORA_OWNERSHIP = {"african-founded-diaspora-based"} MAJOR_EUROPEAN_LANGS = {"English", "French", "Portuguese", "Arabic"} def norm_country(c): c = (c or "").strip() return COUNTRY_FIX.get(c, c) def region_of(country): return REGION.get(norm_country(country), "Unrecorded") def decade_of(year): if year is None or pd.isna(year): return "Undated" y = int(year) return f"{y // 10 * 10}s" def clean_str(v): """Chaine propre. Une cellule CSV vide arrive en float NaN : str(nan) donne « nan », qui est vrai - c'est ainsi qu'une notice sans source a franchi le controle « au moins une source ». Le NaN est donc traite comme le vide.""" if v is None: return "" if isinstance(v, float) and v != v: # NaN return "" t = str(v).strip() return "" if t.lower() in ("nan", "none") else t def sources_to_cols(rec): """Flatten the sources array. Records with zero sources are dropped upstream.""" srcs = rec.get("sources") or [] srcs = [s for s in srcs if isinstance(s, dict) and clean_str(s.get("url"))] return { "source_count": len(srcs), "source_titles": " | ".join(clean_str(s.get("title")) for s in srcs), "source_urls": " | ".join(clean_str(s.get("url")) for s in srcs), "primary_source_url": srcs[0]["url"] if srcs else "", } # -------------------------------------------------------------------------- # Presses # -------------------------------------------------------------------------- def build_presses() -> pd.DataFrame: rows = [] dropped = 0 for slug, batch_label in BATCHES: path = RESEARCH / f"{slug}_presses.json" for rec in json.loads(path.read_text(encoding="utf-8")): s = sources_to_cols(rec) if s["source_count"] == 0: dropped += 1 continue country = norm_country(rec.get("country")) ownership = clean_str(rec.get("ownership")) or "unknown" rows.append( { "press_id": rec["press_id"], "name": clean_str(rec.get("name")), "city": canon_city(rec.get("city")), "city_raw": clean_str(rec.get("city")), "country": country, "map_country": map_name(country), "region": region_of(country), "research_batch": batch_label, "founded": rec.get("founded"), "ceased": rec.get("ceased"), "active_span": span_label(rec.get("founded"), rec.get("ceased")), "ownership": ownership, "based_in_africa": ownership not in DIASPORA_OWNERSHIP and region_of(country) != "Diaspora (Europe)", "type": clean_str(rec.get("type")) or "unknown", "languages": ", ".join(rec.get("languages") or []), "key_figures": ", ".join(rec.get("key_figures") or []), "notable_series": ", ".join(rec.get("notable_series") or []), "description": clean_str(rec.get("description")), "significance": clean_str(rec.get("significance")), "archive_holdings": ", ".join(rec.get("archive_holdings") or []), "confidence": clean_str(rec.get("confidence")) or "low", **s, } ) df = pd.DataFrame(rows) if dropped: print(f" presses: dropped {dropped} unsourced record(s)") assert df["press_id"].is_unique, "duplicate press_id in source data" return df def span_label(founded, ceased): if founded and ceased: return f"{int(founded)}-{int(ceased)}" if founded: return f"{int(founded)}-" if ceased: return f"?-{int(ceased)}" return "undated" # -------------------------------------------------------------------------- # Titles # -------------------------------------------------------------------------- def build_titles(presses: pd.DataFrame) -> pd.DataFrame: pmap = presses.set_index("press_id").to_dict("index") rows = [] dropped = 0 for slug, batch_label in BATCHES: path = RESEARCH / f"{slug}_titles.json" for rec in json.loads(path.read_text(encoding="utf-8")): s = sources_to_cols(rec) if s["source_count"] == 0: dropped += 1 continue pid = rec.get("press_id") press = pmap.get(pid, {}) dig = rec.get("digitization") or {} dig_status = clean_str(dig.get("status")) or "unknown" year = rec.get("year") city = canon_city(rec.get("city")) or clean_str(press.get("city")) country = norm_country(press.get("country")) lat, lon, _ = CITY_COORD.get(city, (None, None, None)) lang = clean_str(rec.get("language")) or "unknown" rows.append( { "title": clean_str(rec.get("title")), "author": clean_str(rec.get("author")) or "Unattributed", "press_id": pid, "press_name": clean_str(rec.get("press_name")) or clean_str(press.get("name")), "press_ownership": clean_str(press.get("ownership")) or "unknown", "city": city, "lat": lat, "lon": lon, "country": country, "map_country": map_name(country), "region": region_of(country), "research_batch": batch_label, "year": year, "decade": decade_of(year), "language": lang, "african_language": lang not in MAJOR_EUROPEAN_LANGS and lang != "unknown", "language_group": language_group(lang), "genre": clean_str(rec.get("genre")) or "unrecorded", "genre_group": genre_group(rec.get("genre")), "pages": rec.get("pages"), "banned": rec.get("banned"), "notes": clean_str(rec.get("notes")), "digitization_status": dig_status, "digitization_holder": clean_str(dig.get("holder")), "digitization_url": clean_str(dig.get("url")), "in_print": clean_str(rec.get("in_print")) or "unknown", "confidence": clean_str(rec.get("confidence")) or "low", "provenance": "verified", **s, } ) df = pd.DataFrame(rows) if dropped: print(f" titles: dropped {dropped} unsourced record(s)") # ---- recovery stages ------------------------------------------------- # Each stage is an INDEPENDENT flag, not a monotonic funnel: a handful of # titles are commercially in print (stage 5) without the project having # traced rights (stage 3) itself. The dashboard says so explicitly. df["stage1_located"] = True # a catalogue record exists - that is this table df["stage2_preserved"] = df["digitization_status"].isin( ["full_text_open", "scanned_restricted"] ) df["stage3_rights_identified"] = False # no estate or successor traced yet df["stage4_cleared"] = False # no permission obtained yet df["stage5_republished"] = df["in_print"].eq("yes") df["stage6_curriculum_ready"] = False # no teaching apparatus written yet df["furthest_stage"] = ( df[["stage1_located", "stage2_preserved", "stage5_republished"]] .apply( lambda r: "Republished" if r["stage5_republished"] else ("Preserved" if r["stage2_preserved"] else "Located"), axis=1, ) ) df["title_id"] = [f"T{i:04d}" for i in range(1, len(df) + 1)] df = assign_pilot(df) return df # Regroupement des langues pour le filtre de l'interface. Vingt-sept langues # forment une liste que personne ne parcourt ; cinq groupes repondent a la # question qu'on se pose vraiment. Les langues africaines ne sont PAS fondues # entre elles dans les donnees : la colonne `language` garde le detail, et le # catalogue l'affiche. Ce regroupement ne sert qu'a rendre un filtre cliquable. LANGUAGE_GROUPS = ["English", "French", "Portuguese", "Arabic", "African languages", "Unrecorded"] def language_group(lang): if lang in MAJOR_EUROPEAN_LANGS: return lang if not lang or lang == "unknown": return "Unrecorded" return "African languages" def merge_harvested(verified: pd.DataFrame) -> pd.DataFrame: """Ajoute harvested.csv, s'il existe, SANS toucher aux notices verifiees. Deux niveaux qui ne se melangent pas : provenance="verified" les notices etablies a la main, avec leurs sources provenance="harvested" les pistes moissonnees par harvest.py Une piste qui double une notice verifiee est ecartee : la verification l'emporte toujours sur la machine. Le tableau de bord compte les deux separement et permet de n'afficher que le verifie.""" src = ROOT / "harvested.csv" if not src.exists(): print(" harvested.csv absent - catalogue verifie seul") return verified h = pd.read_csv(src) if h.empty: return verified def cle(t, a, y): t = re.sub(r"[^a-z0-9 ]", "", clean_str(t).lower())[:80] a = re.sub(r"[^a-z0-9 ]", "", clean_str(a).split(";")[0].lower())[:40] # 1956.0 cote verifie, 1956 cote moissonne : sans cette normalisation # la meme notice passait deux fois. try: y = str(int(float(y))) except (TypeError, ValueError): y = "" return (t.strip(), a.strip(), y) connues = {cle(r.title, r.author, r.year) for r in verified.itertuples()} rows, doublons = [], 0 for r in h.itertuples(): an = int(r.year) if pd.notna(r.year) and str(r.year).strip() else None if cle(r.title, r.author, an) in connues: doublons += 1 continue country = norm_country(getattr(r, "country", "") or "") lang = clean_str(getattr(r, "language", "")) or "unknown" scan = clean_str(getattr(r, "scan_hint", "")) urls = [u for u in [clean_str(getattr(r, "record_url", "")), clean_str(getattr(r, "query_url", ""))] if u] if not urls: continue # la regle des sources vaut aussi ici rows.append({ "title": clean_str(r.title), "author": clean_str(r.author) or "Unattributed", "press_id": "", "press_name": clean_str(getattr(r, "publisher", "")), "press_ownership": "unknown", "city": canon_city(getattr(r, "place", "")), "lat": None, "lon": None, "country": country, "map_country": map_name(country), "region": region_of(country), "research_batch": "harvested", "year": an, "decade": decade_of(an), "language": lang, "african_language": lang not in MAJOR_EUROPEAN_LANGS and lang != "unknown", "language_group": language_group(lang), "genre": "unrecorded", "genre_group": "Other / unrecorded", "pages": None, "banned": None, "notes": f"Harvested from {r.source}. Not verified by hand.", "digitization_status": ("scanned_restricted" if scan else "unknown"), "digitization_holder": scan, "digitization_url": "", "in_print": "unknown", "confidence": clean_str(getattr(r, "confidence", "")) or "low", "provenance": "harvested", "source_count": len(urls), "source_titles": clean_str(getattr(r, "source", "")), "source_urls": " | ".join(urls), "primary_source_url": urls[0], }) if not rows: print(f" harvested: 0 new record ({doublons} already verified)") return verified hd = pd.DataFrame(rows) for col in ["stage1_located", "stage2_preserved", "stage3_rights_identified", "stage4_cleared", "stage5_republished", "stage6_curriculum_ready"]: hd[col] = False hd["stage1_located"] = True hd["stage2_preserved"] = hd["digitization_status"].isin( ["full_text_open", "scanned_restricted"]) hd["furthest_stage"] = hd["stage2_preserved"].map({True: "Preserved", False: "Located"}) hd["pilot_cluster"] = "" # le pilote reste un choix humain hd["title_id"] = [f"H{i:05d}" for i in range(1, len(hd) + 1)] out = pd.concat([verified, hd], ignore_index=True) print(f" harvested: {len(hd)} added, {doublons} dropped as already verified") return out GENRE_GROUPS = [ ("Children's & folklore", r"child|folk|fable|tale|picture|juvenil|reader"), ("Fiction & popular fiction", r"novel|fiction|novella|short stor|romance|crime|thriller"), ("Poetry", r"poet|verse"), ("Drama", r"drama|play|theatre"), ("Politics & political economy", r"politic|econom|nationalis|liberat|sociolog"), ("History & biography", r"histor|biograph|memoir|autobiograph"), ("Language & reference", r"dictionar|grammar|language|textbook|linguist"), ("Essays & criticism", r"essay|critic|philosoph|theolog"), ("Popular pamphlet & self-help", r"pamphlet|self-help|conduct|advice|letter-writ"), ] def genre_group(g): g = (g or "").lower() for label, pattern in GENRE_GROUPS: if re.search(pattern, g): return label return "Other / unrecorded" # -------------------------------------------------------------------------- # Pilot clusters - the 100-title first phase # -------------------------------------------------------------------------- PILOT_TARGETS = { "A · Onitsha market pamphlets": 25, "B · Ghanaian children's & folklore": 20, "C · East African popular fiction": 20, "D · Francophone (CLE, NEA & related)": 20, "E · African-language works": 15, } CONF_RANK = {"high": 0, "medium": 1, "low": 2} def assign_pilot(df: pd.DataFrame) -> pd.DataFrame: """Assign each title to at most one pilot cluster, in cluster order, taking the best-evidenced candidates first. Deterministic: sort keys are explicit.""" df["pilot_cluster"] = "" taken = set() def pick(mask, cluster): pool = df[mask & ~df.index.isin(taken)].copy() pool["_c"] = pool["confidence"].map(CONF_RANK).fillna(3) pool = pool.sort_values( ["_c", "source_count", "year", "title"], ascending=[True, False, True, True], na_position="last", ) chosen = pool.head(PILOT_TARGETS[cluster]).index df.loc[chosen, "pilot_cluster"] = cluster taken.update(chosen) return len(pool), len(chosen) stats = {} stats["A · Onitsha market pamphlets"] = pick( df["city"].str.contains("Onitsha", case=False, na=False), "A · Onitsha market pamphlets", ) stats["B · Ghanaian children's & folklore"] = pick( (df["country"] == "Ghana") & ( df["genre_group"].eq("Children's & folklore") | df["genre"].str.contains("folk|fable|tale|child", case=False, na=False) ), "B · Ghanaian children's & folklore", ) stats["C · East African popular fiction"] = pick( df["region"].eq("East Africa") & df["genre_group"].isin(["Fiction & popular fiction", "Drama"]), "C · East African popular fiction", ) stats["D · Francophone (CLE, NEA & related)"] = pick( df["language"].str.contains("French", na=False), "D · Francophone (CLE, NEA & related)", ) stats["E · African-language works"] = pick( df["african_language"], "E · African-language works", ) for cluster, (pool, chosen) in stats.items(): flag = "" if chosen >= PILOT_TARGETS[cluster] else " << SHORT" print(f" pilot {cluster}: {chosen}/{PILOT_TARGETS[cluster]} (pool {pool}){flag}") return df # -------------------------------------------------------------------------- # Digitization initiatives & rights # -------------------------------------------------------------------------- def build_digitization() -> pd.DataFrame: rows = [] for rec in json.loads((RESEARCH / "digitization.json").read_text(encoding="utf-8")): s = sources_to_cols(rec) rows.append( { "initiative_id": rec["initiative_id"], "name": clean_str(rec.get("name")), "host_institution": clean_str(rec.get("host_institution")), "country": clean_str(rec.get("country")), "region": clean_str(rec.get("region")) or "Unrecorded", "in_africa": clean_str(rec.get("region")) == "Africa", "scope": clean_str(rec.get("scope")), "relevance": clean_str(rec.get("relevance")), "access": clean_str(rec.get("access")) or "unknown", "formats": ", ".join(rec.get("formats") or []), "api": clean_str(rec.get("api")) or "unknown", "approx_items": rec.get("approx_items"), "url": clean_str(rec.get("url")), "confidence": clean_str(rec.get("confidence")) or "low", **s, } ) return pd.DataFrame(rows) def build_rights() -> pd.DataFrame: rows = [] for rec in json.loads((RESEARCH / "rights.json").read_text(encoding="utf-8")): s = sources_to_cols(rec) rows.append( { "country": norm_country(rec.get("country")), "region": region_of(rec.get("country")), "statute": clean_str(rec.get("statute")), "copyright_term": clean_str(rec.get("copyright_term")), "term_years": rec.get("term_years"), "anonymous_or_corporate_term": clean_str(rec.get("anonymous_or_corporate_term")), "berne_member": bool(rec.get("berne_member")), "public_domain_notes": clean_str(rec.get("public_domain_notes")), "orphan_works_mechanism": clean_str(rec.get("orphan_works_mechanism")), "recovery_implication": clean_str(rec.get("recovery_implication")), "confidence": clean_str(rec.get("confidence")) or "low", **s, } ) return pd.DataFrame(rows) def build_cities(titles: pd.DataFrame, presses: pd.DataFrame) -> pd.DataFrame: """One row per publishing city, with the coordinates used to place its dot. Cities with no gazetteer entry are kept with empty coordinates: they belong in the counts even when they cannot be mapped, and the app names them. """ t = titles.groupby("city").size().rename("titles") pr = presses.groupby("city").size().rename("presses") df = pd.concat([t, pr], axis=1).fillna(0).astype(int).reset_index() df["lat"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, None))[0]) df["lon"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, None))[1]) df["country"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, ""))[2]) df["region"] = df["country"].map(region_of) df["on_map"] = df["lat"].notna() & (df["region"] != "Diaspora (Europe)") unplaced = df[df["lat"].isna()]["city"].tolist() if unplaced: print(f" cities: {len(unplaced)} without coordinates -> {', '.join(unplaced)}") return df.sort_values("titles", ascending=False) def build_countries(titles: pd.DataFrame, presses: pd.DataFrame) -> pd.DataFrame: """Coverage for every African country that has a polygon - zeros included. The zeros are the point of this table. A country with no record is not a country with no publishing history; it is a country nobody has searched yet, and the map has to be able to say so. """ contours = pd.read_csv(ROOT / "contours_afrique.csv") names = sorted(contours["pays"].unique()) v = titles[titles["provenance"] == "verified"] t = titles.groupby("map_country").size().rename("titles") tv = v.groupby("map_country").size().rename("titles_verified") pr = presses.groupby("map_country").size().rename("presses") dg = (titles[titles["stage2_preserved"]].groupby("map_country").size() .rename("preserved")) df = pd.DataFrame({"map_country": names}).set_index("map_country") df = df.join(t).join(tv).join(pr).join(dg).fillna(0).astype(int).reset_index() df["titles_harvested"] = df["titles"] - df["titles_verified"] back = {v: k for k, v in MAP_COUNTRY.items()} df["country"] = df["map_country"].map(lambda n: back.get(n, n)) df["region"] = df["country"].map(region_of) df["searched"] = df["titles"] > 0 df["verified"] = df["titles_verified"] > 0 # Countries the research reached that have no polygon: they must not vanish. orphans = sorted(set(titles["map_country"]) - set(names) - {""}) if orphans: print(f" countries: {len(orphans)} with records but no polygon -> " f"{', '.join(orphans)} (off-continent imprints, shown as a note)") n_zero = int((~df["searched"]).sum()) print(f" countries: {len(df)} on the map, {n_zero} with no record yet") return df.sort_values("titles", ascending=False) def build_holdings() -> pd.DataFrame: """Search channels - where surviving copies may physically be. provenance = 'project brief' : asserted in the project scoping brief provenance = 'press register' : named in a press record's archive_holdings """ rows = json.loads((RESEARCH / "holdings.json").read_text(encoding="utf-8")) df = pd.DataFrame(rows) df["in_africa"] = df["location"].eq("Africa") return df # -------------------------------------------------------------------------- # Static export: one data.js file read by index.html # -------------------------------------------------------------------------- def plain(v): """Replace en and em dashes by a plain hyphen; leave everything else.""" if isinstance(v, str): return (v.replace(" \u2014 ", " - ").replace("\u2014", "-") .replace(" \u2013 ", " - ").replace("\u2013", "-")) return v # Map frame: the west edge goes to -25.5 so that Cabo Verde is drawn. LON0, LON1, LAT0, LAT1 = -25.5, 52.5, -36.0, 38.0 KX = 10 * 0.9962 # cos(5 deg): equirectangular centred near the equator KY = 10 def _xy(lon, lat): return round((lon - LON0) * KX, 1), round((LAT1 - lat) * KY, 1) def map_paths(): """Country outlines as SVG paths, with a label anchor for each country. The anchor is the area-weighted centroid of the largest piece, so that an island group or an exclave never pulls the label into the sea.""" ct = pd.read_csv(ROOT / "contours_afrique.csv").sort_values(["pays", "piece", "ordre"]) out = {} for pays, g in ct.groupby("pays", sort=True): parts, best = [], (0.0, None) total = 0.0 for _, piece in g.groupby("piece", sort=False): pts = [_xy(a, b) for a, b in zip(piece["lon"], piece["lat"])] parts.append("M" + "L".join(f"{x},{y}" for x, y in pts) + "Z") a = cx = cy = 0.0 for (x0, y0), (x1, y1) in zip(pts, pts[1:] + pts[:1]): c = x0 * y1 - x1 * y0 a += c cx += (x0 + x1) * c cy += (y0 + y1) * c a /= 2.0 total += abs(a) if abs(a) > best[0] and a != 0: best = (abs(a), (round(cx / (6 * a), 1), round(cy / (6 * a), 1))) out[pays] = {"d": "".join(parts), "c": best[1], "area": round(total)} w = round((LON1 - LON0) * KX, 1) h = round((LAT1 - LAT0) * KY, 1) return {"w": w, "h": h, "lon0": LON0, "lat1": LAT1, "kx": KX, "ky": KY, "countries": out} def _records(df): df = df.astype(object).where(pd.notna(df), None) return df.to_dict("records") def write_datajs(presses, titles, digi, rights, holdings, cities, countries): tcols = ["title_id", "title", "author", "press_id", "press_name", "city", "country", "map_country", "region", "year", "decade", "language", "language_group", "african_language", "genre", "genre_group", "pages", "banned", "notes", "digitization_status", "digitization_holder", "digitization_url", "in_print", "confidence", "provenance", "source_titles", "source_urls", "pilot_cluster", "stage1_located", "stage2_preserved", "stage3_rights_identified", "stage4_cleared", "stage5_republished", "stage6_curriculum_ready"] t = titles[tcols].copy() t["year"] = t["year"].map(lambda y: int(y) if pd.notna(y) else None) t["pages"] = t["pages"].map(lambda y: int(y) if pd.notna(y) else None) p = presses.copy() for c in ("founded", "ceased"): p[c] = p[c].map(lambda y: int(y) if pd.notna(y) else None) payload = { "built": pd.Timestamp.now().strftime("%Y-%m-%d"), "titles": _records(t), "presses": _records(p), "rights": _records(rights), "digitization": _records(digi), "holdings": _records(holdings), "cities": _records(cities), "countries": _records(countries), "map": map_paths(), } txt = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) txt = plain(txt) (DATA / "data.js").write_text("window.ABRP=" + txt + ";\n", encoding="utf-8") print(f" wrote data.js {(DATA / 'data.js').stat().st_size/1024:7.1f} kB") # -------------------------------------------------------------------------- def main(): print("Building African Books Recovery Project fixtures…") presses = build_presses() titles = build_titles(presses) titles = merge_harvested(titles) digi = build_digitization() rights = build_rights() holdings = build_holdings() cities = build_cities(titles, presses) countries = build_countries(titles, presses) # No en dash or em dash anywhere in what the dashboard shows: the source # JSON keeps its typography, the published tables use a plain hyphen. tables = [presses, titles, digi, rights, holdings, cities, countries] for df in tables: for c in df.columns: if not pd.api.types.is_numeric_dtype(df[c]): df[c] = df[c].map(plain) # CSV, not parquet: the Space then needs streamlit alone - no pyarrow, no # binary format, and the catalogue stays readable in any spreadsheet. for name, df in [ ("presses", presses), ("titles", titles), ("digitization", digi), ("rights", rights), ("holdings", holdings), ("cities", cities), ("countries", countries), ]: out = DATA / f"{name}.csv" df.to_csv(out, index=False) print(f" wrote {out.name:22s} {len(df):4d} rows {out.stat().st_size/1024:7.1f} kB") write_datajs(presses, titles, digi, rights, holdings, cities, countries) nv = int((titles["provenance"] == "verified").sum()) nh = len(titles) - nv print( f"\nTotals: {len(titles)} titles ({nv} verified + {nh} harvested) · " f"{len(presses)} presses · {titles['country'].nunique()} countries · " f"{titles['language'].nunique()} languages" ) return 0 if __name__ == "__main__": sys.exit(main())