Download build_data.py from idiouf/African-Books-Recovery-Project: direct link, hf CLI and curl.
- Browser
- Download file 33.6 kB
-
https://huggingface.co/spaces/idiouf/African-Books-Recovery-Project/resolve/main/build_data.py
- Command line
-
hf download hf://spaces/idiouf/African-Books-Recovery-Project/build_data.py
-
curl -L -o build_data.py https://huggingface.co/spaces/idiouf/African-Books-Recovery-Project/resolve/main/build_data.py
33.6 kB
| """ | |
| build_data.py - African Books Recovery Project | |
| ETL: normalise the hand-researched source JSON (*_presses.json, *_titles.json, | |
| rights.json, digitization.json, holdings.json) into the CSV tables and the | |
| data.js file that the static dashboard (index.html) reads. Everything lives side by side in one folder. | |
| Governance rules (mirrored in METHODS.md and enforced here): | |
| 1. Every record keeps its source URLs. Records with no source are dropped. | |
| 2. No value is invented. Unknown year stays null; it is never estimated. | |
| 3. Every record keeps the researcher confidence flag (high/medium/low) and | |
| the dashboard always exposes it. | |
| 4. Derived fields (region, decade, recovery stage, pilot cluster) are computed | |
| here, deterministically, and documented - never hand-entered. | |
| 5. "Out of print" is never rendered as "out of copyright". Rights status is a | |
| separate, explicitly unresolved field. | |
| Run: python build_data.py (needs pandas; the Space itself needs nothing) | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import pathlib | |
| import re | |
| import sys | |
| import pandas as pd | |
| # Flat layout: source JSON, fixtures and code all live in this one folder. | |
| ROOT = pathlib.Path(__file__).parent | |
| RESEARCH = ROOT | |
| DATA = ROOT | |
| BATCHES = [ | |
| ("nigeria", "Nigeria"), | |
| ("ghana", "Ghana & Anglophone West Africa"), | |
| ("eastafrica", "East Africa & Horn"), | |
| ("franco", "Francophone, Lusophone & North Africa"), | |
| ("southern", "Southern Africa"), | |
| ] | |
| # -------------------------------------------------------------------------- | |
| # Country -> region / normalised country name | |
| # -------------------------------------------------------------------------- | |
| COUNTRY_FIX = { | |
| "eSwatini": "Eswatini", | |
| "Republic of the Congo": "Congo", | |
| "United Republic of Tanzania": "Tanzania", | |
| "Ivory Coast": "Côte d'Ivoire", | |
| "São Tomé and Principe": "São Tomé and Príncipe", | |
| "Zaire (now Democratic Republic of the Congo)": "DR Congo", | |
| "Zimbabwe (Rhodesia)": "Zimbabwe", | |
| "Namibia (South West Africa)": "Namibia", | |
| "Senegal / Côte d'Ivoire / Togo": "Senegal", | |
| "The Gambia": "Gambia", | |
| } | |
| REGION = { | |
| # Les 54 pays, tous assignes : quand le catalogue ne couvrait que cinq | |
| # regions, une table partielle suffisait ; des que le moissonnage rapporte | |
| # du Mali ou du Rwanda, un pays absent d'ici tombe dans « Unrecorded » et | |
| # disparait des cartes comme des graphiques. La table doit etre complete. | |
| "Nigeria": "West Africa", "Ghana": "West Africa", "Sierra Leone": "West Africa", | |
| "Liberia": "West Africa", "Gambia": "West Africa", "Senegal": "West Africa", | |
| "Côte d'Ivoire": "West Africa", "Togo": "West Africa", "Benin": "West Africa", | |
| "Mali": "West Africa", "Burkina Faso": "West Africa", "Niger": "West Africa", | |
| "Guinea": "West Africa", "Guinea-Bissau": "West Africa", | |
| "Mauritania": "West Africa", "Cabo Verde": "West Africa", | |
| "Cameroon": "Central Africa", "DR Congo": "Central Africa", | |
| "Congo": "Central Africa", "Gabon": "Central Africa", "Chad": "Central Africa", | |
| "Central African Republic": "Central Africa", | |
| "Equatorial Guinea": "Central Africa", | |
| "São Tomé and Príncipe": "Central Africa", | |
| "Kenya": "East Africa", "Tanzania": "East Africa", "Uganda": "East Africa", | |
| "Rwanda": "East Africa", "Burundi": "East Africa", | |
| "Ethiopia": "Horn of Africa", "Somalia": "Horn of Africa", | |
| "Somaliland": "Horn of Africa", "Sudan": "Horn of Africa", | |
| "South Sudan": "Horn of Africa", "Eritrea": "Horn of Africa", | |
| "Djibouti": "Horn of Africa", | |
| "Egypt": "North Africa", "Algeria": "North Africa", "Tunisia": "North Africa", | |
| "Morocco": "North Africa", "Libya": "North Africa", | |
| "Western Sahara": "North Africa", | |
| "South Africa": "Southern Africa", "Zimbabwe": "Southern Africa", | |
| "Zambia": "Southern Africa", "Malawi": "Southern Africa", | |
| "Lesotho": "Southern Africa", "Botswana": "Southern Africa", | |
| "Namibia": "Southern Africa", "Mozambique": "Southern Africa", | |
| "Angola": "Southern Africa", "Eswatini": "Southern Africa", | |
| "Madagascar": "Indian Ocean", "Mauritius": "Indian Ocean", | |
| "Seychelles": "Indian Ocean", "Comoros": "Indian Ocean", | |
| "France": "Diaspora (Europe)", "Portugal": "Diaspora (Europe)", | |
| "United Kingdom": "Diaspora (Europe)", | |
| } | |
| # Country name as it appears in contours_afrique.csv (Natural Earth 1:50m). | |
| # Only the three that differ are listed; everything else matches on its own name. | |
| # A country with no entry and no match simply has no polygon, and the map says so | |
| # rather than silently dropping it. | |
| MAP_COUNTRY = { | |
| "DR Congo": "Democratic Republic of the Congo", | |
| "Tanzania": "United Republic of Tanzania", | |
| "Côte d'Ivoire": "Ivory Coast", | |
| "Congo": "Republic of the Congo", | |
| "Eswatini": "eSwatini", | |
| "São Tomé and Príncipe": "São Tomé and Principe", | |
| } | |
| # Cities: canonical name, coordinates and the spelling variants that occur in the | |
| # research files. Loaded once here so that normalisation is a reviewable table | |
| # rather than a regex buried in the app. | |
| _CITIES = json.loads((RESEARCH / "villes.json").read_text(encoding="utf-8"))["cities"] | |
| CITY_COORD = {c["city"]: (c["lat"], c["lon"], c["country"]) for c in _CITIES} | |
| CITY_CANON = {} | |
| for _c in _CITIES: | |
| CITY_CANON[_c["city"].lower()] = _c["city"] | |
| for _v in _c["variants"]: | |
| CITY_CANON[_v.lower()] = _c["city"] | |
| def canon_city(raw): | |
| """Canonical city name. An unmapped spelling is returned unchanged and shows | |
| up in selftest.py as a city with no coordinates - visible, not silently lost.""" | |
| r = (raw or "").strip() | |
| return CITY_CANON.get(r.lower(), r) | |
| def map_name(country): | |
| c = norm_country(country) | |
| return MAP_COUNTRY.get(c, c) | |
| # Presses whose imprint sits outside the continent. Kept in the register because | |
| # they were African-founded and African-directed, but never counted as | |
| # "published in Africa" without the flag being visible. | |
| DIASPORA_OWNERSHIP = {"african-founded-diaspora-based"} | |
| MAJOR_EUROPEAN_LANGS = {"English", "French", "Portuguese", "Arabic"} | |
| def norm_country(c): | |
| c = (c or "").strip() | |
| return COUNTRY_FIX.get(c, c) | |
| def region_of(country): | |
| return REGION.get(norm_country(country), "Unrecorded") | |
| def decade_of(year): | |
| if year is None or pd.isna(year): | |
| return "Undated" | |
| y = int(year) | |
| return f"{y // 10 * 10}s" | |
| def clean_str(v): | |
| """Chaine propre. Une cellule CSV vide arrive en float NaN : str(nan) donne | |
| « nan », qui est vrai - c'est ainsi qu'une notice sans source a franchi le | |
| controle « au moins une source ». Le NaN est donc traite comme le vide.""" | |
| if v is None: | |
| return "" | |
| if isinstance(v, float) and v != v: # NaN | |
| return "" | |
| t = str(v).strip() | |
| return "" if t.lower() in ("nan", "none") else t | |
| def sources_to_cols(rec): | |
| """Flatten the sources array. Records with zero sources are dropped upstream.""" | |
| srcs = rec.get("sources") or [] | |
| srcs = [s for s in srcs if isinstance(s, dict) and clean_str(s.get("url"))] | |
| return { | |
| "source_count": len(srcs), | |
| "source_titles": " | ".join(clean_str(s.get("title")) for s in srcs), | |
| "source_urls": " | ".join(clean_str(s.get("url")) for s in srcs), | |
| "primary_source_url": srcs[0]["url"] if srcs else "", | |
| } | |
| # -------------------------------------------------------------------------- | |
| # Presses | |
| # -------------------------------------------------------------------------- | |
| def build_presses() -> pd.DataFrame: | |
| rows = [] | |
| dropped = 0 | |
| for slug, batch_label in BATCHES: | |
| path = RESEARCH / f"{slug}_presses.json" | |
| for rec in json.loads(path.read_text(encoding="utf-8")): | |
| s = sources_to_cols(rec) | |
| if s["source_count"] == 0: | |
| dropped += 1 | |
| continue | |
| country = norm_country(rec.get("country")) | |
| ownership = clean_str(rec.get("ownership")) or "unknown" | |
| rows.append( | |
| { | |
| "press_id": rec["press_id"], | |
| "name": clean_str(rec.get("name")), | |
| "city": canon_city(rec.get("city")), | |
| "city_raw": clean_str(rec.get("city")), | |
| "country": country, | |
| "map_country": map_name(country), | |
| "region": region_of(country), | |
| "research_batch": batch_label, | |
| "founded": rec.get("founded"), | |
| "ceased": rec.get("ceased"), | |
| "active_span": span_label(rec.get("founded"), rec.get("ceased")), | |
| "ownership": ownership, | |
| "based_in_africa": ownership not in DIASPORA_OWNERSHIP | |
| and region_of(country) != "Diaspora (Europe)", | |
| "type": clean_str(rec.get("type")) or "unknown", | |
| "languages": ", ".join(rec.get("languages") or []), | |
| "key_figures": ", ".join(rec.get("key_figures") or []), | |
| "notable_series": ", ".join(rec.get("notable_series") or []), | |
| "description": clean_str(rec.get("description")), | |
| "significance": clean_str(rec.get("significance")), | |
| "archive_holdings": ", ".join(rec.get("archive_holdings") or []), | |
| "confidence": clean_str(rec.get("confidence")) or "low", | |
| **s, | |
| } | |
| ) | |
| df = pd.DataFrame(rows) | |
| if dropped: | |
| print(f" presses: dropped {dropped} unsourced record(s)") | |
| assert df["press_id"].is_unique, "duplicate press_id in source data" | |
| return df | |
| def span_label(founded, ceased): | |
| if founded and ceased: | |
| return f"{int(founded)}-{int(ceased)}" | |
| if founded: | |
| return f"{int(founded)}-" | |
| if ceased: | |
| return f"?-{int(ceased)}" | |
| return "undated" | |
| # -------------------------------------------------------------------------- | |
| # Titles | |
| # -------------------------------------------------------------------------- | |
| def build_titles(presses: pd.DataFrame) -> pd.DataFrame: | |
| pmap = presses.set_index("press_id").to_dict("index") | |
| rows = [] | |
| dropped = 0 | |
| for slug, batch_label in BATCHES: | |
| path = RESEARCH / f"{slug}_titles.json" | |
| for rec in json.loads(path.read_text(encoding="utf-8")): | |
| s = sources_to_cols(rec) | |
| if s["source_count"] == 0: | |
| dropped += 1 | |
| continue | |
| pid = rec.get("press_id") | |
| press = pmap.get(pid, {}) | |
| dig = rec.get("digitization") or {} | |
| dig_status = clean_str(dig.get("status")) or "unknown" | |
| year = rec.get("year") | |
| city = canon_city(rec.get("city")) or clean_str(press.get("city")) | |
| country = norm_country(press.get("country")) | |
| lat, lon, _ = CITY_COORD.get(city, (None, None, None)) | |
| lang = clean_str(rec.get("language")) or "unknown" | |
| rows.append( | |
| { | |
| "title": clean_str(rec.get("title")), | |
| "author": clean_str(rec.get("author")) or "Unattributed", | |
| "press_id": pid, | |
| "press_name": clean_str(rec.get("press_name")) or clean_str(press.get("name")), | |
| "press_ownership": clean_str(press.get("ownership")) or "unknown", | |
| "city": city, | |
| "lat": lat, | |
| "lon": lon, | |
| "country": country, | |
| "map_country": map_name(country), | |
| "region": region_of(country), | |
| "research_batch": batch_label, | |
| "year": year, | |
| "decade": decade_of(year), | |
| "language": lang, | |
| "african_language": lang not in MAJOR_EUROPEAN_LANGS and lang != "unknown", | |
| "language_group": language_group(lang), | |
| "genre": clean_str(rec.get("genre")) or "unrecorded", | |
| "genre_group": genre_group(rec.get("genre")), | |
| "pages": rec.get("pages"), | |
| "banned": rec.get("banned"), | |
| "notes": clean_str(rec.get("notes")), | |
| "digitization_status": dig_status, | |
| "digitization_holder": clean_str(dig.get("holder")), | |
| "digitization_url": clean_str(dig.get("url")), | |
| "in_print": clean_str(rec.get("in_print")) or "unknown", | |
| "confidence": clean_str(rec.get("confidence")) or "low", | |
| "provenance": "verified", | |
| **s, | |
| } | |
| ) | |
| df = pd.DataFrame(rows) | |
| if dropped: | |
| print(f" titles: dropped {dropped} unsourced record(s)") | |
| # ---- recovery stages ------------------------------------------------- | |
| # Each stage is an INDEPENDENT flag, not a monotonic funnel: a handful of | |
| # titles are commercially in print (stage 5) without the project having | |
| # traced rights (stage 3) itself. The dashboard says so explicitly. | |
| df["stage1_located"] = True # a catalogue record exists - that is this table | |
| df["stage2_preserved"] = df["digitization_status"].isin( | |
| ["full_text_open", "scanned_restricted"] | |
| ) | |
| df["stage3_rights_identified"] = False # no estate or successor traced yet | |
| df["stage4_cleared"] = False # no permission obtained yet | |
| df["stage5_republished"] = df["in_print"].eq("yes") | |
| df["stage6_curriculum_ready"] = False # no teaching apparatus written yet | |
| df["furthest_stage"] = ( | |
| df[["stage1_located", "stage2_preserved", "stage5_republished"]] | |
| .apply( | |
| lambda r: "Republished" | |
| if r["stage5_republished"] | |
| else ("Preserved" if r["stage2_preserved"] else "Located"), | |
| axis=1, | |
| ) | |
| ) | |
| df["title_id"] = [f"T{i:04d}" for i in range(1, len(df) + 1)] | |
| df = assign_pilot(df) | |
| return df | |
| # Regroupement des langues pour le filtre de l'interface. Vingt-sept langues | |
| # forment une liste que personne ne parcourt ; cinq groupes repondent a la | |
| # question qu'on se pose vraiment. Les langues africaines ne sont PAS fondues | |
| # entre elles dans les donnees : la colonne `language` garde le detail, et le | |
| # catalogue l'affiche. Ce regroupement ne sert qu'a rendre un filtre cliquable. | |
| LANGUAGE_GROUPS = ["English", "French", "Portuguese", "Arabic", | |
| "African languages", "Unrecorded"] | |
| def language_group(lang): | |
| if lang in MAJOR_EUROPEAN_LANGS: | |
| return lang | |
| if not lang or lang == "unknown": | |
| return "Unrecorded" | |
| return "African languages" | |
| def merge_harvested(verified: pd.DataFrame) -> pd.DataFrame: | |
| """Ajoute harvested.csv, s'il existe, SANS toucher aux notices verifiees. | |
| Deux niveaux qui ne se melangent pas : | |
| provenance="verified" les notices etablies a la main, avec leurs sources | |
| provenance="harvested" les pistes moissonnees par harvest.py | |
| Une piste qui double une notice verifiee est ecartee : la verification | |
| l'emporte toujours sur la machine. Le tableau de bord compte les deux | |
| separement et permet de n'afficher que le verifie.""" | |
| src = ROOT / "harvested.csv" | |
| if not src.exists(): | |
| print(" harvested.csv absent - catalogue verifie seul") | |
| return verified | |
| h = pd.read_csv(src) | |
| if h.empty: | |
| return verified | |
| def cle(t, a, y): | |
| t = re.sub(r"[^a-z0-9 ]", "", clean_str(t).lower())[:80] | |
| a = re.sub(r"[^a-z0-9 ]", "", clean_str(a).split(";")[0].lower())[:40] | |
| # 1956.0 cote verifie, 1956 cote moissonne : sans cette normalisation | |
| # la meme notice passait deux fois. | |
| try: | |
| y = str(int(float(y))) | |
| except (TypeError, ValueError): | |
| y = "" | |
| return (t.strip(), a.strip(), y) | |
| connues = {cle(r.title, r.author, r.year) for r in verified.itertuples()} | |
| rows, doublons = [], 0 | |
| for r in h.itertuples(): | |
| an = int(r.year) if pd.notna(r.year) and str(r.year).strip() else None | |
| if cle(r.title, r.author, an) in connues: | |
| doublons += 1 | |
| continue | |
| country = norm_country(getattr(r, "country", "") or "") | |
| lang = clean_str(getattr(r, "language", "")) or "unknown" | |
| scan = clean_str(getattr(r, "scan_hint", "")) | |
| urls = [u for u in [clean_str(getattr(r, "record_url", "")), | |
| clean_str(getattr(r, "query_url", ""))] if u] | |
| if not urls: | |
| continue # la regle des sources vaut aussi ici | |
| rows.append({ | |
| "title": clean_str(r.title), "author": clean_str(r.author) or "Unattributed", | |
| "press_id": "", "press_name": clean_str(getattr(r, "publisher", "")), | |
| "press_ownership": "unknown", | |
| "city": canon_city(getattr(r, "place", "")), | |
| "lat": None, "lon": None, | |
| "country": country, "map_country": map_name(country), | |
| "region": region_of(country), "research_batch": "harvested", | |
| "year": an, "decade": decade_of(an), "language": lang, | |
| "african_language": lang not in MAJOR_EUROPEAN_LANGS and lang != "unknown", | |
| "language_group": language_group(lang), | |
| "genre": "unrecorded", "genre_group": "Other / unrecorded", | |
| "pages": None, "banned": None, | |
| "notes": f"Harvested from {r.source}. Not verified by hand.", | |
| "digitization_status": ("scanned_restricted" if scan else "unknown"), | |
| "digitization_holder": scan, "digitization_url": "", | |
| "in_print": "unknown", | |
| "confidence": clean_str(getattr(r, "confidence", "")) or "low", | |
| "provenance": "harvested", | |
| "source_count": len(urls), | |
| "source_titles": clean_str(getattr(r, "source", "")), | |
| "source_urls": " | ".join(urls), "primary_source_url": urls[0], | |
| }) | |
| if not rows: | |
| print(f" harvested: 0 new record ({doublons} already verified)") | |
| return verified | |
| hd = pd.DataFrame(rows) | |
| for col in ["stage1_located", "stage2_preserved", "stage3_rights_identified", | |
| "stage4_cleared", "stage5_republished", "stage6_curriculum_ready"]: | |
| hd[col] = False | |
| hd["stage1_located"] = True | |
| hd["stage2_preserved"] = hd["digitization_status"].isin( | |
| ["full_text_open", "scanned_restricted"]) | |
| hd["furthest_stage"] = hd["stage2_preserved"].map({True: "Preserved", | |
| False: "Located"}) | |
| hd["pilot_cluster"] = "" # le pilote reste un choix humain | |
| hd["title_id"] = [f"H{i:05d}" for i in range(1, len(hd) + 1)] | |
| out = pd.concat([verified, hd], ignore_index=True) | |
| print(f" harvested: {len(hd)} added, {doublons} dropped as already verified") | |
| return out | |
| GENRE_GROUPS = [ | |
| ("Children's & folklore", r"child|folk|fable|tale|picture|juvenil|reader"), | |
| ("Fiction & popular fiction", r"novel|fiction|novella|short stor|romance|crime|thriller"), | |
| ("Poetry", r"poet|verse"), | |
| ("Drama", r"drama|play|theatre"), | |
| ("Politics & political economy", r"politic|econom|nationalis|liberat|sociolog"), | |
| ("History & biography", r"histor|biograph|memoir|autobiograph"), | |
| ("Language & reference", r"dictionar|grammar|language|textbook|linguist"), | |
| ("Essays & criticism", r"essay|critic|philosoph|theolog"), | |
| ("Popular pamphlet & self-help", r"pamphlet|self-help|conduct|advice|letter-writ"), | |
| ] | |
| def genre_group(g): | |
| g = (g or "").lower() | |
| for label, pattern in GENRE_GROUPS: | |
| if re.search(pattern, g): | |
| return label | |
| return "Other / unrecorded" | |
| # -------------------------------------------------------------------------- | |
| # Pilot clusters - the 100-title first phase | |
| # -------------------------------------------------------------------------- | |
| PILOT_TARGETS = { | |
| "A · Onitsha market pamphlets": 25, | |
| "B · Ghanaian children's & folklore": 20, | |
| "C · East African popular fiction": 20, | |
| "D · Francophone (CLE, NEA & related)": 20, | |
| "E · African-language works": 15, | |
| } | |
| CONF_RANK = {"high": 0, "medium": 1, "low": 2} | |
| def assign_pilot(df: pd.DataFrame) -> pd.DataFrame: | |
| """Assign each title to at most one pilot cluster, in cluster order, taking | |
| the best-evidenced candidates first. Deterministic: sort keys are explicit.""" | |
| df["pilot_cluster"] = "" | |
| taken = set() | |
| def pick(mask, cluster): | |
| pool = df[mask & ~df.index.isin(taken)].copy() | |
| pool["_c"] = pool["confidence"].map(CONF_RANK).fillna(3) | |
| pool = pool.sort_values( | |
| ["_c", "source_count", "year", "title"], | |
| ascending=[True, False, True, True], | |
| na_position="last", | |
| ) | |
| chosen = pool.head(PILOT_TARGETS[cluster]).index | |
| df.loc[chosen, "pilot_cluster"] = cluster | |
| taken.update(chosen) | |
| return len(pool), len(chosen) | |
| stats = {} | |
| stats["A · Onitsha market pamphlets"] = pick( | |
| df["city"].str.contains("Onitsha", case=False, na=False), | |
| "A · Onitsha market pamphlets", | |
| ) | |
| stats["B · Ghanaian children's & folklore"] = pick( | |
| (df["country"] == "Ghana") | |
| & ( | |
| df["genre_group"].eq("Children's & folklore") | |
| | df["genre"].str.contains("folk|fable|tale|child", case=False, na=False) | |
| ), | |
| "B · Ghanaian children's & folklore", | |
| ) | |
| stats["C · East African popular fiction"] = pick( | |
| df["region"].eq("East Africa") | |
| & df["genre_group"].isin(["Fiction & popular fiction", "Drama"]), | |
| "C · East African popular fiction", | |
| ) | |
| stats["D · Francophone (CLE, NEA & related)"] = pick( | |
| df["language"].str.contains("French", na=False), | |
| "D · Francophone (CLE, NEA & related)", | |
| ) | |
| stats["E · African-language works"] = pick( | |
| df["african_language"], | |
| "E · African-language works", | |
| ) | |
| for cluster, (pool, chosen) in stats.items(): | |
| flag = "" if chosen >= PILOT_TARGETS[cluster] else " << SHORT" | |
| print(f" pilot {cluster}: {chosen}/{PILOT_TARGETS[cluster]} (pool {pool}){flag}") | |
| return df | |
| # -------------------------------------------------------------------------- | |
| # Digitization initiatives & rights | |
| # -------------------------------------------------------------------------- | |
| def build_digitization() -> pd.DataFrame: | |
| rows = [] | |
| for rec in json.loads((RESEARCH / "digitization.json").read_text(encoding="utf-8")): | |
| s = sources_to_cols(rec) | |
| rows.append( | |
| { | |
| "initiative_id": rec["initiative_id"], | |
| "name": clean_str(rec.get("name")), | |
| "host_institution": clean_str(rec.get("host_institution")), | |
| "country": clean_str(rec.get("country")), | |
| "region": clean_str(rec.get("region")) or "Unrecorded", | |
| "in_africa": clean_str(rec.get("region")) == "Africa", | |
| "scope": clean_str(rec.get("scope")), | |
| "relevance": clean_str(rec.get("relevance")), | |
| "access": clean_str(rec.get("access")) or "unknown", | |
| "formats": ", ".join(rec.get("formats") or []), | |
| "api": clean_str(rec.get("api")) or "unknown", | |
| "approx_items": rec.get("approx_items"), | |
| "url": clean_str(rec.get("url")), | |
| "confidence": clean_str(rec.get("confidence")) or "low", | |
| **s, | |
| } | |
| ) | |
| return pd.DataFrame(rows) | |
| def build_rights() -> pd.DataFrame: | |
| rows = [] | |
| for rec in json.loads((RESEARCH / "rights.json").read_text(encoding="utf-8")): | |
| s = sources_to_cols(rec) | |
| rows.append( | |
| { | |
| "country": norm_country(rec.get("country")), | |
| "region": region_of(rec.get("country")), | |
| "statute": clean_str(rec.get("statute")), | |
| "copyright_term": clean_str(rec.get("copyright_term")), | |
| "term_years": rec.get("term_years"), | |
| "anonymous_or_corporate_term": clean_str(rec.get("anonymous_or_corporate_term")), | |
| "berne_member": bool(rec.get("berne_member")), | |
| "public_domain_notes": clean_str(rec.get("public_domain_notes")), | |
| "orphan_works_mechanism": clean_str(rec.get("orphan_works_mechanism")), | |
| "recovery_implication": clean_str(rec.get("recovery_implication")), | |
| "confidence": clean_str(rec.get("confidence")) or "low", | |
| **s, | |
| } | |
| ) | |
| return pd.DataFrame(rows) | |
| def build_cities(titles: pd.DataFrame, presses: pd.DataFrame) -> pd.DataFrame: | |
| """One row per publishing city, with the coordinates used to place its dot. | |
| Cities with no gazetteer entry are kept with empty coordinates: they belong | |
| in the counts even when they cannot be mapped, and the app names them. | |
| """ | |
| t = titles.groupby("city").size().rename("titles") | |
| pr = presses.groupby("city").size().rename("presses") | |
| df = pd.concat([t, pr], axis=1).fillna(0).astype(int).reset_index() | |
| df["lat"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, None))[0]) | |
| df["lon"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, None))[1]) | |
| df["country"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, ""))[2]) | |
| df["region"] = df["country"].map(region_of) | |
| df["on_map"] = df["lat"].notna() & (df["region"] != "Diaspora (Europe)") | |
| unplaced = df[df["lat"].isna()]["city"].tolist() | |
| if unplaced: | |
| print(f" cities: {len(unplaced)} without coordinates -> {', '.join(unplaced)}") | |
| return df.sort_values("titles", ascending=False) | |
| def build_countries(titles: pd.DataFrame, presses: pd.DataFrame) -> pd.DataFrame: | |
| """Coverage for every African country that has a polygon - zeros included. | |
| The zeros are the point of this table. A country with no record is not a | |
| country with no publishing history; it is a country nobody has searched yet, | |
| and the map has to be able to say so. | |
| """ | |
| contours = pd.read_csv(ROOT / "contours_afrique.csv") | |
| names = sorted(contours["pays"].unique()) | |
| v = titles[titles["provenance"] == "verified"] | |
| t = titles.groupby("map_country").size().rename("titles") | |
| tv = v.groupby("map_country").size().rename("titles_verified") | |
| pr = presses.groupby("map_country").size().rename("presses") | |
| dg = (titles[titles["stage2_preserved"]].groupby("map_country").size() | |
| .rename("preserved")) | |
| df = pd.DataFrame({"map_country": names}).set_index("map_country") | |
| df = df.join(t).join(tv).join(pr).join(dg).fillna(0).astype(int).reset_index() | |
| df["titles_harvested"] = df["titles"] - df["titles_verified"] | |
| back = {v: k for k, v in MAP_COUNTRY.items()} | |
| df["country"] = df["map_country"].map(lambda n: back.get(n, n)) | |
| df["region"] = df["country"].map(region_of) | |
| df["searched"] = df["titles"] > 0 | |
| df["verified"] = df["titles_verified"] > 0 | |
| # Countries the research reached that have no polygon: they must not vanish. | |
| orphans = sorted(set(titles["map_country"]) - set(names) - {""}) | |
| if orphans: | |
| print(f" countries: {len(orphans)} with records but no polygon -> " | |
| f"{', '.join(orphans)} (off-continent imprints, shown as a note)") | |
| n_zero = int((~df["searched"]).sum()) | |
| print(f" countries: {len(df)} on the map, {n_zero} with no record yet") | |
| return df.sort_values("titles", ascending=False) | |
| def build_holdings() -> pd.DataFrame: | |
| """Search channels - where surviving copies may physically be. | |
| provenance = 'project brief' : asserted in the project scoping brief | |
| provenance = 'press register' : named in a press record's archive_holdings | |
| """ | |
| rows = json.loads((RESEARCH / "holdings.json").read_text(encoding="utf-8")) | |
| df = pd.DataFrame(rows) | |
| df["in_africa"] = df["location"].eq("Africa") | |
| return df | |
| # -------------------------------------------------------------------------- | |
| # Static export: one data.js file read by index.html | |
| # -------------------------------------------------------------------------- | |
| def plain(v): | |
| """Replace en and em dashes by a plain hyphen; leave everything else.""" | |
| if isinstance(v, str): | |
| return (v.replace(" \u2014 ", " - ").replace("\u2014", "-") | |
| .replace(" \u2013 ", " - ").replace("\u2013", "-")) | |
| return v | |
| # Map frame: the west edge goes to -25.5 so that Cabo Verde is drawn. | |
| LON0, LON1, LAT0, LAT1 = -25.5, 52.5, -36.0, 38.0 | |
| KX = 10 * 0.9962 # cos(5 deg): equirectangular centred near the equator | |
| KY = 10 | |
| def _xy(lon, lat): | |
| return round((lon - LON0) * KX, 1), round((LAT1 - lat) * KY, 1) | |
| def map_paths(): | |
| """Country outlines as SVG paths, with a label anchor for each country. | |
| The anchor is the area-weighted centroid of the largest piece, so that an | |
| island group or an exclave never pulls the label into the sea.""" | |
| ct = pd.read_csv(ROOT / "contours_afrique.csv").sort_values(["pays", "piece", "ordre"]) | |
| out = {} | |
| for pays, g in ct.groupby("pays", sort=True): | |
| parts, best = [], (0.0, None) | |
| total = 0.0 | |
| for _, piece in g.groupby("piece", sort=False): | |
| pts = [_xy(a, b) for a, b in zip(piece["lon"], piece["lat"])] | |
| parts.append("M" + "L".join(f"{x},{y}" for x, y in pts) + "Z") | |
| a = cx = cy = 0.0 | |
| for (x0, y0), (x1, y1) in zip(pts, pts[1:] + pts[:1]): | |
| c = x0 * y1 - x1 * y0 | |
| a += c | |
| cx += (x0 + x1) * c | |
| cy += (y0 + y1) * c | |
| a /= 2.0 | |
| total += abs(a) | |
| if abs(a) > best[0] and a != 0: | |
| best = (abs(a), (round(cx / (6 * a), 1), round(cy / (6 * a), 1))) | |
| out[pays] = {"d": "".join(parts), "c": best[1], "area": round(total)} | |
| w = round((LON1 - LON0) * KX, 1) | |
| h = round((LAT1 - LAT0) * KY, 1) | |
| return {"w": w, "h": h, "lon0": LON0, "lat1": LAT1, "kx": KX, "ky": KY, | |
| "countries": out} | |
| def _records(df): | |
| df = df.astype(object).where(pd.notna(df), None) | |
| return df.to_dict("records") | |
| def write_datajs(presses, titles, digi, rights, holdings, cities, countries): | |
| tcols = ["title_id", "title", "author", "press_id", "press_name", "city", | |
| "country", "map_country", "region", "year", "decade", "language", | |
| "language_group", "african_language", "genre", "genre_group", "pages", | |
| "banned", "notes", "digitization_status", "digitization_holder", | |
| "digitization_url", "in_print", "confidence", "provenance", | |
| "source_titles", "source_urls", "pilot_cluster", "stage1_located", | |
| "stage2_preserved", "stage3_rights_identified", "stage4_cleared", | |
| "stage5_republished", "stage6_curriculum_ready"] | |
| t = titles[tcols].copy() | |
| t["year"] = t["year"].map(lambda y: int(y) if pd.notna(y) else None) | |
| t["pages"] = t["pages"].map(lambda y: int(y) if pd.notna(y) else None) | |
| p = presses.copy() | |
| for c in ("founded", "ceased"): | |
| p[c] = p[c].map(lambda y: int(y) if pd.notna(y) else None) | |
| payload = { | |
| "built": pd.Timestamp.now().strftime("%Y-%m-%d"), | |
| "titles": _records(t), "presses": _records(p), | |
| "rights": _records(rights), "digitization": _records(digi), | |
| "holdings": _records(holdings), "cities": _records(cities), | |
| "countries": _records(countries), "map": map_paths(), | |
| } | |
| txt = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) | |
| txt = plain(txt) | |
| (DATA / "data.js").write_text("window.ABRP=" + txt + ";\n", encoding="utf-8") | |
| print(f" wrote data.js {(DATA / 'data.js').stat().st_size/1024:7.1f} kB") | |
| # -------------------------------------------------------------------------- | |
| def main(): | |
| print("Building African Books Recovery Project fixtures…") | |
| presses = build_presses() | |
| titles = build_titles(presses) | |
| titles = merge_harvested(titles) | |
| digi = build_digitization() | |
| rights = build_rights() | |
| holdings = build_holdings() | |
| cities = build_cities(titles, presses) | |
| countries = build_countries(titles, presses) | |
| # No en dash or em dash anywhere in what the dashboard shows: the source | |
| # JSON keeps its typography, the published tables use a plain hyphen. | |
| tables = [presses, titles, digi, rights, holdings, cities, countries] | |
| for df in tables: | |
| for c in df.columns: | |
| if not pd.api.types.is_numeric_dtype(df[c]): | |
| df[c] = df[c].map(plain) | |
| # CSV, not parquet: the Space then needs streamlit alone - no pyarrow, no | |
| # binary format, and the catalogue stays readable in any spreadsheet. | |
| for name, df in [ | |
| ("presses", presses), | |
| ("titles", titles), | |
| ("digitization", digi), | |
| ("rights", rights), | |
| ("holdings", holdings), | |
| ("cities", cities), | |
| ("countries", countries), | |
| ]: | |
| out = DATA / f"{name}.csv" | |
| df.to_csv(out, index=False) | |
| print(f" wrote {out.name:22s} {len(df):4d} rows {out.stat().st_size/1024:7.1f} kB") | |
| write_datajs(presses, titles, digi, rights, holdings, cities, countries) | |
| nv = int((titles["provenance"] == "verified").sum()) | |
| nh = len(titles) - nv | |
| print( | |
| f"\nTotals: {len(titles)} titles ({nv} verified + {nh} harvested) · " | |
| f"{len(presses)} presses · {titles['country'].nunique()} countries · " | |
| f"{titles['language'].nunique()} languages" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |