idiouf's picture
Upload 29 files
a074ed2 verified
Raw History Blame Contribute Delete
33.6 kB
"""
build_data.py - African Books Recovery Project
ETL: normalise the hand-researched source JSON (*_presses.json, *_titles.json,
rights.json, digitization.json, holdings.json) into the CSV tables and the
data.js file that the static dashboard (index.html) reads. Everything lives side by side in one folder.
Governance rules (mirrored in METHODS.md and enforced here):
1. Every record keeps its source URLs. Records with no source are dropped.
2. No value is invented. Unknown year stays null; it is never estimated.
3. Every record keeps the researcher confidence flag (high/medium/low) and
the dashboard always exposes it.
4. Derived fields (region, decade, recovery stage, pilot cluster) are computed
here, deterministically, and documented - never hand-entered.
5. "Out of print" is never rendered as "out of copyright". Rights status is a
separate, explicitly unresolved field.
Run: python build_data.py (needs pandas; the Space itself needs nothing)
"""
from __future__ import annotations
import json
import pathlib
import re
import sys
import pandas as pd
# Flat layout: source JSON, fixtures and code all live in this one folder.
ROOT = pathlib.Path(__file__).parent
RESEARCH = ROOT
DATA = ROOT
BATCHES = [
("nigeria", "Nigeria"),
("ghana", "Ghana & Anglophone West Africa"),
("eastafrica", "East Africa & Horn"),
("franco", "Francophone, Lusophone & North Africa"),
("southern", "Southern Africa"),
]
# --------------------------------------------------------------------------
# Country -> region / normalised country name
# --------------------------------------------------------------------------
COUNTRY_FIX = {
"eSwatini": "Eswatini",
"Republic of the Congo": "Congo",
"United Republic of Tanzania": "Tanzania",
"Ivory Coast": "Côte d'Ivoire",
"São Tomé and Principe": "São Tomé and Príncipe",
"Zaire (now Democratic Republic of the Congo)": "DR Congo",
"Zimbabwe (Rhodesia)": "Zimbabwe",
"Namibia (South West Africa)": "Namibia",
"Senegal / Côte d'Ivoire / Togo": "Senegal",
"The Gambia": "Gambia",
}
REGION = {
# Les 54 pays, tous assignes : quand le catalogue ne couvrait que cinq
# regions, une table partielle suffisait ; des que le moissonnage rapporte
# du Mali ou du Rwanda, un pays absent d'ici tombe dans « Unrecorded » et
# disparait des cartes comme des graphiques. La table doit etre complete.
"Nigeria": "West Africa", "Ghana": "West Africa", "Sierra Leone": "West Africa",
"Liberia": "West Africa", "Gambia": "West Africa", "Senegal": "West Africa",
"Côte d'Ivoire": "West Africa", "Togo": "West Africa", "Benin": "West Africa",
"Mali": "West Africa", "Burkina Faso": "West Africa", "Niger": "West Africa",
"Guinea": "West Africa", "Guinea-Bissau": "West Africa",
"Mauritania": "West Africa", "Cabo Verde": "West Africa",
"Cameroon": "Central Africa", "DR Congo": "Central Africa",
"Congo": "Central Africa", "Gabon": "Central Africa", "Chad": "Central Africa",
"Central African Republic": "Central Africa",
"Equatorial Guinea": "Central Africa",
"São Tomé and Príncipe": "Central Africa",
"Kenya": "East Africa", "Tanzania": "East Africa", "Uganda": "East Africa",
"Rwanda": "East Africa", "Burundi": "East Africa",
"Ethiopia": "Horn of Africa", "Somalia": "Horn of Africa",
"Somaliland": "Horn of Africa", "Sudan": "Horn of Africa",
"South Sudan": "Horn of Africa", "Eritrea": "Horn of Africa",
"Djibouti": "Horn of Africa",
"Egypt": "North Africa", "Algeria": "North Africa", "Tunisia": "North Africa",
"Morocco": "North Africa", "Libya": "North Africa",
"Western Sahara": "North Africa",
"South Africa": "Southern Africa", "Zimbabwe": "Southern Africa",
"Zambia": "Southern Africa", "Malawi": "Southern Africa",
"Lesotho": "Southern Africa", "Botswana": "Southern Africa",
"Namibia": "Southern Africa", "Mozambique": "Southern Africa",
"Angola": "Southern Africa", "Eswatini": "Southern Africa",
"Madagascar": "Indian Ocean", "Mauritius": "Indian Ocean",
"Seychelles": "Indian Ocean", "Comoros": "Indian Ocean",
"France": "Diaspora (Europe)", "Portugal": "Diaspora (Europe)",
"United Kingdom": "Diaspora (Europe)",
}
# Country name as it appears in contours_afrique.csv (Natural Earth 1:50m).
# Only the three that differ are listed; everything else matches on its own name.
# A country with no entry and no match simply has no polygon, and the map says so
# rather than silently dropping it.
MAP_COUNTRY = {
"DR Congo": "Democratic Republic of the Congo",
"Tanzania": "United Republic of Tanzania",
"Côte d'Ivoire": "Ivory Coast",
"Congo": "Republic of the Congo",
"Eswatini": "eSwatini",
"São Tomé and Príncipe": "São Tomé and Principe",
}
# Cities: canonical name, coordinates and the spelling variants that occur in the
# research files. Loaded once here so that normalisation is a reviewable table
# rather than a regex buried in the app.
_CITIES = json.loads((RESEARCH / "villes.json").read_text(encoding="utf-8"))["cities"]
CITY_COORD = {c["city"]: (c["lat"], c["lon"], c["country"]) for c in _CITIES}
CITY_CANON = {}
for _c in _CITIES:
CITY_CANON[_c["city"].lower()] = _c["city"]
for _v in _c["variants"]:
CITY_CANON[_v.lower()] = _c["city"]
def canon_city(raw):
"""Canonical city name. An unmapped spelling is returned unchanged and shows
up in selftest.py as a city with no coordinates - visible, not silently lost."""
r = (raw or "").strip()
return CITY_CANON.get(r.lower(), r)
def map_name(country):
c = norm_country(country)
return MAP_COUNTRY.get(c, c)
# Presses whose imprint sits outside the continent. Kept in the register because
# they were African-founded and African-directed, but never counted as
# "published in Africa" without the flag being visible.
DIASPORA_OWNERSHIP = {"african-founded-diaspora-based"}
MAJOR_EUROPEAN_LANGS = {"English", "French", "Portuguese", "Arabic"}
def norm_country(c):
c = (c or "").strip()
return COUNTRY_FIX.get(c, c)
def region_of(country):
return REGION.get(norm_country(country), "Unrecorded")
def decade_of(year):
if year is None or pd.isna(year):
return "Undated"
y = int(year)
return f"{y // 10 * 10}s"
def clean_str(v):
"""Chaine propre. Une cellule CSV vide arrive en float NaN : str(nan) donne
« nan », qui est vrai - c'est ainsi qu'une notice sans source a franchi le
controle « au moins une source ». Le NaN est donc traite comme le vide."""
if v is None:
return ""
if isinstance(v, float) and v != v: # NaN
return ""
t = str(v).strip()
return "" if t.lower() in ("nan", "none") else t
def sources_to_cols(rec):
"""Flatten the sources array. Records with zero sources are dropped upstream."""
srcs = rec.get("sources") or []
srcs = [s for s in srcs if isinstance(s, dict) and clean_str(s.get("url"))]
return {
"source_count": len(srcs),
"source_titles": " | ".join(clean_str(s.get("title")) for s in srcs),
"source_urls": " | ".join(clean_str(s.get("url")) for s in srcs),
"primary_source_url": srcs[0]["url"] if srcs else "",
}
# --------------------------------------------------------------------------
# Presses
# --------------------------------------------------------------------------
def build_presses() -> pd.DataFrame:
rows = []
dropped = 0
for slug, batch_label in BATCHES:
path = RESEARCH / f"{slug}_presses.json"
for rec in json.loads(path.read_text(encoding="utf-8")):
s = sources_to_cols(rec)
if s["source_count"] == 0:
dropped += 1
continue
country = norm_country(rec.get("country"))
ownership = clean_str(rec.get("ownership")) or "unknown"
rows.append(
{
"press_id": rec["press_id"],
"name": clean_str(rec.get("name")),
"city": canon_city(rec.get("city")),
"city_raw": clean_str(rec.get("city")),
"country": country,
"map_country": map_name(country),
"region": region_of(country),
"research_batch": batch_label,
"founded": rec.get("founded"),
"ceased": rec.get("ceased"),
"active_span": span_label(rec.get("founded"), rec.get("ceased")),
"ownership": ownership,
"based_in_africa": ownership not in DIASPORA_OWNERSHIP
and region_of(country) != "Diaspora (Europe)",
"type": clean_str(rec.get("type")) or "unknown",
"languages": ", ".join(rec.get("languages") or []),
"key_figures": ", ".join(rec.get("key_figures") or []),
"notable_series": ", ".join(rec.get("notable_series") or []),
"description": clean_str(rec.get("description")),
"significance": clean_str(rec.get("significance")),
"archive_holdings": ", ".join(rec.get("archive_holdings") or []),
"confidence": clean_str(rec.get("confidence")) or "low",
**s,
}
)
df = pd.DataFrame(rows)
if dropped:
print(f" presses: dropped {dropped} unsourced record(s)")
assert df["press_id"].is_unique, "duplicate press_id in source data"
return df
def span_label(founded, ceased):
if founded and ceased:
return f"{int(founded)}-{int(ceased)}"
if founded:
return f"{int(founded)}-"
if ceased:
return f"?-{int(ceased)}"
return "undated"
# --------------------------------------------------------------------------
# Titles
# --------------------------------------------------------------------------
def build_titles(presses: pd.DataFrame) -> pd.DataFrame:
pmap = presses.set_index("press_id").to_dict("index")
rows = []
dropped = 0
for slug, batch_label in BATCHES:
path = RESEARCH / f"{slug}_titles.json"
for rec in json.loads(path.read_text(encoding="utf-8")):
s = sources_to_cols(rec)
if s["source_count"] == 0:
dropped += 1
continue
pid = rec.get("press_id")
press = pmap.get(pid, {})
dig = rec.get("digitization") or {}
dig_status = clean_str(dig.get("status")) or "unknown"
year = rec.get("year")
city = canon_city(rec.get("city")) or clean_str(press.get("city"))
country = norm_country(press.get("country"))
lat, lon, _ = CITY_COORD.get(city, (None, None, None))
lang = clean_str(rec.get("language")) or "unknown"
rows.append(
{
"title": clean_str(rec.get("title")),
"author": clean_str(rec.get("author")) or "Unattributed",
"press_id": pid,
"press_name": clean_str(rec.get("press_name")) or clean_str(press.get("name")),
"press_ownership": clean_str(press.get("ownership")) or "unknown",
"city": city,
"lat": lat,
"lon": lon,
"country": country,
"map_country": map_name(country),
"region": region_of(country),
"research_batch": batch_label,
"year": year,
"decade": decade_of(year),
"language": lang,
"african_language": lang not in MAJOR_EUROPEAN_LANGS and lang != "unknown",
"language_group": language_group(lang),
"genre": clean_str(rec.get("genre")) or "unrecorded",
"genre_group": genre_group(rec.get("genre")),
"pages": rec.get("pages"),
"banned": rec.get("banned"),
"notes": clean_str(rec.get("notes")),
"digitization_status": dig_status,
"digitization_holder": clean_str(dig.get("holder")),
"digitization_url": clean_str(dig.get("url")),
"in_print": clean_str(rec.get("in_print")) or "unknown",
"confidence": clean_str(rec.get("confidence")) or "low",
"provenance": "verified",
**s,
}
)
df = pd.DataFrame(rows)
if dropped:
print(f" titles: dropped {dropped} unsourced record(s)")
# ---- recovery stages -------------------------------------------------
# Each stage is an INDEPENDENT flag, not a monotonic funnel: a handful of
# titles are commercially in print (stage 5) without the project having
# traced rights (stage 3) itself. The dashboard says so explicitly.
df["stage1_located"] = True # a catalogue record exists - that is this table
df["stage2_preserved"] = df["digitization_status"].isin(
["full_text_open", "scanned_restricted"]
)
df["stage3_rights_identified"] = False # no estate or successor traced yet
df["stage4_cleared"] = False # no permission obtained yet
df["stage5_republished"] = df["in_print"].eq("yes")
df["stage6_curriculum_ready"] = False # no teaching apparatus written yet
df["furthest_stage"] = (
df[["stage1_located", "stage2_preserved", "stage5_republished"]]
.apply(
lambda r: "Republished"
if r["stage5_republished"]
else ("Preserved" if r["stage2_preserved"] else "Located"),
axis=1,
)
)
df["title_id"] = [f"T{i:04d}" for i in range(1, len(df) + 1)]
df = assign_pilot(df)
return df
# Regroupement des langues pour le filtre de l'interface. Vingt-sept langues
# forment une liste que personne ne parcourt ; cinq groupes repondent a la
# question qu'on se pose vraiment. Les langues africaines ne sont PAS fondues
# entre elles dans les donnees : la colonne `language` garde le detail, et le
# catalogue l'affiche. Ce regroupement ne sert qu'a rendre un filtre cliquable.
LANGUAGE_GROUPS = ["English", "French", "Portuguese", "Arabic",
"African languages", "Unrecorded"]
def language_group(lang):
if lang in MAJOR_EUROPEAN_LANGS:
return lang
if not lang or lang == "unknown":
return "Unrecorded"
return "African languages"
def merge_harvested(verified: pd.DataFrame) -> pd.DataFrame:
"""Ajoute harvested.csv, s'il existe, SANS toucher aux notices verifiees.
Deux niveaux qui ne se melangent pas :
provenance="verified" les notices etablies a la main, avec leurs sources
provenance="harvested" les pistes moissonnees par harvest.py
Une piste qui double une notice verifiee est ecartee : la verification
l'emporte toujours sur la machine. Le tableau de bord compte les deux
separement et permet de n'afficher que le verifie."""
src = ROOT / "harvested.csv"
if not src.exists():
print(" harvested.csv absent - catalogue verifie seul")
return verified
h = pd.read_csv(src)
if h.empty:
return verified
def cle(t, a, y):
t = re.sub(r"[^a-z0-9 ]", "", clean_str(t).lower())[:80]
a = re.sub(r"[^a-z0-9 ]", "", clean_str(a).split(";")[0].lower())[:40]
# 1956.0 cote verifie, 1956 cote moissonne : sans cette normalisation
# la meme notice passait deux fois.
try:
y = str(int(float(y)))
except (TypeError, ValueError):
y = ""
return (t.strip(), a.strip(), y)
connues = {cle(r.title, r.author, r.year) for r in verified.itertuples()}
rows, doublons = [], 0
for r in h.itertuples():
an = int(r.year) if pd.notna(r.year) and str(r.year).strip() else None
if cle(r.title, r.author, an) in connues:
doublons += 1
continue
country = norm_country(getattr(r, "country", "") or "")
lang = clean_str(getattr(r, "language", "")) or "unknown"
scan = clean_str(getattr(r, "scan_hint", ""))
urls = [u for u in [clean_str(getattr(r, "record_url", "")),
clean_str(getattr(r, "query_url", ""))] if u]
if not urls:
continue # la regle des sources vaut aussi ici
rows.append({
"title": clean_str(r.title), "author": clean_str(r.author) or "Unattributed",
"press_id": "", "press_name": clean_str(getattr(r, "publisher", "")),
"press_ownership": "unknown",
"city": canon_city(getattr(r, "place", "")),
"lat": None, "lon": None,
"country": country, "map_country": map_name(country),
"region": region_of(country), "research_batch": "harvested",
"year": an, "decade": decade_of(an), "language": lang,
"african_language": lang not in MAJOR_EUROPEAN_LANGS and lang != "unknown",
"language_group": language_group(lang),
"genre": "unrecorded", "genre_group": "Other / unrecorded",
"pages": None, "banned": None,
"notes": f"Harvested from {r.source}. Not verified by hand.",
"digitization_status": ("scanned_restricted" if scan else "unknown"),
"digitization_holder": scan, "digitization_url": "",
"in_print": "unknown",
"confidence": clean_str(getattr(r, "confidence", "")) or "low",
"provenance": "harvested",
"source_count": len(urls),
"source_titles": clean_str(getattr(r, "source", "")),
"source_urls": " | ".join(urls), "primary_source_url": urls[0],
})
if not rows:
print(f" harvested: 0 new record ({doublons} already verified)")
return verified
hd = pd.DataFrame(rows)
for col in ["stage1_located", "stage2_preserved", "stage3_rights_identified",
"stage4_cleared", "stage5_republished", "stage6_curriculum_ready"]:
hd[col] = False
hd["stage1_located"] = True
hd["stage2_preserved"] = hd["digitization_status"].isin(
["full_text_open", "scanned_restricted"])
hd["furthest_stage"] = hd["stage2_preserved"].map({True: "Preserved",
False: "Located"})
hd["pilot_cluster"] = "" # le pilote reste un choix humain
hd["title_id"] = [f"H{i:05d}" for i in range(1, len(hd) + 1)]
out = pd.concat([verified, hd], ignore_index=True)
print(f" harvested: {len(hd)} added, {doublons} dropped as already verified")
return out
GENRE_GROUPS = [
("Children's & folklore", r"child|folk|fable|tale|picture|juvenil|reader"),
("Fiction & popular fiction", r"novel|fiction|novella|short stor|romance|crime|thriller"),
("Poetry", r"poet|verse"),
("Drama", r"drama|play|theatre"),
("Politics & political economy", r"politic|econom|nationalis|liberat|sociolog"),
("History & biography", r"histor|biograph|memoir|autobiograph"),
("Language & reference", r"dictionar|grammar|language|textbook|linguist"),
("Essays & criticism", r"essay|critic|philosoph|theolog"),
("Popular pamphlet & self-help", r"pamphlet|self-help|conduct|advice|letter-writ"),
]
def genre_group(g):
g = (g or "").lower()
for label, pattern in GENRE_GROUPS:
if re.search(pattern, g):
return label
return "Other / unrecorded"
# --------------------------------------------------------------------------
# Pilot clusters - the 100-title first phase
# --------------------------------------------------------------------------
PILOT_TARGETS = {
"A · Onitsha market pamphlets": 25,
"B · Ghanaian children's & folklore": 20,
"C · East African popular fiction": 20,
"D · Francophone (CLE, NEA & related)": 20,
"E · African-language works": 15,
}
CONF_RANK = {"high": 0, "medium": 1, "low": 2}
def assign_pilot(df: pd.DataFrame) -> pd.DataFrame:
"""Assign each title to at most one pilot cluster, in cluster order, taking
the best-evidenced candidates first. Deterministic: sort keys are explicit."""
df["pilot_cluster"] = ""
taken = set()
def pick(mask, cluster):
pool = df[mask & ~df.index.isin(taken)].copy()
pool["_c"] = pool["confidence"].map(CONF_RANK).fillna(3)
pool = pool.sort_values(
["_c", "source_count", "year", "title"],
ascending=[True, False, True, True],
na_position="last",
)
chosen = pool.head(PILOT_TARGETS[cluster]).index
df.loc[chosen, "pilot_cluster"] = cluster
taken.update(chosen)
return len(pool), len(chosen)
stats = {}
stats["A · Onitsha market pamphlets"] = pick(
df["city"].str.contains("Onitsha", case=False, na=False),
"A · Onitsha market pamphlets",
)
stats["B · Ghanaian children's & folklore"] = pick(
(df["country"] == "Ghana")
& (
df["genre_group"].eq("Children's & folklore")
| df["genre"].str.contains("folk|fable|tale|child", case=False, na=False)
),
"B · Ghanaian children's & folklore",
)
stats["C · East African popular fiction"] = pick(
df["region"].eq("East Africa")
& df["genre_group"].isin(["Fiction & popular fiction", "Drama"]),
"C · East African popular fiction",
)
stats["D · Francophone (CLE, NEA & related)"] = pick(
df["language"].str.contains("French", na=False),
"D · Francophone (CLE, NEA & related)",
)
stats["E · African-language works"] = pick(
df["african_language"],
"E · African-language works",
)
for cluster, (pool, chosen) in stats.items():
flag = "" if chosen >= PILOT_TARGETS[cluster] else " << SHORT"
print(f" pilot {cluster}: {chosen}/{PILOT_TARGETS[cluster]} (pool {pool}){flag}")
return df
# --------------------------------------------------------------------------
# Digitization initiatives & rights
# --------------------------------------------------------------------------
def build_digitization() -> pd.DataFrame:
rows = []
for rec in json.loads((RESEARCH / "digitization.json").read_text(encoding="utf-8")):
s = sources_to_cols(rec)
rows.append(
{
"initiative_id": rec["initiative_id"],
"name": clean_str(rec.get("name")),
"host_institution": clean_str(rec.get("host_institution")),
"country": clean_str(rec.get("country")),
"region": clean_str(rec.get("region")) or "Unrecorded",
"in_africa": clean_str(rec.get("region")) == "Africa",
"scope": clean_str(rec.get("scope")),
"relevance": clean_str(rec.get("relevance")),
"access": clean_str(rec.get("access")) or "unknown",
"formats": ", ".join(rec.get("formats") or []),
"api": clean_str(rec.get("api")) or "unknown",
"approx_items": rec.get("approx_items"),
"url": clean_str(rec.get("url")),
"confidence": clean_str(rec.get("confidence")) or "low",
**s,
}
)
return pd.DataFrame(rows)
def build_rights() -> pd.DataFrame:
rows = []
for rec in json.loads((RESEARCH / "rights.json").read_text(encoding="utf-8")):
s = sources_to_cols(rec)
rows.append(
{
"country": norm_country(rec.get("country")),
"region": region_of(rec.get("country")),
"statute": clean_str(rec.get("statute")),
"copyright_term": clean_str(rec.get("copyright_term")),
"term_years": rec.get("term_years"),
"anonymous_or_corporate_term": clean_str(rec.get("anonymous_or_corporate_term")),
"berne_member": bool(rec.get("berne_member")),
"public_domain_notes": clean_str(rec.get("public_domain_notes")),
"orphan_works_mechanism": clean_str(rec.get("orphan_works_mechanism")),
"recovery_implication": clean_str(rec.get("recovery_implication")),
"confidence": clean_str(rec.get("confidence")) or "low",
**s,
}
)
return pd.DataFrame(rows)
def build_cities(titles: pd.DataFrame, presses: pd.DataFrame) -> pd.DataFrame:
"""One row per publishing city, with the coordinates used to place its dot.
Cities with no gazetteer entry are kept with empty coordinates: they belong
in the counts even when they cannot be mapped, and the app names them.
"""
t = titles.groupby("city").size().rename("titles")
pr = presses.groupby("city").size().rename("presses")
df = pd.concat([t, pr], axis=1).fillna(0).astype(int).reset_index()
df["lat"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, None))[0])
df["lon"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, None))[1])
df["country"] = df["city"].map(lambda c: CITY_COORD.get(c, (None, None, ""))[2])
df["region"] = df["country"].map(region_of)
df["on_map"] = df["lat"].notna() & (df["region"] != "Diaspora (Europe)")
unplaced = df[df["lat"].isna()]["city"].tolist()
if unplaced:
print(f" cities: {len(unplaced)} without coordinates -> {', '.join(unplaced)}")
return df.sort_values("titles", ascending=False)
def build_countries(titles: pd.DataFrame, presses: pd.DataFrame) -> pd.DataFrame:
"""Coverage for every African country that has a polygon - zeros included.
The zeros are the point of this table. A country with no record is not a
country with no publishing history; it is a country nobody has searched yet,
and the map has to be able to say so.
"""
contours = pd.read_csv(ROOT / "contours_afrique.csv")
names = sorted(contours["pays"].unique())
v = titles[titles["provenance"] == "verified"]
t = titles.groupby("map_country").size().rename("titles")
tv = v.groupby("map_country").size().rename("titles_verified")
pr = presses.groupby("map_country").size().rename("presses")
dg = (titles[titles["stage2_preserved"]].groupby("map_country").size()
.rename("preserved"))
df = pd.DataFrame({"map_country": names}).set_index("map_country")
df = df.join(t).join(tv).join(pr).join(dg).fillna(0).astype(int).reset_index()
df["titles_harvested"] = df["titles"] - df["titles_verified"]
back = {v: k for k, v in MAP_COUNTRY.items()}
df["country"] = df["map_country"].map(lambda n: back.get(n, n))
df["region"] = df["country"].map(region_of)
df["searched"] = df["titles"] > 0
df["verified"] = df["titles_verified"] > 0
# Countries the research reached that have no polygon: they must not vanish.
orphans = sorted(set(titles["map_country"]) - set(names) - {""})
if orphans:
print(f" countries: {len(orphans)} with records but no polygon -> "
f"{', '.join(orphans)} (off-continent imprints, shown as a note)")
n_zero = int((~df["searched"]).sum())
print(f" countries: {len(df)} on the map, {n_zero} with no record yet")
return df.sort_values("titles", ascending=False)
def build_holdings() -> pd.DataFrame:
"""Search channels - where surviving copies may physically be.
provenance = 'project brief' : asserted in the project scoping brief
provenance = 'press register' : named in a press record's archive_holdings
"""
rows = json.loads((RESEARCH / "holdings.json").read_text(encoding="utf-8"))
df = pd.DataFrame(rows)
df["in_africa"] = df["location"].eq("Africa")
return df
# --------------------------------------------------------------------------
# Static export: one data.js file read by index.html
# --------------------------------------------------------------------------
def plain(v):
"""Replace en and em dashes by a plain hyphen; leave everything else."""
if isinstance(v, str):
return (v.replace(" \u2014 ", " - ").replace("\u2014", "-")
.replace(" \u2013 ", " - ").replace("\u2013", "-"))
return v
# Map frame: the west edge goes to -25.5 so that Cabo Verde is drawn.
LON0, LON1, LAT0, LAT1 = -25.5, 52.5, -36.0, 38.0
KX = 10 * 0.9962 # cos(5 deg): equirectangular centred near the equator
KY = 10
def _xy(lon, lat):
return round((lon - LON0) * KX, 1), round((LAT1 - lat) * KY, 1)
def map_paths():
"""Country outlines as SVG paths, with a label anchor for each country.
The anchor is the area-weighted centroid of the largest piece, so that an
island group or an exclave never pulls the label into the sea."""
ct = pd.read_csv(ROOT / "contours_afrique.csv").sort_values(["pays", "piece", "ordre"])
out = {}
for pays, g in ct.groupby("pays", sort=True):
parts, best = [], (0.0, None)
total = 0.0
for _, piece in g.groupby("piece", sort=False):
pts = [_xy(a, b) for a, b in zip(piece["lon"], piece["lat"])]
parts.append("M" + "L".join(f"{x},{y}" for x, y in pts) + "Z")
a = cx = cy = 0.0
for (x0, y0), (x1, y1) in zip(pts, pts[1:] + pts[:1]):
c = x0 * y1 - x1 * y0
a += c
cx += (x0 + x1) * c
cy += (y0 + y1) * c
a /= 2.0
total += abs(a)
if abs(a) > best[0] and a != 0:
best = (abs(a), (round(cx / (6 * a), 1), round(cy / (6 * a), 1)))
out[pays] = {"d": "".join(parts), "c": best[1], "area": round(total)}
w = round((LON1 - LON0) * KX, 1)
h = round((LAT1 - LAT0) * KY, 1)
return {"w": w, "h": h, "lon0": LON0, "lat1": LAT1, "kx": KX, "ky": KY,
"countries": out}
def _records(df):
df = df.astype(object).where(pd.notna(df), None)
return df.to_dict("records")
def write_datajs(presses, titles, digi, rights, holdings, cities, countries):
tcols = ["title_id", "title", "author", "press_id", "press_name", "city",
"country", "map_country", "region", "year", "decade", "language",
"language_group", "african_language", "genre", "genre_group", "pages",
"banned", "notes", "digitization_status", "digitization_holder",
"digitization_url", "in_print", "confidence", "provenance",
"source_titles", "source_urls", "pilot_cluster", "stage1_located",
"stage2_preserved", "stage3_rights_identified", "stage4_cleared",
"stage5_republished", "stage6_curriculum_ready"]
t = titles[tcols].copy()
t["year"] = t["year"].map(lambda y: int(y) if pd.notna(y) else None)
t["pages"] = t["pages"].map(lambda y: int(y) if pd.notna(y) else None)
p = presses.copy()
for c in ("founded", "ceased"):
p[c] = p[c].map(lambda y: int(y) if pd.notna(y) else None)
payload = {
"built": pd.Timestamp.now().strftime("%Y-%m-%d"),
"titles": _records(t), "presses": _records(p),
"rights": _records(rights), "digitization": _records(digi),
"holdings": _records(holdings), "cities": _records(cities),
"countries": _records(countries), "map": map_paths(),
}
txt = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
txt = plain(txt)
(DATA / "data.js").write_text("window.ABRP=" + txt + ";\n", encoding="utf-8")
print(f" wrote data.js {(DATA / 'data.js').stat().st_size/1024:7.1f} kB")
# --------------------------------------------------------------------------
def main():
print("Building African Books Recovery Project fixtures…")
presses = build_presses()
titles = build_titles(presses)
titles = merge_harvested(titles)
digi = build_digitization()
rights = build_rights()
holdings = build_holdings()
cities = build_cities(titles, presses)
countries = build_countries(titles, presses)
# No en dash or em dash anywhere in what the dashboard shows: the source
# JSON keeps its typography, the published tables use a plain hyphen.
tables = [presses, titles, digi, rights, holdings, cities, countries]
for df in tables:
for c in df.columns:
if not pd.api.types.is_numeric_dtype(df[c]):
df[c] = df[c].map(plain)
# CSV, not parquet: the Space then needs streamlit alone - no pyarrow, no
# binary format, and the catalogue stays readable in any spreadsheet.
for name, df in [
("presses", presses),
("titles", titles),
("digitization", digi),
("rights", rights),
("holdings", holdings),
("cities", cities),
("countries", countries),
]:
out = DATA / f"{name}.csv"
df.to_csv(out, index=False)
print(f" wrote {out.name:22s} {len(df):4d} rows {out.stat().st_size/1024:7.1f} kB")
write_datajs(presses, titles, digi, rights, holdings, cities, countries)
nv = int((titles["provenance"] == "verified").sum())
nh = len(titles) - nv
print(
f"\nTotals: {len(titles)} titles ({nv} verified + {nh} harvested) · "
f"{len(presses)} presses · {titles['country'].nunique()} countries · "
f"{titles['language'].nunique()} languages"
)
return 0
if __name__ == "__main__":
sys.exit(main())