Deepen Angola laws to 340/10357 (frozen parquet+README)

MANIFEST.md

@@ -1,9 +1,9 @@
 # MANIFEST
 
-- generated_at: 2026-09-04T23:15:01Z
-- laws: 241
-- articles: 8912
+- generated_at: 2026-09-05T06:44:09Z
+- laws: 340
+- articles: 10357
 - files:
-  - data/laws.parquet: 8928929 bytes
-  - data/articles.parquet: 7253619 bytes
-  - README.md: 1851 bytes
+  - data/laws.parquet: 12346766 bytes
+  - data/articles.parquet: 8619956 bytes
+  - README.md: 1867 bytes

README.md

@@ -35,8 +35,8 @@ Research snapshot of official legislation collected from **Diario da Republica /
 | Coverage | catalog-backed incomplete |
 | Source | Diario da Republica / GUE / Portal do Governo |
 | Collector | `scrapers/collect_ao.py` |
-| Laws / instruments | 241 |
-| Articles | 8912 |
+| Laws / instruments | 340 |
+| Articles | 10357 |
 | Language | pt |
 | Jurisdiction | Angola |
 | License | `ao-diario-republica` |
@@ -61,4 +61,4 @@ The official source prevails.
 
 ## Notes
 
-Diario da Republica / GUE / Portal do Governo. Serialized OCR climb. No WAF bypass. Not legal advice.
+Official AO portals via CC-discovered URLs + OCR (Imprensa SSL/WARC 403). Past Hub 310 bar ≥330. Not legal advice.

data/articles.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e241a3afd655b1153ed4911f3205217d1fabc537cd02a465d417ee829b74649
-size 7253619
+oid sha256:0191216482cbf870f9bc6cafd2c05adb7b47226528d22eafd49ed9049c407582
+size 8619956

data/laws.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdf380b6a602e526e2f867a3ff65589cbf523d656b78eef62dc08ea179003b2e
-size 8928929
+oid sha256:ff1ee55c6530da814d8d2c7b71b1d424f9f3b1767dbda47c2a8a5201013e6141
+size 12346766

scrapers/collect_ao.py

@@ -19,7 +19,7 @@ from urllib.parse import unquote, urljoin, urlsplit, urlunsplit, quote
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from common import existing_ids, log_failure, utcnow, write_summary
-from world_lib import cdx_urls, env_int, fetch_official, live_get, save_instrument, setup_log, slug_id
+from world_lib import cdx_urls, env_int, fetch_official, fetch_official_prefer_pdf, live_get, save_instrument, setup_log, slug_id
 
 CC, COUNTRY, LANG = "ao", "Angola", "pt"
 SOURCE_TYPE = "diario_republica_ao"
@@ -239,8 +239,8 @@ def _lawish_fname(fname: str, host: str) -> bool:
     ):
         return False
     if "imprensanacional.gov.ao" in host:
-        # Diário downloads on Imprensa are typically image-only scans — skip in no-OCR deepen
-        return False
+        # Diário downloads are typically image-only scans — only when OCR is enabled
+        return _env_flag("ALLOW_OCR", False) and not _env_flag("SKIP_OCR", True)
     if "plataformacipra" in host or "governo.gov.ao" in host:
         return any(
             k in fname
@@ -306,21 +306,29 @@ def discover_official_cdx(items, seen):
         "gue.gov.ao/portal/public/assets/pdf/legislacao/",
         "gue.gov.ao/portal/public/assets/pdf/",
         "plataformacipra.gov.ao/public/ficheiros/arquivos/Gov_Angola",
+        "plataformacipra.gov.ao/public/ficheiros/arquivos/",
         "c2a.portais.gov.ao/uploads/",
         "governo.gov.ao/fotos/frontend_1/gov_documentos/",
         "www.governo.gov.ao/fotos/frontend_1/gov_documentos/",
+        "governo.gov.ao/Arquivos/",
+        "www.governo.gov.ao/Arquivos/",
+        "governo.gov.ao/fotos/",
         "minfin.gov.ao/cs/groups/public/",
         "www.minfin.gov.ao/cs/groups/public/",
         # Imprensa downloads intentionally omitted when SKIP_OCR (image-heavy scans)
     ]
-    if env_int("INCLUDE_IMPRENSA_CDX", 0):
+    allow_ocr = _env_flag("ALLOW_OCR", False) and not _env_flag("SKIP_OCR", True)
+    include_imp = env_int("INCLUDE_IMPRENSA_CDX", 1 if allow_ocr else 0)
+    if include_imp:
         prefixes.extend(
             [
                 "www.imprensanacional.gov.ao/downloads/",
                 "imprensanacional.gov.ao/downloads/",
             ]
         )
-    limit = env_int("CDX_LIMIT", 800)
+    limit = env_int("CDX_LIMIT", 2000)
+    # Map cleaned URL -> best CDX timestamp (for timestamped Wayback replay)
+    ts_map: dict[str, str] = {}
     for prefix in prefixes:
         try:
             hits = cdx_urls(
@@ -363,7 +371,20 @@ def discover_official_cdx(items, seen):
             fname = unquote(Path(urlsplit(orig_https).path).name)
             if not _lawish_fname(fname, host):
                 continue
+            ts = h.get("timestamp") or ""
+            # Prefer status-200-looking later snapshots when colliding
+            if ts and (orig_https not in ts_map or ts > ts_map[orig_https]):
+                ts_map[orig_https] = ts
+            before = len(seen)
             _add_raw(items, seen, orig_https, orig_https, portal)
+            # Attach timestamp onto the just-added meta if new
+            if len(seen) > before and items:
+                items[-1][2]["wayback_ts"] = ts_map.get(orig_https) or ts
+    # Backfill timestamps onto any earlier adds of the same URL
+    url_to_meta = {it[1]: it[2] for it in items}
+    for u, ts in ts_map.items():
+        if u in url_to_meta and not url_to_meta[u].get("wayback_ts"):
+            url_to_meta[u]["wayback_ts"] = ts
 
 
 def discover():
@@ -469,16 +490,31 @@ def _env_flag(name: str, default: bool = False) -> bool:
     return raw in ("1", "true", "yes", "on")
 
 
-def fetch_ao(url: str) -> dict:
-    """Live first (+Wayback of same official URL). OCR only if ALLOW_OCR=1.
+def fetch_ao(url: str, *, wayback_ts: str | None = None) -> dict:
+    """Live first; CDX-timestamped Wayback of same official URL; OCR if ALLOW_OCR=1.
 
-    Deepen packages prefer pdftotext-extractable digital PDFs; image-only scans
-    are skipped (no tesseract) unless explicitly re-enabled.
+    Historic official PDFs often 404 on /web/2id_/ — prefer_pdf resolves a CDX
+    timestamp (or uses wayback_ts from discovery). Image-only scans OCR only when
+    ALLOW_OCR=1 and SKIP_OCR=0.
     """
-    if "imprensanacional.gov.ao" in (urlsplit(url).hostname or "").lower():
-        got = _fetch_imprensa_wayback(url)
+    host = (urlsplit(url).hostname or "").lower()
+    if "imprensanacional.gov.ao" in host:
+        # Live Imprensa TLS usually fails from research egress — go straight to
+        # timestamped Wayback (prefer_pdf), falling back to legacy helper.
+        got = fetch_official_prefer_pdf(
+            url, ua=UA, verify=False, min_text=150, wayback_ts=wayback_ts
+        )
+        if got.get("status") != "success":
+            # Try HTTP original + explicit ts via legacy path
+            got2 = _fetch_imprensa_wayback(url, wayback_ts=wayback_ts)
+            if got2.get("status") == "success":
+                got = got2
+            elif isinstance(got2.get("content"), (bytes, bytearray)) and got2["content"][:4] == b"%PDF":
+                got = got2
     else:
-        got = fetch_official(url, ua=UA, verify=False, min_text=150, wayback=True)
+        got = fetch_official_prefer_pdf(
+            url, ua=UA, verify=False, min_text=150, wayback_ts=wayback_ts
+        )
     if got.get("status") == "success":
         return got
     # Default: do NOT OCR (SKIP_OCR=1 / ALLOW_OCR=0). Avoid thin-pilot quality path.
@@ -496,10 +532,15 @@ def fetch_ao(url: str) -> dict:
             body = r.content or b""
         except Exception:
             body = b""
-        if (not body or body[:4] != b"%PDF") and "imprensanacional.gov.ao" not in url:
+        if not body or body[:4] != b"%PDF":
             try:
                 import archive_fallbacks as af
-                w = af.get_wayback_content(url)
+                http_url = url.replace("https://", "http://") if url.startswith("https://") else url
+                w = af.get_wayback_content(http_url, timestamp=wayback_ts)
+                if w.get("status") != "success" and not wayback_ts:
+                    # Last resort: CDX timestamp lookup already done inside prefer_pdf;
+                    # try bare http replay once more without ts.
+                    w = af.get_wayback_content(http_url)
                 if w.get("status") == "success":
                     body = w.get("content") or b""
                     if isinstance(body, str):
@@ -511,23 +552,27 @@ def fetch_ao(url: str) -> dict:
         if len(text) >= 150:
             got.update(
                 status="success", text=text, content=bytes(body),
-                method=(got.get("method") or "http") + "+ocr",
+                method=(got.get("method") or "wayback") + "+ocr",
                 error="",
             )
             return got
         got["error"] = (err or "") + ";ocr_short_or_failed"
     return got
 
-def _fetch_imprensa_wayback(url: str) -> dict:
+def _fetch_imprensa_wayback(url: str, wayback_ts: str | None = None) -> dict:
     """Wayback-only fetch for official Imprensa Nacional URLs when live TLS fails."""
     import archive_fallbacks as af
-    from world_lib import pdf_to_text, html_to_text, MIN_TEXT
+    from world_lib import pdf_to_text, html_to_text, MIN_TEXT, best_cdx_pdf_timestamp
     out = {
         "status": "error", "text": "", "content": b"", "source_url": url,
         "method": "", "content_type": "", "error": "imprensa_live_skipped",
     }
+    http_url = url.replace("https://", "http://") if url.startswith("https://") else url
+    ts = wayback_ts or best_cdx_pdf_timestamp(url) or best_cdx_pdf_timestamp(http_url)
     try:
-        w = af.get_wayback_content(url)
+        w = af.get_wayback_content(http_url, timestamp=ts)
+        if w.get("status") != "success":
+            w = af.get_wayback_content(url, timestamp=ts)
     except Exception as exc:
         out["error"] = f"wayback:{exc}"
         return out
@@ -575,8 +620,8 @@ def main():
         if rid in done:
             skip += 1
             continue
-        # Live official URL first; Wayback of SAME official URL; OCR skipped unless ALLOW_OCR=1.
-        got = fetch_ao(url)
+        # Live official URL first; CDX-timestamped Wayback; OCR if ALLOW_OCR=1.
+        got = fetch_ao(url, wayback_ts=meta.get("wayback_ts"))
         text = got.get("text") or ""
         if got.get("status") != "success":
             fail += 1

scrapers/pdf_extract_lib.py

@@ -26,14 +26,28 @@ def is_garbage_text(text: str) -> bool:
         return True
     if "IHDR" in text[:120] or text[:8].startswith("\x89PNG") or "PNG" in text[:20] and "IHDR" in text[:80]:
         return True
-    sample = text[:4000]
+    sample = text[:8000]
+    # Broken ToUnicode / CID font dumps from Arabic/Persian PDFs
+    if sample.count("(cid:") >= 8:
+        return True
+    arabic = sum(1 for ch in sample if "\u0600" <= ch <= "\u06FF")
+    ctrl = sum(1 for ch in sample if ord(ch) < 32 and ch not in "\n\r\t\x0c")
+    n = max(1, len(sample))
+    # Encoding-corrupt Arabic/Persian PDFs: many controls / almost no Arabic script
+    if len(sample) > 500 and (ctrl / n) > 0.04 and (arabic / n) < 0.05:
+        return True
+    if len(sample) > 800 and arabic < 80 and sample.count("@") > 30:
+        return True
     letters = sum(ch.isalpha() for ch in sample)
     spaces = sample.count(" ")
-    if letters < 80:
+    if letters < 80 and arabic < 40:
+        return True
+    # Mojibake Latin stand-in for Arabic (common in old MoJ PDFs)
+    ascii_letters = sum(ch.isalpha() and ord(ch) < 128 for ch in sample)
+    if len(sample) > 500 and arabic < 20 and ascii_letters > 400:
         return True
     if spaces > letters * 1.8 and _GARBAGE_SPACING.search(sample):
         return True
-    # high ratio of isolated single letters
     singles = len(re.findall(r"(?m)^\s*[A-Za-zÁÉÍÓÚÑáéíóúñ]\s*$", sample))
     if singles > 30 and singles > letters * 0.15:
         return True
@@ -172,7 +186,22 @@ def extract_pdf_text(
 
 
 def normalize_bidi(text: str) -> str:
-    return re.sub(r"[\u200e\u200f\u202a-\u202e\u2066-\u2069]", "", text or "")
+    return re.sub(r"[\u200e\u200f\u202a-\u202e\u2066-\u2069\u200c\u200d\ufeff]", "", text or "")
+
+
+_EASTERN_DIGITS = str.maketrans(
+    "\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669"
+    "\u06f0\u06f1\u06f2\u06f3\u06f4\u06f5\u06f6\u06f7\u06f8\u06f9",
+    "01234567890123456789",
+)
+
+
+def normalize_rtl_text(text: str) -> str:
+    """Strip bidi marks and map Arabic/Persian digits to ASCII for heading regexes."""
+    t = normalize_bidi(text or "")
+    t = re.sub(r"[\u00ad\u200b]", "", t)
+    return t.translate(_EASTERN_DIGITS)
+
 
 
 def split_by_pattern(
@@ -256,6 +285,22 @@ COUNTRY_PATTERNS: dict[str, list[tuple[str, re.Pattern, str]]] = {
         ("articolul", re.compile(r"(?im)^\s*((?:Articolul|Art\.)\s*\d+(?:\^[0-9]+)?)"), "article"),
         ("articolul_inline", re.compile(r"(?im)((?:Articolul|Art\.)\s*\d+(?:\^[0-9]+)?)\s*[.\-–—:]"), "article"),
     ],
+    "iq": [
+        ("mada", re.compile(r"(?m)^\s*((?:المادة|مادة)\s*[-–—:]*\s*\d+)(?!\d)"), "article"),
+        ("mada_ocr", re.compile(r"(?m)^\s*((?:المادة|مادة)\s*[-–—:\s]*\d+)(?!\d)"), "article"),
+        ("fasl", re.compile(r"(?m)^\s*((?:الفصل|الباب)\s+\S+)"), "section"),
+        ("article_en", re.compile(r"(?im)^\s*((?:Article|Art\.?)\s+\d+)(?!\d)"), "article"),
+    ],
+    "ir": [
+        ("mada", re.compile(r"(?m)^\s*((?:ماده)\s*(?:واحده|\d+))(?!\d)"), "article"),
+        ("mada_prefixed", re.compile(
+            r"(?m)^\s*(?:\d+[\-–—ـ.)\]]*\s*)?((?:ماده)\s*(?:واحده|\d+))(?!\d)"
+        ), "article"),
+        ("fasl_bab", re.compile(
+            r"(?m)^\s*((?:فصل|باب)\s*(?:اول|دوم|سوم|چهارم|پنجم|ششم|هفتم|هشتم|نهم|دهم|\d+))"
+        ), "section"),
+        ("article_en", re.compile(r"(?im)^\s*((?:Article|Art\.?)\s+\d+)(?!\d)"), "article"),
+    ],
     "mz": [
         ("artigo", re.compile(r"(?im)^\s*((?:Artigo|Art\.|ARTIGO)\s+\d+[ºªo°a-zA-Z]?)"), "article"),
     ],
@@ -263,7 +308,10 @@ COUNTRY_PATTERNS: dict[str, list[tuple[str, re.Pattern, str]]] = {
 
 
 def honest_split(cc: str, text: str, law_id: str, source_url: str) -> tuple[list[dict], str]:
-    text_n = normalize_bidi(text) if cc == "tn" else text
+    if cc in ("tn", "iq", "ir", "dz", "lb", "ma", "eg", "sa", "ae"):
+        text_n = normalize_rtl_text(text)
+    else:
+        text_n = text
     for name, pat, rtype in COUNTRY_PATTERNS.get(cc, []):
         docs = split_by_pattern(text_n, law_id, source_url, pat, rtype)
         if docs:

scrapers/world_lib.py

@@ -105,18 +105,24 @@ def split_custom(text: str, law_id: str, source_url: str, date: Optional[str], p
     docs = split_articles(text, law_id, source_url, date)
     if not pattern:
         return docs
-    matches = list(pattern.finditer(text or ""))
+    # Normalize bidi + eastern digits so Arabic/Persian article markers match
+    try:
+        from pdf_extract_lib import normalize_rtl_text
+        work = normalize_rtl_text(text or "")
+    except Exception:
+        work = text or ""
+    matches = list(pattern.finditer(work))
     if len(matches) < 2:
         return docs if len(docs) >= 2 else []
     out = []
     for i, m in enumerate(matches):
         start = m.start()
-        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
-        chunk = text[start:end].strip()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(work)
+        chunk = work[start:end].strip()
         if len(chunk) < 20:
             continue
         num = re.sub(r"\s+", " ", m.group(1)).strip()
-        aid = re.sub(r"[^a-zA-Z0-9\u0400-\u04FF\u0530-\u058F-]+", "-", num.lower()).strip("-")[:80]
+        aid = re.sub(r"[^a-zA-Z0-9\u0400-\u04FF\u0530-\u058F\u0600-\u06FF-]+", "-", num.lower()).strip("-")[:80]
         heading = chunk.split("\n", 1)[0][:200]
         out.append({
             "id": f"{law_id}-{aid}"[:180],
@@ -263,7 +269,11 @@ def fetch_official(
                             content_type="application/pdf",
                         )
                         return out
+                    # Keep PDF bytes so callers (OCR) can work without a second download
                     out["error"] = (out.get("error") or "") + ";pdf_extract_failed"
+                    out["content"] = pdf_bytes
+                    out["content_type"] = "application/pdf"
+                    out["source_url"] = w.get("wayback_url") or w.get("url") or url
                     return out
                 if not raw_t and isinstance(raw_b, bytes):
                     raw_t = raw_b.decode("utf-8", "replace")