Upload 12 files

app.py

@@ -169,10 +169,31 @@ def ingestion_status_fn(secret):
         f"unchanged: {summary.get('unchanged', 0)}, "
         f"skipped: {summary.get('skipped', 0)}, "
         f"failed: {summary.get('failed', 0)}"
+        + _pruned_status_line(summary)
+        + _library_status_line(summary)
         + _tinyfish_status_line(summary)
     )
 
 
+def _pruned_status_line(summary) -> str:
+    pr = summary.get("pruned")
+    if not pr:
+        return ""
+    if "skipped" in pr or "error" in pr:
+        return "\nBill window: " + (pr.get("skipped") or pr.get("error"))
+    return (f"\nBill window: keeping newest {pr.get('kept', 0)} bills; "
+            f"removed {pr.get('bills_removed', 0)} older bills and {pr.get('pdfs_removed', 0)} PDFs")
+
+
+def _library_status_line(summary) -> str:
+    lib = summary.get("library")
+    if not lib:
+        return ""
+    by = ", ".join(f"{k}: {v}" for k, v in sorted(lib.get("by_section", {}).items())) or "none new"
+    more = " — budget reached, run ingestion again to continue" if lib.get("more_remaining") else " — library up to date"
+    return f"\nPRS library: {lib.get('new_documents', 0)} new documents ({by}){more}"
+
+
 def _tinyfish_status_line(summary) -> str:
     by_src = summary.get("ingested_by_source") or {}
     tf = summary.get("tinyfish") or {}

config.py

@@ -73,12 +73,67 @@ CONFIG = {
         "bill_link_pattern": r"^/billtrack/[^/]+$",
         "bill_link_exclude_prefixes": ("/billtrack/category/", "/billtrack/field_bill_category/"),
         "rate_limit_seconds": 2,
-        "max_bills_per_run": 25,
+        # Rolling window: JanMat keeps the newest N bills (PRS lists newest first). Each ingestion
+        # adds any new bills and then deletes the oldest ones beyond N, with their PDFs and text.
+        "max_bills_kept": 25,
+        "listing_pages": 6,                  # how many ?page=N listing pages to read to find N bills
+        "delete_comments_with_bill": False,  # True = also delete user comments on bills that rotate out
         "license_note": (
             "PRS's disclaimer permits reproduction for non-commercial purposes "
             "with due acknowledgement of PRS Legislative Research."
         ),
     },
+    # The rest of the PRS website (everything beyond the first page of bills).
+    # The crawler visits each section's index pages (following ?page=N pagination),
+    # collects links whose path matches `item_pattern`, and ingests each page plus
+    # any PDFs it links to. Add or remove sections freely.
+    "prs_library": {
+        "enabled": True,
+        # Each run ingests at most this many NEW documents, then stops. Documents
+        # already stored are skipped without being fetched, so just click
+        # "Run ingestion now" repeatedly and the library grows until it is complete.
+        # (Free Supabase = 500 MB; roughly 4 KB per chunk, ~100 chunks per document.)
+        "max_new_docs_per_run": 60,
+        "max_index_pages": 5,        # pagination depth per index URL
+        "sections": [
+            {"key": "policy_review", "label": "Policy Reviews",
+             "index_urls": ["https://prsindia.org/policy/monthly-policy-review",
+                            "https://prsindia.org/policy/annual-policy-review"],
+             "item_pattern": r"^/policy/(monthly|annual)-policy-review/[^/]+$"},
+            {"key": "committee_report", "label": "Committee Reports",
+             "index_urls": ["https://prsindia.org/policy/report-summaries",
+                            "https://prsindia.org/parliamenttrack/report-summaries"],
+             "item_pattern": r"^/(policy|parliamenttrack)/report-summaries/[^/]+$"},
+            {"key": "session", "label": "Session Wraps",
+             "index_urls": ["https://prsindia.org/sessiontrack"],
+             "item_pattern": r"^/sessiontrack/[^/]+(/[^/]+)?$"},
+            {"key": "discussion_paper", "label": "Discussion Papers",
+             "index_urls": ["https://prsindia.org/policy/discussion-papers",
+                            "https://prsindia.org/legislatures/discussion-papers",
+                            "https://prsindia.org/budgets/discussionpapers"],
+             "item_pattern": r"^/(policy|legislatures|budgets)/(discussion-papers|discussionpapers)/[^/]+$"},
+            {"key": "blog", "label": "Blog",
+             "index_urls": ["https://prsindia.org/theprsblog"],
+             "item_pattern": r"^/theprsblog/(?!blog-author)[^/]+$"},
+            {"key": "article", "label": "Articles",
+             "index_urls": ["https://prsindia.org/articles-by-prs-team/2026",
+                            "https://prsindia.org/articles-by-prs-team/2025",
+                            "https://prsindia.org/articles-by-prs-team/2024"],
+             "item_pattern": r"^/articles-by-prs-team/(?!\d{4}$)[^/]+$"},
+            {"key": "act", "label": "Acts",
+             "index_urls": ["https://prsindia.org/acts/parliament", "https://prsindia.org/recent-acts"],
+             "item_pattern": r"^/(acts/[^/]+|recent-acts)/[^/]+$"},
+            {"key": "state_bill", "label": "State Bills",
+             "index_urls": ["https://prsindia.org/bills/states",
+                            "https://prsindia.org/bills/state-legislative-briefs"],
+             "item_pattern": r"^/bills/(states|state-legislative-briefs)/[^/]+$"},
+            {"key": "budget", "label": "Budgets",
+             "index_urls": ["https://prsindia.org/budgets/parliament", "https://prsindia.org/budgets/states"],
+             "item_pattern": r"^/budgets/(parliament|states)/[^/]+$"},
+        ],
+        # "Comments invited on ..." notices (bill name, deadline, link to submit).
+        "announcement_urls": ["https://prsindia.org/", "https://prsindia.org/announcements"],
+    },
     # Add more sources here as plain "seed URL lists" — the ingestion pipeline
     # treats any HTTP(S) URL the same way: fetch, detect PDF vs HTML, extract
     # full text, chunk, embed, store. See scraper.py:ArticleSource.

db.py

@@ -119,6 +119,54 @@ def delete_chunks_for_document(document_id: int) -> None:
     _raise_for_status(resp)
 
 
+def list_prs_bill_documents() -> List[dict]:
+    """Every stored PRS bill page and bill PDF (id, url, kind, metadata), for the rolling window."""
+    rows, offset = [], 0
+    while True:
+        resp = requests.get(
+            f"{_base_url()}/documents",
+            headers=_headers(),
+            params={"source": "eq.prs", "kind": "in.(bill_page,bill_pdf)",
+                    "select": "id,url,kind,metadata", "order": "id.asc", "limit": 1000, "offset": offset},
+            timeout=30,
+        )
+        _raise_for_status(resp)
+        batch = resp.json()
+        rows.extend(batch)
+        if len(batch) < 1000:
+            return rows
+        offset += 1000
+
+
+def delete_documents(ids: List[int]) -> int:
+    """Delete documents by id. Their chunks go with them (ON DELETE CASCADE)."""
+    deleted = 0
+    for i in range(0, len(ids), 50):
+        batch = ids[i:i + 50]
+        resp = requests.delete(
+            f"{_base_url()}/documents",
+            headers=_headers(),
+            params={"id": f"in.({','.join(str(int(x)) for x in batch)})"},
+            timeout=60,
+        )
+        _raise_for_status(resp)
+        deleted += len(batch)
+    return deleted
+
+
+def delete_comments_for_topics(topic_keys: List[str]) -> None:
+    """Delete user comments attached to the given topic keys (bill URLs)."""
+    for i in range(0, len(topic_keys), 20):
+        quoted = ",".join('"' + k.replace('"', '\\"') + '"' for k in topic_keys[i:i + 20])
+        resp = requests.delete(
+            f"{_base_url()}/comments",
+            headers=_headers(),
+            params={"topic_key": f"in.({quoted})"},
+            timeout=60,
+        )
+        _raise_for_status(resp)
+
+
 # --------------------------------------------------------------------- chunks
 
 def insert_chunks(document_id: int, chunks: List[dict]) -> int:

ingest_pipeline.py

@@ -14,7 +14,7 @@ from config import CONFIG
 from embeddings import chunk_pages, chunk_text, embed_texts
 from pdf_ingest import ingest_source_document, BlockedHostError
 import tinyfish_client
-from scraper import (PRSDiscovery, DiscoveredDocument, discover_article_seed_urls,
+from scraper import (PRSDiscovery, PRSLibraryCrawler, DiscoveredDocument, discover_article_seed_urls,
                      discover_via_tinyfish_search, discover_comment_documents)
 
 logger = logging.getLogger("janmat.ingest_pipeline")
@@ -23,6 +23,18 @@ logger = logging.getLogger("janmat.ingest_pipeline")
 def _ingest_one(doc: DiscoveredDocument) -> dict:
     """Fetch, extract, chunk, embed, and store a single discovered document."""
     result = {"url": doc.url, "status": "skipped", "chunks": 0, "source": doc.source}
+
+    # Archival PRS material (old reports, past reviews...) does not change. If it is already
+    # stored, skip it without fetching, so repeated runs only spend time on new documents.
+    if doc.metadata.get("archival"):
+        try:
+            stored = db.find_document_by_url(doc.url)
+        except Exception:
+            stored = None
+        if stored and stored.get("status") == "ingested":
+            result["status"] = "unchanged"
+            return result
+
     try:
         if doc.text:
             extracted = {"title": doc.title, "kind_hint": "html", "pages": [], "full_text": doc.text, "pdf_links": []}
@@ -165,6 +177,16 @@ def run_ingestion() -> dict:
             processed_urls.add(article_doc.url)
             _record(_ingest_one(article_doc))
 
+        # 1b. Rolling window: drop bills older than the newest `max_bills_kept`.
+        _prune_old_bills([d.url for d in bill_pages], summary)
+
+        # 2b. The rest of PRS: policy reviews, committee reports, discussion papers, session wraps,
+        #     blog, articles, acts, state bills, budgets, and "Comments invited" notices.
+        #     Bounded per run; stored documents are skipped, so repeated runs finish the library.
+        lib_cfg = CONFIG.get("prs_library", {})
+        if lib_cfg.get("enabled"):
+            _crawl_prs_library(lib_cfg, processed_urls, _record, summary)
+
         # 3. TinyFish (optional): live-web discovery + structured comment extraction.
         if tinyfish_client.enabled():
             tf = CONFIG["tinyfish"]
@@ -187,3 +209,99 @@ def run_ingestion() -> dict:
         db.finish_ingestion_run(run_id, datetime.now(timezone.utc).isoformat(), "failed", summary)
 
     return summary
+
+
+def _crawl_prs_library(lib_cfg: dict, processed_urls: set, record, summary: dict) -> None:
+    """Ingest up to `max_new_docs_per_run` new documents from the PRS sections."""
+    budget = int(lib_cfg.get("max_new_docs_per_run", 60))
+    crawler = PRSLibraryCrawler(CONFIG["prs"], lib_cfg)
+    new_docs = 0
+    per_section = {}
+
+    def _docs():
+        yield from crawler.discover_announcements()
+        for section in lib_cfg.get("sections", []):
+            yield from crawler.discover_section(section)
+
+    try:
+        for doc in _docs():
+            if new_docs >= budget:
+                break
+            if doc.url in processed_urls:
+                continue
+            processed_urls.add(doc.url)
+
+            result = _ingest_one(doc)
+            record(result)
+            section = doc.metadata.get("section", "other")
+            if result["status"] == "ingested":
+                new_docs += 1
+                per_section[section] = per_section.get(section, 0) + 1
+
+            # Always finish a page's own PDFs even if the budget was just reached,
+            # otherwise they would never be found (the page is skipped next run).
+            for pdf_url in result.get("extra_pdf_links", []):
+                if pdf_url in processed_urls:
+                    continue
+                processed_urls.add(pdf_url)
+                pdf_kind = "bill_pdf" if doc.kind == "bill_page" else "article"
+                pdf_doc = DiscoveredDocument(
+                    url=pdf_url, kind=pdf_kind, source="prs", title=f"{doc.title} (PDF)",
+                    metadata={**doc.metadata, "doc_type": "pdf", "parent_url": doc.url},
+                )
+                pr = _ingest_one(pdf_doc)
+                record(pr)
+                if pr["status"] == "ingested":
+                    new_docs += 1
+                    per_section[section] = per_section.get(section, 0) + 1
+    except Exception:
+        logger.exception("PRS library crawl stopped early")
+        summary["library_error"] = True
+
+    summary["library"] = {"new_documents": new_docs, "budget": budget,
+                          "more_remaining": new_docs >= budget, "by_section": per_section}
+
+
+def _prune_old_bills(kept_urls: list, summary: dict) -> None:
+    """
+    Keep only the newest bills. `kept_urls` is this run's listing (newest first). Anything stored
+    as a PRS bill (or a bill PDF) that is not in it gets deleted, text and embeddings included.
+
+    Safety: if PRS returned suspiciously few bills (site change, outage, blocked page), nothing is
+    deleted, so a bad scrape can never wipe the database.
+    """
+    cfg = CONFIG["prs"]
+    keep = int(cfg.get("max_bills_kept") or 0)
+    if keep <= 0:
+        return
+    min_required = min(keep, 10)
+    if len(kept_urls) < min_required:
+        summary["pruned"] = {"skipped": f"PRS listing returned only {len(kept_urls)} bills; not deleting anything"}
+        logger.warning("Skipping bill pruning: listing returned %d bills (need at least %d)", len(kept_urls), min_required)
+        return
+
+    keep_set = set(kept_urls[:keep])
+    try:
+        stored = db.list_prs_bill_documents()
+        old_pages = [d for d in stored if d["kind"] == "bill_page" and d["url"] not in keep_set]
+        old_page_urls = {d["url"] for d in old_pages}
+        old_pdfs = []
+        for d in stored:
+            if d["kind"] != "bill_pdf":
+                continue
+            meta = d.get("metadata") or {}
+            parent = meta.get("parent_bill_url") or meta.get("parent_url")
+            if parent and (parent in old_page_urls or parent not in keep_set):
+                old_pdfs.append(d)
+
+        ids = [d["id"] for d in old_pages + old_pdfs]
+        if ids:
+            db.delete_documents(ids)
+            if cfg.get("delete_comments_with_bill") and old_page_urls:
+                db.delete_comments_for_topics(sorted(old_page_urls))
+        summary["pruned"] = {"bills_removed": len(old_pages), "pdfs_removed": len(old_pdfs), "kept": len(keep_set)}
+        if ids:
+            logger.info("Pruned %d old bills and %d PDFs; keeping the newest %d", len(old_pages), len(old_pdfs), len(keep_set))
+    except Exception:
+        logger.exception("Bill pruning failed (new bills were still ingested)")
+        summary["pruned"] = {"error": "pruning failed; see logs"}

scraper.py

@@ -49,47 +49,59 @@ class PRSDiscovery:
         return resp
 
     def discover_bill_pages(self) -> List[DiscoveredDocument]:
+        """
+        The newest `max_bills_kept` bills, in the order PRS lists them (newest first).
+        Reads ?page=N listing pages until enough bills are found.
+        """
         link_pattern = re.compile(self.cfg["bill_link_pattern"])
         exclude_prefixes = self.cfg["bill_link_exclude_prefixes"]
+        limit = self.cfg.get("max_bills_kept") or self.cfg.get("max_bills_per_run")
+        docs, seen = [], set()
 
-        resp = self._get(self.cfg["listing_url"])
-        soup = BeautifulSoup(resp.text, "lxml")
+        for page_no in range(self.cfg.get("listing_pages", 6)):
+            page_url = self.cfg["listing_url"] if page_no == 0 else f"{self.cfg['listing_url']}?page={page_no}"
+            try:
+                resp = self._get(page_url)
+            except Exception:
+                if page_no == 0:
+                    raise          # first page failing is a real error; later pages just end the walk
+                break
+            soup = BeautifulSoup(resp.text, "lxml")
+            before = len(docs)
 
-        docs, seen = [], set()
-        for heading in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
-            link = heading.find("a", href=True)
-            if not link:
-                continue
-            href = link["href"]
-            path = href if href.startswith("/") else href.replace(self.cfg["base_url"], "", 1)
-            if any(path.startswith(p) for p in exclude_prefixes):
-                continue
-            if not link_pattern.match(path):
-                continue
+            for heading in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
+                link = heading.find("a", href=True)
+                if not link:
+                    continue
+                href = link["href"]
+                path = href if href.startswith("/") else href.replace(self.cfg["base_url"], "", 1)
+                if any(path.startswith(p) for p in exclude_prefixes):
+                    continue
+                if not link_pattern.match(path):
+                    continue
 
-            url = href if href.startswith("http") else self.cfg["base_url"] + href
-            if url in seen:
-                continue
-            seen.add(url)
-
-            status = None
-            node = heading.find_next_sibling()
-            while node is not None and not node.get_text(strip=True):
-                node = node.find_next_sibling()
-            if node is not None and node.name not in ("h1", "h2", "h3", "h4", "h5", "h6"):
-                status = node.get_text(strip=True)
-
-            docs.append(
-                DiscoveredDocument(
-                    url=url,
-                    kind="bill_page",
-                    source="prs",
-                    title=link.get_text(strip=True),
-                    metadata={"bill_status": status} if status else {},
+                url = href if href.startswith("http") else self.cfg["base_url"] + href
+                if url in seen:
+                    continue
+                seen.add(url)
+
+                status = None
+                node = heading.find_next_sibling()
+                while node is not None and not node.get_text(strip=True):
+                    node = node.find_next_sibling()
+                if node is not None and node.name not in ("h1", "h2", "h3", "h4", "h5", "h6"):
+                    status = node.get_text(strip=True)
+
+                docs.append(
+                    DiscoveredDocument(
+                        url=url, kind="bill_page", source="prs", title=link.get_text(strip=True),
+                        metadata={"bill_status": status, "section": "bill", "section_label": "Bills"}
+                        if status else {"section": "bill", "section_label": "Bills"},
+                    )
                 )
-            )
+            if len(docs) >= (limit or 10**9) or len(docs) == before:
+                break
 
-        limit = self.cfg.get("max_bills_per_run")
         return docs[:limit] if limit else docs
 
     def discover_pdfs_from_bill_page(self, bill_doc: DiscoveredDocument, pdf_links: List[str]) -> List[DiscoveredDocument]:
@@ -154,3 +166,143 @@ def discover_comment_documents(pages: List[DiscoveredDocument]) -> List[Discover
             metadata={"parent_bill_url": page.url, "comment_count": len(comments)},
             text=tinyfish_client.comments_to_text(page.title, comments)))
     return docs
+
+
+# ---------------------------------------------------------------------------
+# Whole-site PRS crawler
+# ---------------------------------------------------------------------------
+import re as _re
+from datetime import datetime as _dt
+from typing import Iterator, Tuple
+from urllib.parse import urljoin, urlparse
+
+_MD_LINK_RE = _re.compile(r"\[([^\]]*)\]\((https?://[^)\s]+)\)")
+_DEADLINE_RE = _re.compile(
+    r"^(?P<title>.+?)\s*Deadline for submission:\s*(?P<date>[A-Z][a-z]{2,8} \d{1,2}, \d{4})\s*\|?\s*$", _re.S
+)
+_PRS_HOSTS = {"prsindia.org", "www.prsindia.org"}
+
+
+def _clean(text: str) -> str:
+    return _re.sub(r"\s+", " ", text or "").strip()
+
+
+class PRSLibraryCrawler:
+    """
+    Crawls the PRS sections listed in CONFIG["prs_library"] and yields
+    DiscoveredDocuments (pages and PDFs) tagged with their section, so the website
+    can browse them by section. It never relies on page-specific HTML: it just
+    follows links whose *path* matches the section's pattern, so a redesign of an
+    index page does not break it, and a changed URL scheme is a one-line config edit.
+
+    If PRS refuses a direct request, the page is re-fetched through TinyFish Fetch
+    (a real browser) and its links are read from the returned markdown.
+    """
+
+    def __init__(self, prs_cfg: dict, lib_cfg: dict):
+        self.prs_cfg = prs_cfg
+        self.cfg = lib_cfg
+        self.delay = prs_cfg.get("rate_limit_seconds", 2)
+        self.session = requests.Session()
+        self.session.headers.update({"User-Agent": USER_AGENT})
+
+    # -- fetching -----------------------------------------------------------
+    def _links(self, url: str) -> List[Tuple[str, str]]:
+        """All (absolute_url, anchor_text) links on a page. Empty list on failure."""
+        try:
+            resp = self.session.get(url, timeout=20)
+            resp.raise_for_status()
+            time.sleep(self.delay)
+            soup = BeautifulSoup(resp.text, "lxml")
+            return [(urljoin(url, a["href"]), _clean(a.get_text(" "))) for a in soup.find_all("a", href=True)]
+        except Exception as exc:
+            logger.info("Direct fetch of %s failed (%s); trying TinyFish Fetch", url, exc)
+        try:
+            page = tinyfish_client.fetch_text(url)
+            return [(href, _clean(text)) for text, href in _MD_LINK_RE.findall(page["text"])]
+        except Exception as exc:
+            logger.warning("Could not read index page %s: %s", url, exc)
+            return []
+
+    @staticmethod
+    def _normalise(url: str) -> Optional[str]:
+        """Drop query/fragment; keep only prsindia.org (English site)."""
+        p = urlparse(url)
+        if p.scheme not in ("http", "https") or p.netloc.lower() not in _PRS_HOSTS:
+            return None
+        return f"https://prsindia.org{p.path}".rstrip("/") if p.path not in ("", "/") else None
+
+    # -- sections -----------------------------------------------------------
+    def discover_section(self, section: dict) -> Iterator[DiscoveredDocument]:
+        pattern = _re.compile(section["item_pattern"])
+        seen = set()
+        max_pages = section.get("max_index_pages", self.cfg.get("max_index_pages", 5))
+        page_kind = section.get("kind", "article")
+        pdf_kind = section.get("pdf_kind", "article")
+        base_meta = {
+            "section": section["key"],
+            "section_label": section["label"],
+            "archival": section.get("archival", True),
+        }
+
+        for index_url in section["index_urls"]:
+            for page_no in range(max_pages):
+                page_url = index_url if page_no == 0 else f"{index_url}?page={page_no}"
+                new_on_page = 0
+                for href, text in self._links(page_url):
+                    is_pdf = urlparse(href).path.lower().endswith(".pdf") and "/files/" in href
+                    if is_pdf and urlparse(href).netloc.lower() in _PRS_HOSTS:
+                        url, kind, doc_type = href.split("#")[0], pdf_kind, "pdf"
+                    else:
+                        url = self._normalise(href)
+                        if not url or not pattern.match(urlparse(url).path):
+                            continue
+                        kind, doc_type = page_kind, "page"
+                    if url in seen:
+                        continue
+                    seen.add(url)
+                    new_on_page += 1
+                    title = text or urlparse(url).path.rsplit("/", 1)[-1].replace("-", " ").title()
+                    yield DiscoveredDocument(url=url, kind=kind, source="prs", title=title,
+                                             metadata={**base_meta, "doc_type": doc_type})
+                if new_on_page == 0:   # ran past the last page of this index
+                    break
+
+    # -- "Comments invited on ..." notices ------------------------------------
+    def discover_announcements(self) -> List[DiscoveredDocument]:
+        docs, seen = [], set()
+        for page_url in self.cfg.get("announcement_urls", []):
+            for href, text in self._links(page_url):
+                m = _DEADLINE_RE.match(text)
+                if not m:
+                    continue
+                title = _clean(m.group("title")).rstrip("|").strip()
+                try:
+                    deadline = _dt.strptime(m.group("date"), "%b %d, %Y").date().isoformat()
+                except ValueError:
+                    try:
+                        deadline = _dt.strptime(m.group("date"), "%B %d, %Y").date().isoformat()
+                    except ValueError:
+                        deadline = None
+                key = (title, deadline)
+                if key in seen:
+                    continue
+                seen.add(key)
+                slug = _re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")[:80]
+                text_body = (
+                    f"Public comments are invited on: {title}.\n"
+                    f"Deadline for submission: {m.group('date')}.\n"
+                    f"Official notice and how to submit comments: {href}\n"
+                    "Source: PRS Legislative Research announcements."
+                )
+                docs.append(DiscoveredDocument(
+                    url=f"https://prsindia.org/announcements#{slug}",
+                    kind="article", source="prs", title=f"Comments invited: {title}",
+                    text=text_body,
+                    metadata={"section": "announcement", "section_label": "Comments Invited",
+                              "deadline": deadline, "notice_url": href, "archival": False, "doc_type": "notice"},
+                ))
+                # The notice itself (PDF/page on a ministry site) also goes into the knowledge base.
+                docs.append(DiscoveredDocument(url=href, kind="article", source="prs", title=title,
+                                               metadata={"role": "announcement_notice", "archival": True}))
+        return docs