Create ingest/adapters/cia.py

ingest/adapters/cia.py

@@ -3,11 +3,13 @@ from bs4 import BeautifulSoup
 from ingest.generic_public_foia import GenericFOIAAdapter
 
 class CIAAdapter(GenericFOIAAdapter):
-    source_name = "CIA CREST"
-    agency = "CIA"
+    source_name = "CIA"
     base_url = "https://www.cia.gov/readingroom/search/site"
 
     async def search(self, query: str):
+        if not self.robots_allowed():
+            return []
+
         await self._rate_limit()
         params = {"search_api_fulltext": query}
 
@@ -18,19 +20,18 @@ class CIAAdapter(GenericFOIAAdapter):
         soup = BeautifulSoup(html, "html.parser")
         results = []
 
-        for r in soup.select(".search-result"):
-            title_el = r.select_one("h3")
-            link_el = r.select_one("a")
+        for item in soup.select(".search-result"):
+            title = item.select_one("h3")
+            link = item.select_one("a")
 
-            if not title_el or not link_el:
+            if not title or not link:
                 continue
 
             results.append({
                 "source": self.source_name,
-                "agency": self.agency,
-                "title": title_el.get_text(strip=True),
-                "url": "https://www.cia.gov" + link_el["href"],
-                "snippet": ""
+                "title": title.text.strip(),
+                "url": link["href"],
+                "snippet": title.text.strip()
             })
 
         return results