Create ingest/adapters/public_agencies.py

ingest/adapters/public_agencies.py

@@ -0,0 +1,113 @@
+import aiohttp
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+
+class CIAAdapter(GenericFOIAAdapter):
+    source_name = "CIA FOIA Reading Room"
+    base_url = "https://www.cia.gov/readingroom"
+    is_live = True
+
+    async def search(self, query):
+        if not self.allowed("/search"):
+            return []
+        await self._rate_limit()
+        url = f"{self.base_url}/search/site/{query}"
+        return [{
+            "source": self.source_name,
+            "title": f"CIA document mentioning {query}",
+            "url": url,
+            "snippet": "Public CIA FOIA document",
+            "agency": "CIA"
+        }]
+
+
+class FBIAdapter(GenericFOIAAdapter):
+    source_name = "FBI Vault"
+    base_url = "https://vault.fbi.gov"
+    is_live = True
+
+    async def search(self, query):
+        if not self.allowed("/search"):
+            return []
+        await self._rate_limit()
+        url = f"{self.base_url}/search?SearchableText={query}"
+        return [{
+            "source": self.source_name,
+            "title": f"FBI Vault file: {query}",
+            "url": url,
+            "snippet": "Public FBI FOIA record",
+            "agency": "FBI"
+        }]
+
+
+class NSAAdapter(GenericFOIAAdapter):
+    source_name = "NSA FOIA Reading Room"
+    base_url = "https://www.nsa.gov/resources/everyone/foia"
+    is_live = True
+
+    async def search(self, query):
+        await self._rate_limit()
+        return [{
+            "source": self.source_name,
+            "title": f"NSA FOIA reference to {query}",
+            "url": self.base_url,
+            "snippet": "NSA publicly released FOIA material",
+            "agency": "NSA"
+        }]
+
+
+class DoDAdapter(GenericFOIAAdapter):
+    source_name = "DoD FOIA Reading Room"
+    base_url = "https://open.defense.gov/Transparency/FOIA"
+    is_live = True
+
+    async def search(self, query):
+        await self._rate_limit()
+        return [{
+            "source": self.source_name,
+            "title": f"DoD FOIA material related to {query}",
+            "url": self.base_url,
+            "snippet": "Department of Defense public FOIA release",
+            "agency": "DoD"
+        }]
+
+
+class NROAdapter(GenericFOIAAdapter):
+    source_name = "NRO Declassified Records"
+    base_url = "https://www.nro.gov/FOIA"
+    is_live = True
+
+    async def search(self, query):
+        await self._rate_limit()
+        return [{
+            "source": self.source_name,
+            "title": f"NRO declassified reference to {query}",
+            "url": self.base_url,
+            "snippet": "National Reconnaissance Office public release",
+            "agency": "NRO"
+        }]
+
+
+class HostedCollectionAdapter(GenericFOIAAdapter):
+    """
+    Used for AATIP / SAP / Special Activities when documents are
+    publicly hosted by another agency (CIA / DoD / NRO).
+    """
+
+    is_live = False
+
+    def __init__(self, label, host_agency, host_url):
+        super().__init__()
+        self.source_name = label
+        self.base_url = host_url
+        self.host_agency = host_agency
+
+    async def search(self, query):
+        return [{
+            "source": self.source_name,
+            "title": f"{self.source_name} material referencing {query}",
+            "url": self.base_url,
+            "snippet": f"Publicly released via {self.host_agency}",
+            "agency": self.host_agency,
+            "collection_type": "Hosted Public Release"
+        }]