Spaces:
Running
Running
| import sys | |
| import unittest | |
| from pathlib import Path | |
| from unittest.mock import patch | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from app.crawler_registry import ( | |
| CRAWLERS, | |
| active_crawler_caveats, | |
| bot_user_agents, | |
| crawler_directive_sources, | |
| fetch_probe_names, | |
| robots_scoring_crawlers, | |
| ) | |
| from app.checks.bot_access import check_bot_access | |
| from app.checks.indexing import check_indexing | |
| from app.checks.robots import _classify_bot_access, check_robots | |
| from app.fetching import BROWSER_USER_AGENT, FetchResult | |
| from backend.tests.fixtures import make_audit_context | |
| class CrawlerRegistryTests(unittest.TestCase): | |
| def test_duckduckgo_crawlers_are_active_in_all_three_consumers(self) -> None: | |
| crawlers = {crawler.name: crawler for crawler in CRAWLERS} | |
| scored_names = {crawler.name for crawler in robots_scoring_crawlers()} | |
| probe_names = set(fetch_probe_names()) | |
| directive_sources = crawler_directive_sources() | |
| self.assertEqual(len(scored_names), 15) | |
| self.assertEqual(len(probe_names), 9) | |
| self.assertEqual(len(directive_sources), 17) | |
| for name in ("DuckDuckBot", "DuckAssistBot"): | |
| crawler = crawlers[name] | |
| self.assertTrue(crawler.robots_scored) | |
| self.assertTrue(crawler.fetch_probe) | |
| self.assertTrue(crawler.directive_source) | |
| self.assertIn(name, scored_names) | |
| self.assertIn(name, probe_names) | |
| self.assertIn(name.lower(), directive_sources) | |
| self.assertEqual( | |
| crawlers["DuckDuckBot"].fetch_user_agent, | |
| "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", | |
| ) | |
| self.assertEqual( | |
| crawlers["DuckAssistBot"].fetch_user_agent, | |
| "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)", | |
| ) | |
| self.assertEqual( | |
| crawlers["DuckDuckBot"].official_documentation_url, | |
| "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", | |
| ) | |
| self.assertEqual( | |
| crawlers["DuckAssistBot"].official_ip_list_url, | |
| "https://duckduckgo.com/duckassistbot.json", | |
| ) | |
| self.assertEqual( | |
| crawlers["DuckDuckBot"].official_ip_list_url, | |
| "https://duckduckgo.com/duckduckbot.json", | |
| ) | |
| self.assertEqual( | |
| crawlers["DuckAssistBot"].official_documentation_url, | |
| "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", | |
| ) | |
| def test_consumer_caveats_are_scoped_and_concise(self) -> None: | |
| fetch_caveats = " ".join(active_crawler_caveats("fetch")) | |
| robots_caveats = " ".join(active_crawler_caveats("robots")) | |
| directive_caveats = " ".join( | |
| active_crawler_caveats( | |
| "directive", | |
| crawler_tokens={"duckassistbot"}, | |
| ) | |
| ) | |
| self.assertIn("do not authenticate provider IPs", fetch_caveats) | |
| self.assertIn("real-time retrieval for AI-assisted answers", fetch_caveats) | |
| self.assertIn("not model training", fetch_caveats) | |
| self.assertIn("organic search ranking or result inclusion", fetch_caveats) | |
| self.assertIn("Google-Extended is a robots.txt control token", robots_caveats) | |
| self.assertIn("ChatGPT-User is user-triggered", robots_caveats) | |
| self.assertNotIn("cannot authenticate official DuckDuckGo traffic", robots_caveats) | |
| self.assertNotIn("provider IPs", directive_caveats) | |
| self.assertIn("real-time retrieval for AI-assisted answers", directive_caveats) | |
| class CrawlerBehaviorTests(unittest.IsolatedAsyncioTestCase): | |
| async def test_robots_scores_duckassist_policy_with_purpose_caveat(self) -> None: | |
| robots_text = ( | |
| "User-agent: *\nAllow: /\n" | |
| "User-agent: DuckAssistBot\nDisallow: /" | |
| ) | |
| context = make_audit_context(robots_text=robots_text) | |
| access = _classify_bot_access(robots_text, context.url + "/") | |
| result = await check_robots(context) | |
| self.assertEqual(access["DuckAssistBot"], "explicit_blocked") | |
| self.assertEqual(result.state, "partial") | |
| self.assertEqual(result.score, 5) | |
| self.assertIn("DuckAssistBot", result.finding) | |
| self.assertIn("real-time retrieval for AI-assisted answers", result.finding) | |
| self.assertIn("not model training", result.finding) | |
| self.assertIn("organic search ranking or result inclusion", result.finding) | |
| self.assertNotIn("do not authenticate provider IPs", result.finding) | |
| async def test_fetch_matrix_probes_exact_duckduckgo_user_agents_and_caveats(self) -> None: | |
| body = "<html><body><main>" + ("Public crawler content. " * 100) + "</main></body></html>" | |
| context = make_audit_context(homepage_html=body) | |
| requested_user_agents: list[str] = [] | |
| async def fake_fetch( | |
| url: str, | |
| user_agent: str = "", | |
| **kwargs: object, | |
| ) -> FetchResult: | |
| requested_user_agents.append(user_agent) | |
| status_code = 403 if user_agent.startswith("DuckAssistBot/1.2") else 200 | |
| return FetchResult( | |
| requested_url=url, | |
| final_url=url, | |
| status_code=status_code, | |
| headers={"content-type": "text/html"}, | |
| text=body, | |
| elapsed_ms=1, | |
| redirect_chain=[], | |
| ) | |
| with patch("app.checks.bot_access.fetch_url", new=fake_fetch): | |
| result = await check_bot_access(context) | |
| user_agents = bot_user_agents() | |
| self.assertEqual(requested_user_agents[0], BROWSER_USER_AGENT) | |
| self.assertIn(user_agents["DuckDuckBot"], requested_user_agents) | |
| self.assertIn(user_agents["DuckAssistBot"], requested_user_agents) | |
| self.assertEqual(result.state, "partial") | |
| self.assertEqual(result.score, 5) | |
| self.assertIn("DuckAssistBot (HTTP 403)", result.finding) | |
| self.assertIn("do not authenticate provider IPs", result.finding) | |
| self.assertIn("real-time retrieval for AI-assisted answers", result.finding) | |
| self.assertIn("organic search ranking or result inclusion", result.finding) | |
| async def test_indexing_reads_duckassist_directive_with_scoped_caveat(self) -> None: | |
| context = make_audit_context( | |
| homepage_html=( | |
| '<html><head><meta name="DuckAssistBot" content="noindex"></head>' | |
| "<body>Example</body></html>" | |
| ) | |
| ) | |
| result = await check_indexing(context) | |
| self.assertEqual(result.state, "fail") | |
| self.assertEqual(result.score, 0) | |
| self.assertIn("meta duckassistbot", result.finding) | |
| self.assertIn("real-time retrieval for AI-assisted answers", result.finding) | |
| self.assertIn("not model training", result.finding) | |
| self.assertIn("organic search ranking or result inclusion", result.finding) | |
| self.assertNotIn("provider IPs", result.finding) | |
| if __name__ == "__main__": | |
| unittest.main() | |