Spaces:
Sleeping
Sleeping
File size: 7,093 Bytes
7d7841f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | import sys
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.crawler_registry import (
CRAWLERS,
active_crawler_caveats,
bot_user_agents,
crawler_directive_sources,
fetch_probe_names,
robots_scoring_crawlers,
)
from app.checks.bot_access import check_bot_access
from app.checks.indexing import check_indexing
from app.checks.robots import _classify_bot_access, check_robots
from app.fetching import BROWSER_USER_AGENT, FetchResult
from backend.tests.fixtures import make_audit_context
class CrawlerRegistryTests(unittest.TestCase):
def test_duckduckgo_crawlers_are_active_in_all_three_consumers(self) -> None:
crawlers = {crawler.name: crawler for crawler in CRAWLERS}
scored_names = {crawler.name for crawler in robots_scoring_crawlers()}
probe_names = set(fetch_probe_names())
directive_sources = crawler_directive_sources()
self.assertEqual(len(scored_names), 15)
self.assertEqual(len(probe_names), 9)
self.assertEqual(len(directive_sources), 17)
for name in ("DuckDuckBot", "DuckAssistBot"):
crawler = crawlers[name]
self.assertTrue(crawler.robots_scored)
self.assertTrue(crawler.fetch_probe)
self.assertTrue(crawler.directive_source)
self.assertIn(name, scored_names)
self.assertIn(name, probe_names)
self.assertIn(name.lower(), directive_sources)
self.assertEqual(
crawlers["DuckDuckBot"].fetch_user_agent,
"DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
)
self.assertEqual(
crawlers["DuckAssistBot"].fetch_user_agent,
"DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
)
self.assertEqual(
crawlers["DuckDuckBot"].official_documentation_url,
"https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot",
)
self.assertEqual(
crawlers["DuckAssistBot"].official_ip_list_url,
"https://duckduckgo.com/duckassistbot.json",
)
self.assertEqual(
crawlers["DuckDuckBot"].official_ip_list_url,
"https://duckduckgo.com/duckduckbot.json",
)
self.assertEqual(
crawlers["DuckAssistBot"].official_documentation_url,
"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
)
def test_consumer_caveats_are_scoped_and_concise(self) -> None:
fetch_caveats = " ".join(active_crawler_caveats("fetch"))
robots_caveats = " ".join(active_crawler_caveats("robots"))
directive_caveats = " ".join(
active_crawler_caveats(
"directive",
crawler_tokens={"duckassistbot"},
)
)
self.assertIn("do not authenticate provider IPs", fetch_caveats)
self.assertIn("real-time retrieval for AI-assisted answers", fetch_caveats)
self.assertIn("not model training", fetch_caveats)
self.assertIn("organic search ranking or result inclusion", fetch_caveats)
self.assertIn("Google-Extended is a robots.txt control token", robots_caveats)
self.assertIn("ChatGPT-User is user-triggered", robots_caveats)
self.assertNotIn("cannot authenticate official DuckDuckGo traffic", robots_caveats)
self.assertNotIn("provider IPs", directive_caveats)
self.assertIn("real-time retrieval for AI-assisted answers", directive_caveats)
class CrawlerBehaviorTests(unittest.IsolatedAsyncioTestCase):
async def test_robots_scores_duckassist_policy_with_purpose_caveat(self) -> None:
robots_text = (
"User-agent: *\nAllow: /\n"
"User-agent: DuckAssistBot\nDisallow: /"
)
context = make_audit_context(robots_text=robots_text)
access = _classify_bot_access(robots_text, context.url + "/")
result = await check_robots(context)
self.assertEqual(access["DuckAssistBot"], "explicit_blocked")
self.assertEqual(result.state, "partial")
self.assertEqual(result.score, 5)
self.assertIn("DuckAssistBot", result.finding)
self.assertIn("real-time retrieval for AI-assisted answers", result.finding)
self.assertIn("not model training", result.finding)
self.assertIn("organic search ranking or result inclusion", result.finding)
self.assertNotIn("do not authenticate provider IPs", result.finding)
async def test_fetch_matrix_probes_exact_duckduckgo_user_agents_and_caveats(self) -> None:
body = "<html><body><main>" + ("Public crawler content. " * 100) + "</main></body></html>"
context = make_audit_context(homepage_html=body)
requested_user_agents: list[str] = []
async def fake_fetch(
url: str,
user_agent: str = "",
**kwargs: object,
) -> FetchResult:
requested_user_agents.append(user_agent)
status_code = 403 if user_agent.startswith("DuckAssistBot/1.2") else 200
return FetchResult(
requested_url=url,
final_url=url,
status_code=status_code,
headers={"content-type": "text/html"},
text=body,
elapsed_ms=1,
redirect_chain=[],
)
with patch("app.checks.bot_access.fetch_url", new=fake_fetch):
result = await check_bot_access(context)
user_agents = bot_user_agents()
self.assertEqual(requested_user_agents[0], BROWSER_USER_AGENT)
self.assertIn(user_agents["DuckDuckBot"], requested_user_agents)
self.assertIn(user_agents["DuckAssistBot"], requested_user_agents)
self.assertEqual(result.state, "partial")
self.assertEqual(result.score, 5)
self.assertIn("DuckAssistBot (HTTP 403)", result.finding)
self.assertIn("do not authenticate provider IPs", result.finding)
self.assertIn("real-time retrieval for AI-assisted answers", result.finding)
self.assertIn("organic search ranking or result inclusion", result.finding)
async def test_indexing_reads_duckassist_directive_with_scoped_caveat(self) -> None:
context = make_audit_context(
homepage_html=(
'<html><head><meta name="DuckAssistBot" content="noindex"></head>'
"<body>Example</body></html>"
)
)
result = await check_indexing(context)
self.assertEqual(result.state, "fail")
self.assertEqual(result.score, 0)
self.assertIn("meta duckassistbot", result.finding)
self.assertIn("real-time retrieval for AI-assisted answers", result.finding)
self.assertIn("not model training", result.finding)
self.assertIn("organic search ranking or result inclusion", result.finding)
self.assertNotIn("provider IPs", result.finding)
if __name__ == "__main__":
unittest.main()
|