import asyncio import httpx import pytest from fastapi.testclient import TestClient from app.config import Settings from app.errors import ScrapeError from app.main import create_app SAMPLE_HTML = ( "Sample Title" "

Some real sample body content for the test page.

" ) class FakeBrowserManager: def __init__(self): self.started = False async def start(self): self.started = True async def stop(self): self.started = False def get_browser(self): return object() if self.started else None class FakeScraper: def __init__(self, html=None, error=None, delay=0.0): self.html = html if html is not None else SAMPLE_HTML self.error = error self.delay = delay self.calls: list[str] = [] self.active = 0 self.max_active = 0 async def scrape(self, url: str) -> str: self.calls.append(url) self.active += 1 self.max_active = max(self.max_active, self.active) try: if self.delay: await asyncio.sleep(self.delay) if self.error is not None: raise self.error return self.html finally: self.active -= 1 def base_settings(**overrides) -> Settings: defaults = dict( scrape_timeout_ms=5000, max_concurrent_scrapes=3, user_agent="TestAgent/1.0", port=7860, rate_limit_requests=1000, rate_limit_window_seconds=60, api_key=None, cache_ttl_seconds=300, cache_max_size=256, ) defaults.update(overrides) return Settings(**defaults) def make_client(settings=None, scraper=None): app = create_app( settings=settings or base_settings(), browser_manager=FakeBrowserManager(), scraper=scraper or FakeScraper(), ) return TestClient(app) def test_root_endpoint(): with make_client() as client: r = client.get("/") assert r.status_code == 200 assert "message" in r.json() def test_health_reports_browser_ready_once_lifespan_has_started(): with make_client() as client: r = client.get("/health") assert r.status_code == 200 assert r.json() == {"status": "ok", "browser_ready": True} @pytest.mark.parametrize( "url", ["http://127.0.0.1/", "http://169.254.169.254/latest/meta-data/", "http://10.0.0.5/"], ) def test_scrape_blocks_ssrf_targets(url): with make_client() as client: r = client.get("/scrape", params={"url": url}) assert r.status_code == 400 def test_scrape_rejects_invalid_url(): with make_client() as client: r = client.get("/scrape", params={"url": "not-a-url"}) assert r.status_code == 422 def test_scrape_get_returns_extracted_content(): with make_client() as client: r = client.get("/scrape", params={"url": "https://example.com/"}) assert r.status_code == 200 body = r.json() assert body["status"] == "success" assert body["title"] == "Sample Title" assert "real sample body content" in body["content"] assert body["cached"] is False def test_scrape_post_returns_extracted_content(): with make_client() as client: r = client.post("/scrape", json={"url": "https://example.com/"}) assert r.status_code == 200 assert r.json()["status"] == "success" def test_repeat_requests_are_served_from_cache(): scraper = FakeScraper() with make_client(scraper=scraper) as client: first = client.get("/scrape", params={"url": "https://example.com/"}) second = client.get("/scrape", params={"url": "https://example.com/"}) assert first.status_code == 200 and second.status_code == 200 assert first.json()["cached"] is False assert second.json()["cached"] is True assert scraper.calls == ["https://example.com/"] # scraper only invoked once def test_cache_disabled_when_ttl_is_zero(): scraper = FakeScraper() settings = base_settings(cache_ttl_seconds=0) with make_client(settings=settings, scraper=scraper) as client: client.get("/scrape", params={"url": "https://example.com/"}) client.get("/scrape", params={"url": "https://example.com/"}) assert scraper.calls == ["https://example.com/", "https://example.com/"] def test_rate_limit_blocks_after_the_configured_number_of_requests(): settings = base_settings(rate_limit_requests=2, rate_limit_window_seconds=60, cache_ttl_seconds=0) with make_client(settings=settings) as client: r1 = client.get("/scrape", params={"url": "https://example.com/1"}) r2 = client.get("/scrape", params={"url": "https://example.com/2"}) r3 = client.get("/scrape", params={"url": "https://example.com/3"}) assert r1.status_code == 200 assert r2.status_code == 200 assert r3.status_code == 429 def test_rate_limit_keys_on_x_forwarded_for_not_the_proxy_peer(): # The app is only reachable through Hugging Face's reverse proxy, so # request.client.host is always the proxy's own connecting address, not # the real caller. Without honoring X-Forwarded-For, every caller would # share (or fragment across) the wrong bucket and the limit would never # engage correctly - this reproduces that against a fixed TestClient # peer address, where two distinct forwarded IPs must be tracked # independently and a repeated one must accumulate hits. settings = base_settings(rate_limit_requests=2, rate_limit_window_seconds=60, cache_ttl_seconds=0) with make_client(settings=settings) as client: a1 = client.get( "/scrape", params={"url": "https://example.com/a1"}, headers={"X-Forwarded-For": "1.2.3.4"} ) a2 = client.get( "/scrape", params={"url": "https://example.com/a2"}, headers={"X-Forwarded-For": "1.2.3.4"} ) a3 = client.get( "/scrape", params={"url": "https://example.com/a3"}, headers={"X-Forwarded-For": "1.2.3.4"} ) # A different forwarded client is not affected by 1.2.3.4's limit. b1 = client.get( "/scrape", params={"url": "https://example.com/b1"}, headers={"X-Forwarded-For": "5.6.7.8"} ) assert a1.status_code == 200 assert a2.status_code == 200 assert a3.status_code == 429 assert b1.status_code == 200 def test_api_key_required_when_configured(): settings = base_settings(api_key="secret123") with make_client(settings=settings) as client: unauthenticated = client.get("/scrape", params={"url": "https://example.com/"}) wrong_key = client.get( "/scrape", params={"url": "https://example.com/"}, headers={"X-API-Key": "wrong"} ) correct_key = client.get( "/scrape", params={"url": "https://example.com/"}, headers={"X-API-Key": "secret123"} ) assert unauthenticated.status_code == 401 assert wrong_key.status_code == 401 assert correct_key.status_code == 200 def test_no_api_key_required_when_not_configured(): with make_client(settings=base_settings(api_key=None)) as client: r = client.get("/scrape", params={"url": "https://example.com/"}) assert r.status_code == 200 def test_scrape_timeout_is_mapped_to_504(): scraper = FakeScraper(error=ScrapeError(504, "Timed out loading the page.")) with make_client(scraper=scraper) as client: r = client.get("/scrape", params={"url": "https://example.com/"}) assert r.status_code == 504 def test_unexpected_scraper_error_is_mapped_to_502_without_leaking_details(): scraper = FakeScraper(error=RuntimeError("some internal secret stack detail")) with make_client(scraper=scraper) as client: r = client.get("/scrape", params={"url": "https://example.com/"}) assert r.status_code == 502 assert "some internal secret stack detail" not in r.text def test_concurrent_scrapes_are_limited_by_the_semaphore(): scraper = FakeScraper(delay=0.05) settings = base_settings(max_concurrent_scrapes=2, cache_ttl_seconds=0, rate_limit_requests=1000) app = create_app(settings=settings, browser_manager=FakeBrowserManager(), scraper=scraper) async def run(): transport = httpx.ASGITransport(app=app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: async with app.router.lifespan_context(app): requests = [client.get("/scrape", params={"url": f"https://example.com/{i}"}) for i in range(5)] responses = await asyncio.gather(*requests) return responses responses = asyncio.run(run()) assert all(r.status_code == 200 for r in responses) assert scraper.max_active == 2 # never exceeded the configured limit assert len(scraper.calls) == 5