"""Decoding Bing's result-link redirector. Bing does not put the destination in a result's href. It wraps every link in `https://www.bing.com/ck/a?...&u=a1&ntb=1`, so the URLs this API handed back were Bing tracking links rather than the pages they point at. An agent following one goes through Bing's redirector, and the href is useless for deduplication, citation, or deciding whether a result is worth fetching. Verified against a redirect captured from the live deployment: the `u` parameter is the literal prefix "a1" followed by the destination, base64url encoded with its padding stripped. """ import pytest from serp import decode_bing_redirect # Captured from a real /serp/search response. REAL_REDIRECT = ( "https://www.bing.com/ck/a?!&&p=355fd6a450a5a451290a8e71334947199bc81cf4" "c6084e0f8daa8d5a0debc698JmltdHM9MTc4Nzk2MTYwMA&ptn=3&ver=2&hsh=4" "&fclid=12f2c74a-55e9-6296-1923-d08f542d635f" "&u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy9saWJyYXJ5L2FzeW5jaW8uaHRtbA&ntb=1" ) def test_decodes_a_real_captured_redirect(): assert decode_bing_redirect(REAL_REDIRECT) == "https://docs.python.org/3/library/asyncio.html" def test_decodes_a_url_whose_base64_needs_padding(): """Bing strips base64 padding, so the decoder has to restore it - and the amount needed varies with the length of the destination.""" import base64 for destination in [ "https://example.org/a", "https://example.org/ab", "https://example.org/abc", "https://example.org/abcd", ]: payload = base64.urlsafe_b64encode(destination.encode()).decode().rstrip("=") assert decode_bing_redirect(f"https://www.bing.com/ck/a?u=a1{payload}") == destination # ------------------------------ leaving links alone ------------------------------ # Every failure mode below must return the original URL. A link that can't be # decoded is still a working link; replacing it with None or "" would lose it. def test_a_direct_url_passes_through_unchanged(): assert decode_bing_redirect("https://example.org/page") == "https://example.org/page" def test_a_redirect_without_a_u_parameter_passes_through(): url = "https://www.bing.com/ck/a?!&&p=abc&ntb=1" assert decode_bing_redirect(url) == url def test_a_u_parameter_without_the_a1_prefix_passes_through(): url = "https://www.bing.com/ck/a?u=zzsomethingelse" assert decode_bing_redirect(url) == url def test_undecodable_base64_passes_through(): url = "https://www.bing.com/ck/a?u=a1!!!not-base64!!!" assert decode_bing_redirect(url) == url def test_base64_that_is_not_utf8_passes_through(): import base64 payload = base64.urlsafe_b64encode(b"\xff\xfe\xfd").decode().rstrip("=") url = f"https://www.bing.com/ck/a?u=a1{payload}" assert decode_bing_redirect(url) == url @pytest.mark.parametrize("hostile", [ "javascript:alert(1)", "data:text/html;base64,PHNjcmlwdD4=", "file:///etc/passwd", "not a url at all", ]) def test_a_decoded_non_http_scheme_is_rejected(hostile): """The decoded payload is attacker-influenceable content from a scraped page, so only http(s) destinations are allowed out. Anything else keeps the original Bing link rather than being handed to a caller as if it were a result URL. """ import base64 payload = base64.urlsafe_b64encode(hostile.encode()).decode().rstrip("=") url = f"https://www.bing.com/ck/a?u=a1{payload}" assert decode_bing_redirect(url) == url def test_none_is_handled(): assert decode_bing_redirect(None) is None