fix(scraper): send default_headers with curl_cffi impersonation (fixes 403 on WAF sites) (#7792)

2026-06-23 02:20:16 +00:00 · 2026-06-22 13:42:22 -04:00
parent 6c124a2dbd
commit aaab52326c
1 changed files with 1 additions and 0 deletions
@@ -78,6 +78,7 @@ async def safe_scrape_html(url: str) -> str:

        transport = safehttp.AsyncSafeTransport(
            impersonate=impersonation,
+            default_headers=True,
            verify=False,  # disable SSL verification since we can handle untrusted data and some sites don't have certs
        )
        async with AsyncClient(transport=transport) as client: