This page tests different browser adapters.
User-Agent detection test
"""
# Save test HTML
with open("/tmp/antibot_test.html", "w") as f:
f.write(test_html)
test_url = "file:///tmp/antibot_test.html"
strategies = [
("default", "Default Playwright"),
("stealth", "Stealth Mode"),
]
for strategy, description in strategies:
print(f"\n๐ Testing: {description} (strategy: {strategy})")
print("-" * 40)
try:
# Import adapter based on strategy
if strategy == "stealth":
try:
from crawl4ai import StealthAdapter
adapter = StealthAdapter()
print(f"โ
Using StealthAdapter")
except ImportError:
print(
f"โ ๏ธ StealthAdapter not available, using PlaywrightAdapter"
)
adapter = PlaywrightAdapter()
else:
adapter = PlaywrightAdapter()
print(f"โ
Using PlaywrightAdapter")
# Configure browser
browser_config = BrowserConfig(headless=True, browser_type="chromium")
# Configure crawler
crawler_config = CrawlerRunConfig(cache_mode="bypass")
# Run crawler
async with AsyncWebCrawler(
config=browser_config, browser_adapter=adapter
) as crawler:
result = await crawler.arun(url=test_url, config=crawler_config)
if result.success:
print(f"โ
Crawl successful")
print(f" ๐ Title: {result.metadata.get('title', 'N/A')}")
print(f" ๐ Content length: {len(result.markdown)} chars")
# Check if user agent info is in content
if (
"User-Agent" in result.markdown
or "Browser:" in result.markdown
):
print(f" ๐ User-agent info detected in content")
else:
print(f" โน๏ธ No user-agent info in content")
else:
print(f"โ Crawl failed: {result.error_message}")
except Exception as e:
print(f"โ Error testing {strategy}: {e}")
import traceback
traceback.print_exc()
print(f"\n๐ Anti-bot strategy testing completed!")
except Exception as e:
print(f"โ Setup error: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
asyncio.run(test_antibot_strategies())