This page tests different browser adapters.
User-Agent detection test
"""
# Save test HTML
with open('/tmp/antibot_test.html', 'w') as f:
f.write(test_html)
test_url = 'file:///tmp/antibot_test.html'
strategies = [
('default', 'Default Playwright'),
('stealth', 'Stealth Mode'),
]
for strategy, description in strategies:
print(f"\n๐ Testing: {description} (strategy: {strategy})")
print("-" * 40)
try:
# Import adapter based on strategy
if strategy == 'stealth':
try:
from crawl4ai import StealthAdapter
adapter = StealthAdapter()
print(f"โ
Using StealthAdapter")
except ImportError:
print(f"โ ๏ธ StealthAdapter not available, using PlaywrightAdapter")
adapter = PlaywrightAdapter()
else:
adapter = PlaywrightAdapter()
print(f"โ
Using PlaywrightAdapter")
# Configure browser
browser_config = BrowserConfig(
headless=True,
browser_type="chromium"
)
# Configure crawler
crawler_config = CrawlerRunConfig(
cache_mode="bypass"
)
# Run crawler
async with AsyncWebCrawler(
config=browser_config,
browser_adapter=adapter
) as crawler:
result = await crawler.arun(
url=test_url,
config=crawler_config
)
if result.success:
print(f"โ
Crawl successful")
print(f" ๐ Title: {result.metadata.get('title', 'N/A')}")
print(f" ๐ Content length: {len(result.markdown)} chars")
# Check if user agent info is in content
if 'User-Agent' in result.markdown or 'Browser:' in result.markdown:
print(f" ๐ User-agent info detected in content")
else:
print(f" โน๏ธ No user-agent info in content")
else:
print(f"โ Crawl failed: {result.error_message}")
except Exception as e:
print(f"โ Error testing {strategy}: {e}")
import traceback
traceback.print_exc()
print(f"\n๐ Anti-bot strategy testing completed!")
except Exception as e:
print(f"โ Setup error: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
asyncio.run(test_antibot_strategies())