ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename some flags name, introducing magic flag.

Browse Source
This commit is contained in:
UncleCode
2024-10-18 12:35:09 +08:00
parent 768aa06ceb
commit dd17ed0e63
2 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crawl4ai/async_crawler_strategy.py
View File

@@ -309,7 +309,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
) )
await context.set_extra_http_headers(self.headers) await context.set_extra_http_headers(self.headers)
if kwargs.get("override_navigator", False): if kwargs.get("override_navigator", False) or kwargs.get("simulate_user", False) or kwargs.get("magic", False):
# Inject scripts to override navigator properties # Inject scripts to override navigator properties
await context.add_init_script(""" await context.add_init_script("""
// Pass the Permissions Test. // Pass the Permissions Test.
@@ -344,8 +344,9 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
# await stealth_async(page) #, stealth_config) # await stealth_async(page) #, stealth_config)
# Add console message and error logging # Add console message and error logging
page.on("console", lambda msg: print(f"Console: {msg.text}")) if kwargs.get("log_console", False):
page.on("pageerror", lambda exc: print(f"Page Error: {exc}")) page.on("console", lambda msg: print(f"Console: {msg.text}"))
page.on("pageerror", lambda exc: print(f"Page Error: {exc}"))
try: try:
if self.verbose: if self.verbose:
@@ -403,7 +404,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
# Check for on execution event # Check for on execution event
await self.execute_hook('on_execution_started', page) await self.execute_hook('on_execution_started', page)
if kwargs.get("simulate_user", False): if kwargs.get("simulate_user", False) or kwargs.get("magic", False):
# Simulate user interactions # Simulate user interactions
await page.mouse.move(100, 100) await page.mouse.move(100, 100)
await page.mouse.down() await page.mouse.down()

2
crawl4ai/content_scrapping_strategy.py
View File

@@ -293,7 +293,7 @@ class WebScrappingStrategy(ContentScrappingStrategy):
Suggestions: Suggestions:
- Try calling the crawl function with these parameters: - Try calling the crawl function with these parameters:
simulate_user=True, override_navigator=True magic=True,
- Set headless=False to visualize what's happening on the page. - Set headless=False to visualize what's happening on the page.
If the issue persists, please check the page's structure and any potential anti-crawling measures. If the issue persists, please check the page's structure and any potential anti-crawling measures.