refactor(core): reorganize project structure and remove legacy code
Major reorganization of the project structure: - Moved legacy synchronous crawler code to legacy folder - Removed deprecated CLI and docs manager - Consolidated version manager into utils.py - Added CrawlerHub to __init__.py exports - Fixed type hints in async_webcrawler.py - Fixed minor bugs in chunking and crawler strategies BREAKING CHANGE: Removed synchronous WebCrawler, CLI, and docs management functionality. Users should migrate to AsyncWebCrawler.
This commit is contained in:
@@ -1265,6 +1265,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
|
||||
"""
|
||||
config.url = url
|
||||
response_headers = {}
|
||||
execution_result = None
|
||||
status_code = None
|
||||
redirected_url = url
|
||||
|
||||
@@ -1522,6 +1523,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
|
||||
execution_result = await self.robust_execute_user_script(
|
||||
page, config.js_code
|
||||
)
|
||||
|
||||
if not execution_result["success"]:
|
||||
self.logger.warning(
|
||||
message="User script execution had issues: {error}",
|
||||
|
||||
Reference in New Issue
Block a user