ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to ChromeDriverManager due some issues with download the chrome driver

Browse Source
This commit is contained in:
unclecode
2024-06-26 13:00:17 +08:00
parent a0dff192ae
commit 144cfa0eda
3 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crawl4ai/crawler_strategy.py
View File

@@ -6,6 +6,9 @@ from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import InvalidArgumentException from selenium.common.exceptions import InvalidArgumentException
from selenium.webdriver.chrome.service import Service as ChromeService
from webdriver_manager.chrome import ChromeDriverManager
import logging import logging
import base64 import base64
from PIL import Image, ImageDraw, ImageFont from PIL import Image, ImageDraw, ImageFont
@@ -118,10 +121,15 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
} }
# chromedriver_autoinstaller.install() # chromedriver_autoinstaller.install()
import chromedriver_autoinstaller # import chromedriver_autoinstaller
crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai") # crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai")
chromedriver_path = chromedriver_autoinstaller.utils.download_chromedriver(crawl4ai_folder, False) # driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=self.options)
# chromedriver_path = chromedriver_autoinstaller.install()
# chromedriver_path = chromedriver_autoinstaller.utils.download_chromedriver()
# self.service = Service(chromedriver_autoinstaller.install()) # self.service = Service(chromedriver_autoinstaller.install())
chromedriver_path = ChromeDriverManager().install()
self.service = Service(chromedriver_path) self.service = Service(chromedriver_path)
self.service.log_path = "NUL" self.service.log_path = "NUL"
self.driver = webdriver.Chrome(service=self.service, options=self.options) self.driver = webdriver.Chrome(service=self.service, options=self.options)

4
main.py
View File

@@ -49,7 +49,9 @@ templates = Jinja2Templates(directory=__location__ + "/pages")
@lru_cache() @lru_cache()
def get_crawler(): def get_crawler():
# Initialize and return a WebCrawler instance # Initialize and return a WebCrawler instance
return WebCrawler(verbose = True) crawler = WebCrawler(verbose = True)
crawler.warmup()
return crawler
class CrawlRequest(BaseModel): class CrawlRequest(BaseModel):
urls: List[str] urls: List[str]

1
requirements.txt
View File

@@ -20,3 +20,4 @@ torch==2.3.1
onnxruntime==1.18.0 onnxruntime==1.18.0
tokenizers==0.19.1 tokenizers==0.19.1
pillow==10.3.0 pillow==10.3.0
webdriver-manager==4.0.1