Improve library loading

This commit is contained in:
unclecode
2024-05-16 21:19:02 +08:00
parent d19488a821
commit ea16dec587
6 changed files with 13 additions and 16 deletions

View File

@@ -2,7 +2,7 @@ from functools import lru_cache
from pathlib import Path
import subprocess, os
import shutil
from .config import MODEL_REPO_BRANCH
from crawl4ai.config import MODEL_REPO_BRANCH
import argparse
def get_home_folder():
@@ -40,7 +40,6 @@ def load_spacy_en_core_web_sm():
print("[LOG] ✅ spaCy model loaded successfully")
return nlp
@lru_cache()
def load_spacy_model():
import spacy
@@ -92,11 +91,8 @@ def load_spacy_model():
return spacy.load(model_folder)
def download_all_models(remove_existing=False):
"""Download all models required for Crawl4AI."""
print("[LOG] Welcome to the Crawl4AI Model Downloader!")
print("[LOG] This script will download all the models required for Crawl4AI.")
if remove_existing:
print("[LOG] Removing existing models...")
home_folder = get_home_folder()
@@ -121,6 +117,8 @@ def download_all_models(remove_existing=False):
print("[LOG] ✅ All models downloaded successfully.")
def main():
print("[LOG] Welcome to the Crawl4AI Model Downloader!")
print("[LOG] This script will download all the models required for Crawl4AI.")
parser = argparse.ArgumentParser(description="Crawl4AI Model Downloader")
parser.add_argument('--remove-existing', action='store_true', help="Remove existing models before downloading")
args = parser.parse_args()