The Future of AI
+
+
+ 
+
+
+ Setup Guide
Follow these steps...
{}
'.format(s) for s in filtered_html)
+ filtered_html = "\n".join(
+ "{}
".format(s) for s in filtered_html
+ )
fit_markdown = h.handle(filtered_html)
except Exception as e:
fit_markdown = f"Error generating fit markdown: {str(e)}"
diff --git a/crawl4ai/migrations.py b/crawl4ai/migrations.py
index 3386b0fb..d6da292f 100644
--- a/crawl4ai/migrations.py
+++ b/crawl4ai/migrations.py
@@ -1,13 +1,11 @@
import os
import asyncio
-import logging
from pathlib import Path
import aiosqlite
from typing import Optional
import xxhash
import aiofiles
import shutil
-import time
from datetime import datetime
from .async_logger import AsyncLogger, LogLevel
@@ -17,18 +15,19 @@ logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
# logging.basicConfig(level=logging.INFO)
# logger = logging.getLogger(__name__)
+
class DatabaseMigration:
def __init__(self, db_path: str):
self.db_path = db_path
self.content_paths = self._ensure_content_dirs(os.path.dirname(db_path))
-
+
def _ensure_content_dirs(self, base_path: str) -> dict:
dirs = {
- 'html': 'html_content',
- 'cleaned': 'cleaned_html',
- 'markdown': 'markdown_content',
- 'extracted': 'extracted_content',
- 'screenshots': 'screenshots'
+ "html": "html_content",
+ "cleaned": "cleaned_html",
+ "markdown": "markdown_content",
+ "extracted": "extracted_content",
+ "screenshots": "screenshots",
}
content_paths = {}
for key, dirname in dirs.items():
@@ -47,43 +46,55 @@ class DatabaseMigration:
async def _store_content(self, content: str, content_type: str) -> str:
if not content:
return ""
-
+
content_hash = self._generate_content_hash(content)
file_path = os.path.join(self.content_paths[content_type], content_hash)
-
+
if not os.path.exists(file_path):
- async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+ async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
await f.write(content)
-
+
return content_hash
async def migrate_database(self):
"""Migrate existing database to file-based storage"""
# logger.info("Starting database migration...")
logger.info("Starting database migration...", tag="INIT")
-
+
try:
async with aiosqlite.connect(self.db_path) as db:
# Get all rows
async with db.execute(
- '''SELECT url, html, cleaned_html, markdown,
- extracted_content, screenshot FROM crawled_data'''
+ """SELECT url, html, cleaned_html, markdown,
+ extracted_content, screenshot FROM crawled_data"""
) as cursor:
rows = await cursor.fetchall()
migrated_count = 0
for row in rows:
- url, html, cleaned_html, markdown, extracted_content, screenshot = row
-
+ (
+ url,
+ html,
+ cleaned_html,
+ markdown,
+ extracted_content,
+ screenshot,
+ ) = row
+
# Store content in files and get hashes
- html_hash = await self._store_content(html, 'html')
- cleaned_hash = await self._store_content(cleaned_html, 'cleaned')
- markdown_hash = await self._store_content(markdown, 'markdown')
- extracted_hash = await self._store_content(extracted_content, 'extracted')
- screenshot_hash = await self._store_content(screenshot, 'screenshots')
+ html_hash = await self._store_content(html, "html")
+ cleaned_hash = await self._store_content(cleaned_html, "cleaned")
+ markdown_hash = await self._store_content(markdown, "markdown")
+ extracted_hash = await self._store_content(
+ extracted_content, "extracted"
+ )
+ screenshot_hash = await self._store_content(
+ screenshot, "screenshots"
+ )
# Update database with hashes
- await db.execute('''
+ await db.execute(
+ """
UPDATE crawled_data
SET html = ?,
cleaned_html = ?,
@@ -91,40 +102,51 @@ class DatabaseMigration:
extracted_content = ?,
screenshot = ?
WHERE url = ?
- ''', (html_hash, cleaned_hash, markdown_hash,
- extracted_hash, screenshot_hash, url))
-
+ """,
+ (
+ html_hash,
+ cleaned_hash,
+ markdown_hash,
+ extracted_hash,
+ screenshot_hash,
+ url,
+ ),
+ )
+
migrated_count += 1
if migrated_count % 100 == 0:
logger.info(f"Migrated {migrated_count} records...", tag="INIT")
-
await db.commit()
- logger.success(f"Migration completed. {migrated_count} records processed.", tag="COMPLETE")
+ logger.success(
+ f"Migration completed. {migrated_count} records processed.",
+ tag="COMPLETE",
+ )
except Exception as e:
# logger.error(f"Migration failed: {e}")
logger.error(
message="Migration failed: {error}",
tag="ERROR",
- params={"error": str(e)}
+ params={"error": str(e)},
)
raise e
+
async def backup_database(db_path: str) -> str:
"""Create backup of existing database"""
if not os.path.exists(db_path):
logger.info("No existing database found. Skipping backup.", tag="INIT")
return None
-
+
# Create backup with timestamp
- timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = f"{db_path}.backup_{timestamp}"
-
+
try:
# Wait for any potential write operations to finish
await asyncio.sleep(1)
-
+
# Create backup
shutil.copy2(db_path, backup_path)
logger.info(f"Database backup created at: {backup_path}", tag="COMPLETE")
@@ -132,37 +154,41 @@ async def backup_database(db_path: str) -> str:
except Exception as e:
# logger.error(f"Backup failed: {e}")
logger.error(
- message="Migration failed: {error}",
- tag="ERROR",
- params={"error": str(e)}
- )
+ message="Migration failed: {error}", tag="ERROR", params={"error": str(e)}
+ )
raise e
-
+
+
async def run_migration(db_path: Optional[str] = None):
"""Run database migration"""
if db_path is None:
db_path = os.path.join(Path.home(), ".crawl4ai", "crawl4ai.db")
-
+
if not os.path.exists(db_path):
logger.info("No existing database found. Skipping migration.", tag="INIT")
return
-
+
# Create backup first
backup_path = await backup_database(db_path)
if not backup_path:
return
-
+
migration = DatabaseMigration(db_path)
await migration.migrate_database()
-
+
+
def main():
"""CLI entry point for migration"""
import argparse
- parser = argparse.ArgumentParser(description='Migrate Crawl4AI database to file-based storage')
- parser.add_argument('--db-path', help='Custom database path')
+
+ parser = argparse.ArgumentParser(
+ description="Migrate Crawl4AI database to file-based storage"
+ )
+ parser.add_argument("--db-path", help="Custom database path")
args = parser.parse_args()
-
+
asyncio.run(run_migration(args.db_path))
+
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/crawl4ai/model_loader.py b/crawl4ai/model_loader.py
index d1872d7e..aa80f673 100644
--- a/crawl4ai/model_loader.py
+++ b/crawl4ai/model_loader.py
@@ -2,109 +2,125 @@ from functools import lru_cache
from pathlib import Path
import subprocess, os
import shutil
-import tarfile
from .model_loader import *
import argparse
-import urllib.request
from crawl4ai.config import MODEL_REPO_BRANCH
+
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
@lru_cache()
def get_available_memory(device):
import torch
- if device.type == 'cuda':
+
+ if device.type == "cuda":
return torch.cuda.get_device_properties(device).total_memory
- elif device.type == 'mps':
- return 48 * 1024 ** 3 # Assuming 8GB for MPS, as a conservative estimate
+ elif device.type == "mps":
+ return 48 * 1024**3 # Assuming 8GB for MPS, as a conservative estimate
else:
return 0
+
@lru_cache()
def calculate_batch_size(device):
available_memory = get_available_memory(device)
-
- if device.type == 'cpu':
+
+ if device.type == "cpu":
return 16
- elif device.type in ['cuda', 'mps']:
+ elif device.type in ["cuda", "mps"]:
# Adjust these thresholds based on your model size and available memory
- if available_memory >= 31 * 1024 ** 3: # > 32GB
+ if available_memory >= 31 * 1024**3: # > 32GB
return 256
- elif available_memory >= 15 * 1024 ** 3: # > 16GB to 32GB
+ elif available_memory >= 15 * 1024**3: # > 16GB to 32GB
return 128
- elif available_memory >= 8 * 1024 ** 3: # 8GB to 16GB
+ elif available_memory >= 8 * 1024**3: # 8GB to 16GB
return 64
else:
return 32
else:
- return 16 # Default batch size
-
+ return 16 # Default batch size
+
+
@lru_cache()
def get_device():
import torch
+
if torch.cuda.is_available():
- device = torch.device('cuda')
+ device = torch.device("cuda")
elif torch.backends.mps.is_available():
- device = torch.device('mps')
+ device = torch.device("mps")
else:
- device = torch.device('cpu')
- return device
-
+ device = torch.device("cpu")
+ return device
+
+
def set_model_device(model):
device = get_device()
- model.to(device)
+ model.to(device)
return model, device
+
@lru_cache()
def get_home_folder():
- home_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+ home_folder = os.path.join(
+ os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+ )
os.makedirs(home_folder, exist_ok=True)
os.makedirs(f"{home_folder}/cache", exist_ok=True)
os.makedirs(f"{home_folder}/models", exist_ok=True)
- return home_folder
+ return home_folder
+
@lru_cache()
def load_bert_base_uncased():
- from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
- tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', resume_download=None)
- model = BertModel.from_pretrained('bert-base-uncased', resume_download=None)
+ from transformers import BertTokenizer, BertModel
+
+ tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", resume_download=None)
+ model = BertModel.from_pretrained("bert-base-uncased", resume_download=None)
model.eval()
model, device = set_model_device(model)
return tokenizer, model
+
@lru_cache()
def load_HF_embedding_model(model_name="BAAI/bge-small-en-v1.5") -> tuple:
"""Load the Hugging Face model for embedding.
-
+
Args:
model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5".
-
+
Returns:
tuple: The tokenizer and model.
"""
- from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
+ from transformers import AutoTokenizer, AutoModel
+
tokenizer = AutoTokenizer.from_pretrained(model_name, resume_download=None)
model = AutoModel.from_pretrained(model_name, resume_download=None)
model.eval()
model, device = set_model_device(model)
return tokenizer, model
+
@lru_cache()
def load_text_classifier():
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from transformers import pipeline
- import torch
- tokenizer = AutoTokenizer.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
- model = AutoModelForSequenceClassification.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
+ tokenizer = AutoTokenizer.from_pretrained(
+ "dstefa/roberta-base_topic_classification_nyt_news"
+ )
+ model = AutoModelForSequenceClassification.from_pretrained(
+ "dstefa/roberta-base_topic_classification_nyt_news"
+ )
model.eval()
model, device = set_model_device(model)
pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
return pipe
+
@lru_cache()
def load_text_multilabel_classifier():
from transformers import AutoModelForSequenceClassification, AutoTokenizer
- import numpy as np
from scipy.special import expit
import torch
@@ -116,18 +132,27 @@ def load_text_multilabel_classifier():
# else:
# device = torch.device("cpu")
# # return load_spacy_model(), torch.device("cpu")
-
MODEL = "cardiffnlp/tweet-topic-21-multi"
tokenizer = AutoTokenizer.from_pretrained(MODEL, resume_download=None)
- model = AutoModelForSequenceClassification.from_pretrained(MODEL, resume_download=None)
+ model = AutoModelForSequenceClassification.from_pretrained(
+ MODEL, resume_download=None
+ )
model.eval()
model, device = set_model_device(model)
class_mapping = model.config.id2label
def _classifier(texts, threshold=0.5, max_length=64):
- tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=max_length)
- tokens = {key: val.to(device) for key, val in tokens.items()} # Move tokens to the selected device
+ tokens = tokenizer(
+ texts,
+ return_tensors="pt",
+ padding=True,
+ truncation=True,
+ max_length=max_length,
+ )
+ tokens = {
+ key: val.to(device) for key, val in tokens.items()
+ } # Move tokens to the selected device
with torch.no_grad():
output = model(**tokens)
@@ -138,35 +163,41 @@ def load_text_multilabel_classifier():
batch_labels = []
for prediction in predictions:
- labels = [class_mapping[i] for i, value in enumerate(prediction) if value == 1]
+ labels = [
+ class_mapping[i] for i, value in enumerate(prediction) if value == 1
+ ]
batch_labels.append(labels)
return batch_labels
return _classifier, device
+
@lru_cache()
def load_nltk_punkt():
import nltk
+
try:
- nltk.data.find('tokenizers/punkt')
+ nltk.data.find("tokenizers/punkt")
except LookupError:
- nltk.download('punkt')
- return nltk.data.find('tokenizers/punkt')
+ nltk.download("punkt")
+ return nltk.data.find("tokenizers/punkt")
+
@lru_cache()
def load_spacy_model():
import spacy
+
name = "models/reuters"
home_folder = get_home_folder()
model_folder = Path(home_folder) / name
-
+
# Check if the model directory already exists
if not (model_folder.exists() and any(model_folder.iterdir())):
repo_url = "https://github.com/unclecode/crawl4ai.git"
- branch = MODEL_REPO_BRANCH
+ branch = MODEL_REPO_BRANCH
repo_folder = Path(home_folder) / "crawl4ai"
-
+
print("[LOG] β¬ Downloading Spacy model for the first time...")
# Remove existing repo folder if it exists
@@ -176,7 +207,9 @@ def load_spacy_model():
if model_folder.exists():
shutil.rmtree(model_folder)
except PermissionError:
- print("[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:")
+ print(
+ "[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:"
+ )
print(f"- {repo_folder}")
print(f"- {model_folder}")
return None
@@ -187,7 +220,7 @@ def load_spacy_model():
["git", "clone", "-b", branch, repo_url, str(repo_folder)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
- check=True
+ check=True,
)
# Create the models directory if it doesn't exist
@@ -215,6 +248,7 @@ def load_spacy_model():
print(f"Error loading spacy model: {e}")
return None
+
def download_all_models(remove_existing=False):
"""Download all models required for Crawl4AI."""
if remove_existing:
@@ -243,14 +277,20 @@ def download_all_models(remove_existing=False):
load_nltk_punkt()
print("[LOG] β
All models downloaded successfully.")
+
def main():
print("[LOG] Welcome to the Crawl4AI Model Downloader!")
print("[LOG] This script will download all the models required for Crawl4AI.")
parser = argparse.ArgumentParser(description="Crawl4AI Model Downloader")
- parser.add_argument('--remove-existing', action='store_true', help="Remove existing models before downloading")
+ parser.add_argument(
+ "--remove-existing",
+ action="store_true",
+ help="Remove existing models before downloading",
+ )
args = parser.parse_args()
-
+
download_all_models(remove_existing=args.remove_existing)
+
if __name__ == "__main__":
main()
diff --git a/crawl4ai/models.py b/crawl4ai/models.py
index 6fb362a3..81e08b0c 100644
--- a/crawl4ai/models.py
+++ b/crawl4ai/models.py
@@ -1,21 +1,83 @@
from pydantic import BaseModel, HttpUrl
from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
+from enum import Enum
from dataclasses import dataclass
from .ssl_certificate import SSLCertificate
+from datetime import datetime
+from datetime import timedelta
+
+###############################
+# Dispatcher Models
+###############################
+@dataclass
+class DomainState:
+ last_request_time: float = 0
+ current_delay: float = 0
+ fail_count: int = 0
+
+
+@dataclass
+class CrawlerTaskResult:
+ task_id: str
+ url: str
+ result: "CrawlResult"
+ memory_usage: float
+ peak_memory: float
+ start_time: datetime
+ end_time: datetime
+ error_message: str = ""
+
+
+class CrawlStatus(Enum):
+ QUEUED = "QUEUED"
+ IN_PROGRESS = "IN_PROGRESS"
+ COMPLETED = "COMPLETED"
+ FAILED = "FAILED"
+
+
+@dataclass
+class CrawlStats:
+ task_id: str
+ url: str
+ status: CrawlStatus
+ start_time: Optional[datetime] = None
+ end_time: Optional[datetime] = None
+ memory_usage: float = 0.0
+ peak_memory: float = 0.0
+ error_message: str = ""
+
+ @property
+ def duration(self) -> str:
+ if not self.start_time:
+ return "0:00"
+ end = self.end_time or datetime.now()
+ duration = end - self.start_time
+ return str(timedelta(seconds=int(duration.total_seconds())))
+
+
+class DisplayMode(Enum):
+ DETAILED = "DETAILED"
+ AGGREGATED = "AGGREGATED"
+
+
+###############################
+# Crawler Models
+###############################
@dataclass
class TokenUsage:
completion_tokens: int = 0
- prompt_tokens: int = 0
+ prompt_tokens: int = 0
total_tokens: int = 0
completion_tokens_details: Optional[dict] = None
prompt_tokens_details: Optional[dict] = None
-
+
class UrlModel(BaseModel):
url: HttpUrl
forced: bool = False
+
class MarkdownGenerationResult(BaseModel):
raw_markdown: str
markdown_with_citations: str
@@ -23,6 +85,16 @@ class MarkdownGenerationResult(BaseModel):
fit_markdown: Optional[str] = None
fit_html: Optional[str] = None
+
+class DispatchResult(BaseModel):
+ task_id: str
+ memory_usage: float
+ peak_memory: float
+ start_time: datetime
+ end_time: datetime
+ error_message: str = ""
+
+
class CrawlResult(BaseModel):
url: str
html: str
@@ -32,7 +104,7 @@ class CrawlResult(BaseModel):
links: Dict[str, List[Dict]] = {}
downloaded_files: Optional[List[str]] = None
screenshot: Optional[str] = None
- pdf : Optional[bytes] = None
+ pdf: Optional[bytes] = None
markdown: Optional[Union[str, MarkdownGenerationResult]] = None
markdown_v2: Optional[MarkdownGenerationResult] = None
fit_markdown: Optional[str] = None
@@ -44,9 +116,13 @@ class CrawlResult(BaseModel):
response_headers: Optional[dict] = None
status_code: Optional[int] = None
ssl_certificate: Optional[SSLCertificate] = None
+ dispatch_result: Optional[DispatchResult] = None
+ redirected_url: Optional[str] = None
+
class Config:
arbitrary_types_allowed = True
+
class AsyncCrawlResponse(BaseModel):
html: str
response_headers: Dict[str, str]
@@ -56,6 +132,51 @@ class AsyncCrawlResponse(BaseModel):
get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
downloaded_files: Optional[List[str]] = None
ssl_certificate: Optional[SSLCertificate] = None
+ final_url: Optional[str] = None
class Config:
arbitrary_types_allowed = True
+
+
+###############################
+# Scraping Models
+###############################
+class MediaItem(BaseModel):
+ src: Optional[str] = ""
+ alt: Optional[str] = ""
+ desc: Optional[str] = ""
+ score: Optional[int] = 0
+ type: str = "image"
+ group_id: Optional[int] = 0
+ format: Optional[str] = None
+ width: Optional[int] = None
+
+
+class Link(BaseModel):
+ href: Optional[str] = ""
+ text: Optional[str] = ""
+ title: Optional[str] = ""
+ base_domain: Optional[str] = ""
+
+
+class Media(BaseModel):
+ images: List[MediaItem] = []
+ videos: List[
+ MediaItem
+ ] = [] # Using MediaItem model for now, can be extended with Video model if needed
+ audios: List[
+ MediaItem
+ ] = [] # Using MediaItem model for now, can be extended with Audio model if needed
+
+
+class Links(BaseModel):
+ internal: List[Link] = []
+ external: List[Link] = []
+
+
+class ScrapingResult(BaseModel):
+ cleaned_html: str
+ success: bool
+ media: Media = Media()
+ links: Links = Links()
+ metadata: Dict[str, Any] = {}
diff --git a/crawl4ai/prompts.py b/crawl4ai/prompts.py
index 7a963e6d..be5e0310 100644
--- a/crawl4ai/prompts.py
+++ b/crawl4ai/prompts.py
@@ -202,3 +202,808 @@ Avoid Common Mistakes:
Result
Output the final list of JSON objects, wrapped in
+
+
+Example Output:
+# Setup Guide
+Follow these steps...
+
+IMPORTANT: If specific instruction is provided above, prioritize those requirements over these general guidelines.
+
+OUTPUT FORMAT:
+Wrap your response in
+ 
+
+
+
+Generated Schema:
+{
+ "name": "Product Cards",
+ "baseSelector": ".product-card",
+ "baseFields": [
+ {"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
+ {"name": "data_subcat_id", "type": "attribute", "attribute": "data-subcat-id"}
+ ],
+ "fields": [
+ {
+ "name": "title",
+ "selector": ".product-title",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".price",
+ "type": "text"
+ },
+ {
+ "name": "image_url",
+ "selector": "img",
+ "type": "attribute",
+ "attribute": "src"
+ }
+ ]
+}
+
+2. Article with Author Details Example:
+
+Gaming Laptop
+ $999.99 +
+
+
+
+
+Generated Schema:
+{
+ "name": "E-commerce Categories",
+ "baseSelector": ".category-section",
+ "baseFields": [
+ {"name": "data_category", "type": "attribute", "attribute": "data-category"}
+ ],
+ "fields": [
+ {
+ "name": "category_name",
+ "selector": "h2",
+ "type": "text"
+ },
+ {
+ "name": "subcategories",
+ "type": "nested_list",
+ "selector": ".subcategory",
+ "fields": [
+ {
+ "name": "name",
+ "selector": "h3",
+ "type": "text"
+ },
+ {
+ "name": "products",
+ "type": "list",
+ "selector": ".product",
+ "fields": [
+ {
+ "name": "name",
+ "selector": ".product-name",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".price",
+ "type": "text"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
+
+5. Job Listings with Transformations Example:
+
+Electronics
+
+
+Laptops
+
+ MacBook Pro
+ $1299
+
+
+ Dell XPS
+ $999
+
+
+
+
+
+Generated Schema:
+{
+ "name": "Job Listings",
+ "baseSelector": ".job-post",
+ "fields": [
+ {
+ "name": "title",
+ "selector": ".job-title",
+ "type": "text",
+ "transform": "uppercase"
+ },
+ {
+ "name": "salary",
+ "selector": ".salary-text",
+ "type": "regex",
+ "pattern": "\\$([\\d,]+)"
+ },
+ {
+ "name": "location",
+ "selector": ".location",
+ "type": "text",
+ "transform": "strip"
+ }
+ ]
+}
+
+6. Skyscanner Place Card Example:
+
+
+
+
+Generated Schema:
+{
+ "name": "Skyscanner Place Cards",
+ "baseSelector": "div[class^='PlaceCard_descriptionContainer__']",
+ "baseFields": [
+ {"name": "data_testid", "type": "attribute", "attribute": "data-testid"}
+ ],
+ "fields": [
+ {
+ "name": "city_name",
+ "selector": "div[class^='PlaceCard_nameContent__'] .BpkText_bpk-text--heading-4__",
+ "type": "text"
+ },
+ {
+ "name": "country_name",
+ "selector": "span[class*='PlaceCard_subName__']",
+ "type": "text"
+ },
+ {
+ "name": "description",
+ "selector": "span[class*='PlaceCard_advertLabel__']",
+ "type": "text"
+ },
+ {
+ "name": "flight_price",
+ "selector": "a[data-testid='flights-link'] .BpkText_bpk-text--heading-5__",
+ "type": "text"
+ },
+ {
+ "name": "flight_url",
+ "selector": "a[data-testid='flights-link']",
+ "type": "attribute",
+ "attribute": "href"
+ }
+ ]
+}
+Senior Developer
+ Salary: $120,000/year + New York, NY +
+
+
+
+
+Generated Schema:
+{
+ "name": "Product Cards",
+ "baseSelector": "//div[@class='product-card']",
+ "baseFields": [
+ {"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
+ {"name": "data_subcat_id", "type": "attribute", "attribute": "data-subcat-id"}
+ ],
+ "fields": [
+ {
+ "name": "title",
+ "selector": ".//h2[@class='product-title']",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".//span[@class='price']",
+ "type": "text"
+ },
+ {
+ "name": "image_url",
+ "selector": ".//img",
+ "type": "attribute",
+ "attribute": "src"
+ }
+ ]
+}
+
+2. Article with Author Details Example:
+
+Gaming Laptop
+ $999.99 +
+The Future of AI
+
+
+
+
+
+
+
+
+
+Generated Schema:
+{
+ "name": "Comment Section",
+ "baseSelector": "//div[@class='comments-container']",
+ "fields": [
+ {
+ "name": "comments",
+ "type": "list",
+ "selector": ".//div[@class='comment']",
+ "baseFields": [
+ {"name": "data_user_id", "type": "attribute", "attribute": "data-user-id"}
+ ],
+ "fields": [
+ {
+ "name": "user",
+ "selector": ".//div[@class='user-name']",
+ "type": "text"
+ },
+ {
+ "name": "content",
+ "selector": ".//p[@class='comment-text']",
+ "type": "text"
+ }
+ ]
+ }
+ ]
+}
+
+4. E-commerce Categories Example:
+
+
+
+ John123
+ Great article!
+
+
+Alice456
+ Thanks for sharing.
+
+
+
+
+Generated Schema:
+{
+ "name": "E-commerce Categories",
+ "baseSelector": "//div[@class='category-section']",
+ "baseFields": [
+ {"name": "data_category", "type": "attribute", "attribute": "data-category"}
+ ],
+ "fields": [
+ {
+ "name": "category_name",
+ "selector": ".//h2",
+ "type": "text"
+ },
+ {
+ "name": "subcategories",
+ "type": "nested_list",
+ "selector": ".//div[@class='subcategory']",
+ "fields": [
+ {
+ "name": "name",
+ "selector": ".//h3",
+ "type": "text"
+ },
+ {
+ "name": "products",
+ "type": "list",
+ "selector": ".//div[@class='product']",
+ "fields": [
+ {
+ "name": "name",
+ "selector": ".//span[@class='product-name']",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".//span[@class='price']",
+ "type": "text"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
+
+5. Job Listings with Transformations Example:
+
+Electronics
+
+
+Laptops
+
+ MacBook Pro
+ $1299
+
+
+ Dell XPS
+ $999
+
+
+
+
+
+Generated Schema:
+{
+ "name": "Job Listings",
+ "baseSelector": "//div[@class='job-post']",
+ "fields": [
+ {
+ "name": "title",
+ "selector": ".//h3[@class='job-title']",
+ "type": "text",
+ "transform": "uppercase"
+ },
+ {
+ "name": "salary",
+ "selector": ".//span[@class='salary-text']",
+ "type": "regex",
+ "pattern": "\\$([\\d,]+)"
+ },
+ {
+ "name": "location",
+ "selector": ".//span[@class='location']",
+ "type": "text",
+ "transform": "strip"
+ }
+ ]
+}
+
+6. Skyscanner Place Card Example:
+
+
+
+
+Generated Schema:
+{
+ "name": "Skyscanner Place Cards",
+ "baseSelector": "//div[contains(@class, 'PlaceCard_descriptionContainer__')]",
+ "baseFields": [
+ {"name": "data_testid", "type": "attribute", "attribute": "data-testid"}
+ ],
+ "fields": [
+ {
+ "name": "city_name",
+ "selector": ".//div[contains(@class, 'PlaceCard_nameContent__')]//span[contains(@class, 'BpkText_bpk-text--heading-4__')]",
+ "type": "text"
+ },
+ {
+ "name": "country_name",
+ "selector": ".//span[contains(@class, 'PlaceCard_subName__')]",
+ "type": "text"
+ },
+ {
+ "name": "description",
+ "selector": ".//span[contains(@class, 'PlaceCard_advertLabel__')]",
+ "type": "text"
+ },
+ {
+ "name": "flight_price",
+ "selector": ".//a[@data-testid='flights-link']//span[contains(@class, 'BpkText_bpk-text--heading-5__')]",
+ "type": "text"
+ },
+ {
+ "name": "flight_url",
+ "selector": ".//a[@data-testid='flights-link']",
+ "type": "attribute",
+ "attribute": "href"
+ }
+ ]
+}
+Senior Developer
+ Salary: $120,000/year + New York, NY +Test Content
This is a test paragraph.
- """) - + """ + ) + async with AsyncWebCrawler(verbose=True) as crawler: # Process local file - local_result = await crawler.arun( - url=f"file://{os.path.abspath(sample_file)}" - ) - + local_result = await crawler.arun(url=f"file://{os.path.abspath(sample_file)}") + # Process raw HTML raw_html = """ @@ -78,16 +79,15 @@ async def local_and_raw_html_example():This is a test of raw HTML processing.
""" - raw_result = await crawler.arun( - url=f"raw:{raw_html}" - ) - + raw_result = await crawler.arun(url=f"raw:{raw_html}") + # Clean up os.remove(sample_file) - + print("Local file content:", local_result.markdown) print("\nRaw HTML content:", raw_result.markdown) + # 3. Enhanced Markdown Generation Example async def markdown_generation_example(): """Example of enhanced markdown generation with citations and LLM-friendly features""" @@ -97,58 +97,66 @@ async def markdown_generation_example(): # user_query="History and cultivation", bm25_threshold=1.0 ) - + result = await crawler.arun( url="https://en.wikipedia.org/wiki/Apple", css_selector="main div#bodyContent", content_filter=content_filter, - cache_mode=CacheMode.BYPASS + cache_mode=CacheMode.BYPASS, ) - - from crawl4ai import AsyncWebCrawler + from crawl4ai.content_filter_strategy import BM25ContentFilter - + result = await crawler.arun( url="https://en.wikipedia.org/wiki/Apple", css_selector="main div#bodyContent", - content_filter=BM25ContentFilter() + content_filter=BM25ContentFilter(), ) print(result.markdown_v2.fit_markdown) - + print("\nMarkdown Generation Results:") print(f"1. Original markdown length: {len(result.markdown)}") - print(f"2. New markdown versions (markdown_v2):") + print("2. New markdown versions (markdown_v2):") print(f" - Raw markdown length: {len(result.markdown_v2.raw_markdown)}") - print(f" - Citations markdown length: {len(result.markdown_v2.markdown_with_citations)}") - print(f" - References section length: {len(result.markdown_v2.references_markdown)}") + print( + f" - Citations markdown length: {len(result.markdown_v2.markdown_with_citations)}" + ) + print( + f" - References section length: {len(result.markdown_v2.references_markdown)}" + ) if result.markdown_v2.fit_markdown: - print(f" - Filtered markdown length: {len(result.markdown_v2.fit_markdown)}") - + print( + f" - Filtered markdown length: {len(result.markdown_v2.fit_markdown)}" + ) + # Save examples to files output_dir = os.path.join(__data__, "markdown_examples") os.makedirs(output_dir, exist_ok=True) - + # Save different versions with open(os.path.join(output_dir, "1_raw_markdown.md"), "w") as f: f.write(result.markdown_v2.raw_markdown) - + with open(os.path.join(output_dir, "2_citations_markdown.md"), "w") as f: f.write(result.markdown_v2.markdown_with_citations) - + with open(os.path.join(output_dir, "3_references.md"), "w") as f: f.write(result.markdown_v2.references_markdown) - + if result.markdown_v2.fit_markdown: with open(os.path.join(output_dir, "4_filtered_markdown.md"), "w") as f: f.write(result.markdown_v2.fit_markdown) - + print(f"\nMarkdown examples saved to: {output_dir}") - + # Show a sample of citations and references print("\nSample of markdown with citations:") print(result.markdown_v2.markdown_with_citations[:500] + "...\n") print("Sample of references:") - print('\n'.join(result.markdown_v2.references_markdown.split('\n')[:10]) + "...") + print( + "\n".join(result.markdown_v2.references_markdown.split("\n")[:10]) + "..." + ) + # 4. Browser Management Example async def browser_management_example(): @@ -156,38 +164,38 @@ async def browser_management_example(): # Use the specified user directory path user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile") os.makedirs(user_data_dir, exist_ok=True) - + print(f"Browser profile will be saved to: {user_data_dir}") - + async with AsyncWebCrawler( use_managed_browser=True, user_data_dir=user_data_dir, headless=False, - verbose=True + verbose=True, ) as crawler: - result = await crawler.arun( url="https://crawl4ai.com", # session_id="persistent_session_1", - cache_mode=CacheMode.BYPASS - ) + cache_mode=CacheMode.BYPASS, + ) # Use GitHub as an example - it's a good test for browser management # because it requires proper browser handling result = await crawler.arun( url="https://github.com/trending", # session_id="persistent_session_1", - cache_mode=CacheMode.BYPASS + cache_mode=CacheMode.BYPASS, ) - + print("\nBrowser session result:", result.success) if result.success: - print("Page title:", result.metadata.get('title', 'No title found')) + print("Page title:", result.metadata.get("title", "No title found")) + # 5. API Usage Example async def api_example(): """Example of using the new API endpoints""" - api_token = os.getenv('CRAWL4AI_API_TOKEN') or "test_api_code" - headers = {'Authorization': f'Bearer {api_token}'} + api_token = os.getenv("CRAWL4AI_API_TOKEN") or "test_api_code" + headers = {"Authorization": f"Bearer {api_token}"} async with aiohttp.ClientSession() as session: # Submit crawl job crawl_request = { @@ -199,25 +207,17 @@ async def api_example(): "name": "Hacker News Articles", "baseSelector": ".athing", "fields": [ - { - "name": "title", - "selector": ".title a", - "type": "text" - }, - { - "name": "score", - "selector": ".score", - "type": "text" - }, + {"name": "title", "selector": ".title a", "type": "text"}, + {"name": "score", "selector": ".score", "type": "text"}, { "name": "url", "selector": ".title a", "type": "attribute", - "attribute": "href" - } - ] + "attribute": "href", + }, + ], } - } + }, }, "crawler_params": { "headless": True, @@ -227,51 +227,50 @@ async def api_example(): # "screenshot": True, # "magic": True } - + async with session.post( - "http://localhost:11235/crawl", - json=crawl_request, - headers=headers + "http://localhost:11235/crawl", json=crawl_request, headers=headers ) as response: task_data = await response.json() task_id = task_data["task_id"] - + # Check task status while True: async with session.get( - f"http://localhost:11235/task/{task_id}", - headers=headers + f"http://localhost:11235/task/{task_id}", headers=headers ) as status_response: result = await status_response.json() print(f"Task status: {result['status']}") - + if result["status"] == "completed": print("Task completed!") print("Results:") - news = json.loads(result["results"][0]['extracted_content']) + news = json.loads(result["results"][0]["extracted_content"]) print(json.dumps(news[:4], indent=2)) break else: await asyncio.sleep(1) + # Main execution async def main(): # print("Running Crawl4AI feature examples...") - + # print("\n1. Running Download Example:") # await download_example() - + # print("\n2. Running Markdown Generation Example:") # await markdown_generation_example() - + # # print("\n3. Running Local and Raw HTML Example:") # await local_and_raw_html_example() - + # # print("\n4. Running Browser Management Example:") await browser_management_example() - + # print("\n5. Running API Example:") await api_example() + if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/docs/examples/v0_4_24_walkthrough.py b/docs/examples/v0_4_24_walkthrough.py index 135ac29c..996e7b04 100644 --- a/docs/examples/v0_4_24_walkthrough.py +++ b/docs/examples/v0_4_24_walkthrough.py @@ -10,18 +10,17 @@ import asyncio import os import json import re -from typing import List, Optional, Dict, Any -from pydantic import BaseModel, Field +from typing import List from crawl4ai import ( AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMExtractionStrategy, - JsonCssExtractionStrategy + JsonCssExtractionStrategy, ) from crawl4ai.content_filter_strategy import RelevantContentFilter -from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator from bs4 import BeautifulSoup # Sample HTML for demonstrations @@ -52,17 +51,18 @@ SAMPLE_HTML = """ """ + async def demo_ssl_features(): """ Enhanced SSL & Security Features Demo ----------------------------------- - + This example demonstrates the new SSL certificate handling and security features: 1. Custom certificate paths 2. SSL verification options 3. HTTPS error handling 4. Certificate validation configurations - + These features are particularly useful when: - Working with self-signed certificates - Dealing with corporate proxies @@ -76,14 +76,11 @@ async def demo_ssl_features(): run_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, - fetch_ssl_certificate=True # Enable SSL certificate fetching + fetch_ssl_certificate=True, # Enable SSL certificate fetching ) async with AsyncWebCrawler(config=browser_config) as crawler: - result = await crawler.arun( - url="https://example.com", - config=run_config - ) + result = await crawler.arun(url="https://example.com", config=run_config) print(f"SSL Crawl Success: {result.success}") result.ssl_certificate.to_json( os.path.join(os.getcwd(), "ssl_certificate.json") @@ -91,11 +88,12 @@ async def demo_ssl_features(): if not result.success: print(f"SSL Error: {result.error_message}") + async def demo_content_filtering(): """ Smart Content Filtering Demo ---------------------- - + Demonstrates advanced content filtering capabilities: 1. Custom filter to identify and extract specific content 2. Integration with markdown generation @@ -110,87 +108,90 @@ async def demo_content_filtering(): super().__init__() # Add news-specific patterns self.negative_patterns = re.compile( - r'nav|footer|header|sidebar|ads|comment|share|related|recommended|popular|trending', - re.I + r"nav|footer|header|sidebar|ads|comment|share|related|recommended|popular|trending", + re.I, ) self.min_word_count = 30 # Higher threshold for news content - def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]: + def filter_content( + self, html: str, min_word_threshold: int = None + ) -> List[str]: """ Implements news-specific content filtering logic. - + Args: html (str): HTML content to be filtered min_word_threshold (int, optional): Minimum word count threshold - + Returns: List[str]: List of filtered HTML content blocks """ if not html or not isinstance(html, str): return [] - - soup = BeautifulSoup(html, 'lxml') + + soup = BeautifulSoup(html, "lxml") if not soup.body: - soup = BeautifulSoup(f'{html}', 'lxml') - - body = soup.find('body') - + soup = BeautifulSoup(f"{html}", "lxml") + + body = soup.find("body") + # Extract chunks with metadata - chunks = self.extract_text_chunks(body, min_word_threshold or self.min_word_count) - + chunks = self.extract_text_chunks( + body, min_word_threshold or self.min_word_count + ) + # Filter chunks based on news-specific criteria filtered_chunks = [] for _, text, tag_type, element in chunks: # Skip if element has negative class/id if self.is_excluded(element): continue - + # Headers are important in news articles - if tag_type == 'header': + if tag_type == "header": filtered_chunks.append(self.clean_element(element)) continue - + # For content, check word count and link density text = element.get_text(strip=True) if len(text.split()) >= (min_word_threshold or self.min_word_count): # Calculate link density - links_text = ' '.join(a.get_text(strip=True) for a in element.find_all('a')) + links_text = " ".join( + a.get_text(strip=True) for a in element.find_all("a") + ) link_density = len(links_text) / len(text) if text else 1 - + # Accept if link density is reasonable if link_density < 0.5: filtered_chunks.append(self.clean_element(element)) - + return filtered_chunks # Create markdown generator with custom filter - markdown_gen = DefaultMarkdownGenerator( - content_filter=CustomNewsFilter() - ) + markdown_gen = DefaultMarkdownGenerator(content_filter=CustomNewsFilter()) run_config = CrawlerRunConfig( - markdown_generator=markdown_gen, - cache_mode=CacheMode.BYPASS + markdown_generator=markdown_gen, cache_mode=CacheMode.BYPASS ) async with AsyncWebCrawler() as crawler: result = await crawler.arun( - url="https://news.ycombinator.com", - config=run_config + url="https://news.ycombinator.com", config=run_config ) print("Filtered Content Sample:") print(result.markdown[:500]) # Show first 500 chars + async def demo_json_extraction(): """ Improved JSON Extraction Demo --------------------------- - + Demonstrates the enhanced JSON extraction capabilities: 1. Base element attributes extraction 2. Complex nested structures 3. Multiple extraction patterns - + Key features shown: - Extracting attributes from base elements (href, data-* attributes) - Processing repeated patterns @@ -206,7 +207,7 @@ async def demo_json_extraction(): "baseSelector": "div.article-list", "baseFields": [ {"name": "list_id", "type": "attribute", "attribute": "data-list-id"}, - {"name": "category", "type": "attribute", "attribute": "data-category"} + {"name": "category", "type": "attribute", "attribute": "data-category"}, ], "fields": [ { @@ -214,8 +215,16 @@ async def demo_json_extraction(): "selector": "article.post", "type": "nested_list", "baseFields": [ - {"name": "post_id", "type": "attribute", "attribute": "data-post-id"}, - {"name": "author_id", "type": "attribute", "attribute": "data-author"} + { + "name": "post_id", + "type": "attribute", + "attribute": "data-post-id", + }, + { + "name": "author_id", + "type": "attribute", + "attribute": "data-author", + }, ], "fields": [ { @@ -223,60 +232,68 @@ async def demo_json_extraction(): "selector": "h2.title a", "type": "text", "baseFields": [ - {"name": "url", "type": "attribute", "attribute": "href"} - ] + { + "name": "url", + "type": "attribute", + "attribute": "href", + } + ], }, { "name": "author", "selector": "div.meta a.author", "type": "text", "baseFields": [ - {"name": "profile_url", "type": "attribute", "attribute": "href"} - ] - }, - { - "name": "date", - "selector": "span.date", - "type": "text" + { + "name": "profile_url", + "type": "attribute", + "attribute": "href", + } + ], }, + {"name": "date", "selector": "span.date", "type": "text"}, { "name": "read_more", "selector": "a.read-more", "type": "nested", "fields": [ {"name": "text", "type": "text"}, - {"name": "url", "type": "attribute", "attribute": "href"} - ] - } - ] + { + "name": "url", + "type": "attribute", + "attribute": "href", + }, + ], + }, + ], } - ] + ], } ) # Demonstrate extraction from raw HTML run_config = CrawlerRunConfig( - extraction_strategy=json_strategy, - cache_mode=CacheMode.BYPASS + extraction_strategy=json_strategy, cache_mode=CacheMode.BYPASS ) async with AsyncWebCrawler() as crawler: result = await crawler.arun( url="raw:" + SAMPLE_HTML, # Use raw: prefix for raw HTML - config=run_config + config=run_config, ) print("Extracted Content:") print(result.extracted_content) + async def demo_input_formats(): """ Input Format Handling Demo ---------------------- - + Demonstrates how LLM extraction can work with different input formats: 1. Markdown (default) - Good for simple text extraction 2. HTML - Better when you need structure and attributes - + This example shows how HTML input can be beneficial when: - You need to understand the DOM structure - You want to extract both visible text and HTML attributes @@ -350,7 +367,7 @@ async def demo_input_formats(): """ - + # Use raw:// prefix to pass HTML content directly url = f"raw://{dummy_html}" @@ -359,18 +376,30 @@ async def demo_input_formats(): # Define our schema using Pydantic class JobRequirement(BaseModel): - category: str = Field(description="Category of the requirement (e.g., Technical, Soft Skills)") - items: List[str] = Field(description="List of specific requirements in this category") - priority: str = Field(description="Priority level (Required/Preferred) based on the HTML class or context") + category: str = Field( + description="Category of the requirement (e.g., Technical, Soft Skills)" + ) + items: List[str] = Field( + description="List of specific requirements in this category" + ) + priority: str = Field( + description="Priority level (Required/Preferred) based on the HTML class or context" + ) class JobPosting(BaseModel): title: str = Field(description="Job title") department: str = Field(description="Department or team") location: str = Field(description="Job location, including remote options") salary_range: Optional[str] = Field(description="Salary range if specified") - requirements: List[JobRequirement] = Field(description="Categorized job requirements") - application_deadline: Optional[str] = Field(description="Application deadline if specified") - contact_info: Optional[dict] = Field(description="Contact information from footer or contact section") + requirements: List[JobRequirement] = Field( + description="Categorized job requirements" + ) + application_deadline: Optional[str] = Field( + description="Application deadline if specified" + ) + contact_info: Optional[dict] = Field( + description="Contact information from footer or contact section" + ) # First try with markdown (default) markdown_strategy = LLMExtractionStrategy( @@ -382,7 +411,7 @@ async def demo_input_formats(): Extract job posting details into structured data. Focus on the visible text content and organize requirements into categories. """, - input_format="markdown" # default + input_format="markdown", # default ) # Then with HTML for better structure understanding @@ -400,34 +429,25 @@ async def demo_input_formats(): Use HTML attributes and classes to enhance extraction accuracy. """, - input_format="html" # explicitly use HTML + input_format="html", # explicitly use HTML ) async with AsyncWebCrawler() as crawler: # Try with markdown first - markdown_config = CrawlerRunConfig( - extraction_strategy=markdown_strategy - ) - markdown_result = await crawler.arun( - url=url, - config=markdown_config - ) + markdown_config = CrawlerRunConfig(extraction_strategy=markdown_strategy) + markdown_result = await crawler.arun(url=url, config=markdown_config) print("\nMarkdown-based Extraction Result:") items = json.loads(markdown_result.extracted_content) print(json.dumps(items, indent=2)) # Then with HTML for better structure understanding - html_config = CrawlerRunConfig( - extraction_strategy=html_strategy - ) - html_result = await crawler.arun( - url=url, - config=html_config - ) + html_config = CrawlerRunConfig(extraction_strategy=html_strategy) + html_result = await crawler.arun(url=url, config=html_config) print("\nHTML-based Extraction Result:") items = json.loads(html_result.extracted_content) print(json.dumps(items, indent=2)) + # Main execution async def main(): print("Crawl4AI v0.4.24 Feature Walkthrough") @@ -439,5 +459,6 @@ async def main(): await demo_json_extraction() # await demo_input_formats() + if __name__ == "__main__": asyncio.run(main()) diff --git a/docs/md_v3/tutorials/advanced-features.md b/docs/md_v2/advanced/advanced-features.md similarity index 88% rename from docs/md_v3/tutorials/advanced-features.md rename to docs/md_v2/advanced/advanced-features.md index 16f85874..1f402948 100644 --- a/docs/md_v3/tutorials/advanced-features.md +++ b/docs/md_v2/advanced/advanced-features.md @@ -1,15 +1,16 @@ -# Advanced Features (Proxy, PDF, Screenshot, SSL, Headers, & Storage State) +# Overview of Some Important Advanced Features +(Proxy, PDF, Screenshot, SSL, Headers, & Storage State) Crawl4AI offers multiple power-user features that go beyond simple crawling. This tutorial covers: -1. **Proxy Usage** -2. **Capturing PDFs & Screenshots** -3. **Handling SSL Certificates** -4. **Custom Headers** -5. **Session Persistence & Local Storage** +1.β**Proxy Usage** +2.β**Capturing PDFs & Screenshots** +3.β**Handling SSL Certificates** +4.β**Custom Headers** +5.β**Session Persistence & Local Storage** > **Prerequisites** -> - You have a basic grasp of [AsyncWebCrawler Basics](./async-webcrawler-basics.md) +> - You have a basic grasp of [AsyncWebCrawler Basics](../core/simple-crawling.md) > - You know how to run or configure your Python environment with Playwright installed --- @@ -84,7 +85,7 @@ async def main(): # Save PDF if result.pdf: with open("wikipedia_page.pdf", "wb") as f: - f.write(b64decode(result.pdf)) + f.write(result.pdf) print("[OK] PDF & screenshot captured.") else: @@ -186,7 +187,7 @@ if __name__ == "__main__": **Notes** - Some sites may react differently to certain headers (e.g., `Accept-Language`). -- If you need advanced user-agent randomization or client hints, see [Identity-Based Crawling (Anti-Bot)](./identity-anti-bot.md) or use `UserAgentGenerator`. +- If you need advanced user-agent randomization or client hints, see [Identity-Based Crawling (Anti-Bot)](./identity-based-crawling.md) or use `UserAgentGenerator`. --- @@ -246,7 +247,7 @@ You can sign in once, export the browser context, and reuse it laterβwithout r - **`await context.storage_state(path="my_storage.json")`**: Exports cookies, localStorage, etc. to a file. - Provide `storage_state="my_storage.json"` on subsequent runs to skip the login step. -**See**: [Detailed session management tutorial](./hooks-custom.md#using-storage_state) or [Explanations β Browser Context & Managed Browser](../../explanations/browser-management.md) for more advanced scenarios (like multi-step logins, or capturing after interactive pages). +**See**: [Detailed session management tutorial](./session-management.md) or [Explanations β Browser Context & Managed Browser](./identity-based-crawling.md) for more advanced scenarios (like multi-step logins, or capturing after interactive pages). --- @@ -283,7 +284,10 @@ async def main(): # 3. Crawl async with AsyncWebCrawler(config=browser_cfg) as crawler: - result = await crawler.arun("https://secure.example.com/protected", config=crawler_cfg) + result = await crawler.arun( + url = "https://secure.example.com/protected", + config=crawler_cfg + ) if result.success: print("[OK] Crawled the secure page. Links found:", len(result.links.get("internal", []))) @@ -318,12 +322,6 @@ Youβve now explored several **advanced** features: - **Custom Headers** for language or specialized requests - **Session Persistence** via storage state -**Where to go next**: - -- **[Hooks & Custom Code](./hooks-custom.md)**: For multi-step interactions (clicking βLoad More,β performing logins, etc.) -- **[Identity-Based Crawling & Anti-Bot](./identity-anti-bot.md)**: If you need more sophisticated user simulation or stealth. -- **[Reference β BrowserConfig & CrawlerRunConfig](../../reference/configuration.md)**: Detailed param descriptions for everything youβve seen here and more. - With these power tools, you can build robust scraping workflows that mimic real user behavior, handle secure sites, capture detailed snapshots, and manage sessions across multiple runsβstreamlining your entire data collection pipeline. -**Last Updated**: 2024-XX-XX \ No newline at end of file +**Last Updated**: 2025-01-01 \ No newline at end of file diff --git a/docs/md_v2/advanced/content-processing.md b/docs/md_v2/advanced/content-processing.md deleted file mode 100644 index 25ed6172..00000000 --- a/docs/md_v2/advanced/content-processing.md +++ /dev/null @@ -1,136 +0,0 @@ -# Content Processing - -Crawl4AI provides powerful content processing capabilities that help you extract clean, relevant content from web pages. This guide covers content cleaning, media handling, link analysis, and metadata extraction. - -## Media Processing - -Crawl4AI provides comprehensive media extraction and analysis capabilities. It automatically detects and processes various types of media elements while maintaining their context and relevance. - -### Image Processing - -The library handles various image scenarios, including: -- Regular images -- Lazy-loaded images -- Background images -- Responsive images -- Image metadata and context - -```python -from crawl4ai.async_configs import CrawlerRunConfig - -config = CrawlerRunConfig() -result = await crawler.arun(url="https://example.com", config=config) - -for image in result.media["images"]: - # Each image includes rich metadata - print(f"Source: {image['src']}") - print(f"Alt text: {image['alt']}") - print(f"Description: {image['desc']}") - print(f"Context: {image['context']}") # Surrounding text - print(f"Relevance score: {image['score']}") # 0-10 score -``` - -### Handling Lazy-Loaded Content - -Crawl4AI already handles lazy loading for media elements. You can customize the wait time for lazy-loaded content with `CrawlerRunConfig`: - -```python -config = CrawlerRunConfig( - wait_for="css:img[data-src]", # Wait for lazy images - delay_before_return_html=2.0 # Additional wait time -) -result = await crawler.arun(url="https://example.com", config=config) -``` - -### Video and Audio Content - -The library extracts video and audio elements with their metadata: - -```python -from crawl4ai.async_configs import CrawlerRunConfig - -config = CrawlerRunConfig() -result = await crawler.arun(url="https://example.com", config=config) - -# Process videos -for video in result.media["videos"]: - print(f"Video source: {video['src']}") - print(f"Type: {video['type']}") - print(f"Duration: {video.get('duration')}") - print(f"Thumbnail: {video.get('poster')}") - -# Process audio -for audio in result.media["audios"]: - print(f"Audio source: {audio['src']}") - print(f"Type: {audio['type']}") - print(f"Duration: {audio.get('duration')}") -``` - -## Link Analysis - -Crawl4AI provides sophisticated link analysis capabilities, helping you understand the relationship between pages and identify important navigation patterns. - -### Link Classification - -The library automatically categorizes links into: -- Internal links (same domain) -- External links (different domains) -- Social media links -- Navigation links -- Content links - -```python -from crawl4ai.async_configs import CrawlerRunConfig - -config = CrawlerRunConfig() -result = await crawler.arun(url="https://example.com", config=config) - -# Analyze internal links -for link in result.links["internal"]: - print(f"Internal: {link['href']}") - print(f"Link text: {link['text']}") - print(f"Context: {link['context']}") # Surrounding text - print(f"Type: {link['type']}") # nav, content, etc. - -# Analyze external links -for link in result.links["external"]: - print(f"External: {link['href']}") - print(f"Domain: {link['domain']}") - print(f"Type: {link['type']}") -``` - -### Smart Link Filtering - -Control which links are included in the results with `CrawlerRunConfig`: - -```python -config = CrawlerRunConfig( - exclude_external_links=True, # Remove external links - exclude_social_media_links=True, # Remove social media links - exclude_social_media_domains=[ # Custom social media domains - "facebook.com", "twitter.com", "instagram.com" - ], - exclude_domains=["ads.example.com"] # Exclude specific domains -) -result = await crawler.arun(url="https://example.com", config=config) -``` - -## Metadata Extraction - -Crawl4AI automatically extracts and processes page metadata, providing valuable information about the content: - -```python -from crawl4ai.async_configs import CrawlerRunConfig - -config = CrawlerRunConfig() -result = await crawler.arun(url="https://example.com", config=config) - -metadata = result.metadata -print(f"Title: {metadata['title']}") -print(f"Description: {metadata['description']}") -print(f"Keywords: {metadata['keywords']}") -print(f"Author: {metadata['author']}") -print(f"Published Date: {metadata['published_date']}") -print(f"Modified Date: {metadata['modified_date']}") -print(f"Language: {metadata['language']}") -``` diff --git a/docs/md_v2/advanced/crawl-dispatcher.md b/docs/md_v2/advanced/crawl-dispatcher.md new file mode 100644 index 00000000..e4059f25 --- /dev/null +++ b/docs/md_v2/advanced/crawl-dispatcher.md @@ -0,0 +1,12 @@ +# Crawl Dispatcher + +Weβre excited to announce a **Crawl Dispatcher** module that can handle **thousands** of crawling tasks simultaneously. By efficiently managing system resources (memory, CPU, network), this dispatcher ensures high-performance data extraction at scale. It also provides **real-time monitoring** of each crawlerβs status, memory usage, and overall progress. + +Stay tunedβthis feature is **coming soon** in an upcoming release of Crawl4AI! For the latest news, keep an eye on our changelogs and follow [@unclecode](https://twitter.com/unclecode) on X. + +Below is a **sample** of how the dispatcherβs performance monitor might look in action: + + + + +We canβt wait to bring you this streamlined, **scalable** approach to multi-URL crawlingβ**watch this space** for updates! \ No newline at end of file diff --git a/docs/md_v2/basic/file-download.md b/docs/md_v2/advanced/file-downloading.md similarity index 92% rename from docs/md_v2/basic/file-download.md rename to docs/md_v2/advanced/file-downloading.md index eac0f5cb..2fa3759f 100644 --- a/docs/md_v2/basic/file-download.md +++ b/docs/md_v2/advanced/file-downloading.md @@ -17,18 +17,6 @@ async def main(): asyncio.run(main()) ``` -Or, enable it for a specific crawl by using `CrawlerRunConfig`: - -```python -from crawl4ai.async_configs import CrawlerRunConfig - -async def main(): - async with AsyncWebCrawler() as crawler: - config = CrawlerRunConfig(accept_downloads=True) - result = await crawler.arun(url="https://example.com", config=config) - # ... -``` - ## Specifying Download Location Specify the download directory using the `downloads_path` attribute in the `BrowserConfig` object. If not provided, Crawl4AI defaults to creating a "downloads" directory inside the `.crawl4ai` folder in your home directory. @@ -98,7 +86,8 @@ async def download_multiple_files(url: str, download_path: str): const downloadLinks = document.querySelectorAll('a[download]'); for (const link of downloadLinks) { link.click(); - await new Promise(r => setTimeout(r, 2000)); // Delay between clicks + // Delay between clicks + await new Promise(r => setTimeout(r, 2000)); } """, wait_for=10 # Wait for all downloads to start diff --git a/docs/md_v2/advanced/hooks-auth.md b/docs/md_v2/advanced/hooks-auth.md index 66042229..6787abd9 100644 --- a/docs/md_v2/advanced/hooks-auth.md +++ b/docs/md_v2/advanced/hooks-auth.md @@ -1,121 +1,254 @@ -# Hooks & Auth for AsyncWebCrawler +# Hooks & Auth in AsyncWebCrawler -Crawl4AI's `AsyncWebCrawler` allows you to customize the behavior of the web crawler using hooks. Hooks are asynchronous functions called at specific points in the crawling process, allowing you to modify the crawler's behavior or perform additional actions. This updated documentation demonstrates how to use hooks, including the new `on_page_context_created` hook, and ensures compatibility with `BrowserConfig` and `CrawlerRunConfig`. +Crawl4AIβs **hooks** let you customize the crawler at specific points in the pipeline: -## Example: Using Crawler Hooks with AsyncWebCrawler +1.β**`on_browser_created`** β After browser creation. +2.β**`on_page_context_created`** β After a new context & page are created. +3.β**`before_goto`** β Just before navigating to a page. +4.β**`after_goto`** β Right after navigation completes. +5.β**`on_user_agent_updated`** β Whenever the user agent changes. +6.β**`on_execution_started`** β Once custom JavaScript execution begins. +7.β**`before_retrieve_html`** β Just before the crawler retrieves final HTML. +8.β**`before_return_html`** β Right before returning the HTML content. -In this example, we'll: +**Important**: Avoid heavy tasks in `on_browser_created` since you donβt yet have a page context. If you need to *log in*, do so in **`on_page_context_created`**. -1. Configure the browser and set up authentication when it's created. -2. Apply custom routing and initial actions when the page context is created. -3. Add custom headers before navigating to the URL. -4. Log the current URL after navigation. -5. Perform actions after JavaScript execution. -6. Log the length of the HTML before returning it. +> note "Important Hook Usage Warning" + **Avoid Misusing Hooks**: Do not manipulate page objects in the wrong hook or at the wrong time, as it can crash the pipeline or produce incorrect results. A common mistake is attempting to handle authentication prematurelyβsuch as creating or closing pages in `on_browser_created`. -### Hook Definitions +> **Use the Right Hook for Auth**: If you need to log in or set tokens, use `on_page_context_created`. This ensures you have a valid page/context to work with, without disrupting the main crawling flow. + +> **Identity-Based Crawling**: For robust auth, consider identity-based crawling (or passing a session ID) to preserve state. Run your initial login steps in a separate, well-defined process, then feed that session to your main crawlβrather than shoehorning complex authentication into early hooks. Check out [Identity-Based Crawling](../advanced/identity-based-crawling.md) for more details. + +> **Be Cautious**: Overwriting or removing elements in the wrong hook can compromise the final crawl. Keep hooks focused on smaller tasks (like route filters, custom headers), and let your main logic (crawling, data extraction) proceed normally. + + +Below is an example demonstration. + +--- + +## Example: Using Hooks in AsyncWebCrawler ```python import asyncio -from crawl4ai import AsyncWebCrawler -from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig -from playwright.async_api import Page, Browser, BrowserContext +import json +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from playwright.async_api import Page, BrowserContext -def log_routing(route): - # Example: block loading images - if route.request.resource_type == "image": - print(f"[HOOK] Blocking image request: {route.request.url}") - asyncio.create_task(route.abort()) - else: - asyncio.create_task(route.continue_()) - -async def on_browser_created(browser: Browser, **kwargs): - print("[HOOK] on_browser_created") - # Example: Set browser viewport size and log in - context = await browser.new_context(viewport={"width": 1920, "height": 1080}) - page = await context.new_page() - await page.goto("https://example.com/login") - await page.fill("input[name='username']", "testuser") - await page.fill("input[name='password']", "password123") - await page.click("button[type='submit']") - await page.wait_for_selector("#welcome") - await context.add_cookies([{"name": "auth_token", "value": "abc123", "url": "https://example.com"}]) - await page.close() - await context.close() - -async def on_page_context_created(context: BrowserContext, page: Page, **kwargs): - print("[HOOK] on_page_context_created") - await context.route("**", log_routing) - -async def before_goto(page: Page, context: BrowserContext, **kwargs): - print("[HOOK] before_goto") - await page.set_extra_http_headers({"X-Test-Header": "test"}) - -async def after_goto(page: Page, context: BrowserContext, **kwargs): - print("[HOOK] after_goto") - print(f"Current URL: {page.url}") - -async def on_execution_started(page: Page, context: BrowserContext, **kwargs): - print("[HOOK] on_execution_started") - await page.evaluate("console.log('Custom JS executed')") - -async def before_return_html(page: Page, context: BrowserContext, html: str, **kwargs): - print("[HOOK] before_return_html") - print(f"HTML length: {len(html)}") - return page -``` - -### Using the Hooks with AsyncWebCrawler - -```python async def main(): - print("\nπ Using Crawler Hooks: Customize AsyncWebCrawler with hooks!") + print("π Hooks Example: Demonstrating recommended usage") - # Configure browser and crawler settings + # 1) Configure the browser browser_config = BrowserConfig( headless=True, - viewport_width=1920, - viewport_height=1080 + verbose=True ) - + + # 2) Configure the crawler run crawler_run_config = CrawlerRunConfig( js_code="window.scrollTo(0, document.body.scrollHeight);", - wait_for="footer" + wait_for="body", + cache_mode=CacheMode.BYPASS ) - # Initialize crawler - async with AsyncWebCrawler(config=browser_config) as crawler: - crawler.crawler_strategy.set_hook("on_browser_created", on_browser_created) - crawler.crawler_strategy.set_hook("on_page_context_created", on_page_context_created) - crawler.crawler_strategy.set_hook("before_goto", before_goto) - crawler.crawler_strategy.set_hook("after_goto", after_goto) - crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started) - crawler.crawler_strategy.set_hook("before_return_html", before_return_html) + # 3) Create the crawler instance + crawler = AsyncWebCrawler(config=browser_config) - # Run the crawler - result = await crawler.arun(url="https://example.com", config=crawler_run_config) + # + # Define Hook Functions + # - print("\nπ¦ Crawler Hooks Result:") - print(result) + async def on_browser_created(browser, **kwargs): + # Called once the browser instance is created (but no pages or contexts yet) + print("[HOOK] on_browser_created - Browser created successfully!") + # Typically, do minimal setup here if needed + return browser -asyncio.run(main()) + async def on_page_context_created(page: Page, context: BrowserContext, **kwargs): + # Called right after a new page + context are created (ideal for auth or route config). + print("[HOOK] on_page_context_created - Setting up page & context.") + + # Example 1: Route filtering (e.g., block images) + async def route_filter(route): + if route.request.resource_type == "image": + print(f"[HOOK] Blocking image request: {route.request.url}") + await route.abort() + else: + await route.continue_() + + await context.route("**", route_filter) + + # Example 2: (Optional) Simulate a login scenario + # (We do NOT create or close pages here, just do quick steps if needed) + # e.g., await page.goto("https://example.com/login") + # e.g., await page.fill("input[name='username']", "testuser") + # e.g., await page.fill("input[name='password']", "password123") + # e.g., await page.click("button[type='submit']") + # e.g., await page.wait_for_selector("#welcome") + # e.g., await context.add_cookies([...]) + # Then continue + + # Example 3: Adjust the viewport + await page.set_viewport_size({"width": 1080, "height": 600}) + return page + + async def before_goto( + page: Page, context: BrowserContext, url: str, **kwargs + ): + # Called before navigating to each URL. + print(f"[HOOK] before_goto - About to navigate: {url}") + # e.g., inject custom headers + await page.set_extra_http_headers({ + "Custom-Header": "my-value" + }) + return page + + async def after_goto( + page: Page, context: BrowserContext, + url: str, response, **kwargs + ): + # Called after navigation completes. + print(f"[HOOK] after_goto - Successfully loaded: {url}") + # e.g., wait for a certain element if we want to verify + try: + await page.wait_for_selector('.content', timeout=1000) + print("[HOOK] Found .content element!") + except: + print("[HOOK] .content not found, continuing anyway.") + return page + + async def on_user_agent_updated( + page: Page, context: BrowserContext, + user_agent: str, **kwargs + ): + # Called whenever the user agent updates. + print(f"[HOOK] on_user_agent_updated - New user agent: {user_agent}") + return page + + async def on_execution_started(page: Page, context: BrowserContext, **kwargs): + # Called after custom JavaScript execution begins. + print("[HOOK] on_execution_started - JS code is running!") + return page + + async def before_retrieve_html(page: Page, context: BrowserContext, **kwargs): + # Called before final HTML retrieval. + print("[HOOK] before_retrieve_html - We can do final actions") + # Example: Scroll again + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + return page + + async def before_return_html( + page: Page, context: BrowserContext, html: str, **kwargs + ): + # Called just before returning the HTML in the result. + print(f"[HOOK] before_return_html - HTML length: {len(html)}") + return page + + # + # Attach Hooks + # + + crawler.crawler_strategy.set_hook("on_browser_created", on_browser_created) + crawler.crawler_strategy.set_hook( + "on_page_context_created", on_page_context_created + ) + crawler.crawler_strategy.set_hook("before_goto", before_goto) + crawler.crawler_strategy.set_hook("after_goto", after_goto) + crawler.crawler_strategy.set_hook( + "on_user_agent_updated", on_user_agent_updated + ) + crawler.crawler_strategy.set_hook( + "on_execution_started", on_execution_started + ) + crawler.crawler_strategy.set_hook( + "before_retrieve_html", before_retrieve_html + ) + crawler.crawler_strategy.set_hook( + "before_return_html", before_return_html + ) + + await crawler.start() + + # 4) Run the crawler on an example page + url = "https://example.com" + result = await crawler.arun(url, config=crawler_run_config) + + if result.success: + print("\nCrawled URL:", result.url) + print("HTML length:", len(result.html)) + else: + print("Error:", result.error_message) + + await crawler.close() + +if __name__ == "__main__": + asyncio.run(main()) ``` -### Explanation of Hooks +--- -- **`on_browser_created`**: Called when the browser is created. Use this to configure the browser or handle authentication (e.g., logging in and setting cookies). -- **`on_page_context_created`**: Called when a new page context is created. Use this to apply routing, block resources, or inject custom logic before navigating to the URL. -- **`before_goto`**: Called before navigating to the URL. Use this to add custom headers or perform other pre-navigation actions. -- **`after_goto`**: Called after navigation. Use this to verify content or log the URL. -- **`on_execution_started`**: Called after executing custom JavaScript. Use this to perform additional actions. -- **`before_return_html`**: Called before returning the HTML content. Use this to log details or preprocess the content. +## Hook Lifecycle Summary -### Additional Customizations +1.β**`on_browser_created`**: + - Browser is up, but **no** pages or contexts yet. + - Light setup onlyβdonβt try to open or close pages here (that belongs in `on_page_context_created`). -- **Resource Management**: Use `on_page_context_created` to block or modify requests (e.g., block images, fonts, or third-party scripts). -- **Dynamic Headers**: Use `before_goto` to add or modify headers dynamically based on the URL. -- **Authentication**: Use `on_browser_created` to handle login processes and set authentication cookies or tokens. -- **Content Analysis**: Use `before_return_html` to analyze or modify the extracted HTML content. +2.β**`on_page_context_created`**: + - Perfect for advanced **auth** or route blocking. + - You have a **page** + **context** ready but havenβt navigated to the target URL yet. -These hooks provide powerful customization options for tailoring the crawling process to your needs. +3.β**`before_goto`**: + - Right before navigation. Typically used for setting **custom headers** or logging the target URL. + +4.β**`after_goto`**: + - After page navigation is done. Good place for verifying content or waiting on essential elements. + +5.β**`on_user_agent_updated`**: + - Whenever the user agent changes (for stealth or different UA modes). + +6.β**`on_execution_started`**: + - If you set `js_code` or run custom scripts, this runs once your JS is about to start. + +7.β**`before_retrieve_html`**: + - Just before the final HTML snapshot is taken. Often you do a final scroll or lazy-load triggers here. + +8.β**`before_return_html`**: + - The last hook before returning HTML to the `CrawlResult`. Good for logging HTML length or minor modifications. + +--- + +## When to Handle Authentication + +**Recommended**: Use **`on_page_context_created`** if you need to: + +- Navigate to a login page or fill forms +- Set cookies or localStorage tokens +- Block resource routes to avoid ads + +This ensures the newly created context is under your control **before** `arun()` navigates to the main URL. + +--- + +## Additional Considerations + +- **Session Management**: If you want multiple `arun()` calls to reuse a single session, pass `session_id=` in your `CrawlerRunConfig`. Hooks remain the same. +- **Performance**: Hooks can slow down crawling if they do heavy tasks. Keep them concise. +- **Error Handling**: If a hook fails, the overall crawl might fail. Catch exceptions or handle them gracefully. +- **Concurrency**: If you run `arun_many()`, each URL triggers these hooks in parallel. Ensure your hooks are thread/async-safe. + +--- + +## Conclusion + +Hooks provide **fine-grained** control over: + +- **Browser** creation (light tasks only) +- **Page** and **context** creation (auth, route blocking) +- **Navigation** phases +- **Final HTML** retrieval + +Follow the recommended usage: +- **Login** or advanced tasks in `on_page_context_created` +- **Custom headers** or logs in `before_goto` / `after_goto` +- **Scrolling** or final checks in `before_retrieve_html` / `before_return_html` diff --git a/docs/md_v2/advanced/identity-based-crawling.md b/docs/md_v2/advanced/identity-based-crawling.md new file mode 100644 index 00000000..702d9475 --- /dev/null +++ b/docs/md_v2/advanced/identity-based-crawling.md @@ -0,0 +1,180 @@ +# Preserve Your Identity with Crawl4AI + +Crawl4AI empowers you to navigate and interact with the web using your **authentic digital identity**, ensuring youβre recognized as a human and not mistaken for a bot. This tutorial covers: + +1.β**Managed Browsers** β The recommended approach for persistent profiles and identity-based crawling. +2.β**Magic Mode** β A simplified fallback solution for quick automation without persistent identity. + +--- + +## 1. Managed Browsers: Your Digital Identity Solution + +**Managed Browsers** let developers create and use **persistent browser profiles**. These profiles store local storage, cookies, and other session data, letting you browse as your **real self**βcomplete with logins, preferences, and cookies. + +### Key Benefits + +- **Authentic Browsing Experience**: Retain session data and browser fingerprints as though youβre a normal user. +- **Effortless Configuration**: Once you log in or solve CAPTCHAs in your chosen data directory, you can re-run crawls without repeating those steps. +- **Empowered Data Access**: If you can see the data in your own browser, you can automate its retrieval with your genuine identity. + +--- + +Below is a **partial update** to your **Managed Browsers** tutorial, specifically the section about **creating a user-data directory** using **Playwrightβs Chromium** binary rather than a system-wide Chrome/Edge. Weβll show how to **locate** that binary and launch it with a `--user-data-dir` argument to set up your profile. You can then point `BrowserConfig.user_data_dir` to that folder for subsequent crawls. + +--- + +### Creating a User Data Directory (Command-Line Approach via Playwright) + +If you installed Crawl4AI (which installs Playwright under the hood), you already have a Playwright-managed Chromium on your system. Follow these steps to launch that **Chromium** from your command line, specifying a **custom** data directory: + +1.β**Find** the Playwright Chromium binary: + - On most systems, installed browsers go under a `~/.cache/ms-playwright/` folder or similar path. + - To see an overview of installed browsers, run: + ```bash + python -m playwright install --dry-run + ``` + or + ```bash + playwright install --dry-run + ``` + (depending on your environment). This shows where Playwright keeps Chromium. + + - For instance, you might see a path like: + ``` + ~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome + ``` + on Linux, or a corresponding folder on macOS/Windows. + +2.β**Launch** the Playwright Chromium binary with a **custom** user-data directory: + ```bash + # Linux example + ~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome \ + --user-data-dir=/home/
Great article!
+Thanks for sharing.
+