diff --git a/README.md b/README.md index 069c02b8..9c3796cd 100644 --- a/README.md +++ b/README.md @@ -11,21 +11,19 @@ Crawl4AI simplifies asynchronous web crawling and data extraction, making it accessible for large language models (LLMs) and AI applications. ๐Ÿ†“๐ŸŒ -## ๐ŸŒŸ Meet the Crawl4AI Assistant: Your Copilot for Crawling +## New in 0.3.74 โœจ -Use the [Crawl4AI GPT Assistant](https://tinyurl.com/crawl4ai-gpt) as your AI-powered copilot! With this assistant, you can: +- ๐Ÿš€ **Blazing Fast Scraping:** The scraping process is now significantly faster, often completing in under 100 milliseconds (excluding web fetch time)! +- ๐Ÿ“ฅ **Download Mastery:** Control downloads, specify folders, and track files within the `CrawlResult` object. +- ๐Ÿ”Ž **Relevance Filtering:** Extract the most important content with the new `RelevanceContentFilter` and BM25 algorithm. Control filtering with the `fit_markdown` flag. +- ๐Ÿ—‚๏ธ **Local & Raw HTML:** Crawl local files (`file://`) and raw HTML strings (`raw:`) directly. +- ๐Ÿค– **Browser Boss:** Manage browser sessions with persistent contexts, process monitoring, and tf-playwright-stealth integration. Configure using `use_managed_browser`, `user_data_dir`, and `use_persistent_context` parameters. +- โ˜๏ธ **API & Cache Boost:** CORS support, static file serving, and a new filesystem-based cache for blazing-fast performance. Fine-tune caching with the `CacheMode` enum (ENABLED, DISABLED, READ_ONLY, WRITE_ONLY, BYPASS) and the `always_bypass_cache` parameter. +- ๐Ÿ”’ **API Security:** Protect your API server with token-based authentication using the `CRAWL4AI_API_TOKEN` environment variable. +- ๐Ÿ”„ **Synchronous & Direct Crawling:** Get immediate results with `/crawl_sync` or bypass the task queue with `/crawl_direct`. +- ๐Ÿ› ๏ธ **Database Migration:** A new `crawl4ai-migrate` command ensures smooth upgrades and data integrity between versions. +- ๐Ÿ› **Squashed Bugs:** Fixed browser context issues in Docker, memory leaks, enhanced error handling, and improved HTML parsing. -- ๐Ÿง‘โ€๐Ÿ’ป Generate code for complex crawling and extraction tasks -- ๐Ÿ’ก Get tailored support and examples -- ๐Ÿ“˜ Learn Crawl4AI faster with step-by-step guidance - -## New in 0.3.73 โœจ - -- ๐Ÿณ Docker Ready: Full API server with seamless deployment & scaling -- ๐ŸŽฏ Browser Takeover: Use your own browser with cookies & history intact (CDP support) -- ๐Ÿ“ Mockdown+: Enhanced tag preservation & content extraction -- โšก๏ธ Parallel Power: Supercharged multi-URL crawling performance -- ๐ŸŒŸ And many more exciting updates... ## Try it Now! diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index d554576d..d22e3b1f 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -10,14 +10,14 @@ import asyncio from .models import CrawlResult from .async_database import async_db_manager from .chunking_strategy import * +from .content_filter_strategy import * from .extraction_strategy import * from .async_crawler_strategy import AsyncCrawlerStrategy, AsyncPlaywrightCrawlerStrategy, AsyncCrawlResponse from .cache_context import CacheMode, CacheContext, _legacy_to_cache_mode from .content_scrapping_strategy import WebScrapingStrategy from .config import ( MIN_WORD_THRESHOLD, - IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, - SHOW_DEPRECATION_WARNINGS # New import + IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD ) from .utils import ( sanitize_input_encode, @@ -49,7 +49,7 @@ class AsyncWebCrawler: ) To disable deprecation warnings: - Set SHOW_DEPRECATION_WARNINGS = False in config.py + Pass warning=False to suppress the warning. """ def __init__( @@ -85,11 +85,11 @@ class AsyncWebCrawler: # Handle deprecated parameter if always_by_pass_cache is not None: - if SHOW_DEPRECATION_WARNINGS: + if kwargs.get("warning", True): warnings.warn( "'always_by_pass_cache' is deprecated and will be removed in version X.X.X. " "Use 'always_bypass_cache' instead. " - "Set SHOW_DEPRECATION_WARNINGS=False in config.py to suppress this warning.", + "Pass warning=False to suppress this warning.", DeprecationWarning, stacklevel=2 ) @@ -126,6 +126,7 @@ class AsyncWebCrawler: word_count_threshold=MIN_WORD_THRESHOLD, extraction_strategy: ExtractionStrategy = None, chunking_strategy: ChunkingStrategy = RegexChunking(), + content_filter: RelevantContentFilter = None, cache_mode: Optional[CacheMode] = None, # Deprecated parameters bypass_cache: bool = False, @@ -172,7 +173,7 @@ class AsyncWebCrawler: try: # Handle deprecated parameters if any([bypass_cache, disable_cache, no_cache_read, no_cache_write]): - if SHOW_DEPRECATION_WARNINGS: + if kwargs.get("warning", True): warnings.warn( "Cache control boolean flags are deprecated and will be removed in version X.X.X. " "Use 'cache_mode' parameter instead. Examples:\n" @@ -180,7 +181,7 @@ class AsyncWebCrawler: "- For disable_cache=True, use cache_mode=CacheMode.DISABLED\n" "- For no_cache_read=True, use cache_mode=CacheMode.WRITE_ONLY\n" "- For no_cache_write=True, use cache_mode=CacheMode.READ_ONLY\n" - "Set SHOW_DEPRECATION_WARNINGS=False in config.py to suppress this warning.", + "Pass warning=False to suppress this warning.", DeprecationWarning, stacklevel=2 ) @@ -257,6 +258,7 @@ class AsyncWebCrawler: word_count_threshold=word_count_threshold, extraction_strategy=extraction_strategy, chunking_strategy=chunking_strategy, + content_filter=content_filter, css_selector=css_selector, screenshot=screenshot_data, verbose=verbose, @@ -308,6 +310,7 @@ class AsyncWebCrawler: word_count_threshold=MIN_WORD_THRESHOLD, extraction_strategy: ExtractionStrategy = None, chunking_strategy: ChunkingStrategy = RegexChunking(), + content_filter: RelevantContentFilter = None, cache_mode: Optional[CacheMode] = None, # Deprecated parameters bypass_cache: bool = False, @@ -335,14 +338,15 @@ class AsyncWebCrawler: Returns: List[CrawlResult]: Results for each URL """ - if bypass_cache and SHOW_DEPRECATION_WARNINGS: - warnings.warn( - "'bypass_cache' is deprecated and will be removed in version X.X.X. " - "Use 'cache_mode=CacheMode.BYPASS' instead. " - "Set SHOW_DEPRECATION_WARNINGS=False in config.py to suppress this warning.", - DeprecationWarning, - stacklevel=2 - ) + if bypass_cache: + if kwargs.get("warning", True): + warnings.warn( + "'bypass_cache' is deprecated and will be removed in version X.X.X. " + "Use 'cache_mode=CacheMode.BYPASS' instead. " + "Pass warning=False to suppress this warning.", + DeprecationWarning, + stacklevel=2 + ) if cache_mode is None: cache_mode = CacheMode.BYPASS @@ -356,6 +360,7 @@ class AsyncWebCrawler: word_count_threshold=word_count_threshold, extraction_strategy=extraction_strategy, chunking_strategy=chunking_strategy, + content_filter=content_filter, cache_mode=cache_mode, css_selector=css_selector, screenshot=screenshot, @@ -377,6 +382,7 @@ class AsyncWebCrawler: word_count_threshold: int, extraction_strategy: ExtractionStrategy, chunking_strategy: ChunkingStrategy, + content_filter: RelevantContentFilter, css_selector: str, screenshot: str, verbose: bool, @@ -397,6 +403,7 @@ class AsyncWebCrawler: image_description_min_word_threshold=kwargs.get( "image_description_min_word_threshold", IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD ), + content_filter = content_filter, **kwargs, ) diff --git a/crawl4ai/content_scrapping_strategy.py b/crawl4ai/content_scrapping_strategy.py index 9c81638c..d16b0680 100644 --- a/crawl4ai/content_scrapping_strategy.py +++ b/crawl4ai/content_scrapping_strategy.py @@ -532,14 +532,13 @@ class WebScrapingStrategy(ContentScrapingStrategy): fit_markdown = "Set flag 'fit_markdown' to True to get cleaned HTML content." fit_html = "Set flag 'fit_markdown' to True to get cleaned HTML content." - if kwargs.get('fit_markdown', False): - # cleaner = ContentCleaningStrategy() - # fit_html = cleaner.clean(cleaned_html) - # fit_markdown = h.handle(fit_html) - content_filter = BM25ContentFilter( - user_query= kwargs.get('fit_markdown_user_query', None), - bm25_threshold= kwargs.get('fit_markdown_bm25_threshold', 1.0) - ) + if kwargs.get('content_filter', None) or kwargs.get('fit_markdown', False): + content_filter = kwargs.get('content_filter', None) + if not content_filter: + content_filter = BM25ContentFilter( + user_query= kwargs.get('fit_markdown_user_query', None), + bm25_threshold= kwargs.get('fit_markdown_bm25_threshold', 1.0) + ) fit_html = content_filter.filter_content(html) fit_html = '\n'.join('
{}
'.format(s) for s in fit_html) fit_markdown = h.handle(fit_html) diff --git a/docs/examples/v0.3.74.overview.py b/docs/examples/v0.3.74.overview.py index 579d05dd..ec3a7d73 100644 --- a/docs/examples/v0.3.74.overview.py +++ b/docs/examples/v0.3.74.overview.py @@ -56,7 +56,7 @@ async def content_filtering_example(): result = await crawler.arun( url="https://openai.com/blog", - extraction_strategy=content_filter + content_filter=content_filter ) print(f"Filtered content: {result.extracted_content}") diff --git a/docs/md_v2/advanced/managed_browser.md b/docs/md_v2/advanced/managed_browser.md new file mode 100644 index 00000000..80d6fc1a --- /dev/null +++ b/docs/md_v2/advanced/managed_browser.md @@ -0,0 +1,84 @@ +# Content Filtering in Crawl4AI + +This guide explains how to use content filtering strategies in Crawl4AI to extract the most relevant information from crawled web pages. You'll learn how to use the built-in `BM25ContentFilter` and how to create your own custom content filtering strategies. + +## Relevance Content Filter + +The `RelevanceContentFilter` is an abstract class that provides a common interface for content filtering strategies. Specific filtering algorithms, like `BM25ContentFilter`, inherit from this class and implement the `filter_content` method. This method takes the HTML content as input and returns a list of filtered text blocks. + +## BM25 Algorithm + +The `BM25ContentFilter` uses the BM25 algorithm, a ranking function used in information retrieval to estimate the relevance of documents to a given search query. In Crawl4AI, this algorithm helps to identify and extract text chunks that are most relevant to the page's metadata or a user-specified query. + +### Usage + +To use the `BM25ContentFilter`, initialize it and then pass it as the `extraction_strategy` parameter to the `arun` method of the crawler. + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.content_filter_strategy import BM25ContentFilter + +async def filter_content(url, query=None): + async with AsyncWebCrawler() as crawler: + content_filter = BM25ContentFilter(user_query=query) + result = await crawler.arun(url=url, extraction_strategy=content_filter, fit_markdown=True) # Set fit_markdown flag to True to trigger BM25 filtering + if result.success: + print(f"Filtered Content (JSON):\n{result.extracted_content}") + print(f"\nFiltered Markdown:\n{result.fit_markdown}") # New field in CrawlResult object + print(f"\nFiltered HTML:\n{result.fit_html}") # New field in CrawlResult object. Note that raw HTML may have tags re-organized due to internal parsing. + else: + print("Error:", result.error_message) + +# Example usage: +asyncio.run(filter_content("https://en.wikipedia.org/wiki/Apple", "fruit nutrition health")) # with query +asyncio.run(filter_content("https://en.wikipedia.org/wiki/Apple")) # without query, metadata will be used as the query. + +``` + +### Parameters + +- **`user_query`**: (Optional) A string representing the search query. If not provided, the filter extracts relevant metadata (title, description, keywords) from the page and uses that as the query. +- **`bm25_threshold`**: (Optional, default 1.0) A float value that controls the threshold for relevance. Higher values result in stricter filtering, returning only the most relevant text chunks. Lower values result in more lenient filtering. + + +## Fit Markdown Flag + +Setting the `fit_markdown` flag to `True` in the `arun` method activates the BM25 content filtering during the crawl. The `fit_markdown` parameter instructs the scraper to extract and clean the HTML, primarily to prepare for a Large Language Model that cannot process large amounts of data. Setting this flag not only improves the quality of the extracted content but also adds the filtered content to two new attributes in the returned `CrawlResult` object: `fit_markdown` and `fit_html`. + + +## Custom Content Filtering Strategies + +You can create your own custom filtering strategies by inheriting from the `RelevantContentFilter` class and implementing the `filter_content` method. This allows you to tailor the filtering logic to your specific needs. + +```python +from crawl4ai.content_filter_strategy import RelevantContentFilter +from bs4 import BeautifulSoup, Tag +from typing import List + +class MyCustomFilter(RelevantContentFilter): + def filter_content(self, html: str) -> List[str]: + soup = BeautifulSoup(html, 'lxml') + # Implement custom filtering logic here + # Example: extract all paragraphs within divs with class "article-body" + filtered_paragraphs = [] + for tag in soup.select("div.article-body p"): + if isinstance(tag, Tag): + filtered_paragraphs.append(str(tag)) # Add the cleaned HTML element. + return filtered_paragraphs + + + +async def custom_filter_demo(url: str): + async with AsyncWebCrawler() as crawler: + custom_filter = MyCustomFilter() + result = await crawler.arun(url, extraction_strategy=custom_filter) + if result.success: + print(result.extracted_content) + +``` + +This example demonstrates extracting paragraphs from a specific div class. You can customize this logic to implement different filtering strategies, use regular expressions, analyze text density, or apply other relevant techniques. + +## Conclusion + +Content filtering strategies provide a powerful way to refine the output of your crawls. By using `BM25ContentFilter` or creating custom strategies, you can focus on the most pertinent information and improve the efficiency of your data processing pipeline. diff --git a/docs/md_v2/advanced/session-management-advanced.md b/docs/md_v2/advanced/session-management-advanced.md index f8c81da2..908828f7 100644 --- a/docs/md_v2/advanced/session-management-advanced.md +++ b/docs/md_v2/advanced/session-management-advanced.md @@ -30,7 +30,7 @@ Let's start with a basic example of session-based crawling: ```python import asyncio -from crawl4ai import AsyncWebCrawler +from crawl4ai import AsyncWebCrawler, CacheMode async def basic_session_crawl(): async with AsyncWebCrawler(verbose=True) as crawler: @@ -43,7 +43,7 @@ async def basic_session_crawl(): session_id=session_id, js_code="document.querySelector('.load-more-button').click();" if page > 0 else None, css_selector=".content-item", - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(f"Page {page + 1}: Found {result.extracted_content.count('.content-item')} items") @@ -102,7 +102,7 @@ async def advanced_session_crawl_with_hooks(): session_id=session_id, css_selector="li.commit-item", js_code=js_next_page if page > 0 else None, - bypass_cache=True, + cache_mode=CacheMode.BYPASS, js_only=page > 0 ) @@ -174,7 +174,7 @@ async def integrated_js_and_wait_crawl(): extraction_strategy=extraction_strategy, js_code=js_next_page_and_wait if page > 0 else None, js_only=page > 0, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) commits = json.loads(result.extracted_content) @@ -241,7 +241,7 @@ async def wait_for_parameter_crawl(): js_code=js_next_page if page > 0 else None, wait_for=wait_for if page > 0 else None, js_only=page > 0, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) commits = json.loads(result.extracted_content) diff --git a/docs/md_v2/advanced/session-management.md b/docs/md_v2/advanced/session-management.md index c38ed852..eae4cf7b 100644 --- a/docs/md_v2/advanced/session-management.md +++ b/docs/md_v2/advanced/session-management.md @@ -75,7 +75,7 @@ async def crawl_dynamic_content(): js_code=js_next_page if page > 0 else None, wait_for=wait_for if page > 0 else None, js_only=page > 0, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) if result.success: diff --git a/docs/md_v2/api/arun.md b/docs/md_v2/api/arun.md index 9ef73aef..509991e5 100644 --- a/docs/md_v2/api/arun.md +++ b/docs/md_v2/api/arun.md @@ -8,11 +8,26 @@ The following parameters can be passed to the `arun()` method. They are organize await crawler.arun( url="https://example.com", # Required: URL to crawl verbose=True, # Enable detailed logging - bypass_cache=False, # Skip cache for this request + cache_mode=CacheMode.ENABLED, # Control cache behavior warmup=True # Whether to run warmup check ) ``` +## Cache Control + +```python +from crawl4ai import CacheMode + +await crawler.arun( + cache_mode=CacheMode.ENABLED, # Normal caching (read/write) + # Other cache modes: + # cache_mode=CacheMode.DISABLED # No caching at all + # cache_mode=CacheMode.READ_ONLY # Only read from cache + # cache_mode=CacheMode.WRITE_ONLY # Only write to cache + # cache_mode=CacheMode.BYPASS # Skip cache for this operation +) +``` + ## Content Processing Parameters ### Text Processing @@ -162,14 +177,13 @@ await crawler.arun( ## Parameter Interactions and Notes -1. **Magic Mode Combinations** +1. **Cache and Performance Setup** ```python - # Full anti-detection setup + # Optimal caching for repeated crawls await crawler.arun( - magic=True, - headless=False, - simulate_user=True, - override_navigator=True + cache_mode=CacheMode.ENABLED, + word_count_threshold=10, + process_iframes=False ) ``` @@ -179,7 +193,8 @@ await crawler.arun( await crawler.arun( js_code="window.scrollTo(0, document.body.scrollHeight);", wait_for="css:.lazy-content", - delay_before_return_html=2.0 + delay_before_return_html=2.0, + cache_mode=CacheMode.WRITE_ONLY # Cache results after dynamic load ) ``` @@ -192,7 +207,8 @@ await crawler.arun( extraction_strategy=my_strategy, chunking_strategy=my_chunking, process_iframes=True, - remove_overlay_elements=True + remove_overlay_elements=True, + cache_mode=CacheMode.ENABLED ) ``` @@ -201,7 +217,7 @@ await crawler.arun( 1. **Performance Optimization** ```python await crawler.arun( - bypass_cache=False, # Use cache when possible + cache_mode=CacheMode.ENABLED, # Use full caching word_count_threshold=10, # Filter out noise process_iframes=False # Skip iframes if not needed ) @@ -212,7 +228,8 @@ await crawler.arun( await crawler.arun( magic=True, # Enable anti-detection delay_before_return_html=1.0, # Wait for dynamic content - page_timeout=60000 # Longer timeout for slow pages + page_timeout=60000, # Longer timeout for slow pages + cache_mode=CacheMode.WRITE_ONLY # Cache results after successful crawl ) ``` @@ -221,6 +238,7 @@ await crawler.arun( await crawler.arun( remove_overlay_elements=True, # Remove popups excluded_tags=['nav', 'aside'],# Remove unnecessary elements - keep_data_attributes=False # Remove data attributes + keep_data_attributes=False, # Remove data attributes + cache_mode=CacheMode.ENABLED # Use cache for faster processing ) ``` \ No newline at end of file diff --git a/docs/md_v2/api/crawl-result.md b/docs/md_v2/api/crawl-result.md index 06998af3..7e3bda98 100644 --- a/docs/md_v2/api/crawl-result.md +++ b/docs/md_v2/api/crawl-result.md @@ -20,6 +20,7 @@ class CrawlResult(BaseModel): fit_html: Optional[str] = None # Most relevant HTML content markdown: Optional[str] = None # HTML converted to markdown fit_markdown: Optional[str] = None # Most relevant markdown content + downloaded_files: Optional[List[str]] = None # Downloaded files # Extracted Data extracted_content: Optional[str] = None # Content from extraction strategy diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md index 6c7960d2..c1c4d2ea 100644 --- a/docs/md_v2/api/parameters.md +++ b/docs/md_v2/api/parameters.md @@ -32,4 +32,5 @@ | async_webcrawler.py | warmup | `kwargs.get("warmup", True)` | AsyncWebCrawler | Initialize crawler with warmup request | | async_webcrawler.py | session_id | `kwargs.get("session_id", None)` | AsyncWebCrawler | Session identifier for browser reuse | | async_webcrawler.py | only_text | `kwargs.get("only_text", False)` | AsyncWebCrawler | Extract only text content | -| async_webcrawler.py | bypass_cache | `kwargs.get("bypass_cache", False)` | AsyncWebCrawler | Skip cache and force fresh crawl | \ No newline at end of file +| async_webcrawler.py | bypass_cache | `kwargs.get("bypass_cache", False)` | AsyncWebCrawler | Skip cache and force fresh crawl | +| async_webcrawler.py | cache_mode | `kwargs.get("cache_mode", CacheMode.ENABLE)` | AsyncWebCrawler | Cache handling mode for request | \ No newline at end of file diff --git a/docs/md_v2/basic/content_filtering.md b/docs/md_v2/basic/content_filtering.md new file mode 100644 index 00000000..9506c075 --- /dev/null +++ b/docs/md_v2/basic/content_filtering.md @@ -0,0 +1,84 @@ +# Content Filtering in Crawl4AI + +This guide explains how to use content filtering strategies in Crawl4AI to extract the most relevant information from crawled web pages. You'll learn how to use the built-in `BM25ContentFilter` and how to create your own custom content filtering strategies. + +## Relevance Content Filter + +The `RelevanceContentFilter` is an abstract class that provides a common interface for content filtering strategies. Specific filtering algorithms, like `BM25ContentFilter`, inherit from this class and implement the `filter_content` method. This method takes the HTML content as input and returns a list of filtered text blocks. + +## BM25 Algorithm + +The `BM25ContentFilter` uses the BM25 algorithm, a ranking function used in information retrieval to estimate the relevance of documents to a given search query. In Crawl4AI, this algorithm helps to identify and extract text chunks that are most relevant to the page's metadata or a user-specified query. + +### Usage + +To use the `BM25ContentFilter`, initialize it and then pass it as the `extraction_strategy` parameter to the `arun` method of the crawler. + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.content_filter_strategy import BM25ContentFilter + +async def filter_content(url, query=None): + async with AsyncWebCrawler() as crawler: + content_filter = BM25ContentFilter(user_query=query) + result = await crawler.arun(url=url, content_filter=content_filter, fit_markdown=True) # Set fit_markdown flag to True to trigger BM25 filtering + if result.success: + print(f"Filtered Content (JSON):\n{result.extracted_content}") + print(f"\nFiltered Markdown:\n{result.fit_markdown}") # New field in CrawlResult object + print(f"\nFiltered HTML:\n{result.fit_html}") # New field in CrawlResult object. Note that raw HTML may have tags re-organized due to internal parsing. + else: + print("Error:", result.error_message) + +# Example usage: +asyncio.run(filter_content("https://en.wikipedia.org/wiki/Apple", "fruit nutrition health")) # with query +asyncio.run(filter_content("https://en.wikipedia.org/wiki/Apple")) # without query, metadata will be used as the query. + +``` + +### Parameters + +- **`user_query`**: (Optional) A string representing the search query. If not provided, the filter extracts relevant metadata (title, description, keywords) from the page and uses that as the query. +- **`bm25_threshold`**: (Optional, default 1.0) A float value that controls the threshold for relevance. Higher values result in stricter filtering, returning only the most relevant text chunks. Lower values result in more lenient filtering. + + +## Fit Markdown Flag + +Setting the `fit_markdown` flag to `True` in the `arun` method activates the BM25 content filtering during the crawl. The `fit_markdown` parameter instructs the scraper to extract and clean the HTML, primarily to prepare for a Large Language Model that cannot process large amounts of data. Setting this flag not only improves the quality of the extracted content but also adds the filtered content to two new attributes in the returned `CrawlResult` object: `fit_markdown` and `fit_html`. + + +## Custom Content Filtering Strategies + +You can create your own custom filtering strategies by inheriting from the `RelevantContentFilter` class and implementing the `filter_content` method. This allows you to tailor the filtering logic to your specific needs. + +```python +from crawl4ai.content_filter_strategy import RelevantContentFilter +from bs4 import BeautifulSoup, Tag +from typing import List + +class MyCustomFilter(RelevantContentFilter): + def filter_content(self, html: str) -> List[str]: + soup = BeautifulSoup(html, 'lxml') + # Implement custom filtering logic here + # Example: extract all paragraphs within divs with class "article-body" + filtered_paragraphs = [] + for tag in soup.select("div.article-body p"): + if isinstance(tag, Tag): + filtered_paragraphs.append(str(tag)) # Add the cleaned HTML element. + return filtered_paragraphs + + + +async def custom_filter_demo(url: str): + async with AsyncWebCrawler() as crawler: + custom_filter = MyCustomFilter() + result = await crawler.arun(url, content_filter=custom_filter) + if result.success: + print(result.extracted_content) + +``` + +This example demonstrates extracting paragraphs from a specific div class. You can customize this logic to implement different filtering strategies, use regular expressions, analyze text density, or apply other relevant techniques. + +## Conclusion + +Content filtering strategies provide a powerful way to refine the output of your crawls. By using `BM25ContentFilter` or creating custom strategies, you can focus on the most pertinent information and improve the efficiency of your data processing pipeline. diff --git a/docs/md_v2/basic/file-download.md b/docs/md_v2/basic/file-download.md new file mode 100644 index 00000000..c37e8812 --- /dev/null +++ b/docs/md_v2/basic/file-download.md @@ -0,0 +1,148 @@ +# Download Handling in Crawl4AI + +This guide explains how to use Crawl4AI to handle file downloads during crawling. You'll learn how to trigger downloads, specify download locations, and access downloaded files. + +## Enabling Downloads + +By default, Crawl4AI does not download files. To enable downloads, set the `accept_downloads` parameter to `True` in either the `AsyncWebCrawler` constructor or the `arun` method. + +```python +from crawl4ai import AsyncWebCrawler + +async def main(): + async with AsyncWebCrawler(accept_downloads=True) as crawler: # Globally enable downloads + # ... your crawling logic ... + +asyncio.run(main()) +``` + +Or, enable it for a specific crawl: + +```python +async def main(): + async with AsyncWebCrawler() as crawler: + result = await crawler.arun(url="...", accept_downloads=True) + # ... +``` + +## Specifying Download Location + +You can specify the download directory using the `downloads_path` parameter. If not provided, Crawl4AI creates a "downloads" directory inside the `.crawl4ai` folder in your home directory. + +```python +import os +from pathlib import Path + +# ... inside your crawl function: + +downloads_path = os.path.join(os.getcwd(), "my_downloads") # Custom download path +os.makedirs(downloads_path, exist_ok=True) + +result = await crawler.arun(url="...", downloads_path=downloads_path, accept_downloads=True) + +# ... +``` + +If you are setting it globally, provide the path to the AsyncWebCrawler: +```python +async def crawl_with_downloads(url: str, download_path: str): + async with AsyncWebCrawler( + accept_downloads=True, + downloads_path=download_path, # or set it on arun + verbose=True + ) as crawler: + result = await crawler.arun(url=url) # you still need to enable downloads per call. + # ... +``` + + + +## Triggering Downloads + +Downloads are typically triggered by user interactions on a web page (e.g., clicking a download button). You can simulate these actions with the `js_code` parameter, injecting JavaScript code to be executed within the browser context. The `wait_for` parameter might also be crucial to allowing sufficient time for downloads to initiate before the crawler proceeds. + +```python +result = await crawler.arun( + url="https://www.python.org/downloads/", + js_code=""" + // Find and click the first Windows installer link + const downloadLink = document.querySelector('a[href$=".exe"]'); + if (downloadLink) { + downloadLink.click(); + } + """, + wait_for=5 # Wait for 5 seconds for the download to start +) +``` + +## Accessing Downloaded Files + +Downloaded file paths are stored in the `downloaded_files` attribute of the returned `CrawlResult` object. This is a list of strings, with each string representing the absolute path to a downloaded file. + +```python +if result.downloaded_files: + print("Downloaded files:") + for file_path in result.downloaded_files: + print(f"- {file_path}") + # Perform operations with downloaded files, e.g., check file size + file_size = os.path.getsize(file_path) + print(f"- File size: {file_size} bytes") +else: + print("No files downloaded.") +``` + + +## Example: Downloading Multiple Files + +```python +import asyncio +import os +from pathlib import Path +from crawl4ai import AsyncWebCrawler + +async def download_multiple_files(url: str, download_path: str): + + async with AsyncWebCrawler( + accept_downloads=True, + downloads_path=download_path, + verbose=True + ) as crawler: + result = await crawler.arun( + url=url, + js_code=""" + // Trigger multiple downloads (example) + const downloadLinks = document.querySelectorAll('a[download]'); // Or a more specific selector + for (const link of downloadLinks) { + link.click(); + await new Promise(r => setTimeout(r, 2000)); // Add a small delay between clicks if needed + } + """, + wait_for=10 # Adjust the timeout to match the expected time for all downloads to start + ) + + if result.downloaded_files: + print("Downloaded files:") + for file in result.downloaded_files: + print(f"- {file}") + else: + print("No files downloaded.") + + +# Example usage +download_path = os.path.join(Path.home(), ".crawl4ai", "downloads") +os.makedirs(download_path, exist_ok=True) # Create directory if it doesn't exist + + +asyncio.run(download_multiple_files("https://www.python.org/downloads/windows/", download_path)) +``` + +## Important Considerations + +- **Browser Context:** Downloads are managed within the browser context. Ensure your `js_code` correctly targets the download triggers on the specific web page. +- **Waiting:** Use `wait_for` to manage the timing of the crawl process if immediate download might not occur. +- **Error Handling:** Implement proper error handling to gracefully manage failed downloads or incorrect file paths. +- **Security:** Downloaded files should be scanned for potential security threats before use. + + + +This guide provides a foundation for handling downloads with Crawl4AI. You can adapt these techniques to manage downloads in various scenarios and integrate them into more complex crawling workflows. diff --git a/docs/md_v2/basic/quickstart.md b/docs/md_v2/basic/quickstart.md index f4904915..95b8a397 100644 --- a/docs/md_v2/basic/quickstart.md +++ b/docs/md_v2/basic/quickstart.md @@ -8,7 +8,7 @@ First, let's import the necessary modules and create an instance of `AsyncWebCra ```python import asyncio -from crawl4ai import AsyncWebCrawler +from crawl4ai import AsyncWebCrawler, CasheMode async def main(): async with AsyncWebCrawler(verbose=True) as crawler: @@ -42,7 +42,7 @@ async def capture_and_save_screenshot(url: str, output_path: str): result = await crawler.arun( url=url, screenshot=True, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) if result.success and result.screenshot: @@ -62,15 +62,15 @@ Crawl4AI supports multiple browser engines. Here's how to use different browsers ```python # Use Firefox async with AsyncWebCrawler(browser_type="firefox", verbose=True, headless=True) as crawler: - result = await crawler.arun(url="https://www.example.com", bypass_cache=True) + result = await crawler.arun(url="https://www.example.com", cache_mode=CacheMode.BYPASS) # Use WebKit async with AsyncWebCrawler(browser_type="webkit", verbose=True, headless=True) as crawler: - result = await crawler.arun(url="https://www.example.com", bypass_cache=True) + result = await crawler.arun(url="https://www.example.com", cache_mode=CacheMode.BYPASS) # Use Chromium (default) async with AsyncWebCrawler(verbose=True, headless=True) as crawler: - result = await crawler.arun(url="https://www.example.com", bypass_cache=True) + result = await crawler.arun(url="https://www.example.com", cache_mode=CacheMode.BYPASS) ``` ### User Simulation ๐ŸŽญ @@ -81,7 +81,7 @@ Simulate real user behavior to avoid detection: async with AsyncWebCrawler(verbose=True, headless=True) as crawler: result = await crawler.arun( url="YOUR-URL-HERE", - bypass_cache=True, + cache_mode=CacheMode.BYPASS, simulate_user=True, # Causes random mouse movements and clicks override_navigator=True # Makes the browser appear more like a real user ) @@ -99,7 +99,7 @@ async def main(): print(f"First crawl result: {result1.markdown[:100]}...") # Force to crawl again - result2 = await crawler.arun(url="https://www.nbcnews.com/business", bypass_cache=True) + result2 = await crawler.arun(url="https://www.nbcnews.com/business", cache_mode=CacheMode.BYPASS) print(f"Second crawl result: {result2.markdown[:100]}...") asyncio.run(main()) @@ -189,7 +189,7 @@ extraction_strategy = LLMExtractionStrategy( async with AsyncWebCrawler() as crawler: result = await crawler.arun( url="https://paulgraham.com/love.html", - bypass_cache=True, + cache_mode=CacheMode.BYPASS, extraction_strategy=extraction_strategy ) ``` @@ -239,7 +239,7 @@ async def crawl_dynamic_content(): js_code=js_next_page if page > 0 else None, wait_for=wait_for if page > 0 else None, js_only=page > 0, - bypass_cache=True, + cache_mode=CacheMode.BYPASS, headless=False, ) @@ -254,7 +254,7 @@ Remove overlay elements and fit content appropriately: async with AsyncWebCrawler(headless=False) as crawler: result = await crawler.arun( url="your-url-here", - bypass_cache=True, + cache_mode=CacheMode.BYPASS, word_count_threshold=10, remove_overlay_elements=True, screenshot=True @@ -282,7 +282,7 @@ async with AsyncWebCrawler() as crawler: result = await crawler.arun( url="https://www.nbcnews.com/business", word_count_threshold=0, - bypass_cache=True, + cache_mode=CacheMode.BYPASS, verbose=False, ) end = time.time() diff --git a/docs/md_v2/basic/simple-crawling.md b/docs/md_v2/basic/simple-crawling.md index 097d5e61..871fa64c 100644 --- a/docs/md_v2/basic/simple-crawling.md +++ b/docs/md_v2/basic/simple-crawling.md @@ -12,7 +12,9 @@ from crawl4ai import AsyncWebCrawler async def main(): async with AsyncWebCrawler() as crawler: - result = await crawler.arun(url="https://example.com") + result = await crawler.arun( + url="https://example.com" + ) print(result.markdown) # Print clean markdown content if __name__ == "__main__": @@ -24,7 +26,7 @@ if __name__ == "__main__": The `arun()` method returns a `CrawlResult` object with several useful properties. Here's a quick overview (see [CrawlResult](../api/crawl-result.md) for complete details): ```python -result = await crawler.arun(url="https://example.com") +result = await crawler.arun(url="https://example.com", fit_markdown=True) # Different content formats print(result.html) # Raw HTML @@ -81,7 +83,7 @@ Here's a more comprehensive example showing common usage patterns: ```python import asyncio -from crawl4ai import AsyncWebCrawler +from crawl4ai import AsyncWebCrawler, CacheMode async def main(): async with AsyncWebCrawler(verbose=True) as crawler: @@ -97,7 +99,7 @@ async def main(): remove_overlay_elements=True, # Cache control - bypass_cache=False # Use cache if available + cache_mode=CacheMode.ENABLE # Use cache if available ) if result.success: diff --git a/docs/md_v2/tutorial/episode_11_2_Extraction_Strategies_LLM.md b/docs/md_v2/tutorial/episode_11_2_Extraction_Strategies_LLM.md index 3682425f..a9f00e92 100644 --- a/docs/md_v2/tutorial/episode_11_2_Extraction_Strategies_LLM.md +++ b/docs/md_v2/tutorial/episode_11_2_Extraction_Strategies_LLM.md @@ -52,7 +52,7 @@ Hereโ€™s a comprehensive outline for the **LLM Extraction Strategy** video, cove extraction_type="schema", instruction="Extract model names and fees for input and output tokens from the page." ), - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` @@ -98,7 +98,7 @@ Hereโ€™s a comprehensive outline for the **LLM Extraction Strategy** video, cove result = await crawler.arun( url="https://example.com/some-article", extraction_strategy=extraction_strategy, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` diff --git a/docs/md_v2/tutorial/episode_11_3_Extraction_Strategies_Cosine.md b/docs/md_v2/tutorial/episode_11_3_Extraction_Strategies_Cosine.md index 9f1c00ea..6100ae4c 100644 --- a/docs/md_v2/tutorial/episode_11_3_Extraction_Strategies_Cosine.md +++ b/docs/md_v2/tutorial/episode_11_3_Extraction_Strategies_Cosine.md @@ -55,7 +55,7 @@ Hereโ€™s a structured outline for the **Cosine Similarity Strategy** video, cove result = await crawler.arun( url=url, extraction_strategy=extraction_strategy, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` @@ -103,7 +103,7 @@ Hereโ€™s a structured outline for the **Cosine Similarity Strategy** video, cove result = await crawler.arun( url=url, extraction_strategy=extraction_strategy, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` diff --git a/docs/md_v2/tutorial/tutorial.md b/docs/md_v2/tutorial/tutorial.md index bf355ed0..7bead842 100644 --- a/docs/md_v2/tutorial/tutorial.md +++ b/docs/md_v2/tutorial/tutorial.md @@ -26,7 +26,7 @@ Here's a condensed outline of the **Installation and Setup** video content: - Walk through a simple test script to confirm the setup: ```python import asyncio - from crawl4ai import AsyncWebCrawler + from crawl4ai import AsyncWebCrawler, CacheMode async def main(): async with AsyncWebCrawler(verbose=True) as crawler: @@ -1093,7 +1093,7 @@ Hereโ€™s a comprehensive outline for the **LLM Extraction Strategy** video, cove extraction_type="schema", instruction="Extract model names and fees for input and output tokens from the page." ), - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` @@ -1139,7 +1139,7 @@ Hereโ€™s a comprehensive outline for the **LLM Extraction Strategy** video, cove result = await crawler.arun( url="https://example.com/some-article", extraction_strategy=extraction_strategy, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` @@ -1248,7 +1248,7 @@ Hereโ€™s a structured outline for the **Cosine Similarity Strategy** video, cove result = await crawler.arun( url=url, extraction_strategy=extraction_strategy, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ``` @@ -1296,7 +1296,7 @@ Hereโ€™s a structured outline for the **Cosine Similarity Strategy** video, cove result = await crawler.arun( url=url, extraction_strategy=extraction_strategy, - bypass_cache=True + cache_mode=CacheMode.BYPASS ) print(result.extracted_content) ```