-
+
-
-
-
-
-
+
🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & Scrapper
+ +
+
+ 🔥🕷️ Crawl4AI: Web Data for your Thoughts
+
+ 📊 Total Website Processed
+ 2
- 📊 Total Website Procceced
- 0
-
- Try It Now
-
-
-
-
-
-
-
-
-
+
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- - Depends on the selected model, it may take up to 1 or 2 minutes to process the request. - Loading... -
-
+
+
+
+
+
+
+
+
+
+
+
-
+
-
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/test_web_crawler.py b/tests/test_web_crawler.py
new file mode 100644
index 00000000..99360f42
--- /dev/null
+++ b/tests/test_web_crawler.py
@@ -0,0 +1,111 @@
+import unittest, os
+from crawl4ai.web_crawler import WebCrawler
+from crawl4ai.chunking_strategy import RegexChunking, FixedLengthWordChunking, SlidingWindowChunking
+from crawl4ai.extraction_strategy import CosineStrategy, LLMExtractionStrategy, TopicExtractionStrategy, NoExtractionStrategy
+
+class TestWebCrawler(unittest.TestCase):
+
+ def setUp(self):
+ self.crawler = WebCrawler()
+
+ def test_warmup(self):
+ self.crawler.warmup()
+ self.assertTrue(self.crawler.ready, "WebCrawler failed to warm up")
+
+ def test_run_default_strategies(self):
+ result = self.crawler.run(
+ url='https://www.nbcnews.com/business',
+ word_count_threshold=5,
+ chunking_strategy=RegexChunking(),
+ extraction_strategy=CosineStrategy(), bypass_cache=True
+ )
+ self.assertTrue(result.success, "Failed to crawl and extract using default strategies")
+
+ def test_run_different_strategies(self):
+ url = 'https://www.nbcnews.com/business'
+
+ # Test with FixedLengthWordChunking and LLMExtractionStrategy
+ result = self.crawler.run(
+ url=url,
+ word_count_threshold=5,
+ chunking_strategy=FixedLengthWordChunking(chunk_size=100),
+ extraction_strategy=LLMExtractionStrategy(provider="openai/gpt-3.5-turbo", api_token=os.getenv('OPENAI_API_KEY')), bypass_cache=True
+ )
+ self.assertTrue(result.success, "Failed to crawl and extract with FixedLengthWordChunking and LLMExtractionStrategy")
+
+ # Test with SlidingWindowChunking and TopicExtractionStrategy
+ result = self.crawler.run(
+ url=url,
+ word_count_threshold=5,
+ chunking_strategy=SlidingWindowChunking(window_size=100, step=50),
+ extraction_strategy=TopicExtractionStrategy(num_keywords=5), bypass_cache=True
+ )
+ self.assertTrue(result.success, "Failed to crawl and extract with SlidingWindowChunking and TopicExtractionStrategy")
+
+ def test_invalid_url(self):
+ with self.assertRaises(Exception) as context:
+ self.crawler.run(url='invalid_url', bypass_cache=True)
+ self.assertIn("Invalid URL", str(context.exception))
+
+ def test_unsupported_extraction_strategy(self):
+ with self.assertRaises(Exception) as context:
+ self.crawler.run(url='https://www.nbcnews.com/business', extraction_strategy="UnsupportedStrategy", bypass_cache=True)
+ self.assertIn("Unsupported extraction strategy", str(context.exception))
+
+ def test_invalid_css_selector(self):
+ with self.assertRaises(ValueError) as context:
+ self.crawler.run(url='https://www.nbcnews.com/business', css_selector="invalid_selector", bypass_cache=True)
+ self.assertIn("Invalid CSS selector", str(context.exception))
+
+
+ def test_crawl_with_cache_and_bypass_cache(self):
+ url = 'https://www.nbcnews.com/business'
+
+ # First crawl with cache enabled
+ result = self.crawler.run(url=url, bypass_cache=False)
+ self.assertTrue(result.success, "Failed to crawl and cache the result")
+
+ # Second crawl with bypass_cache=True
+ result = self.crawler.run(url=url, bypass_cache=True)
+ self.assertTrue(result.success, "Failed to bypass cache and fetch fresh data")
+
+ def test_fetch_multiple_pages(self):
+ urls = [
+ 'https://www.nbcnews.com/business',
+ 'https://www.bbc.com/news'
+ ]
+ results = []
+ for url in urls:
+ result = self.crawler.run(
+ url=url,
+ word_count_threshold=5,
+ chunking_strategy=RegexChunking(),
+ extraction_strategy=CosineStrategy(),
+ bypass_cache=True
+ )
+ results.append(result)
+
+ self.assertEqual(len(results), 2, "Failed to crawl and extract multiple pages")
+ for result in results:
+ self.assertTrue(result.success, "Failed to crawl and extract a page in the list")
+
+ def test_run_fixed_length_word_chunking_and_no_extraction(self):
+ result = self.crawler.run(
+ url='https://www.nbcnews.com/business',
+ word_count_threshold=5,
+ chunking_strategy=FixedLengthWordChunking(chunk_size=100),
+ extraction_strategy=NoExtractionStrategy(), bypass_cache=True
+ )
+ self.assertTrue(result.success, "Failed to crawl and extract with FixedLengthWordChunking and NoExtractionStrategy")
+
+ def test_run_sliding_window_and_no_extraction(self):
+ result = self.crawler.run(
+ url='https://www.nbcnews.com/business',
+ word_count_threshold=5,
+ chunking_strategy=SlidingWindowChunking(window_size=100, step=50),
+ extraction_strategy=NoExtractionStrategy(), bypass_cache=True
+ )
+ self.assertTrue(result.success, "Failed to crawl and extract with SlidingWindowChunking and NoExtractionStrategy")
+
+if __name__ == '__main__':
+ unittest.main()
+
- Loading... Please wait.
+
-
-
+
+
+
+
-
+
-
+
-
+
-
+
+
Installation 💻
+There are two ways to use Crawl4AI: as a library in your Python projects or as a standalone local server.
+ +You can also try Crawl4AI in a Google Colab 
Using Crawl4AI as a Library 📚
+To install Crawl4AI as a library, follow these steps:
+ +-
+
-
+ Install the package from GitHub:
+
+pip install git+https://github.com/unclecode/crawl4ai.git
+ -
+ Alternatively, you can clone the repository and install the package locally:
+
+virtualenv venv +source venv/bin/activate +git clone https://github.com/unclecode/crawl4ai.git +cd crawl4ai +pip install -e . +
+ -
+ Import the necessary modules in your Python script:
+
+from crawl4ai.web_crawler import WebCrawler +from crawl4ai.chunking_strategy import * +from crawl4ai.extraction_strategy import * +import os -+crawler = WebCrawler() + +# Single page crawl +single_url = UrlModel(url='https://www.nbcnews.com/business', forced=False) +result = crawl4ai.fetch_page( + url='https://www.nbcnews.com/business', + word_count_threshold=5, # Minimum word count for a HTML tag to be considered as a worthy block + chunking_strategy= RegexChunking( patterns = ["\\n\\n"]), # Default is RegexChunking + extraction_strategy= CosineStrategy(word_count_threshold=20, max_dist=0.2, linkage_method='ward', top_k=3) # Default is CosineStrategy + # extraction_strategy= LLMExtractionStrategy(provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY')), + bypass_cache=False, + extract_blocks =True, # Whether to extract semantical blocks of text from the HTML + css_selector = "", # Eg: "div.article-body" + verbose=True, + include_raw_html=True, # Whether to include the raw HTML content in the response +) +print(result.model_dump()) +
+
For more information about how to run Crawl4AI as a local server, please refer to the GitHub repository.
+ +📖 Parameters
+
+
+
+
+
+ | Parameter | +Description | +Required | +Default Value | +
|---|---|---|---|
| urls | ++ A list of URLs to crawl and extract data from. + | +Yes | +- | +
| include_raw_html | ++ Whether to include the raw HTML content in the response. + | +No | +false | +
| bypass_cache | ++ Whether to force a fresh crawl even if the URL has been previously crawled. + | +No | +false | +
| extract_blocks | ++ Whether to extract semantical blocks of text from the HTML. + | +No | +true | +
| word_count_threshold | ++ The minimum number of words a block must contain to be considered meaningful (minimum + value is 5). + | +No | +5 | +
| extraction_strategy | ++ The strategy to use for extracting content from the HTML (e.g., "CosineStrategy"). + | +No | +CosineStrategy | +
| chunking_strategy | ++ The strategy to use for chunking the text before processing (e.g., "RegexChunking"). + | +No | +RegexChunking | +
| css_selector | ++ The CSS selector to target specific parts of the HTML for extraction. + | +No | +None | +
| verbose | +Whether to enable verbose logging. | +No | +true | +
+
+ Extraction Strategies
+ +
+
+ Chunking Strategies
+ +🤔 Why building this?
@@ -192,7 +504,7 @@
⚙️ Installation
@@ -202,7 +514,7 @@