diff --git a/CHANGELOG.md b/CHANGELOG.md index 08705a20..90722b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [v0.2.73] - 2024-07-03 + +πŸ’‘ In this release, we've bumped the version to v0.2.73 and refreshed our documentation to ensure you have the best experience with our project. + +* Supporting website need "with-head" mode to crawl the website with head. +* Fixing the installation issues for setup.py and dockerfile. +* Resolve multiple issues. + ## [v0.2.72] - 2024-06-30 This release brings exciting updates and improvements to our project! πŸŽ‰ diff --git a/README.md b/README.md index ed3c67f0..cf4e4760 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Crawl4AI v0.2.72 πŸ•·οΈπŸ€– +# Crawl4AI v0.2.73 πŸ•·οΈπŸ€– [![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers) [![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members) diff --git a/crawl4ai/crawler_strategy.py b/crawl4ai/crawler_strategy.py index cd94e9e7..21de883e 100644 --- a/crawl4ai/crawler_strategy.py +++ b/crawl4ai/crawler_strategy.py @@ -9,7 +9,8 @@ from selenium.common.exceptions import InvalidArgumentException, WebDriverExcept from selenium.webdriver.chrome.service import Service as ChromeService from webdriver_manager.chrome import ChromeDriverManager -import logging +from .config import * +import logging, time import base64 from PIL import Image, ImageDraw, ImageFont from io import BytesIO @@ -177,8 +178,20 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy): # Set extra HTTP headers self.driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {'headers': headers}) + def _ensure_page_load(self, max_checks=6, check_interval=0.01): + initial_length = len(self.driver.page_source) + + for ix in range(max_checks): + # print(f"Checking page load: {ix}") + time.sleep(check_interval) + current_length = len(self.driver.page_source) + + if current_length != initial_length: + break - def crawl(self, url: str) -> str: + return self.driver.page_source + + def crawl(self, url: str, **kwargs) -> str: # Create md5 hash of the URL import hashlib url_hash = hashlib.md5(url.encode()).hexdigest() @@ -194,18 +207,24 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy): if self.verbose: print(f"[LOG] πŸ•ΈοΈ Crawling {url} using LocalSeleniumCrawlerStrategy...") self.driver.get(url) # - html = self.driver.page_source + + WebDriverWait(self.driver, 20).until( + lambda d: d.execute_script('return document.readyState') == 'complete' + ) WebDriverWait(self.driver, 10).until( EC.presence_of_all_elements_located((By.TAG_NAME, "body")) ) + self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") + html = self._ensure_page_load() # self.driver.page_source can_not_be_done_headless = False # Look at my creativity for naming variables # TODO: Very ugly way for now but it works - if html == "": + if not kwargs.get('bypass_headless', False) and html == "": + print("[LOG] πŸ™Œ Page could not be loaded in headless mode. Trying non-headless mode...") can_not_be_done_headless = True options = Options() options.headless = False # set window size very small - options.add_argument("--window-size=10,10") + options.add_argument("--window-size=5,5") driver = webdriver.Chrome(service=self.service, options=options) driver.get(url) html = driver.page_source diff --git a/crawl4ai/extraction_strategy.py b/crawl4ai/extraction_strategy.py index f635f60b..d4415c88 100644 --- a/crawl4ai/extraction_strategy.py +++ b/crawl4ai/extraction_strategy.py @@ -101,7 +101,7 @@ class LLMExtractionStrategy(ExtractionStrategy): prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION if self.extract_type == "schema": - variable_values["SCHEMA"] = json.dumps(self.schema) + variable_values["SCHEMA"] = json.dumps(self.schema, indent=2) prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION for variable in variable_values: @@ -109,7 +109,7 @@ class LLMExtractionStrategy(ExtractionStrategy): "{" + variable + "}", variable_values[variable] ) - response = perform_completion_with_backoff(self.provider, prompt_with_variables, self.api_token) + response = perform_completion_with_backoff(self.provider, prompt_with_variables, self.api_token) # , json_response=self.extract_type == "schema") try: blocks = extract_xml_data(["blocks"], response.choices[0].message.content)['blocks'] blocks = json.loads(blocks) @@ -196,6 +196,10 @@ class LLMExtractionStrategy(ExtractionStrategy): time.sleep(0.5) # 500 ms delay between each processing else: # Parallel processing using ThreadPoolExecutor + # extract_func = partial(self.extract, url) + # for ix, section in enumerate(merged_sections): + # extracted_content.append(extract_func(ix, section)) + with ThreadPoolExecutor(max_workers=4) as executor: extract_func = partial(self.extract, url) futures = [executor.submit(extract_func, ix, section) for ix, section in enumerate(merged_sections)] diff --git a/crawl4ai/prompts.py b/crawl4ai/prompts.py index 39de7e3b..323c4774 100644 --- a/crawl4ai/prompts.py +++ b/crawl4ai/prompts.py @@ -186,7 +186,7 @@ The user has made the following request for what information to extract from the Please carefully read the URL content and the user's request. If the user provided a desired JSON schema in the above, extract the requested information from the URL content according to that schema. If no schema was provided, infer an appropriate JSON schema based on the user's request that will best capture the key information they are looking for. Extraction instructions: -Return the extracted information as a list of JSON objects, with each object in the list corresponding to a block of content from the URL, in the same order as it appears on the page. Wrap the entire JSON list in tags. +Return the extracted information as a list of JSON objects, with each object in the list corresponding to a block of content from the URL, in the same order as it appears on the page. Wrap the entire JSON list in ... XML tags. Quality Reflection: Before outputting your final answer, double check that the JSON you are returning is complete, containing all the information requested by the user, and is valid JSON that could be parsed by json.loads() with no errors or omissions. The outputted JSON objects should fully match the schema, either provided or inferred. @@ -194,5 +194,11 @@ Before outputting your final answer, double check that the JSON you are returnin Quality Score: After reflecting, score the quality and completeness of the JSON data you are about to return on a scale of 1 to 5. Write the score inside tags. +Avoid Common Mistakes: +- Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors. +- Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places. +- Do not miss closing tag at the end of the JSON output. +- Do not generate the Python coee show me how to do the task, this is your task to extract the information and return it in JSON format. + Result -Output the final list of JSON objects, wrapped in tags.""" \ No newline at end of file +Output the final list of JSON objects, wrapped in ... XML tags. Make sure to close the tag properly.""" \ No newline at end of file diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index c468c49a..474ce395 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -419,7 +419,6 @@ def get_content_of_website(url, html, word_count_threshold = MIN_WORD_THRESHOLD, print('Error processing HTML content:', str(e)) raise InvalidCSSSelectorError(f"Invalid CSS selector: {css_selector}") from e - def get_content_of_website_optimized(url: str, html: str, word_count_threshold: int = MIN_WORD_THRESHOLD, css_selector: str = None, **kwargs) -> Dict[str, Any]: if not html: return None @@ -439,71 +438,75 @@ def get_content_of_website_optimized(url: str, html: str, word_count_threshold: media = {'images': [], 'videos': [], 'audios': []} def process_element(element: element.PageElement) -> bool: - if isinstance(element, NavigableString): - if isinstance(element, Comment): - element.extract() - return False + try: + if isinstance(element, NavigableString): + if isinstance(element, Comment): + element.extract() + return False - if element.name in ['script', 'style', 'link', 'meta', 'noscript']: - element.decompose() - return False + if element.name in ['script', 'style', 'link', 'meta', 'noscript']: + element.decompose() + return False - keep_element = False + keep_element = False - if element.name == 'a' and element.get('href'): - href = element['href'] - url_base = url.split('/')[2] - link_data = {'href': href, 'text': element.get_text()} - if href.startswith('http') and url_base not in href: - links['external'].append(link_data) - else: - links['internal'].append(link_data) - keep_element = True - - elif element.name == 'img': - media['images'].append({ - 'src': element.get('src'), - 'alt': element.get('alt'), - 'type': 'image' - }) - return True # Always keep image elements - - elif element.name in ['video', 'audio']: - media[f"{element.name}s"].append({ - 'src': element.get('src'), - 'alt': element.get('alt'), - 'type': element.name - }) - return True # Always keep video and audio elements - - if element.name != 'pre': - if element.name in ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark']: - if kwargs.get('only_text', False): - element.replace_with(element.get_text()) + if element.name == 'a' and element.get('href'): + href = element['href'] + url_base = url.split('/')[2] + link_data = {'href': href, 'text': element.get_text()} + if href.startswith('http') and url_base not in href: + links['external'].append(link_data) else: - element.unwrap() - elif element.name != 'img': - element.attrs = {} + links['internal'].append(link_data) + keep_element = True - # Process children - for child in list(element.children): - if isinstance(child, NavigableString) and not isinstance(child, Comment): - if len(child.strip()) > 0: - keep_element = True - else: - if process_element(child): - keep_element = True - + elif element.name == 'img': + media['images'].append({ + 'src': element.get('src'), + 'alt': element.get('alt'), + 'type': 'image' + }) + return True # Always keep image elements - # Check word count - if not keep_element: - word_count = len(element.get_text(strip=True).split()) - keep_element = word_count >= word_count_threshold + elif element.name in ['video', 'audio']: + media[f"{element.name}s"].append({ + 'src': element.get('src'), + 'alt': element.get('alt'), + 'type': element.name + }) + return True # Always keep video and audio elements - if not keep_element: - element.decompose() + if element.name != 'pre': + if element.name in ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark']: + if kwargs.get('only_text', False): + element.replace_with(element.get_text()) + else: + element.unwrap() + elif element.name != 'img': + element.attrs = {} - return keep_element + # Process children + for child in list(element.children): + if isinstance(child, NavigableString) and not isinstance(child, Comment): + if len(child.strip()) > 0: + keep_element = True + else: + if process_element(child): + keep_element = True + + + # Check word count + if not keep_element: + word_count = len(element.get_text(strip=True).split()) + keep_element = word_count >= word_count_threshold + + if not keep_element: + element.decompose() + + return keep_element + except Exception as e: + print('Error processing element:', str(e)) + return False process_element(body) @@ -540,7 +543,6 @@ def get_content_of_website_optimized(url: str, html: str, word_count_threshold: 'metadata': meta } - def extract_metadata(html, soup = None): metadata = {} @@ -599,12 +601,16 @@ def extract_xml_data(tags, string): return data # Function to perform the completion with exponential backoff -def perform_completion_with_backoff(provider, prompt_with_variables, api_token): +def perform_completion_with_backoff(provider, prompt_with_variables, api_token, json_response = False): from litellm import completion from litellm.exceptions import RateLimitError max_attempts = 3 base_delay = 2 # Base delay in seconds, you can adjust this based on your needs + extra_args = {} + if json_response: + extra_args["response_format"] = { "type": "json_object" } + for attempt in range(max_attempts): try: response =completion( @@ -613,7 +619,8 @@ def perform_completion_with_backoff(provider, prompt_with_variables, api_token): {"role": "user", "content": prompt_with_variables} ], temperature=0.01, - api_key=api_token + api_key=api_token, + **extra_args ) return response # Return the successful response except RateLimitError as e: diff --git a/crawl4ai/web_crawler.py b/crawl4ai/web_crawler.py index ef85066e..954e9b84 100644 --- a/crawl4ai/web_crawler.py +++ b/crawl4ai/web_crawler.py @@ -11,6 +11,8 @@ from .crawler_strategy import * from typing import List from concurrent.futures import ThreadPoolExecutor from .config import * +import warnings +warnings.filterwarnings("ignore", message='Field "model_name" has conflict with protected namespace "model_".') class WebCrawler: @@ -164,7 +166,7 @@ class WebCrawler: if user_agent: self.crawler_strategy.update_user_agent(user_agent) t1 = time.time() - html = self.crawler_strategy.crawl(url) + html = self.crawler_strategy.crawl(url, **kwargs) t2 = time.time() if verbose: print(f"[LOG] πŸš€ Crawling done for {url}, success: {bool(html)}, time taken: {t2 - t1} seconds") diff --git a/docs/md/changelog.md b/docs/md/changelog.md index 8a2e929e..3796d309 100644 --- a/docs/md/changelog.md +++ b/docs/md/changelog.md @@ -1,5 +1,13 @@ # Changelog +## [v0.2.73] - 2024-07-03 + +πŸ’‘ In this release, we've bumped the version to v0.2.73 and refreshed our documentation to ensure you have the best experience with our project. + +* Supporting website need "with-head" mode to crawl the website with head. +* Fixing the installation issues for setup.py and dockerfile. +* Resolve multiple issues. + ## [v0.2.72] - 2024-06-30 This release brings exciting updates and improvements to our project! πŸŽ‰ diff --git a/docs/md/index.md b/docs/md/index.md index 21fcdeb0..b08fdd12 100644 --- a/docs/md/index.md +++ b/docs/md/index.md @@ -1,4 +1,4 @@ -# Crawl4AI v0.2.72 +# Crawl4AI v0.2.73 Welcome to the official documentation for Crawl4AI! πŸ•·οΈπŸ€– Crawl4AI is an open-source Python library designed to simplify web crawling and extract useful information from web pages. This documentation will guide you through the features, usage, and customization of Crawl4AI. diff --git a/setup.py b/setup.py index cfbeaddf..468dc56e 100644 --- a/setup.py +++ b/setup.py @@ -18,17 +18,11 @@ default_requirements = [req for req in requirements if not req.startswith(("torc torch_requirements = [req for req in requirements if req.startswith(("torch", "nltk", "spacy", "scikit-learn", "numpy"))] transformer_requirements = [req for req in requirements if req.startswith(("transformers", "tokenizers", "onnxruntime"))] -class CustomInstallCommand(install): - """Customized setuptools install command to install spacy without dependencies.""" - def run(self): - install.run(self) - subprocess.check_call([os.sys.executable, '-m', 'pip', 'install', 'spacy', '--no-deps']) - setup( name="Crawl4AI", - version="0.2.72", + version="0.2.73", description="πŸ”₯πŸ•·οΈ Crawl4AI: Open-source LLM Friendly Web Crawler & Scrapper", - long_description=open("README.md").read(), + long_description=open("README.md", encoding="utf-8").read(), long_description_content_type="text/markdown", url="https://github.com/unclecode/crawl4ai", author="Unclecode", @@ -41,9 +35,6 @@ setup( "transformer": transformer_requirements, "all": requirements, }, - cmdclass={ - 'install': CustomInstallCommand, - }, entry_points={ 'console_scripts': [ 'crawl4ai-download-models=crawl4ai.model_loader:main',