From 14894b4d7033d619ef616f8b6cae19e896dd6a64 Mon Sep 17 00:00:00 2001 From: UncleCode Date: Thu, 3 Apr 2025 20:34:19 +0800 Subject: [PATCH] feat(config): set DefaultMarkdownGenerator as the default markdown generator in CrawlerRunConfig feat(logger): add color mapping for log message formatting options --- crawl4ai/async_configs.py | 4 ++-- crawl4ai/async_logger.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 8833eea5..2f0efe90 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -15,7 +15,7 @@ from .user_agent_generator import UAGen, ValidUAGenerator # , OnlineUAGenerator from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy from .chunking_strategy import ChunkingStrategy, RegexChunking -from .markdown_generation_strategy import MarkdownGenerationStrategy +from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator from .content_scraping_strategy import ContentScrapingStrategy, WebScrapingStrategy from .deep_crawling import DeepCrawlStrategy @@ -722,7 +722,7 @@ class CrawlerRunConfig(): word_count_threshold: int = MIN_WORD_THRESHOLD, extraction_strategy: ExtractionStrategy = None, chunking_strategy: ChunkingStrategy = RegexChunking(), - markdown_generator: MarkdownGenerationStrategy = None, + markdown_generator: MarkdownGenerationStrategy = DefaultMarkdownGenerator(), only_text: bool = False, css_selector: str = None, target_elements: List[str] = None, diff --git a/crawl4ai/async_logger.py b/crawl4ai/async_logger.py index 6f89c217..273ef53b 100644 --- a/crawl4ai/async_logger.py +++ b/crawl4ai/async_logger.py @@ -156,9 +156,22 @@ class AsyncLogger(AsyncLoggerBase): formatted_message = message.format(**params) # Then apply colors if specified + color_map = { + "green": Fore.GREEN, + "red": Fore.RED, + "yellow": Fore.YELLOW, + "blue": Fore.BLUE, + "cyan": Fore.CYAN, + "magenta": Fore.MAGENTA, + "white": Fore.WHITE, + "black": Fore.BLACK, + "reset": Style.RESET_ALL, + } if colors: for key, color in colors.items(): # Find the formatted value in the message and wrap it with color + if color in color_map: + color = color_map[color] if key in params: value_str = str(params[key]) formatted_message = formatted_message.replace(