ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(config): set DefaultMarkdownGenerator as the default markdown generator in CrawlerRunConfig

Browse Source 
feat(logger): add color mapping for log message formatting options
This commit is contained in:
UncleCode
2025-04-03 20:34:19 +08:00
parent 86df20234b
commit 14894b4d70
2 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl4ai/async_configs.py
View File

@@ -15,7 +15,7 @@ from .user_agent_generator import UAGen, ValidUAGenerator # , OnlineUAGenerator
from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
from .chunking_strategy import ChunkingStrategy, RegexChunking from .chunking_strategy import ChunkingStrategy, RegexChunking
from .markdown_generation_strategy import MarkdownGenerationStrategy from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator
from .content_scraping_strategy import ContentScrapingStrategy, WebScrapingStrategy from .content_scraping_strategy import ContentScrapingStrategy, WebScrapingStrategy
from .deep_crawling import DeepCrawlStrategy from .deep_crawling import DeepCrawlStrategy
@@ -722,7 +722,7 @@ class CrawlerRunConfig():
word_count_threshold: int = MIN_WORD_THRESHOLD, word_count_threshold: int = MIN_WORD_THRESHOLD,
extraction_strategy: ExtractionStrategy = None, extraction_strategy: ExtractionStrategy = None,
chunking_strategy: ChunkingStrategy = RegexChunking(), chunking_strategy: ChunkingStrategy = RegexChunking(),
markdown_generator: MarkdownGenerationStrategy = None, markdown_generator: MarkdownGenerationStrategy = DefaultMarkdownGenerator(),
only_text: bool = False, only_text: bool = False,
css_selector: str = None, css_selector: str = None,
target_elements: List[str] = None, target_elements: List[str] = None,

13
crawl4ai/async_logger.py
View File

@@ -156,9 +156,22 @@ class AsyncLogger(AsyncLoggerBase):
formatted_message = message.format(**params) formatted_message = message.format(**params)
# Then apply colors if specified # Then apply colors if specified
color_map = {
"green": Fore.GREEN,
"red": Fore.RED,
"yellow": Fore.YELLOW,
"blue": Fore.BLUE,
"cyan": Fore.CYAN,
"magenta": Fore.MAGENTA,
"white": Fore.WHITE,
"black": Fore.BLACK,
"reset": Style.RESET_ALL,
}
if colors: if colors:
for key, color in colors.items(): for key, color in colors.items():
# Find the formatted value in the message and wrap it with color # Find the formatted value in the message and wrap it with color
if color in color_map:
color = color_map[color]
if key in params: if key in params:
value_str = str(params[key]) value_str = str(params[key])
formatted_message = formatted_message.replace( formatted_message = formatted_message.replace(