ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor content_scrapping_strategy.py to remove excluded tags

Browse Source
This commit is contained in:
unclecode
2024-09-12 17:35:45 +08:00
parent b179aa9b6f
commit d628bc4034
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl4ai/content_scrapping_strategy.py
View File

@@ -43,6 +43,10 @@ class WebScrappingStrategy(ContentScrappingStrategy):
image_description_min_word_threshold = kwargs.get('image_description_min_word_threshold', IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD) image_description_min_word_threshold = kwargs.get('image_description_min_word_threshold', IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD)
for tag in kwargs.get('excluded_tags', []) or []:
for el in body.select(tag):
el.decompose()
if css_selector: if css_selector:
selected_elements = body.select(css_selector) selected_elements = body.select(css_selector)
if not selected_elements: if not selected_elements: