ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: handled the cases where markdown_with_citations, references_markdown, and filtered_html might not be defined. (#293)

Browse Source 
Thanks, dear Farhan, for the changes you made in the code. I accepted and merged them into the main branch. Also, I will add your name to our contributor list. Thank you so much.
This commit is contained in:
Hamza Farhan
2024-11-27 16:20:54 +05:00
committed by GitHub
parent 73661f7d1f
commit f998e9e949
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crawl4ai/markdown_generation_strategy.py
View File

@@ -84,6 +84,8 @@ class DefaultMarkdownGenerationStrategy(MarkdownGenerationStrategy):
raw_markdown = raw_markdown.replace(' ```', '```') raw_markdown = raw_markdown.replace(' ```', '```')
# Convert links to citations # Convert links to citations
markdown_with_citations: str = ""
references_markdown: str = ""
if citations: if citations:
markdown_with_citations, references_markdown = self.convert_links_to_citations( markdown_with_citations, references_markdown = self.convert_links_to_citations(
raw_markdown, base_url raw_markdown, base_url
@@ -91,9 +93,9 @@ class DefaultMarkdownGenerationStrategy(MarkdownGenerationStrategy):
# Generate fit markdown if content filter is provided # Generate fit markdown if content filter is provided
fit_markdown: Optional[str] = None fit_markdown: Optional[str] = None
filtered_html: Optional[str] = None
if content_filter: if content_filter:
filtered_html = content_filter.filter_content(cleaned_html) filtered_html = '\n'.join('<div>{}</div>'.format(s) for s in content_filter.filter_content(cleaned_html))
filtered_html = '\n'.join('<div>{}</div>'.format(s) for s in filtered_html)
fit_markdown = h.handle(filtered_html) fit_markdown = h.handle(filtered_html)
return MarkdownGenerationResult( return MarkdownGenerationResult(
@@ -101,7 +103,7 @@ class DefaultMarkdownGenerationStrategy(MarkdownGenerationStrategy):
markdown_with_citations=markdown_with_citations, markdown_with_citations=markdown_with_citations,
references_markdown=references_markdown, references_markdown=references_markdown,
fit_markdown=fit_markdown, fit_markdown=fit_markdown,
fit_html=filtered_html fit_html=filtered_html,
) )
def fast_urljoin(base: str, url: str) -> str: def fast_urljoin(base: str, url: str) -> str: