s > 9 correctly.
+ parent_list = None
+ for list in self.list:
+ self.o(
+ " " if parent_list == "ol" and list.name == "ul" else " "
+ )
+ parent_list = list.name
+
+ if li.name == "ul":
+ self.o(self.ul_item_mark + " ")
+ elif li.name == "ol":
+ li.num += 1
+ self.o(str(li.num) + ". ")
+ self.start = True
+
+ if tag in ["table", "tr", "td", "th"]:
+ if self.ignore_tables:
+ if tag == "tr":
+ if start:
+ pass
+ else:
+ self.soft_br()
+ else:
+ pass
+
+ elif self.bypass_tables:
+ if start:
+ self.soft_br()
+ if tag in ["td", "th"]:
+ if start:
+ self.o("<{}>\n\n".format(tag))
+ else:
+ self.o("\n".format(tag))
+ else:
+ if start:
+ self.o("<{}>".format(tag))
+ else:
+ self.o("".format(tag))
+
+ else:
+ if tag == "table":
+ if start:
+ self.table_start = True
+ if self.pad_tables:
+ self.o("<" + config.TABLE_MARKER_FOR_PAD + ">")
+ self.o(" \n")
+ else:
+ if self.pad_tables:
+ # add break in case the table is empty or its 1 row table
+ self.soft_br()
+ self.o("")
+ self.o(" \n")
+ if tag in ["td", "th"] and start:
+ if self.split_next_td:
+ self.o("| ")
+ self.split_next_td = True
+
+ if tag == "tr" and start:
+ self.td_count = 0
+ if tag == "tr" and not start:
+ self.split_next_td = False
+ self.soft_br()
+ if tag == "tr" and not start and self.table_start:
+ # Underline table header
+ self.o("|".join(["---"] * self.td_count))
+ self.soft_br()
+ self.table_start = False
+ if tag in ["td", "th"] and start:
+ self.td_count += 1
+
+ if tag == "pre":
+ if start:
+ self.startpre = True
+ self.pre = True
+ else:
+ self.pre = False
+ if self.mark_code:
+ self.out("\n[/code]")
+ self.p()
+
+ if tag in ["sup", "sub"] and self.include_sup_sub:
+ if start:
+ self.o("<{}>".format(tag))
+ else:
+ self.o("".format(tag))
+
+ # TODO: Add docstring for these one letter functions
+ def pbr(self) -> None:
+ "Pretty print has a line break"
+ if self.p_p == 0:
+ self.p_p = 1
+
+ def p(self) -> None:
+ "Set pretty print to 1 or 2 lines"
+ self.p_p = 1 if self.single_line_break else 2
+
+ def soft_br(self) -> None:
+ "Soft breaks"
+ self.pbr()
+ self.br_toggle = " "
+
+ def o(
+ self, data: str, puredata: bool = False, force: Union[bool, str] = False
+ ) -> None:
+ """
+ Deal with indentation and whitespace
+ """
+ if self.abbr_data is not None:
+ self.abbr_data += data
+
+ if not self.quiet:
+ if self.google_doc:
+ # prevent white space immediately after 'begin emphasis'
+ # marks ('**' and '_')
+ lstripped_data = data.lstrip()
+ if self.drop_white_space and not (self.pre or self.code):
+ data = lstripped_data
+ if lstripped_data != "":
+ self.drop_white_space = 0
+
+ if puredata and not self.pre:
+ # This is a very dangerous call ... it could mess up
+ # all handling of when not handled properly
+ # (see entityref)
+ data = re.sub(r"\s+", r" ", data)
+ if data and data[0] == " ":
+ self.space = True
+ data = data[1:]
+ if not data and not force:
+ return
+
+ if self.startpre:
+ # self.out(" :") #TODO: not output when already one there
+ if not data.startswith("\n") and not data.startswith("\r\n"):
+ # stuff...
+ data = "\n" + data
+ if self.mark_code:
+ self.out("\n[code]")
+ self.p_p = 0
+
+ bq = ">" * self.blockquote
+ if not (force and data and data[0] == ">") and self.blockquote:
+ bq += " "
+
+ if self.pre:
+ if not self.list:
+ bq += " "
+ # else: list content is already partially indented
+ bq += " " * len(self.list)
+ data = data.replace("\n", "\n" + bq)
+
+ if self.startpre:
+ self.startpre = False
+ if self.list:
+ # use existing initial indentation
+ data = data.lstrip("\n")
+
+ if self.start:
+ self.space = False
+ self.p_p = 0
+ self.start = False
+
+ if force == "end":
+ # It's the end.
+ self.p_p = 0
+ self.out("\n")
+ self.space = False
+
+ if self.p_p:
+ self.out((self.br_toggle + "\n" + bq) * self.p_p)
+ self.space = False
+ self.br_toggle = ""
+
+ if self.space:
+ if not self.lastWasNL:
+ self.out(" ")
+ self.space = False
+
+ if self.a and (
+ (self.p_p == 2 and self.links_each_paragraph) or force == "end"
+ ):
+ if force == "end":
+ self.out("\n")
+
+ newa = []
+ for link in self.a:
+ if self.outcount > link.outcount:
+ self.out(
+ " ["
+ + str(link.count)
+ + "]: "
+ + urlparse.urljoin(self.baseurl, link.attrs["href"])
+ )
+ if "title" in link.attrs and link.attrs["title"] is not None:
+ self.out(" (" + link.attrs["title"] + ")")
+ self.out("\n")
+ else:
+ newa.append(link)
+
+ # Don't need an extra line when nothing was done.
+ if self.a != newa:
+ self.out("\n")
+
+ self.a = newa
+
+ if self.abbr_list and force == "end":
+ for abbr, definition in self.abbr_list.items():
+ self.out(" *[" + abbr + "]: " + definition + "\n")
+
+ self.p_p = 0
+ self.out(data)
+ self.outcount += 1
+
+ def handle_data(self, data: str, entity_char: bool = False) -> None:
+ if not data:
+ # Data may be empty for some HTML entities. For example,
+ # LEFT-TO-RIGHT MARK.
+ return
+
+ if self.stressed:
+ data = data.strip()
+ self.stressed = False
+ self.preceding_stressed = True
+ elif self.preceding_stressed:
+ if (
+ re.match(r"[^][(){}\s.!?]", data[0])
+ and not hn(self.current_tag)
+ and self.current_tag not in ["a", "code", "pre"]
+ ):
+ # should match a letter or common punctuation
+ data = " " + data
+ self.preceding_stressed = False
+
+ if self.style:
+ self.style_def.update(dumb_css_parser(data))
+
+ if self.maybe_automatic_link is not None:
+ href = self.maybe_automatic_link
+ if (
+ href == data
+ and self.absolute_url_matcher.match(href)
+ and self.use_automatic_links
+ ):
+ self.o("<" + data + ">")
+ self.empty_link = False
+ return
+ else:
+ self.o("[")
+ self.maybe_automatic_link = None
+ self.empty_link = False
+
+ if not self.code and not self.pre and not entity_char:
+ data = escape_md_section(data, snob=self.escape_snob, escape_dot=self.escape_dot, escape_plus=self.escape_plus, escape_dash=self.escape_dash)
+ self.preceding_data = data
+ self.o(data, puredata=True)
+
+ def charref(self, name: str) -> str:
+ if name[0] in ["x", "X"]:
+ c = int(name[1:], 16)
+ else:
+ c = int(name)
+
+ if not self.unicode_snob and c in unifiable_n:
+ return unifiable_n[c]
+ else:
+ try:
+ return chr(c)
+ except ValueError: # invalid unicode
+ return ""
+
+ def entityref(self, c: str) -> str:
+ if not self.unicode_snob and c in config.UNIFIABLE:
+ return config.UNIFIABLE[c]
+ try:
+ ch = html.entities.html5[c + ";"]
+ except KeyError:
+ return "&" + c + ";"
+ return config.UNIFIABLE[c] if c == "nbsp" else ch
+
+ def google_nest_count(self, style: Dict[str, str]) -> int:
+ """
+ Calculate the nesting count of google doc lists
+
+ :type style: dict
+
+ :rtype: int
+ """
+ nest_count = 0
+ if "margin-left" in style:
+ nest_count = int(style["margin-left"][:-2]) // self.google_list_indent
+
+ return nest_count
+
+ def optwrap(self, text: str) -> str:
+ """
+ Wrap all paragraphs in the provided text.
+
+ :type text: str
+
+ :rtype: str
+ """
+ if not self.body_width:
+ return text
+
+ result = ""
+ newlines = 0
+ # I cannot think of a better solution for now.
+ # To avoid the non-wrap behaviour for entire paras
+ # because of the presence of a link in it
+ if not self.wrap_links:
+ self.inline_links = False
+ for para in text.split("\n"):
+ if len(para) > 0:
+ if not skipwrap(
+ para, self.wrap_links, self.wrap_list_items, self.wrap_tables
+ ):
+ indent = ""
+ if para.startswith(" " + self.ul_item_mark):
+ # list item continuation: add a double indent to the
+ # new lines
+ indent = " "
+ elif para.startswith("> "):
+ # blockquote continuation: add the greater than symbol
+ # to the new lines
+ indent = "> "
+ wrapped = wrap(
+ para,
+ self.body_width,
+ break_long_words=False,
+ subsequent_indent=indent,
+ )
+ result += "\n".join(wrapped)
+ if para.endswith(" "):
+ result += " \n"
+ newlines = 1
+ elif indent:
+ result += "\n"
+ newlines = 1
+ else:
+ result += "\n\n"
+ newlines = 2
+ else:
+ # Warning for the tempted!!!
+ # Be aware that obvious replacement of this with
+ # line.isspace()
+ # DOES NOT work! Explanations are welcome.
+ if not config.RE_SPACE.match(para):
+ result += para + "\n"
+ newlines = 1
+ else:
+ if newlines < 2:
+ result += "\n"
+ newlines += 1
+ return result
+
+
+def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) -> str:
+ if bodywidth is None:
+ bodywidth = config.BODY_WIDTH
+ h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
+
+ return h.handle(html)
diff --git a/crawl4ai/html2text/__main__.py b/crawl4ai/html2text/__main__.py
new file mode 100644
index 00000000..4e28416e
--- /dev/null
+++ b/crawl4ai/html2text/__main__.py
@@ -0,0 +1,3 @@
+from .cli import main
+
+main()
diff --git a/crawl4ai/html2text/_typing.py b/crawl4ai/html2text/_typing.py
new file mode 100644
index 00000000..eed83251
--- /dev/null
+++ b/crawl4ai/html2text/_typing.py
@@ -0,0 +1,2 @@
+class OutCallback:
+ def __call__(self, s: str) -> None: ...
diff --git a/crawl4ai/html2text/cli.py b/crawl4ai/html2text/cli.py
new file mode 100644
index 00000000..01532274
--- /dev/null
+++ b/crawl4ai/html2text/cli.py
@@ -0,0 +1,330 @@
+import argparse
+import sys
+
+from . import HTML2Text, __version__, config
+
+
+def main() -> None:
+ baseurl = ""
+
+ class bcolors:
+ HEADER = "\033[95m"
+ OKBLUE = "\033[94m"
+ OKGREEN = "\033[92m"
+ WARNING = "\033[93m"
+ FAIL = "\033[91m"
+ ENDC = "\033[0m"
+ BOLD = "\033[1m"
+ UNDERLINE = "\033[4m"
+
+ p = argparse.ArgumentParser()
+ p.add_argument(
+ "--default-image-alt",
+ dest="default_image_alt",
+ default=config.DEFAULT_IMAGE_ALT,
+ help="The default alt string for images with missing ones",
+ )
+ p.add_argument(
+ "--pad-tables",
+ dest="pad_tables",
+ action="store_true",
+ default=config.PAD_TABLES,
+ help="pad the cells to equal column width in tables",
+ )
+ p.add_argument(
+ "--no-wrap-links",
+ dest="wrap_links",
+ action="store_false",
+ default=config.WRAP_LINKS,
+ help="don't wrap links during conversion",
+ )
+ p.add_argument(
+ "--wrap-list-items",
+ dest="wrap_list_items",
+ action="store_true",
+ default=config.WRAP_LIST_ITEMS,
+ help="wrap list items during conversion",
+ )
+ p.add_argument(
+ "--wrap-tables",
+ dest="wrap_tables",
+ action="store_true",
+ default=config.WRAP_TABLES,
+ help="wrap tables",
+ )
+ p.add_argument(
+ "--ignore-emphasis",
+ dest="ignore_emphasis",
+ action="store_true",
+ default=config.IGNORE_EMPHASIS,
+ help="don't include any formatting for emphasis",
+ )
+ p.add_argument(
+ "--reference-links",
+ dest="inline_links",
+ action="store_false",
+ default=config.INLINE_LINKS,
+ help="use reference style links instead of inline links",
+ )
+ p.add_argument(
+ "--ignore-links",
+ dest="ignore_links",
+ action="store_true",
+ default=config.IGNORE_ANCHORS,
+ help="don't include any formatting for links",
+ )
+ p.add_argument(
+ "--ignore-mailto-links",
+ action="store_true",
+ dest="ignore_mailto_links",
+ default=config.IGNORE_MAILTO_LINKS,
+ help="don't include mailto: links",
+ )
+ p.add_argument(
+ "--protect-links",
+ dest="protect_links",
+ action="store_true",
+ default=config.PROTECT_LINKS,
+ help="protect links from line breaks surrounding them with angle brackets",
+ )
+ p.add_argument(
+ "--ignore-images",
+ dest="ignore_images",
+ action="store_true",
+ default=config.IGNORE_IMAGES,
+ help="don't include any formatting for images",
+ )
+ p.add_argument(
+ "--images-as-html",
+ dest="images_as_html",
+ action="store_true",
+ default=config.IMAGES_AS_HTML,
+ help=(
+ "Always write image tags as raw html; preserves `height`, `width` and "
+ "`alt` if possible."
+ ),
+ )
+ p.add_argument(
+ "--images-to-alt",
+ dest="images_to_alt",
+ action="store_true",
+ default=config.IMAGES_TO_ALT,
+ help="Discard image data, only keep alt text",
+ )
+ p.add_argument(
+ "--images-with-size",
+ dest="images_with_size",
+ action="store_true",
+ default=config.IMAGES_WITH_SIZE,
+ help=(
+ "Write image tags with height and width attrs as raw html to retain "
+ "dimensions"
+ ),
+ )
+ p.add_argument(
+ "-g",
+ "--google-doc",
+ action="store_true",
+ dest="google_doc",
+ default=False,
+ help="convert an html-exported Google Document",
+ )
+ p.add_argument(
+ "-d",
+ "--dash-unordered-list",
+ action="store_true",
+ dest="ul_style_dash",
+ default=False,
+ help="use a dash rather than a star for unordered list items",
+ )
+ p.add_argument(
+ "-e",
+ "--asterisk-emphasis",
+ action="store_true",
+ dest="em_style_asterisk",
+ default=False,
+ help="use an asterisk rather than an underscore for emphasized text",
+ )
+ p.add_argument(
+ "-b",
+ "--body-width",
+ dest="body_width",
+ type=int,
+ default=config.BODY_WIDTH,
+ help="number of characters per output line, 0 for no wrap",
+ )
+ p.add_argument(
+ "-i",
+ "--google-list-indent",
+ dest="list_indent",
+ type=int,
+ default=config.GOOGLE_LIST_INDENT,
+ help="number of pixels Google indents nested lists",
+ )
+ p.add_argument(
+ "-s",
+ "--hide-strikethrough",
+ action="store_true",
+ dest="hide_strikethrough",
+ default=False,
+ help="hide strike-through text. only relevant when -g is " "specified as well",
+ )
+ p.add_argument(
+ "--escape-all",
+ action="store_true",
+ dest="escape_snob",
+ default=False,
+ help=(
+ "Escape all special characters. Output is less readable, but avoids "
+ "corner case formatting issues."
+ ),
+ )
+ p.add_argument(
+ "--bypass-tables",
+ action="store_true",
+ dest="bypass_tables",
+ default=config.BYPASS_TABLES,
+ help="Format tables in HTML rather than Markdown syntax.",
+ )
+ p.add_argument(
+ "--ignore-tables",
+ action="store_true",
+ dest="ignore_tables",
+ default=config.IGNORE_TABLES,
+ help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
+ )
+ p.add_argument(
+ "--single-line-break",
+ action="store_true",
+ dest="single_line_break",
+ default=config.SINGLE_LINE_BREAK,
+ help=(
+ "Use a single line break after a block element rather than two line "
+ "breaks. NOTE: Requires --body-width=0"
+ ),
+ )
+ p.add_argument(
+ "--unicode-snob",
+ action="store_true",
+ dest="unicode_snob",
+ default=config.UNICODE_SNOB,
+ help="Use unicode throughout document",
+ )
+ p.add_argument(
+ "--no-automatic-links",
+ action="store_false",
+ dest="use_automatic_links",
+ default=config.USE_AUTOMATIC_LINKS,
+ help="Do not use automatic links wherever applicable",
+ )
+ p.add_argument(
+ "--no-skip-internal-links",
+ action="store_false",
+ dest="skip_internal_links",
+ default=config.SKIP_INTERNAL_LINKS,
+ help="Do not skip internal links",
+ )
+ p.add_argument(
+ "--links-after-para",
+ action="store_true",
+ dest="links_each_paragraph",
+ default=config.LINKS_EACH_PARAGRAPH,
+ help="Put links after each paragraph instead of document",
+ )
+ p.add_argument(
+ "--mark-code",
+ action="store_true",
+ dest="mark_code",
+ default=config.MARK_CODE,
+ help="Mark program code blocks with [code]...[/code]",
+ )
+ p.add_argument(
+ "--decode-errors",
+ dest="decode_errors",
+ default=config.DECODE_ERRORS,
+ help=(
+ "What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
+ "acceptable values"
+ ),
+ )
+ p.add_argument(
+ "--open-quote",
+ dest="open_quote",
+ default=config.OPEN_QUOTE,
+ help="The character used to open quotes",
+ )
+ p.add_argument(
+ "--close-quote",
+ dest="close_quote",
+ default=config.CLOSE_QUOTE,
+ help="The character used to close quotes",
+ )
+ p.add_argument(
+ "--version", action="version", version=".".join(map(str, __version__))
+ )
+ p.add_argument("filename", nargs="?")
+ p.add_argument("encoding", nargs="?", default="utf-8")
+ p.add_argument(
+ "--include-sup-sub",
+ dest="include_sup_sub",
+ action="store_true",
+ default=config.INCLUDE_SUP_SUB,
+ help="Include the sup and sub tags",
+ )
+ args = p.parse_args()
+
+ if args.filename and args.filename != "-":
+ with open(args.filename, "rb") as fp:
+ data = fp.read()
+ else:
+ data = sys.stdin.buffer.read()
+
+ try:
+ html = data.decode(args.encoding, args.decode_errors)
+ except UnicodeDecodeError as err:
+ warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
+ warning += " Use the " + bcolors.OKGREEN
+ warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
+ print(warning)
+ raise err
+
+ h = HTML2Text(baseurl=baseurl)
+ # handle options
+ if args.ul_style_dash:
+ h.ul_item_mark = "-"
+ if args.em_style_asterisk:
+ h.emphasis_mark = "*"
+ h.strong_mark = "__"
+
+ h.body_width = args.body_width
+ h.google_list_indent = args.list_indent
+ h.ignore_emphasis = args.ignore_emphasis
+ h.ignore_links = args.ignore_links
+ h.ignore_mailto_links = args.ignore_mailto_links
+ h.protect_links = args.protect_links
+ h.ignore_images = args.ignore_images
+ h.images_as_html = args.images_as_html
+ h.images_to_alt = args.images_to_alt
+ h.images_with_size = args.images_with_size
+ h.google_doc = args.google_doc
+ h.hide_strikethrough = args.hide_strikethrough
+ h.escape_snob = args.escape_snob
+ h.bypass_tables = args.bypass_tables
+ h.ignore_tables = args.ignore_tables
+ h.single_line_break = args.single_line_break
+ h.inline_links = args.inline_links
+ h.unicode_snob = args.unicode_snob
+ h.use_automatic_links = args.use_automatic_links
+ h.skip_internal_links = args.skip_internal_links
+ h.links_each_paragraph = args.links_each_paragraph
+ h.mark_code = args.mark_code
+ h.wrap_links = args.wrap_links
+ h.wrap_list_items = args.wrap_list_items
+ h.wrap_tables = args.wrap_tables
+ h.pad_tables = args.pad_tables
+ h.default_image_alt = args.default_image_alt
+ h.open_quote = args.open_quote
+ h.close_quote = args.close_quote
+ h.include_sup_sub = args.include_sup_sub
+
+ sys.stdout.write(h.handle(html))
diff --git a/crawl4ai/html2text/config.py b/crawl4ai/html2text/config.py
new file mode 100644
index 00000000..d14ed64f
--- /dev/null
+++ b/crawl4ai/html2text/config.py
@@ -0,0 +1,172 @@
+import re
+
+# Use Unicode characters instead of their ascii pseudo-replacements
+UNICODE_SNOB = False
+
+# Marker to use for marking tables for padding post processing
+TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
+# Escape all special characters. Output is less readable, but avoids
+# corner case formatting issues.
+ESCAPE_SNOB = False
+ESCAPE_BACKSLASH = False
+ESCAPE_DOT = False
+ESCAPE_PLUS = False
+ESCAPE_DASH = False
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = False
+
+# Wrap long lines at position. 0 for no wrapping.
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link
+# targets won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = True
+
+# Use inline, rather than reference, formatting for images and links
+INLINE_LINKS = True
+
+# Protect links from line breaks surrounding them with angle brackets (in
+# addition to their square brackets)
+PROTECT_LINKS = False
+# WRAP_LINKS = True
+WRAP_LINKS = True
+
+# Wrap list items.
+WRAP_LIST_ITEMS = False
+
+# Wrap tables
+WRAP_TABLES = False
+
+# Number of pixels Google indents nested lists
+GOOGLE_LIST_INDENT = 36
+
+# Values Google and others may use to indicate bold text
+BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
+
+IGNORE_ANCHORS = False
+IGNORE_MAILTO_LINKS = False
+IGNORE_IMAGES = False
+IMAGES_AS_HTML = False
+IMAGES_TO_ALT = False
+IMAGES_WITH_SIZE = False
+IGNORE_EMPHASIS = False
+MARK_CODE = False
+DECODE_ERRORS = "strict"
+DEFAULT_IMAGE_ALT = ""
+PAD_TABLES = False
+
+# Convert links with same href and text to format
+# if they are absolute links
+USE_AUTOMATIC_LINKS = True
+
+# For checking space-only lines on line 771
+RE_SPACE = re.compile(r"\s\+")
+
+RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
+RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
+RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
+RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
+
+# to find links in the text
+RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
+
+# to find table separators
+RE_TABLE = re.compile(r" \| ")
+
+RE_MD_DOT_MATCHER = re.compile(
+ r"""
+ ^ # start of line
+ (\s*\d+) # optional whitespace and a number
+ (\.) # dot
+ (?=\s) # lookahead assert whitespace
+ """,
+ re.MULTILINE | re.VERBOSE,
+)
+RE_MD_PLUS_MATCHER = re.compile(
+ r"""
+ ^
+ (\s*)
+ (\+)
+ (?=\s)
+ """,
+ flags=re.MULTILINE | re.VERBOSE,
+)
+RE_MD_DASH_MATCHER = re.compile(
+ r"""
+ ^
+ (\s*)
+ (-)
+ (?=\s|\-) # followed by whitespace (bullet list, or spaced out hr)
+ # or another dash (header or hr)
+ """,
+ flags=re.MULTILINE | re.VERBOSE,
+)
+RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
+RE_MD_BACKSLASH_MATCHER = re.compile(
+ r"""
+ (\\) # match one slash
+ (?=[%s]) # followed by a char that requires escaping
+ """
+ % re.escape(RE_SLASH_CHARS),
+ flags=re.VERBOSE,
+)
+
+UNIFIABLE = {
+ "rsquo": "'",
+ "lsquo": "'",
+ "rdquo": '"',
+ "ldquo": '"',
+ "copy": "(C)",
+ "mdash": "--",
+ "nbsp": " ",
+ "rarr": "->",
+ "larr": "<-",
+ "middot": "*",
+ "ndash": "-",
+ "oelig": "oe",
+ "aelig": "ae",
+ "agrave": "a",
+ "aacute": "a",
+ "acirc": "a",
+ "atilde": "a",
+ "auml": "a",
+ "aring": "a",
+ "egrave": "e",
+ "eacute": "e",
+ "ecirc": "e",
+ "euml": "e",
+ "igrave": "i",
+ "iacute": "i",
+ "icirc": "i",
+ "iuml": "i",
+ "ograve": "o",
+ "oacute": "o",
+ "ocirc": "o",
+ "otilde": "o",
+ "ouml": "o",
+ "ugrave": "u",
+ "uacute": "u",
+ "ucirc": "u",
+ "uuml": "u",
+ "lrm": "",
+ "rlm": "",
+}
+
+# Format tables in HTML rather than Markdown syntax
+BYPASS_TABLES = False
+# Ignore table-related tags (table, th, td, tr) while keeping rows
+IGNORE_TABLES = False
+
+
+# Use a single line break after a block element rather than two line breaks.
+# NOTE: Requires body width setting to be 0.
+SINGLE_LINE_BREAK = False
+
+
+# Use double quotation marks when converting the tag.
+OPEN_QUOTE = '"'
+CLOSE_QUOTE = '"'
+
+# Include the and tags
+INCLUDE_SUP_SUB = False
diff --git a/crawl4ai/html2text/elements.py b/crawl4ai/html2text/elements.py
new file mode 100644
index 00000000..2533ec08
--- /dev/null
+++ b/crawl4ai/html2text/elements.py
@@ -0,0 +1,18 @@
+from typing import Dict, Optional
+
+
+class AnchorElement:
+ __slots__ = ["attrs", "count", "outcount"]
+
+ def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
+ self.attrs = attrs
+ self.count = count
+ self.outcount = outcount
+
+
+class ListElement:
+ __slots__ = ["name", "num"]
+
+ def __init__(self, name: str, num: int):
+ self.name = name
+ self.num = num
diff --git a/crawl4ai/html2text/utils.py b/crawl4ai/html2text/utils.py
new file mode 100644
index 00000000..1909d2cf
--- /dev/null
+++ b/crawl4ai/html2text/utils.py
@@ -0,0 +1,303 @@
+import html.entities
+from typing import Dict, List, Optional
+
+from . import config
+
+unifiable_n = {
+ html.entities.name2codepoint[k]: v
+ for k, v in config.UNIFIABLE.items()
+ if k != "nbsp"
+}
+
+
+def hn(tag: str) -> int:
+ if tag[0] == "h" and len(tag) == 2:
+ n = tag[1]
+ if "0" < n <= "9":
+ return int(n)
+ return 0
+
+
+def dumb_property_dict(style: str) -> Dict[str, str]:
+ """
+ :returns: A hash of css attributes
+ """
+ return {
+ x.strip().lower(): y.strip().lower()
+ for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
+ }
+
+
+def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
+ """
+ :type data: str
+
+ :returns: A hash of css selectors, each of which contains a hash of
+ css attributes.
+ :rtype: dict
+ """
+ # remove @import sentences
+ data += ";"
+ importIndex = data.find("@import")
+ while importIndex != -1:
+ data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
+ importIndex = data.find("@import")
+
+ # parse the css. reverted from dictionary comprehension in order to
+ # support older pythons
+ pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
+ try:
+ elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
+ except ValueError:
+ elements = {} # not that important
+
+ return elements
+
+
+def element_style(
+ attrs: Dict[str, Optional[str]],
+ style_def: Dict[str, Dict[str, str]],
+ parent_style: Dict[str, str],
+) -> Dict[str, str]:
+ """
+ :type attrs: dict
+ :type style_def: dict
+ :type style_def: dict
+
+ :returns: A hash of the 'final' style attributes of the element
+ :rtype: dict
+ """
+ style = parent_style.copy()
+ if "class" in attrs:
+ assert attrs["class"] is not None
+ for css_class in attrs["class"].split():
+ css_style = style_def.get("." + css_class, {})
+ style.update(css_style)
+ if "style" in attrs:
+ assert attrs["style"] is not None
+ immediate_style = dumb_property_dict(attrs["style"])
+ style.update(immediate_style)
+
+ return style
+
+
+def google_list_style(style: Dict[str, str]) -> str:
+ """
+ Finds out whether this is an ordered or unordered list
+
+ :type style: dict
+
+ :rtype: str
+ """
+ if "list-style-type" in style:
+ list_style = style["list-style-type"]
+ if list_style in ["disc", "circle", "square", "none"]:
+ return "ul"
+
+ return "ol"
+
+
+def google_has_height(style: Dict[str, str]) -> bool:
+ """
+ Check if the style of the element has the 'height' attribute
+ explicitly defined
+
+ :type style: dict
+
+ :rtype: bool
+ """
+ return "height" in style
+
+
+def google_text_emphasis(style: Dict[str, str]) -> List[str]:
+ """
+ :type style: dict
+
+ :returns: A list of all emphasis modifiers of the element
+ :rtype: list
+ """
+ emphasis = []
+ if "text-decoration" in style:
+ emphasis.append(style["text-decoration"])
+ if "font-style" in style:
+ emphasis.append(style["font-style"])
+ if "font-weight" in style:
+ emphasis.append(style["font-weight"])
+
+ return emphasis
+
+
+def google_fixed_width_font(style: Dict[str, str]) -> bool:
+ """
+ Check if the css of the current element defines a fixed width font
+
+ :type style: dict
+
+ :rtype: bool
+ """
+ font_family = ""
+ if "font-family" in style:
+ font_family = style["font-family"]
+ return "courier new" == font_family or "consolas" == font_family
+
+
+def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
+ """
+ Extract numbering from list element attributes
+
+ :type attrs: dict
+
+ :rtype: int or None
+ """
+ if "start" in attrs:
+ assert attrs["start"] is not None
+ try:
+ return int(attrs["start"]) - 1
+ except ValueError:
+ pass
+
+ return 0
+
+
+def skipwrap(
+ para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
+) -> bool:
+ # If it appears to contain a link
+ # don't wrap
+ if not wrap_links and config.RE_LINK.search(para):
+ return True
+ # If the text begins with four spaces or one tab, it's a code block;
+ # don't wrap
+ if para[0:4] == " " or para[0] == "\t":
+ return True
+
+ # If the text begins with only two "--", possibly preceded by
+ # whitespace, that's an emdash; so wrap.
+ stripped = para.lstrip()
+ if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
+ return False
+
+ # I'm not sure what this is for; I thought it was to detect lists,
+ # but there's a
-inside- case in one of the tests that
+ # also depends upon it.
+ if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
+ return not wrap_list_items
+
+ # If text contains a pipe character it is likely a table
+ if not wrap_tables and config.RE_TABLE.search(para):
+ return True
+
+ # If the text begins with a single -, *, or +, followed by a space,
+ # or an integer, followed by a ., followed by a space (in either
+ # case optionally proceeded by whitespace), it's a list; don't wrap.
+ return bool(
+ config.RE_ORDERED_LIST_MATCHER.match(stripped)
+ or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
+ )
+
+
+def escape_md(text: str) -> str:
+ """
+ Escapes markdown-sensitive characters within other markdown
+ constructs.
+ """
+ return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
+
+
+def escape_md_section(
+ text: str,
+ escape_backslash: bool = True,
+ snob: bool = False,
+ escape_dot: bool = True,
+ escape_plus: bool = True,
+ escape_dash: bool = True
+) -> str:
+ """
+ Escapes markdown-sensitive characters across whole document sections.
+ Each escaping operation can be controlled individually.
+ """
+ if escape_backslash:
+ text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
+
+ if snob:
+ text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
+
+ if escape_dot:
+ text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
+
+ if escape_plus:
+ text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
+
+ if escape_dash:
+ text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
+
+ return text
+
+def reformat_table(lines: List[str], right_margin: int) -> List[str]:
+ """
+ Given the lines of a table
+ padds the cells and returns the new lines
+ """
+ # find the maximum width of the columns
+ max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
+ max_cols = len(max_width)
+ for line in lines:
+ cols = [x.rstrip() for x in line.split("|")]
+ num_cols = len(cols)
+
+ # don't drop any data if colspan attributes result in unequal lengths
+ if num_cols < max_cols:
+ cols += [""] * (max_cols - num_cols)
+ elif max_cols < num_cols:
+ max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
+ max_cols = num_cols
+
+ max_width = [
+ max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
+ ]
+
+ # reformat
+ new_lines = []
+ for line in lines:
+ cols = [x.rstrip() for x in line.split("|")]
+ if set(line.strip()) == set("-|"):
+ filler = "-"
+ new_cols = [
+ x.rstrip() + (filler * (M - len(x.rstrip())))
+ for x, M in zip(cols, max_width)
+ ]
+ new_lines.append("|-" + "|".join(new_cols) + "|")
+ else:
+ filler = " "
+ new_cols = [
+ x.rstrip() + (filler * (M - len(x.rstrip())))
+ for x, M in zip(cols, max_width)
+ ]
+ new_lines.append("| " + "|".join(new_cols) + "|")
+ return new_lines
+
+
+def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
+ """
+ Provide padding for tables in the text
+ """
+ lines = text.split("\n")
+ table_buffer = [] # type: List[str]
+ table_started = False
+ new_lines = []
+ for line in lines:
+ # Toggle table started
+ if config.TABLE_MARKER_FOR_PAD in line:
+ table_started = not table_started
+ if not table_started:
+ table = reformat_table(table_buffer, right_margin)
+ new_lines.extend(table)
+ table_buffer = []
+ new_lines.append("")
+ continue
+ # Process lines
+ if table_started:
+ table_buffer.append(line)
+ else:
+ new_lines.append(line)
+ return "\n".join(new_lines)
diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py
index 711fd2c4..34ab219b 100644
--- a/crawl4ai/utils.py
+++ b/crawl4ai/utils.py
@@ -1,13 +1,12 @@
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup, Comment, element, Tag, NavigableString
-import html2text
import json
import html
import re
import os
import platform
-from html2text import HTML2Text
+from .html2text import HTML2Text
from .prompts import PROMPT_EXTRACT_BLOCKS
from .config import *
from pathlib import Path
@@ -182,9 +181,22 @@ def escape_json_string(s):
class CustomHTML2Text(HTML2Text):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.ignore_links = True
self.inside_pre = False
self.inside_code = False
+
+ self.skip_internal_links = False
+ self.single_line_break = False
+ self.mark_code = False
+ self.include_sup_sub = False
+ self.body_width = 0
+ self.ignore_mailto_links = True
+ self.ignore_links = False
+ self.escape_backslash = False
+ self.escape_dot = False
+ self.escape_plus = False
+ self.escape_dash = False
+ self.escape_snob = False
+
def handle_tag(self, tag, attrs, start):
if tag == 'pre':
@@ -194,6 +206,10 @@ class CustomHTML2Text(HTML2Text):
else:
self.o('\n```')
self.inside_pre = False
+ elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+ pass
+
+
# elif tag == 'code' and not self.inside_pre:
# if start:
# if not self.inside_pre: