修改为东南天坐标系
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
from unstructured.partition.html.partition import partition_html
|
||||
|
||||
__all__ = ["partition_html"]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,320 @@
|
||||
import logging
|
||||
from abc import ABC
|
||||
from collections import defaultdict
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from unstructured.documents.elements import Element, ElementType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HTML_PARSER = "html.parser"
|
||||
HTML_TEMPLATE = """
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title></title>
|
||||
</head>
|
||||
<body>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
TABLE_BORDER_STYLE = "border: 1px solid black;"
|
||||
TABLE_BORDER_COLLAPSE_STYLE = "border-collapse: collapse;"
|
||||
|
||||
|
||||
class ElementHtml(ABC):
|
||||
element: Element
|
||||
children: list["ElementHtml"]
|
||||
_html_tag: str = "div"
|
||||
|
||||
def __init__(self, element: Element, children: Optional[list["ElementHtml"]] = None):
|
||||
self.element = element
|
||||
self.children = children or []
|
||||
|
||||
@property
|
||||
def html_tag(self) -> str:
|
||||
return self._html_tag
|
||||
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
return None
|
||||
|
||||
def _inject_html_element_content(self, element_html: Tag, **kwargs: Any) -> None:
|
||||
element_html.string = self.element.text
|
||||
|
||||
def get_text_as_html(self) -> Union[Tag, None]:
|
||||
element_html = BeautifulSoup(self.element.metadata.text_as_html or "", HTML_PARSER).find()
|
||||
if not isinstance(element_html, Tag):
|
||||
return None
|
||||
return element_html
|
||||
|
||||
def _get_children_html(self, soup: BeautifulSoup, element_html: Tag, **kwargs: Any) -> Tag:
|
||||
wrapper = soup.new_tag(name="div")
|
||||
wrapper.append(element_html)
|
||||
for child in self.children:
|
||||
child_html = child.get_html_element(_soup=soup, **kwargs)
|
||||
wrapper.append(child_html)
|
||||
return wrapper
|
||||
|
||||
def get_html_element(self, **kwargs: Any) -> Tag:
|
||||
soup: Optional[BeautifulSoup] = kwargs.pop("_soup", None)
|
||||
if soup is None:
|
||||
soup = BeautifulSoup("", HTML_PARSER)
|
||||
|
||||
element_html = self.get_text_as_html()
|
||||
if element_html is None:
|
||||
element_html = soup.new_tag(name=self.html_tag)
|
||||
self._inject_html_element_content(element_html, **kwargs)
|
||||
element_html["class"] = self.element.category
|
||||
element_html["id"] = self.element.id
|
||||
self._inject_html_element_attrs(element_html)
|
||||
if self.children: # if element has children wrap it with a 'div' tag
|
||||
return self._get_children_html(soup, element_html, **kwargs)
|
||||
return element_html
|
||||
|
||||
def set_children(self, children: list["ElementHtml"]) -> None:
|
||||
self.children = children
|
||||
|
||||
|
||||
class TitleElementHtml(ElementHtml):
|
||||
_html_tag = "h%d"
|
||||
|
||||
@property
|
||||
def html_tag(self) -> str:
|
||||
return self._html_tag % (self.element.metadata.category_depth or 1)
|
||||
|
||||
|
||||
class ImageElementHtml(ElementHtml):
|
||||
_html_tag = "img"
|
||||
|
||||
def _inject_html_element_content(self, element_html: Tag, **kwargs: Any) -> None:
|
||||
exclude_binary_image_data = kwargs.get("exclude_binary_image_data", False)
|
||||
if self.element.metadata.image_base64 and not exclude_binary_image_data:
|
||||
image_mime_type = self.element.metadata.image_mime_type or "image/png"
|
||||
element_html["src"] = (
|
||||
f"data:{image_mime_type};base64,{self.element.metadata.image_base64}"
|
||||
)
|
||||
element_html["alt"] = self.element.text
|
||||
|
||||
|
||||
class TableElementHtml(ElementHtml):
|
||||
_html_tag = "table"
|
||||
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
element_html["style"] = f"{TABLE_BORDER_STYLE} {TABLE_BORDER_COLLAPSE_STYLE}"
|
||||
for tag in element_html.find_all(["tr", "th", "td"]):
|
||||
tag["style"] = TABLE_BORDER_STYLE
|
||||
|
||||
|
||||
class LinkElementHtml(ElementHtml):
|
||||
_html_tag = "a"
|
||||
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
element_html["href"] = self.element.metadata.url or ""
|
||||
|
||||
|
||||
class TextElementHtml(ElementHtml):
|
||||
_html_tag = "p"
|
||||
|
||||
|
||||
class UnorderedListElementHtml(ElementHtml):
|
||||
_html_tag = "ul"
|
||||
|
||||
def _get_children_html(self, soup: BeautifulSoup, element_html: Tag, **kwargs: Any) -> Tag:
|
||||
for child in self.children:
|
||||
child_html = child.get_html_element(**kwargs)
|
||||
element_html.append(child_html)
|
||||
return element_html
|
||||
|
||||
|
||||
class OrderedListElementHtml(UnorderedListElementHtml):
|
||||
_html_tag = "ol"
|
||||
|
||||
|
||||
class ListItemElementHtml(UnorderedListElementHtml):
|
||||
_html_tag = "li"
|
||||
|
||||
|
||||
class LabelElementHtml(ElementHtml):
|
||||
_html_tag = "label"
|
||||
|
||||
|
||||
class FormElementHtml(ElementHtml):
|
||||
_html_tag = "form"
|
||||
|
||||
|
||||
class InputElementHtml(ElementHtml):
|
||||
_html_tag = "input"
|
||||
|
||||
|
||||
class CheckboxElementHtml(InputElementHtml):
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
element_html["type"] = "checkbox"
|
||||
|
||||
|
||||
class CheckboxCheckedElementHtml(InputElementHtml):
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
element_html["type"] = "checkbox"
|
||||
element_html["checked"] = "true"
|
||||
|
||||
|
||||
class RadioElementHtml(InputElementHtml):
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
element_html["type"] = "radio"
|
||||
|
||||
|
||||
class RadioCheckedElementHtml(InputElementHtml):
|
||||
def _inject_html_element_attrs(self, element_html: Tag) -> None:
|
||||
element_html["type"] = "radio"
|
||||
element_html["checked"] = "true"
|
||||
|
||||
|
||||
LIST_ELEMENTS = [ElementType.LIST_ITEM, ElementType.LIST_ITEM_OTHER]
|
||||
|
||||
TYPE_TO_HTML_MAP = {
|
||||
ElementType.UNCATEGORIZED_TEXT: TextElementHtml,
|
||||
ElementType.TITLE: TitleElementHtml,
|
||||
ElementType.IMAGE: ImageElementHtml,
|
||||
ElementType.TABLE: TableElementHtml,
|
||||
ElementType.LINK: LinkElementHtml,
|
||||
ElementType.TEXT: TextElementHtml,
|
||||
ElementType.PARAGRAPH: TextElementHtml,
|
||||
ElementType.LIST: OrderedListElementHtml,
|
||||
ElementType.LIST_ITEM: ListItemElementHtml,
|
||||
ElementType.LIST_ITEM_OTHER: ListItemElementHtml,
|
||||
ElementType.FIELD_NAME: LabelElementHtml,
|
||||
ElementType.BULLETED_TEXT: ListItemElementHtml,
|
||||
ElementType.FORM: FormElementHtml,
|
||||
ElementType.CAPTION: TextElementHtml,
|
||||
ElementType.CHECKED: CheckboxCheckedElementHtml,
|
||||
ElementType.UNCHECKED: CheckboxElementHtml,
|
||||
ElementType.CHECK_BOX_CHECKED: CheckboxCheckedElementHtml,
|
||||
ElementType.CHECK_BOX_UNCHECKED: CheckboxElementHtml,
|
||||
ElementType.RADIO_BUTTON_CHECKED: RadioCheckedElementHtml,
|
||||
ElementType.RADIO_BUTTON_UNCHECKED: RadioElementHtml,
|
||||
ElementType.NARRATIVE_TEXT: TextElementHtml,
|
||||
ElementType.FIGURE_CAPTION: TextElementHtml,
|
||||
ElementType.VALUE: InputElementHtml,
|
||||
ElementType.ABSTRACT: ElementHtml,
|
||||
ElementType.THREADING: ElementHtml,
|
||||
ElementType.COMPOSITE_ELEMENT: ElementHtml,
|
||||
ElementType.PICTURE: ElementHtml,
|
||||
ElementType.FIGURE: ElementHtml,
|
||||
ElementType.ADDRESS: ElementHtml,
|
||||
ElementType.EMAIL_ADDRESS: ElementHtml,
|
||||
ElementType.PAGE_BREAK: ElementHtml,
|
||||
ElementType.FORMULA: ElementHtml,
|
||||
ElementType.HEADER: ElementHtml,
|
||||
ElementType.HEADLINE: ElementHtml,
|
||||
ElementType.SUB_HEADLINE: ElementHtml,
|
||||
ElementType.PAGE_HEADER: ElementHtml,
|
||||
ElementType.SECTION_HEADER: ElementHtml,
|
||||
ElementType.FOOTER: ElementHtml,
|
||||
ElementType.FOOTNOTE: ElementHtml,
|
||||
ElementType.PAGE_FOOTER: ElementHtml,
|
||||
ElementType.PAGE_NUMBER: ElementHtml,
|
||||
ElementType.CODE_SNIPPET: ElementHtml,
|
||||
ElementType.FORM_KEYS_VALUES: ElementHtml,
|
||||
ElementType.DOCUMENT_DATA: ElementHtml,
|
||||
}
|
||||
|
||||
|
||||
def _group_element_children(children: list[ElementHtml]) -> list[ElementHtml]:
|
||||
grouped_children: list[ElementHtml] = []
|
||||
temp_group: list["ElementHtml"] = []
|
||||
prev_grouping = False
|
||||
for child in children:
|
||||
grouping = child.element.category in LIST_ELEMENTS
|
||||
if grouping:
|
||||
temp_group.append(child)
|
||||
elif prev_grouping:
|
||||
grouped_children.append(OrderedListElementHtml(Element(), temp_group))
|
||||
grouped_children.append(child)
|
||||
temp_group = []
|
||||
else:
|
||||
grouped_children.append(child)
|
||||
prev_grouping = grouping
|
||||
if temp_group:
|
||||
grouped_children.append(OrderedListElementHtml(Element(), temp_group))
|
||||
return grouped_children
|
||||
|
||||
|
||||
def _elements_to_html_tags_by_parent(elements: list[ElementHtml]) -> list[ElementHtml]:
|
||||
parent_to_children_map: dict[str, list[ElementHtml]] = defaultdict(list)
|
||||
for element in elements:
|
||||
if element.element.metadata.parent_id is not None:
|
||||
parent_to_children_map[element.element.metadata.parent_id].append(element)
|
||||
for parent_id, children in parent_to_children_map.items():
|
||||
grouped_children = _group_element_children(children)
|
||||
parent = next((el for el in elements if el.element.id == parent_id), None)
|
||||
if parent is None:
|
||||
logger.warning(f"Parent element with id {parent_id} not found. Skipping.")
|
||||
continue
|
||||
parent.set_children(grouped_children)
|
||||
return [el for el in elements if el.element.metadata.parent_id is None]
|
||||
|
||||
|
||||
def _elements_to_html_tags(
|
||||
elements: list[Element], exclude_binary_image_data: bool = False
|
||||
) -> list[Tag]:
|
||||
elements_html = [
|
||||
TYPE_TO_HTML_MAP.get(element.category, ElementHtml)(element) for element in elements
|
||||
]
|
||||
elements_html = _elements_to_html_tags_by_parent(elements_html)
|
||||
return [
|
||||
element_html.get_html_element(exclude_binary_image_data=exclude_binary_image_data)
|
||||
for element_html in elements_html
|
||||
]
|
||||
|
||||
|
||||
def _elements_to_html_tags_by_page(
|
||||
elements: list[Element], exclude_binary_image_data: bool = False
|
||||
) -> list[Tag]:
|
||||
soup = BeautifulSoup("", HTML_PARSER)
|
||||
pages_tags: list[Tag] = []
|
||||
grouped_elements = group_elements_by_page(elements)
|
||||
for page, g_elements in enumerate(grouped_elements, start=1):
|
||||
page_html = soup.new_tag(name="div", attrs={"data-page_number": page})
|
||||
elements_html = _elements_to_html_tags(g_elements, exclude_binary_image_data)
|
||||
for element_html in elements_html:
|
||||
page_html.append(element_html)
|
||||
pages_tags.append(page_html)
|
||||
return pages_tags
|
||||
|
||||
|
||||
def group_elements_by_page(
|
||||
unstructured_elements: list[Element],
|
||||
) -> list[list[Element]]:
|
||||
pages_dict: defaultdict[int, list[Element]] = defaultdict(list)
|
||||
|
||||
for element in unstructured_elements:
|
||||
page_number = element.metadata.page_number
|
||||
if page_number is None:
|
||||
logger.warning(f"Page number is not set for an element {element.id}. Skipping.")
|
||||
continue
|
||||
pages_dict[page_number].append(element)
|
||||
|
||||
pages_list = list(pages_dict.values())
|
||||
return pages_list
|
||||
|
||||
|
||||
def elements_to_html(
|
||||
elements: list[Element],
|
||||
exclude_binary_image_data: bool = False,
|
||||
no_group_by_page: bool = False,
|
||||
) -> str:
|
||||
soup = BeautifulSoup(HTML_TEMPLATE, HTML_PARSER)
|
||||
if soup.body is None:
|
||||
raise ValueError("Body tag not found in the HTML template")
|
||||
elements_html = (
|
||||
_elements_to_html_tags(elements, exclude_binary_image_data)
|
||||
if no_group_by_page
|
||||
else _elements_to_html_tags_by_page(elements, exclude_binary_image_data)
|
||||
)
|
||||
for element_html in elements_html:
|
||||
soup.body.append(element_html)
|
||||
return soup.prettify()
|
||||
@@ -0,0 +1,25 @@
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def indent_html(html_string: str, html_parser="html.parser") -> str:
|
||||
"""
|
||||
Formats / indents HTML.
|
||||
|
||||
This function takes an HTML string and formats it using the specified HTML parser.
|
||||
It parses the HTML content and returns a prettified version of it.
|
||||
|
||||
Args:
|
||||
html_string (str): The HTML content to be formatted.
|
||||
html_parser (str, optional): The parser to use for parsing the HTML. Defaults to 'html5lib':
|
||||
- 'html.parser': The built-in HTML parser. Use when you need just parsing
|
||||
- 'html5lib': The slowest. Use when you expect valid HTML parsed
|
||||
the same way a browser does. It adds some extra
|
||||
tags and attributes like <html>, <head>, <body>
|
||||
More in docs https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser
|
||||
|
||||
Returns:
|
||||
str: The formatted and indented HTML content.
|
||||
"""
|
||||
soup = BeautifulSoup(html_string, html_parser)
|
||||
pretty_html = soup.prettify()
|
||||
return pretty_html
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,281 @@
|
||||
# pyright: reportPrivateUsage=false
|
||||
|
||||
"""Provides `partition_html()."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import IO, Any, Iterator, List, Literal, Optional, cast
|
||||
|
||||
import requests
|
||||
from lxml import etree
|
||||
|
||||
from unstructured.chunking import add_chunking_strategy
|
||||
from unstructured.documents.elements import Element, ElementType
|
||||
from unstructured.file_utils.encoding import read_txt_file
|
||||
from unstructured.file_utils.model import FileType
|
||||
from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
|
||||
from unstructured.partition.html.parser import Flow, html_parser
|
||||
from unstructured.partition.html.transformations import (
|
||||
ontology_to_unstructured_elements,
|
||||
parse_html_to_ontology,
|
||||
)
|
||||
from unstructured.utils import is_temp_file_path, lazyproperty
|
||||
|
||||
|
||||
@apply_metadata(FileType.HTML)
|
||||
@add_chunking_strategy
|
||||
def partition_html(
|
||||
filename: Optional[str] = None,
|
||||
*,
|
||||
file: Optional[IO[bytes]] = None,
|
||||
text: Optional[str] = None,
|
||||
encoding: Optional[str] = None,
|
||||
url: Optional[str] = None,
|
||||
headers: dict[str, str] = {},
|
||||
ssl_verify: bool = True,
|
||||
skip_headers_and_footers: bool = False,
|
||||
detection_origin: Optional[str] = None,
|
||||
html_parser_version: Literal["v1", "v2"] = "v1",
|
||||
image_alt_mode: Optional[Literal["to_text"]] = "to_text",
|
||||
extract_image_block_to_payload: bool = False,
|
||||
extract_image_block_types: Optional[list[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> list[Element]:
|
||||
"""Partitions an HTML document into its constituent elements.
|
||||
|
||||
HTML source parameters
|
||||
----------------------
|
||||
The HTML to be partitioned can be specified four different ways:
|
||||
|
||||
filename
|
||||
A string defining the target filename path.
|
||||
file
|
||||
A file-like object using "r" mode --> open(filename, "r").
|
||||
text
|
||||
The string representation of the HTML document.
|
||||
url
|
||||
The URL of a webpage to parse. Only for URLs that return an HTML document.
|
||||
headers
|
||||
The HTTP headers to be used in the HTTP request when `url` is specified.
|
||||
ssl_verify
|
||||
If the URL parameter is set, determines whether or not SSL verification is performed
|
||||
on the HTTP request.
|
||||
encoding
|
||||
The encoding method used to decode the text input. If None, utf-8 will be used.
|
||||
skip_headers_and_footers
|
||||
If True, ignores any content that is within <header> or <footer> tags
|
||||
|
||||
html_parser_version (Literal['v1', 'v2']):
|
||||
The version of the HTML parser to use. The default is 'v1'. For 'v2' the parser will
|
||||
use the ontology schema to parse the HTML document.
|
||||
|
||||
image_alt_mode (Literal['to_text']):
|
||||
When set 'to_text', the v2 parser will include the alternative text of images in the output.
|
||||
"""
|
||||
# -- parser rejects an empty str, nip that edge-case in the bud here --
|
||||
if text is not None and text.strip() == "" and not file and not filename and not url:
|
||||
return []
|
||||
|
||||
opts = HtmlPartitionerOptions(
|
||||
file_path=filename,
|
||||
file=file,
|
||||
text=text,
|
||||
encoding=encoding,
|
||||
url=url,
|
||||
headers=headers,
|
||||
ssl_verify=ssl_verify,
|
||||
skip_headers_and_footers=skip_headers_and_footers,
|
||||
detection_origin=detection_origin,
|
||||
html_parser_version=html_parser_version,
|
||||
image_alt_mode=image_alt_mode,
|
||||
extract_image_block_types=extract_image_block_types,
|
||||
extract_image_block_to_payload=extract_image_block_to_payload,
|
||||
)
|
||||
|
||||
return list(_HtmlPartitioner.iter_elements(opts))
|
||||
|
||||
|
||||
class HtmlPartitionerOptions:
|
||||
"""Encapsulates partitioning option validation, computation, and application of defaults."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
file_path: str | None,
|
||||
file: IO[bytes] | None,
|
||||
text: str | None,
|
||||
encoding: str | None,
|
||||
url: str | None,
|
||||
headers: dict[str, str],
|
||||
ssl_verify: bool,
|
||||
skip_headers_and_footers: bool,
|
||||
detection_origin: str | None,
|
||||
html_parser_version: Literal["v1", "v2"] = "v1",
|
||||
image_alt_mode: Optional[Literal["to_text"]] = "to_text",
|
||||
extract_image_block_types: Optional[list[str]] = None,
|
||||
extract_image_block_to_payload: bool = False,
|
||||
):
|
||||
self._file_path = file_path
|
||||
self._file = file
|
||||
self._text = text
|
||||
self._encoding = encoding
|
||||
self._url = url
|
||||
self._headers = headers
|
||||
self._ssl_verify = ssl_verify
|
||||
self._skip_headers_and_footers = skip_headers_and_footers
|
||||
self._detection_origin = detection_origin
|
||||
self._html_parser_version = html_parser_version
|
||||
self._image_alt_mode = image_alt_mode
|
||||
self._extract_image_block_types = extract_image_block_types
|
||||
self._extract_image_block_to_payload = extract_image_block_to_payload
|
||||
|
||||
@lazyproperty
|
||||
def detection_origin(self) -> str | None:
|
||||
"""Trace of initial partitioner to be included in metadata for debugging purposes."""
|
||||
return self._detection_origin
|
||||
|
||||
@lazyproperty
|
||||
def html_text(self) -> str:
|
||||
"""The HTML document as a string, loaded from wherever the caller specified."""
|
||||
if self._file_path:
|
||||
return read_txt_file(filename=self._file_path, encoding=self._encoding)[1]
|
||||
|
||||
if self._file:
|
||||
return read_txt_file(file=self._file, encoding=self._encoding)[1]
|
||||
|
||||
if self._text:
|
||||
return str(self._text)
|
||||
|
||||
if self._url:
|
||||
response = requests.get(self._url, headers=self._headers, verify=self._ssl_verify)
|
||||
if not response.ok:
|
||||
raise ValueError(
|
||||
f"Error status code on GET of provided URL: {response.status_code}"
|
||||
)
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not content_type.startswith("text/html"):
|
||||
raise ValueError(f"Expected content type text/html. Got {content_type}.")
|
||||
|
||||
return response.text
|
||||
|
||||
raise ValueError("Exactly one of filename, file, text, or url must be specified.")
|
||||
|
||||
@lazyproperty
|
||||
def last_modified(self) -> str | None:
|
||||
"""The best last-modified date available, None if no sources are available."""
|
||||
return (
|
||||
None
|
||||
if not self._file_path or is_temp_file_path(self._file_path)
|
||||
else get_last_modified_date(self._file_path)
|
||||
)
|
||||
|
||||
@lazyproperty
|
||||
def skip_headers_and_footers(self) -> bool:
|
||||
"""When True, elements located within a header or footer are pruned."""
|
||||
return self._skip_headers_and_footers
|
||||
|
||||
@lazyproperty
|
||||
def html_parser_version(self) -> Literal["v1", "v2"]:
|
||||
"""When html_parser_version=='v2', HTML elements follow ontology schema."""
|
||||
return self._html_parser_version
|
||||
|
||||
@lazyproperty
|
||||
def add_img_alt_text(self) -> bool:
|
||||
"""When True, the alternative text of images is included in the output."""
|
||||
return self._image_alt_mode == "to_text"
|
||||
|
||||
|
||||
class _HtmlPartitioner:
|
||||
"""Partition HTML document into document-elements."""
|
||||
|
||||
def __init__(self, opts: HtmlPartitionerOptions):
|
||||
self._opts = opts
|
||||
|
||||
def _should_include_image_base64(self, element: Element) -> bool:
|
||||
"""Determines if an image_base64 element should be included in the output."""
|
||||
return (
|
||||
element.category == ElementType.IMAGE
|
||||
and self._opts._extract_image_block_to_payload
|
||||
and self._opts._extract_image_block_types is not None
|
||||
and "Image" in self._opts._extract_image_block_types
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def iter_elements(cls, opts: HtmlPartitionerOptions) -> Iterator[Element]:
|
||||
"""Partition HTML document provided by `opts` into document-elements."""
|
||||
yield from cls(opts)._iter_elements()
|
||||
|
||||
def _iter_elements(self) -> Iterator[Element]:
|
||||
"""Generated document-elements (e.g. Title, NarrativeText, etc.) parsed from document.
|
||||
|
||||
Elements appear in document order.
|
||||
"""
|
||||
# -- handle empty or whitespace-only HTML content --
|
||||
html_text = self._opts.html_text
|
||||
if not html_text or html_text.strip() == "":
|
||||
return
|
||||
|
||||
elements_iter = (
|
||||
self._main.iter_elements()
|
||||
if self._opts.html_parser_version == "v1"
|
||||
else self._from_ontology
|
||||
)
|
||||
|
||||
for e in elements_iter:
|
||||
e.metadata.last_modified = self._opts.last_modified
|
||||
e.metadata.detection_origin = self._opts.detection_origin
|
||||
|
||||
# -- remove <image_base64> if not requested --
|
||||
if not self._should_include_image_base64(e):
|
||||
e.metadata.image_base64 = None
|
||||
e.metadata.image_mime_type = None
|
||||
yield e
|
||||
|
||||
@lazyproperty
|
||||
def _main(self) -> Flow:
|
||||
"""The root HTML element."""
|
||||
# NOTE(scanny) - get `html_text` first so any encoding error raised is not confused with a
|
||||
# recoverable parsing error.
|
||||
html_text = self._opts.html_text
|
||||
|
||||
# NOTE(scanny) - `lxml` will not parse a `str` that includes an XML encoding declaration
|
||||
# and will raise the following error:
|
||||
# ValueError: Unicode strings with encoding declaration are not supported. ...
|
||||
# This is not valid HTML (would be in XHTML), but Chrome accepts it so we work around it
|
||||
# by UTF-8 encoding the str bytes and parsing those.
|
||||
try:
|
||||
root = etree.fromstring(html_text, html_parser)
|
||||
except ValueError:
|
||||
root = etree.fromstring(html_text.encode("utf-8"), html_parser)
|
||||
|
||||
# -- remove a variety of HTML element types like <script> and <style> that we prefer not
|
||||
# -- to encounter while parsing.
|
||||
etree.strip_elements(
|
||||
root, ["del", "link", "meta", "noscript", "script", "style"], with_tail=False
|
||||
)
|
||||
|
||||
# -- remove <header> and <footer> tags if the caller doesn't want their contents --
|
||||
if self._opts.skip_headers_and_footers:
|
||||
etree.strip_elements(root, ["header", "footer"], with_tail=False)
|
||||
|
||||
# -- jump to the core content if the document indicates where it is --
|
||||
if (main := root.find(".//main")) is not None:
|
||||
return cast(Flow, main)
|
||||
if (body := root.find(".//body")) is not None:
|
||||
return cast(Flow, body)
|
||||
return cast(Flow, root)
|
||||
|
||||
@lazyproperty
|
||||
def _from_ontology(self) -> List[Element]:
|
||||
"""Convert an ontology elements represented in HTML to an ontology element."""
|
||||
html_text = self._opts.html_text
|
||||
|
||||
# -- handle empty or whitespace-only HTML content --
|
||||
if not html_text or html_text.strip() == "":
|
||||
return []
|
||||
|
||||
ontology = parse_html_to_ontology(html_text)
|
||||
unstructured_elements = ontology_to_unstructured_elements(
|
||||
ontology, add_img_alt_text=self._opts.add_img_alt_text
|
||||
)
|
||||
return unstructured_elements
|
||||
@@ -0,0 +1,480 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
from collections import OrderedDict
|
||||
from itertools import chain
|
||||
from typing import Sequence, Type
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from unstructured.documents import elements, ontology
|
||||
from unstructured.documents.mappings import (
|
||||
CSS_CLASS_TO_ELEMENT_TYPE_MAP,
|
||||
HTML_TAG_AND_CSS_NAME_TO_ELEMENT_TYPE_MAP,
|
||||
HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP,
|
||||
ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE,
|
||||
)
|
||||
|
||||
RECURSION_LIMIT = 50
|
||||
|
||||
|
||||
def ontology_to_unstructured_elements(
|
||||
ontology_element: ontology.OntologyElement,
|
||||
parent_id: str | None = None,
|
||||
page_number: int | None = None,
|
||||
depth: int = 0,
|
||||
filename: str | None = None,
|
||||
add_img_alt_text: bool = True,
|
||||
) -> list[elements.Element]:
|
||||
"""
|
||||
Converts an OntologyElement object to a list of unstructured Element objects.
|
||||
|
||||
To preserve the structure of the ontology, the function is recursive
|
||||
and the tree structure is represented in flatten list by the parent_id
|
||||
attribute in the metadata of each Element object.
|
||||
To preserve all the attributes of the ontology element, the HTML code
|
||||
is injected to unstructured Element in ElementMetadata.text_as_html attribute.
|
||||
|
||||
For Layout elements, the function creates an empty Text Element (with the
|
||||
HTML code injected the same way).
|
||||
|
||||
TODO (Pluto): Better way would be to have special Element type in Unstructured
|
||||
|
||||
Args:
|
||||
ontology_element (OntologyElement): The ontology element to be converted.
|
||||
parent_id (str, optional): The ID of the parent element. Defaults to None.
|
||||
page_number (int, optional): The page number of the element. Defaults to None.
|
||||
depth (int, optional): The depth of the element in the hierarchy. Defaults to 0.
|
||||
filename (str, optional): The name of the file the element comes from. Defaults to None.
|
||||
add_img_alt_text (bool): Whether to include the alternative text of images
|
||||
in the output. Defaults to True.
|
||||
Returns:
|
||||
list[Element]: A list of unstructured Element objects.
|
||||
"""
|
||||
elements_to_return: list[elements.Element] = []
|
||||
if ontology_element.elementType == ontology.ElementTypeEnum.layout and depth <= RECURSION_LIMIT:
|
||||
if page_number is None and isinstance(ontology_element, ontology.Page):
|
||||
page_number = ontology_element.page_number
|
||||
|
||||
if not isinstance(ontology_element, ontology.Document):
|
||||
elements_to_return += [
|
||||
elements.Text(
|
||||
text="",
|
||||
element_id=ontology_element.id,
|
||||
detection_origin="vlm_partitioner",
|
||||
metadata=elements.ElementMetadata(
|
||||
parent_id=parent_id,
|
||||
text_as_html=ontology_element.to_html(add_children=False),
|
||||
page_number=page_number,
|
||||
category_depth=depth,
|
||||
filename=filename,
|
||||
),
|
||||
)
|
||||
]
|
||||
children: list[elements.Element] = []
|
||||
for child in ontology_element.children:
|
||||
child = ontology_to_unstructured_elements(
|
||||
child,
|
||||
parent_id=ontology_element.id,
|
||||
page_number=page_number,
|
||||
depth=0 if isinstance(ontology_element, ontology.Document) else depth + 1,
|
||||
filename=filename,
|
||||
add_img_alt_text=add_img_alt_text,
|
||||
)
|
||||
children += child
|
||||
|
||||
combined_children = combine_inline_elements(children)
|
||||
elements_to_return += combined_children
|
||||
else:
|
||||
element_class: type[elements.Element] = ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE[
|
||||
ontology_element.__class__
|
||||
]
|
||||
html_code_of_ontology_element = ontology_element.to_html()
|
||||
element_text = ontology_element.to_text(add_img_alt_text=add_img_alt_text)
|
||||
|
||||
unstructured_element = element_class(
|
||||
text=element_text, # type: ignore
|
||||
element_id=ontology_element.id,
|
||||
detection_origin="vlm_partitioner",
|
||||
metadata=elements.ElementMetadata(
|
||||
parent_id=parent_id,
|
||||
text_as_html=html_code_of_ontology_element,
|
||||
page_number=page_number,
|
||||
category_depth=depth,
|
||||
filename=filename,
|
||||
),
|
||||
)
|
||||
elements_to_return = [unstructured_element]
|
||||
|
||||
return elements_to_return
|
||||
|
||||
|
||||
def combine_inline_elements(elements: list[elements.Element]) -> list[elements.Element]:
|
||||
"""
|
||||
Combines consecutive inline elements into a single element. Inline elements
|
||||
can be also combined with text elements.
|
||||
|
||||
Combined elements contains multiple HTML tags together eg.
|
||||
{
|
||||
'text': "Text from element 1 Text from element 2",
|
||||
'metadata': {
|
||||
'text_as_html': "<p>Text from element 1</p><a>Text from element 2</a>"
|
||||
}
|
||||
}
|
||||
|
||||
Args:
|
||||
elements (list[Element]): A list of elements to be combined.
|
||||
|
||||
Returns:
|
||||
list[Element]: A list of combined elements.
|
||||
"""
|
||||
result_elements: list[elements.Element] = []
|
||||
|
||||
current_element: elements.Element | None = None
|
||||
for next_element in elements:
|
||||
if current_element is None:
|
||||
current_element = next_element
|
||||
continue
|
||||
|
||||
if can_unstructured_elements_be_merged(current_element, next_element):
|
||||
current_element.text += " " + next_element.text
|
||||
current_element.metadata.text_as_html += next_element.metadata.text_as_html
|
||||
else:
|
||||
result_elements.append(current_element)
|
||||
current_element = next_element
|
||||
|
||||
if current_element is not None:
|
||||
result_elements.append(current_element)
|
||||
|
||||
return result_elements
|
||||
|
||||
|
||||
def can_unstructured_elements_be_merged(
|
||||
current_element: elements.Element, next_element: elements.Element
|
||||
) -> bool:
|
||||
"""
|
||||
Elements can be merged when:
|
||||
- They are on the same level in the HTML tree
|
||||
- Neither of them has children
|
||||
- All elements are inline elements or text element
|
||||
"""
|
||||
if current_element.metadata.category_depth != next_element.metadata.category_depth:
|
||||
return False
|
||||
|
||||
current_html_tags = BeautifulSoup(
|
||||
current_element.metadata.text_as_html, "html.parser"
|
||||
).find_all(recursive=False)
|
||||
next_html_tags = BeautifulSoup(next_element.metadata.text_as_html, "html.parser").find_all(
|
||||
recursive=False
|
||||
)
|
||||
|
||||
ontology_elements = [
|
||||
parse_html_to_ontology_element(html_tag)
|
||||
for html_tag in chain(current_html_tags, next_html_tags)
|
||||
]
|
||||
|
||||
for ontology_element in ontology_elements:
|
||||
if ontology_element.children:
|
||||
return False
|
||||
|
||||
if not (is_inline_element(ontology_element) or is_text_element(ontology_element)):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_text_element(ontology_element: ontology.OntologyElement) -> bool:
|
||||
"""Categories or classes that we want to combine with inline text"""
|
||||
|
||||
text_classes = [
|
||||
ontology.NarrativeText,
|
||||
ontology.Quote,
|
||||
ontology.Paragraph,
|
||||
ontology.Footnote,
|
||||
ontology.FootnoteReference,
|
||||
ontology.Citation,
|
||||
ontology.Bibliography,
|
||||
ontology.Glossary,
|
||||
]
|
||||
text_categories = [ontology.ElementTypeEnum.metadata]
|
||||
|
||||
if any(isinstance(ontology_element, class_) for class_ in text_classes):
|
||||
return True
|
||||
|
||||
return any(ontology_element.elementType == category for category in text_categories)
|
||||
|
||||
|
||||
def is_inline_element(ontology_element: ontology.OntologyElement) -> bool:
|
||||
"""Categories or classes that we want to combine with text elements"""
|
||||
|
||||
inline_classes = [ontology.Hyperlink]
|
||||
inline_categories = [
|
||||
ontology.ElementTypeEnum.specialized_text,
|
||||
ontology.ElementTypeEnum.annotation,
|
||||
]
|
||||
|
||||
if any(isinstance(ontology_element, class_) for class_ in inline_classes):
|
||||
return True
|
||||
|
||||
return any(ontology_element.elementType == category for category in inline_categories)
|
||||
|
||||
|
||||
def unstructured_elements_to_ontology(
|
||||
unstructured_elements: Sequence[elements.Element],
|
||||
) -> ontology.OntologyElement:
|
||||
"""
|
||||
Converts a sequence of unstructured Element objects to an OntologyElement object.
|
||||
|
||||
The function caches the elements in a dictionary and each element is assigned to its parent.
|
||||
At the end the root element is popped from the dictionary and returned.
|
||||
|
||||
Such approach comes with limitations:
|
||||
- The parent element has to be in the list before the child element
|
||||
|
||||
Args:
|
||||
unstructured_elements (Sequence[Element]): The sequence of unstructured Element objects.
|
||||
|
||||
Returns:
|
||||
OntologyElement: The converted OntologyElement object.
|
||||
"""
|
||||
id_to_element_mapping: OrderedDict[str, ontology.OntologyElement] = OrderedDict()
|
||||
|
||||
root_element_id = unstructured_elements[0].metadata.parent_id
|
||||
|
||||
if root_element_id is None:
|
||||
root_element_id = ontology.OntologyElement.generate_unique_id()
|
||||
unstructured_elements[0].metadata.parent_id = root_element_id
|
||||
|
||||
id_to_element_mapping[root_element_id] = ontology.Document(
|
||||
additional_attributes={"id": root_element_id}
|
||||
)
|
||||
|
||||
for element in unstructured_elements:
|
||||
html_as_tags = BeautifulSoup(element.metadata.text_as_html, "html.parser").find_all(
|
||||
recursive=False
|
||||
)
|
||||
element_id = element.id
|
||||
parent_id = element.metadata.parent_id
|
||||
|
||||
if parent_id is None:
|
||||
# Make sure that no element is lost
|
||||
parent_id = root_element_id
|
||||
|
||||
for html_as_tag in html_as_tags:
|
||||
ontology_element = parse_html_to_ontology_element(html_as_tag)
|
||||
id_to_element_mapping[element_id] = ontology_element
|
||||
id_to_element_mapping[parent_id].children.append(ontology_element)
|
||||
|
||||
root_id, root_element = id_to_element_mapping.popitem(last=False)
|
||||
return root_element
|
||||
|
||||
|
||||
def parse_html_to_ontology(html_code: str) -> ontology.OntologyElement:
|
||||
"""
|
||||
Parses the given HTML code and converts it into an Element object.
|
||||
|
||||
Args:
|
||||
html_code (str): The HTML code to be parsed.
|
||||
Parsing HTML will start from <div class="Page">.
|
||||
Returns:
|
||||
OntologyElement: The parsed Element object.
|
||||
|
||||
Raises:
|
||||
ValueError: If no <body class="Document"> element is found in the HTML.
|
||||
"""
|
||||
html_code = remove_empty_divs_from_html_content(html_code)
|
||||
html_code = remove_empty_tags_from_html_content(html_code)
|
||||
soup = BeautifulSoup(html_code, "html.parser")
|
||||
document = soup.find("body", class_="Document")
|
||||
if not document:
|
||||
document = soup.find("div", class_="Page")
|
||||
|
||||
if not document:
|
||||
raise ValueError(
|
||||
"No <body class='Document'> or <div class='Page'> element found in the HTML."
|
||||
)
|
||||
|
||||
document_element = parse_html_to_ontology_element(document)
|
||||
return document_element
|
||||
|
||||
|
||||
def remove_empty_divs_from_html_content(html_content: str) -> str:
|
||||
soup = BeautifulSoup(html_content, "html.parser")
|
||||
divs = soup.find_all("div")
|
||||
for div in reversed(divs):
|
||||
if not div.attrs:
|
||||
div.unwrap()
|
||||
return str(soup)
|
||||
|
||||
|
||||
def remove_empty_tags_from_html_content(html_content: str) -> str:
|
||||
soup = BeautifulSoup(html_content, "html.parser")
|
||||
|
||||
def is_empty(tag):
|
||||
# Remove only specific tags, omit self-closing ones
|
||||
if tag.name not in ["p", "span", "div", "h1", "h2", "h3", "h4", "h5", "h6"]:
|
||||
return False
|
||||
|
||||
if tag.find():
|
||||
return False
|
||||
|
||||
if tag.attrs:
|
||||
return False
|
||||
|
||||
return bool(not tag.get_text(strip=True))
|
||||
|
||||
def remove_empty_tags(soup):
|
||||
for tag in soup.find_all():
|
||||
if is_empty(tag):
|
||||
tag.decompose()
|
||||
|
||||
remove_empty_tags(soup)
|
||||
|
||||
return str(soup)
|
||||
|
||||
|
||||
def parse_html_to_ontology_element(soup: Tag, recursion_depth: int = 1) -> ontology.OntologyElement:
|
||||
"""
|
||||
Converts a BeautifulSoup Tag object into an OntologyElement object. This function is recursive.
|
||||
First tries to recognize a class from Unstructured Ontology, then if class is matched tries
|
||||
to go deeper inside HTML tree. The recursive parsing is ended if the class is not recognized or
|
||||
there are no HTML Tags inside HTML - just text. Then it is parsed to
|
||||
Paragraph or UncategorizedText object.
|
||||
|
||||
Args:
|
||||
soup (Tag): The BeautifulSoup Tag object to be converted.
|
||||
recursion_depth (int): Flag to control limit of recursion depth.
|
||||
Returns:
|
||||
OntologyElement: The converted OntologyElement object.
|
||||
"""
|
||||
ontology_html_tag, ontology_class = extract_tag_and_ontology_class_from_tag(soup)
|
||||
escaped_attrs = get_escaped_attributes(soup)
|
||||
|
||||
if soup.name == "br": # Note(Pluto) should it be <br class="UncategorizedText">?
|
||||
return ontology.Paragraph(
|
||||
text="",
|
||||
css_class_name=None,
|
||||
html_tag_name="br",
|
||||
additional_attributes=escaped_attrs,
|
||||
)
|
||||
|
||||
has_children = (
|
||||
(ontology_class != ontology.UncategorizedText)
|
||||
and any(isinstance(content, Tag) for content in soup.contents)
|
||||
or ontology_class().elementType == ontology.ElementTypeEnum.layout
|
||||
)
|
||||
should_unwrap_html = has_children and recursion_depth <= RECURSION_LIMIT
|
||||
|
||||
if should_unwrap_html:
|
||||
text = ""
|
||||
children = [
|
||||
(
|
||||
parse_html_to_ontology_element(child, recursion_depth=recursion_depth + 1)
|
||||
if isinstance(child, Tag)
|
||||
else ontology.Paragraph(text=str(child).strip())
|
||||
)
|
||||
for child in soup.children
|
||||
if str(child).strip()
|
||||
]
|
||||
else:
|
||||
text = "\n".join([str(content).strip() for content in soup.contents]).strip()
|
||||
children = []
|
||||
|
||||
output_element = ontology_class(
|
||||
text=text,
|
||||
children=children,
|
||||
html_tag_name=ontology_html_tag,
|
||||
additional_attributes=escaped_attrs,
|
||||
)
|
||||
# TODO (Pluto): <input class="FormFieldValue"/> requires being wrapped in <label> tags
|
||||
return output_element
|
||||
|
||||
|
||||
def extract_tag_and_ontology_class_from_tag(
|
||||
soup: Tag,
|
||||
) -> tuple[str, Type[ontology.OntologyElement]]:
|
||||
"""
|
||||
Extracts the HTML tag and corresponding ontology class
|
||||
from a BeautifulSoup Tag object. The CSS class is prioritized over
|
||||
the HTML tag. If not recognized soup.name and UnstructuredText is returned.
|
||||
|
||||
Args:
|
||||
soup (Tag): The BeautifulSoup Tag object to extract information from.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing the HTML tag (str) and the ontology class (Type[OntologyElement]).
|
||||
"""
|
||||
html_tag, element_class = None, None
|
||||
|
||||
# Scenario 1: Valid Ontology Element
|
||||
if soup.attrs.get("class"):
|
||||
html_tag, element_class = (
|
||||
soup.name,
|
||||
HTML_TAG_AND_CSS_NAME_TO_ELEMENT_TYPE_MAP.get((soup.name, soup.attrs["class"][0])),
|
||||
)
|
||||
|
||||
# Scenario 2: HTML tag incorrect, CSS class correct
|
||||
# Fallback to css name selector and overwrite html tag
|
||||
if (
|
||||
not element_class
|
||||
and soup.attrs.get("class")
|
||||
and soup.attrs["class"][0] in CSS_CLASS_TO_ELEMENT_TYPE_MAP
|
||||
):
|
||||
element_class = CSS_CLASS_TO_ELEMENT_TYPE_MAP.get(soup.attrs["class"][0])
|
||||
html_tag = element_class().allowed_tags[0]
|
||||
|
||||
# Scenario 3: <input> elements, handled explicitly based on their 'type' attribute
|
||||
if not element_class and soup.name == "input":
|
||||
input_type = (str(soup.get("type")) or "").lower()
|
||||
if input_type == "checkbox":
|
||||
element_class = ontology.Checkbox
|
||||
elif input_type == "radio":
|
||||
element_class = ontology.RadioButton
|
||||
else:
|
||||
# Any other input (including missing type or text/number/etc.) is considered
|
||||
# a generic form field value.
|
||||
element_class = ontology.FormFieldValue
|
||||
html_tag = "input"
|
||||
|
||||
# Scenario 4: CSS class incorrect, but HTML tag correct and exclusive in ontology
|
||||
if not element_class and soup.name in HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP:
|
||||
html_tag, element_class = soup.name, HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP[soup.name]
|
||||
|
||||
# Scenario 5: CSS class incorrect, HTML tag incorrect
|
||||
# Fallback to default UncategorizedText
|
||||
if not element_class:
|
||||
# TODO (Pluto): Sometimes we could infer that from parent type and soup.name
|
||||
# e.g. parent=FormField soup.name=input -> element=FormFieldInput
|
||||
|
||||
html_tag = "span"
|
||||
element_class = ontology.UncategorizedText
|
||||
|
||||
# Scenario 6: UncategorizedText has image and no text
|
||||
# Typically, this happens with a span or div tag with an image inside
|
||||
if element_class == ontology.UncategorizedText and soup.find("img") and not soup.text.strip():
|
||||
element_class = ontology.Image
|
||||
|
||||
return html_tag, element_class
|
||||
|
||||
|
||||
def get_escaped_attributes(soup: Tag) -> dict[str, str | list[str]]:
|
||||
"""
|
||||
Escapes the attributes of a BeautifulSoup Tag object.
|
||||
|
||||
Args:
|
||||
soup (Tag): The BeautifulSoup Tag object whose attributes need to be escaped.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary with escaped attribute names and values.
|
||||
"""
|
||||
escaped_attrs: dict[str, str | list[str]] = {}
|
||||
for key, value in soup.attrs.items():
|
||||
escaped_key = html.escape(key)
|
||||
escaped_value = None
|
||||
if value:
|
||||
if isinstance(value, list):
|
||||
escaped_value = [html.escape(v) for v in value]
|
||||
else:
|
||||
escaped_value = html.escape(value)
|
||||
escaped_attrs[escaped_key] = escaped_value
|
||||
return escaped_attrs
|
||||
Reference in New Issue
Block a user