修改为东南天坐标系
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
class UnsupportedFileFormatError(Exception):
|
||||
"""File-type is not supported for this operation.
|
||||
|
||||
For example, when receiving a file for auto-partitioning where its file-formatt cannot be
|
||||
identified or there is no partitioner available for that file-format.
|
||||
"""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,460 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numbers
|
||||
import subprocess
|
||||
from enum import Enum
|
||||
from io import BufferedReader, BytesIO, TextIOWrapper
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from time import sleep
|
||||
from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, cast
|
||||
|
||||
import emoji
|
||||
import psutil
|
||||
|
||||
from unstructured.documents.coordinates import CoordinateSystem, PixelSpace
|
||||
from unstructured.documents.elements import (
|
||||
TYPE_TO_TEXT_ELEMENT_MAP,
|
||||
CheckBox,
|
||||
CoordinatesMetadata,
|
||||
Element,
|
||||
ElementMetadata,
|
||||
ElementType,
|
||||
ListItem,
|
||||
PageBreak,
|
||||
Text,
|
||||
)
|
||||
from unstructured.logger import logger
|
||||
from unstructured.nlp.patterns import ENUMERATED_BULLETS_RE, UNICODE_BULLETS_RE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from unstructured_inference.inference.layout import PageLayout
|
||||
from unstructured_inference.inference.layoutelement import LayoutElement
|
||||
|
||||
|
||||
def normalize_layout_element(
|
||||
layout_element: LayoutElement | Element | dict[str, Any],
|
||||
coordinate_system: Optional[CoordinateSystem] = None,
|
||||
infer_list_items: bool = True,
|
||||
source_format: Optional[str] = "html",
|
||||
) -> Element | list[Element]:
|
||||
"""Converts an unstructured_inference LayoutElement object to an unstructured Element."""
|
||||
|
||||
if isinstance(layout_element, Element) and source_format == "html":
|
||||
return layout_element
|
||||
|
||||
# NOTE(alan): Won't the lines above ensure this never runs (PageBreak is a subclass of Element)?
|
||||
if isinstance(layout_element, PageBreak):
|
||||
return PageBreak(text="")
|
||||
|
||||
if not isinstance(layout_element, dict):
|
||||
layout_dict = layout_element.to_dict()
|
||||
else:
|
||||
layout_dict = layout_element
|
||||
|
||||
text = layout_dict.get("text", "")
|
||||
# Both `coordinates` and `coordinate_system` must be present
|
||||
# in order to add coordinates metadata to the element.
|
||||
coordinates = layout_dict.get("coordinates") if coordinate_system else None
|
||||
element_type = layout_dict.get("type")
|
||||
prob = layout_dict.get("prob")
|
||||
aux_origin = layout_dict.get("source", None)
|
||||
origin = None
|
||||
if isinstance(layout_dict.get("is_extracted"), Enum):
|
||||
is_extracted = layout_dict["is_extracted"].value
|
||||
else:
|
||||
is_extracted = None
|
||||
if aux_origin:
|
||||
origin = aux_origin.value
|
||||
if prob and isinstance(prob, (int, str, float, numbers.Number)):
|
||||
class_prob_metadata = ElementMetadata(detection_class_prob=float(prob)) # type: ignore
|
||||
else:
|
||||
class_prob_metadata = ElementMetadata()
|
||||
class_prob_metadata.is_extracted = is_extracted
|
||||
common_kwargs = {
|
||||
"coordinates": coordinates,
|
||||
"coordinate_system": coordinate_system,
|
||||
"metadata": class_prob_metadata,
|
||||
"detection_origin": origin,
|
||||
}
|
||||
if element_type == ElementType.LIST:
|
||||
if infer_list_items:
|
||||
return layout_list_to_list_items(
|
||||
text,
|
||||
**common_kwargs,
|
||||
)
|
||||
else:
|
||||
return ListItem(
|
||||
text=text,
|
||||
**common_kwargs,
|
||||
)
|
||||
|
||||
elif element_type in TYPE_TO_TEXT_ELEMENT_MAP:
|
||||
assert isinstance(element_type, str) # Added to resolve type-error
|
||||
_element_class = TYPE_TO_TEXT_ELEMENT_MAP[element_type]
|
||||
_element_class = _element_class(
|
||||
text=text,
|
||||
**common_kwargs,
|
||||
)
|
||||
if element_type == ElementType.HEADLINE:
|
||||
_element_class.metadata.category_depth = 1
|
||||
elif element_type == ElementType.SUB_HEADLINE:
|
||||
_element_class.metadata.category_depth = 2
|
||||
return _element_class
|
||||
elif element_type in [
|
||||
ElementType.CHECK_BOX_CHECKED,
|
||||
ElementType.CHECK_BOX_UNCHECKED,
|
||||
ElementType.RADIO_BUTTON_CHECKED,
|
||||
ElementType.RADIO_BUTTON_UNCHECKED,
|
||||
ElementType.CHECKED,
|
||||
ElementType.UNCHECKED,
|
||||
]:
|
||||
checked = element_type in [
|
||||
ElementType.CHECK_BOX_CHECKED,
|
||||
ElementType.RADIO_BUTTON_CHECKED,
|
||||
ElementType.CHECKED,
|
||||
]
|
||||
return CheckBox(
|
||||
checked=checked,
|
||||
**common_kwargs,
|
||||
)
|
||||
else:
|
||||
return Text(
|
||||
text=text,
|
||||
**common_kwargs,
|
||||
)
|
||||
|
||||
|
||||
def layout_list_to_list_items(
|
||||
text: Optional[str],
|
||||
coordinates: Optional[tuple[tuple[float, float], ...]],
|
||||
coordinate_system: Optional[CoordinateSystem],
|
||||
metadata: Optional[ElementMetadata],
|
||||
detection_origin: Optional[str],
|
||||
) -> list[Element]:
|
||||
"""Converts a list LayoutElement to a list of ListItem elements."""
|
||||
split_items = ENUMERATED_BULLETS_RE.split(text) if text else []
|
||||
# NOTE(robinson) - this means there wasn't a match for the enumerated bullets
|
||||
if len(split_items) == 1:
|
||||
split_items = UNICODE_BULLETS_RE.split(text) if text else []
|
||||
|
||||
list_items: list[Element] = []
|
||||
for text_segment in split_items:
|
||||
if len(text_segment.strip()) > 0:
|
||||
# Both `coordinates` and `coordinate_system` must be present
|
||||
# in order to add coordinates metadata to the element.
|
||||
item = ListItem(
|
||||
text=text_segment.strip(),
|
||||
coordinates=coordinates,
|
||||
coordinate_system=coordinate_system,
|
||||
metadata=metadata,
|
||||
detection_origin=detection_origin,
|
||||
)
|
||||
list_items.append(item)
|
||||
|
||||
return list_items
|
||||
|
||||
|
||||
def add_element_metadata(
|
||||
element: Element,
|
||||
filename: Optional[str] = None,
|
||||
filetype: Optional[str] = None,
|
||||
page_number: Optional[int] = None,
|
||||
url: Optional[str] = None,
|
||||
text_as_html: Optional[str] = None,
|
||||
coordinates: Optional[tuple[tuple[float, float], ...]] = None,
|
||||
coordinate_system: Optional[CoordinateSystem] = None,
|
||||
image_path: Optional[str] = None,
|
||||
detection_origin: Optional[str] = None,
|
||||
languages: Optional[list[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Element:
|
||||
"""Adds document metadata to the document element.
|
||||
|
||||
Document metadata includes information like the filename, source url, and page number.
|
||||
"""
|
||||
|
||||
coordinates_metadata = (
|
||||
CoordinatesMetadata(
|
||||
points=coordinates,
|
||||
system=coordinate_system,
|
||||
)
|
||||
if coordinates is not None and coordinate_system is not None
|
||||
else None
|
||||
)
|
||||
links = element.links if hasattr(element, "links") and len(element.links) > 0 else None
|
||||
link_urls = [link.get("url") for link in links] if links else None
|
||||
link_texts = [link.get("text") for link in links] if links else None
|
||||
link_start_indexes = [link.get("start_index") for link in links] if links else None
|
||||
emphasized_texts = (
|
||||
element.emphasized_texts
|
||||
if hasattr(element, "emphasized_texts") and len(element.emphasized_texts) > 0
|
||||
else None
|
||||
)
|
||||
emphasized_text_contents = (
|
||||
[emphasized_text.get("text") for emphasized_text in emphasized_texts]
|
||||
if emphasized_texts
|
||||
else None
|
||||
)
|
||||
emphasized_text_tags = (
|
||||
[emphasized_text.get("tag") for emphasized_text in emphasized_texts]
|
||||
if emphasized_texts
|
||||
else None
|
||||
)
|
||||
depth = element.metadata.category_depth if element.metadata.category_depth else None
|
||||
|
||||
metadata = ElementMetadata(
|
||||
coordinates=coordinates_metadata,
|
||||
filename=filename,
|
||||
filetype=filetype,
|
||||
page_number=page_number,
|
||||
url=url,
|
||||
text_as_html=text_as_html,
|
||||
link_urls=link_urls,
|
||||
link_texts=link_texts,
|
||||
link_start_indexes=link_start_indexes,
|
||||
emphasized_text_contents=emphasized_text_contents,
|
||||
emphasized_text_tags=emphasized_text_tags,
|
||||
category_depth=depth,
|
||||
image_path=image_path,
|
||||
languages=languages,
|
||||
)
|
||||
element.metadata.update(metadata)
|
||||
if detection_origin is not None:
|
||||
element.metadata.detection_origin = detection_origin
|
||||
return element
|
||||
|
||||
|
||||
def remove_element_metadata(layout_elements: list[Element]) -> list[Element]:
|
||||
"""Removes document metadata from the document element.
|
||||
|
||||
Document metadata includes information like the filename, source url, and page number.
|
||||
"""
|
||||
elements: list[Element] = []
|
||||
metadata = ElementMetadata()
|
||||
for layout_element in layout_elements:
|
||||
element = normalize_layout_element(layout_element)
|
||||
if isinstance(element, list):
|
||||
for _element in element:
|
||||
_element.metadata = metadata
|
||||
elements.extend(element)
|
||||
else:
|
||||
element.metadata = metadata
|
||||
elements.append(element)
|
||||
return elements
|
||||
|
||||
|
||||
def _is_soffice_running():
|
||||
for proc in psutil.process_iter():
|
||||
try:
|
||||
if "soffice" in proc.name().lower():
|
||||
return True
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def convert_office_doc(
|
||||
input_filename: str,
|
||||
output_directory: str,
|
||||
target_format: str = "docx",
|
||||
target_filter: Optional[str] = None,
|
||||
wait_for_soffice_ready_time_out: int = 10,
|
||||
):
|
||||
"""Converts a .doc/.ppt file to a .docx/.pptx file using the libreoffice CLI.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_filename: str
|
||||
The name of the .doc file to convert to .docx
|
||||
output_directory: str
|
||||
The output directory for the convert .docx file
|
||||
target_format: str
|
||||
The desired output format
|
||||
target_filter: str
|
||||
The output filter name to use when converting. See references below
|
||||
for details.
|
||||
wait_for_soffice_ready_time_out: int
|
||||
The max wait time in seconds for soffice to become available to run
|
||||
|
||||
References
|
||||
----------
|
||||
https://stackoverflow.com/questions/52277264/convert-doc-to-docx-using-soffice-not-working
|
||||
https://git.libreoffice.org/core/+/refs/heads/master/filter/source/config/fragments/filters
|
||||
|
||||
"""
|
||||
if target_filter is not None:
|
||||
target_format = f"{target_format}:{target_filter}"
|
||||
# NOTE(robinson) - In the future can also include win32com client as a fallback for windows
|
||||
# users who do not have LibreOffice installed
|
||||
# ref: https://stackoverflow.com/questions/38468442/
|
||||
# multiple-doc-to-docx-file-conversion-using-python
|
||||
command = [
|
||||
"soffice",
|
||||
"--headless",
|
||||
"--convert-to",
|
||||
target_format,
|
||||
"--outdir",
|
||||
output_directory,
|
||||
input_filename,
|
||||
]
|
||||
try:
|
||||
# only one soffice process can be ran
|
||||
wait_time = 0
|
||||
sleep_time = 0.1
|
||||
output = subprocess.run(command, capture_output=True)
|
||||
message = output.stdout.decode().strip()
|
||||
# we can't rely on returncode unfortunately because on macOS it would return 0 even when the
|
||||
# command failed to run; instead we have to rely on the stdout being empty as a sign of the
|
||||
# process failed
|
||||
while (wait_time < wait_for_soffice_ready_time_out) and (message == ""):
|
||||
wait_time += sleep_time
|
||||
if _is_soffice_running():
|
||||
sleep(sleep_time)
|
||||
else:
|
||||
output = subprocess.run(command, capture_output=True)
|
||||
message = output.stdout.decode().strip()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(
|
||||
"""soffice command was not found. Please install libreoffice
|
||||
on your system and try again.
|
||||
|
||||
- Install instructions: https://www.libreoffice.org/get-help/install-howto/
|
||||
- Mac: https://formulae.brew.sh/cask/libreoffice
|
||||
- Debian: https://wiki.debian.org/LibreOffice""",
|
||||
)
|
||||
|
||||
logger.info(message)
|
||||
if output.returncode != 0 or message == "":
|
||||
logger.error(
|
||||
"soffice failed to convert to format %s with code %i", target_format, output.returncode
|
||||
)
|
||||
logger.error(output.stderr.decode().strip())
|
||||
|
||||
|
||||
def exactly_one(**kwargs: Any) -> None:
|
||||
"""
|
||||
Verify arguments; exactly one of all keyword arguments must not be None.
|
||||
|
||||
Example:
|
||||
>>> exactly_one(filename=filename, file=file, text=text, url=url)
|
||||
"""
|
||||
if sum([(arg is not None and arg != "") for arg in kwargs.values()]) != 1:
|
||||
names = list(kwargs.keys())
|
||||
if len(names) > 1:
|
||||
message = f"Exactly one of {', '.join(names[:-1])} and {names[-1]} must be specified."
|
||||
else:
|
||||
message = f"{names[0]} must be specified."
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def spooled_to_bytes_io_if_needed(file: _T | SpooledTemporaryFile[bytes]) -> _T | BytesIO:
|
||||
"""Convert `file` to `BytesIO` when it is a `SpooledTemporaryFile`.
|
||||
|
||||
Note that `file` does not need to be IO[bytes]. It can be `None` or `bytes` and this function
|
||||
will not complain.
|
||||
|
||||
In Python <3.11, `SpooledTemporaryFile` does not implement `.readable()` or `.seekable()` which
|
||||
triggers an exception when the file is loaded by certain packages. In particular, the stdlib
|
||||
`zipfile.Zipfile` raises on opening a `SpooledTemporaryFile` as does `Pandas.read_csv()`.
|
||||
"""
|
||||
if isinstance(file, SpooledTemporaryFile):
|
||||
file.seek(0)
|
||||
return BytesIO(cast(bytes, file.read()))
|
||||
|
||||
# -- return `file` unchanged otherwise --
|
||||
return file
|
||||
|
||||
|
||||
def convert_to_bytes(file: bytes | IO[bytes]) -> bytes:
|
||||
"""Extract the bytes from `file` without preventing it from being read again later.
|
||||
|
||||
As a convenience to simplify client code, also returns `file` unchanged if it is already bytes.
|
||||
"""
|
||||
if isinstance(file, bytes):
|
||||
return file
|
||||
|
||||
if isinstance(file, SpooledTemporaryFile):
|
||||
file.seek(0)
|
||||
f_bytes = file.read()
|
||||
file.seek(0)
|
||||
return f_bytes
|
||||
|
||||
if isinstance(file, BytesIO):
|
||||
return file.getvalue()
|
||||
|
||||
if isinstance(file, (TextIOWrapper, BufferedReader)):
|
||||
with open(file.name, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
raise ValueError("Invalid file-like object type")
|
||||
|
||||
|
||||
def contains_emoji(s: str) -> bool:
|
||||
"""
|
||||
Check if the input string contains any emoji characters.
|
||||
|
||||
Parameters:
|
||||
- s (str): The input string to check.
|
||||
|
||||
Returns:
|
||||
- bool: True if the string contains any emoji, False otherwise.
|
||||
"""
|
||||
|
||||
return bool(emoji.emoji_count(s))
|
||||
|
||||
|
||||
def get_page_image_metadata(page: PageLayout) -> dict[str, Any]:
|
||||
"""Retrieve image metadata and coordinate system from a page."""
|
||||
|
||||
image = getattr(page, "image", None)
|
||||
image_metadata = getattr(page, "image_metadata", None)
|
||||
|
||||
if image:
|
||||
image_format = image.format
|
||||
image_width = image.width
|
||||
image_height = image.height
|
||||
elif image_metadata:
|
||||
image_format = image_metadata.get("format")
|
||||
image_width = image_metadata.get("width")
|
||||
image_height = image_metadata.get("height")
|
||||
else:
|
||||
image_format = None
|
||||
image_width = None
|
||||
image_height = None
|
||||
|
||||
return {
|
||||
"format": image_format,
|
||||
"width": image_width,
|
||||
"height": image_height,
|
||||
}
|
||||
|
||||
|
||||
def ocr_data_to_elements(
|
||||
ocr_data: list["LayoutElement"],
|
||||
image_size: tuple[int | float, int | float],
|
||||
common_metadata: Optional[ElementMetadata] = None,
|
||||
infer_list_items: bool = True,
|
||||
source_format: Optional[str] = None,
|
||||
) -> list[Element]:
|
||||
"""Convert OCR layout data into `unstructured` elements with associated metadata."""
|
||||
|
||||
image_width, image_height = image_size
|
||||
coordinate_system = PixelSpace(width=image_width, height=image_height)
|
||||
elements: list[Element] = []
|
||||
for layout_element in ocr_data:
|
||||
element = normalize_layout_element(
|
||||
layout_element,
|
||||
coordinate_system=coordinate_system,
|
||||
infer_list_items=infer_list_items,
|
||||
source_format=source_format if source_format else "html",
|
||||
)
|
||||
|
||||
if common_metadata:
|
||||
element.metadata.update(common_metadata)
|
||||
|
||||
elements.append(element)
|
||||
|
||||
return elements
|
||||
@@ -0,0 +1,538 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from typing import Iterable, Iterator, Optional
|
||||
|
||||
import iso639 # pyright: ignore[reportMissingTypeStubs]
|
||||
from langdetect import ( # pyright: ignore[reportMissingTypeStubs]
|
||||
DetectorFactory,
|
||||
detect_langs, # pyright: ignore[reportUnknownVariableType]
|
||||
lang_detect_exception,
|
||||
)
|
||||
|
||||
from unstructured.documents.elements import Element
|
||||
from unstructured.logger import logger
|
||||
from unstructured.partition.utils.constants import (
|
||||
TESSERACT_LANGUAGES_AND_CODES,
|
||||
TESSERACT_LANGUAGES_SPLITTER,
|
||||
)
|
||||
|
||||
_ASCII_RE = re.compile(r"^[\x00-\x7F]+$")
|
||||
|
||||
# pytesseract.get_languages(config="") only shows user installed language packs,
|
||||
# so manually include the list of all currently supported Tesseract languages
|
||||
PYTESSERACT_LANG_CODES = [
|
||||
"afr",
|
||||
"amh",
|
||||
"ara",
|
||||
"asm",
|
||||
"aze",
|
||||
"aze_cyrl",
|
||||
"bel",
|
||||
"ben",
|
||||
"bod",
|
||||
"bos",
|
||||
"bre",
|
||||
"bul",
|
||||
"cat",
|
||||
"ceb",
|
||||
"ces",
|
||||
"chi_sim",
|
||||
"chi_sim_vert",
|
||||
"chi_tra",
|
||||
"chi_tra_vert",
|
||||
"chr",
|
||||
"cos",
|
||||
"cym",
|
||||
"dan",
|
||||
"deu",
|
||||
"div",
|
||||
"dzo",
|
||||
"ell",
|
||||
"eng",
|
||||
"enm",
|
||||
"epo",
|
||||
"equ",
|
||||
"est",
|
||||
"eus",
|
||||
"fao",
|
||||
"fas",
|
||||
"fil",
|
||||
"fin",
|
||||
"fra",
|
||||
"frk",
|
||||
"frm",
|
||||
"fry",
|
||||
"gla",
|
||||
"gle",
|
||||
"glg",
|
||||
"grc",
|
||||
"guj",
|
||||
"hat",
|
||||
"heb",
|
||||
"hin",
|
||||
"hrv",
|
||||
"hun",
|
||||
"hye",
|
||||
"iku",
|
||||
"ind",
|
||||
"isl",
|
||||
"ita",
|
||||
"ita_old",
|
||||
"jav",
|
||||
"jpn",
|
||||
"jpn_vert",
|
||||
"kan",
|
||||
"kat",
|
||||
"kat_old",
|
||||
"kaz",
|
||||
"khm",
|
||||
"kir",
|
||||
"kmr",
|
||||
"kor",
|
||||
"kor_vert",
|
||||
"lao",
|
||||
"lat",
|
||||
"lav",
|
||||
"lit",
|
||||
"ltz",
|
||||
"mal",
|
||||
"mar",
|
||||
"mkd",
|
||||
"mlt",
|
||||
"mon",
|
||||
"mri",
|
||||
"msa",
|
||||
"mya",
|
||||
"nep",
|
||||
"nld",
|
||||
"nor",
|
||||
"oci",
|
||||
"ori",
|
||||
"osd",
|
||||
"pan",
|
||||
"pol",
|
||||
"por",
|
||||
"pus",
|
||||
"que",
|
||||
"ron",
|
||||
"rus",
|
||||
"san",
|
||||
"sin",
|
||||
"slk",
|
||||
"slv",
|
||||
"snd",
|
||||
"snum",
|
||||
"spa",
|
||||
"spa_old",
|
||||
"sqi",
|
||||
"srp",
|
||||
"srp_latn",
|
||||
"sun",
|
||||
"swa",
|
||||
"swe",
|
||||
"syr",
|
||||
"tam",
|
||||
"tat",
|
||||
"tel",
|
||||
"tgk",
|
||||
"tha",
|
||||
"tir",
|
||||
"ton",
|
||||
"tur",
|
||||
"uig",
|
||||
"ukr",
|
||||
"urd",
|
||||
"uzb",
|
||||
"uzb_cyrl",
|
||||
"vie",
|
||||
"yid",
|
||||
"yor",
|
||||
]
|
||||
|
||||
PYTESSERACT_TO_PADDLE_LANG_CODE_MAP = {
|
||||
"afr": "af", # Afrikaans
|
||||
"ara": "ar", # Arabic
|
||||
"aze": "az", # Azerbaijani
|
||||
"bel": "be", # Belarusian
|
||||
"bos": "bs", # Bosnian
|
||||
"bul": "bg", # Bulgarian
|
||||
"ces": "cs", # Czech
|
||||
"chi_sim": "ch", # Simplified Chinese
|
||||
"chi_tra": "chinese_cht", # Traditional Chinese
|
||||
"cym": "cy", # Welsh
|
||||
"dan": "da", # Danish
|
||||
"deu": "german", # German
|
||||
"eng": "en", # English
|
||||
"est": "et", # Estonian
|
||||
"fas": "fa", # Persian
|
||||
"fra": "fr", # French
|
||||
"gle": "ga", # Irish
|
||||
"hin": "hi", # Hindi
|
||||
"hrv": "hr", # Croatian
|
||||
"hun": "hu", # Hungarian
|
||||
"ind": "id", # Indonesian
|
||||
"isl": "is", # Icelandic
|
||||
"ita": "it", # Italian
|
||||
"jpn": "japan", # Japanese
|
||||
"kor": "korean", # Korean
|
||||
"kmr": "ku", # Kurdish
|
||||
"lat": "rs_latin", # Latin
|
||||
"lav": "lv", # Latvian
|
||||
"lit": "lt", # Lithuanian
|
||||
"mar": "mr", # Marathi
|
||||
"mlt": "mt", # Maltese
|
||||
"msa": "ms", # Malay
|
||||
"nep": "ne", # Nepali
|
||||
"nld": "nl", # Dutch
|
||||
"nor": "no", # Norwegian
|
||||
"pol": "pl", # Polish
|
||||
"por": "pt", # Portuguese
|
||||
"ron": "ro", # Romanian
|
||||
"rus": "ru", # Russian
|
||||
"slk": "sk", # Slovak
|
||||
"slv": "sl", # Slovenian
|
||||
"spa": "es", # Spanish
|
||||
"sqi": "sq", # Albanian
|
||||
"srp": "rs_cyrillic", # Serbian
|
||||
"swa": "sw", # Swahili
|
||||
"swe": "sv", # Swedish
|
||||
"tam": "ta", # Tamil
|
||||
"tel": "te", # Telugu
|
||||
"tur": "tr", # Turkish
|
||||
"uig": "ug", # Uyghur
|
||||
"ukr": "uk", # Ukrainian
|
||||
"urd": "ur", # Urdu
|
||||
"uzb": "uz", # Uzbek
|
||||
"vie": "vi", # Vietnamese
|
||||
}
|
||||
|
||||
|
||||
def prepare_languages_for_tesseract(languages: Optional[list[str]] = ["eng"]) -> str:
|
||||
"""
|
||||
Entry point: convert languages (list of strings) into tesseract ocr langcode format (uses +)
|
||||
"""
|
||||
if languages is None:
|
||||
raise ValueError("`languages` can not be `None`")
|
||||
converted_languages = [
|
||||
lang_code
|
||||
for lang_code in (
|
||||
_convert_language_code_to_pytesseract_lang_code(lang) for lang in languages
|
||||
)
|
||||
if lang_code
|
||||
]
|
||||
# Remove duplicates from the list but keep the original order
|
||||
converted_languages = list(dict.fromkeys(converted_languages))
|
||||
if len(converted_languages) == 0:
|
||||
logger.warning(
|
||||
"Failed to find any valid standard language code from "
|
||||
f"languages: {languages}, proceed with `eng` instead.",
|
||||
)
|
||||
return "eng"
|
||||
|
||||
return TESSERACT_LANGUAGES_SPLITTER.join(converted_languages)
|
||||
|
||||
|
||||
def tesseract_to_paddle_language(tesseract_language: str) -> str:
|
||||
"""
|
||||
Convert TesseractOCR language code to PaddleOCR language code.
|
||||
|
||||
:param tesseract_language: str, language code used in TesseractOCR
|
||||
:return: str, corresponding language code for PaddleOCR or None if not found
|
||||
"""
|
||||
|
||||
lang = PYTESSERACT_TO_PADDLE_LANG_CODE_MAP.get(tesseract_language.lower())
|
||||
if not lang:
|
||||
logger.warning(
|
||||
f"{tesseract_language} is not a language code supported by PaddleOCR, "
|
||||
f"proceeding with `en` instead."
|
||||
)
|
||||
return "en"
|
||||
|
||||
return lang
|
||||
|
||||
|
||||
def check_language_args(
|
||||
languages: list[str], ocr_languages: str | list[str] | None
|
||||
) -> list[str] | None:
|
||||
"""Handle users defining both `ocr_languages` and `languages`.
|
||||
|
||||
Give preference to `languages` and convert `ocr_languages` if needed, but default to `None`.
|
||||
|
||||
`ocr_languages` is only a parameter for `auto.partition`, `partition_image`, & `partition_pdf`.
|
||||
`ocr_languages` should not be defined as 'auto' since 'auto' is intended for language detection
|
||||
which is not supported by `partition_image` or `partition_pdf`.
|
||||
"""
|
||||
# --- Clean and update defaults
|
||||
if ocr_languages:
|
||||
ocr_languages = _clean_ocr_languages_arg(ocr_languages)
|
||||
logger.warning(
|
||||
"The ocr_languages kwarg will be deprecated in a future version of unstructured. "
|
||||
"Please use languages instead.",
|
||||
)
|
||||
assert ocr_languages is None or isinstance(ocr_languages, str)
|
||||
|
||||
if ocr_languages and "auto" in ocr_languages:
|
||||
raise ValueError(
|
||||
"`ocr_languages` is deprecated but was used to extract text from pdfs and images."
|
||||
" The 'auto' argument is only for language *detection* when it is assigned"
|
||||
" to `languages` and partitioning documents other than pdfs or images."
|
||||
" Language detection is not currently supported in pdfs or images."
|
||||
)
|
||||
|
||||
if not isinstance(languages, list): # pyright: ignore[reportUnnecessaryIsInstance]
|
||||
raise TypeError(
|
||||
"The language parameter must be a list of language codes as strings, ex. ['eng']",
|
||||
)
|
||||
|
||||
# --- If `languages` is a null/default value and `ocr_languages` is defined, use `ocr_languages`
|
||||
if ocr_languages and (languages == ["auto"] or languages == [""] or not languages):
|
||||
languages = ocr_languages.split(TESSERACT_LANGUAGES_SPLITTER)
|
||||
logger.warning(
|
||||
"Only one of languages and ocr_languages should be specified. "
|
||||
"languages is preferred. ocr_languages is marked for deprecation.",
|
||||
)
|
||||
|
||||
# --- Clean `languages`
|
||||
# If "auto" is included in the list of inputs, language detection will be triggered downstream.
|
||||
# The rest of the inputted languages are ignored.
|
||||
if languages:
|
||||
if "auto" not in languages:
|
||||
for i, lang in enumerate(languages):
|
||||
languages[i] = TESSERACT_LANGUAGES_AND_CODES.get(lang.lower(), lang)
|
||||
|
||||
str_languages = _clean_ocr_languages_arg(languages)
|
||||
if not str_languages:
|
||||
return None
|
||||
languages = str_languages.split(TESSERACT_LANGUAGES_SPLITTER)
|
||||
# else, remove the extraneous languages.
|
||||
# NOTE (jennings): "auto" should only be used for partitioners OTHER THAN `_pdf` or `_image`
|
||||
else:
|
||||
# define as 'auto' for language detection when partitioning non-pdfs or -images
|
||||
languages = ["auto"]
|
||||
return languages
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def convert_old_ocr_languages_to_languages(ocr_languages: str) -> list[str]:
|
||||
"""
|
||||
Convert ocr_languages parameter to list of langcode strings.
|
||||
Assumption: ocr_languages is in tesseract plus sign format
|
||||
"""
|
||||
|
||||
return ocr_languages.split(TESSERACT_LANGUAGES_SPLITTER)
|
||||
|
||||
|
||||
def _convert_language_code_to_pytesseract_lang_code(lang: str) -> str:
|
||||
"""
|
||||
Convert a single language code to its tesseract formatted and recognized
|
||||
langcode(s), if supported.
|
||||
"""
|
||||
# if language is already tesseract langcode, return it immediately
|
||||
# this will catch the tesseract special cases equ and osd
|
||||
# NOTE(shreya): this may catch some cases of choosing between tesseract code variants for a lang
|
||||
if lang in PYTESSERACT_LANG_CODES:
|
||||
return lang
|
||||
|
||||
lang_iso639 = _get_iso639_language_object(lang)
|
||||
|
||||
# tesseract uses 3 digit codes (639-3, 639-2b, etc) as prefixes, with suffixes for orthography
|
||||
# use first 3 letters of tesseract codes for matching to standard codes
|
||||
pytesseract_langs_3 = {lang[:3] for lang in PYTESSERACT_LANG_CODES}
|
||||
|
||||
if lang_iso639:
|
||||
# try to match ISO 639-3 code
|
||||
if lang_iso639.part3 in pytesseract_langs_3:
|
||||
matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part3)
|
||||
return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
|
||||
|
||||
# try to match ISO 639-2b
|
||||
elif lang_iso639.part2b in pytesseract_langs_3:
|
||||
matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part2b)
|
||||
return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
|
||||
|
||||
# try to match ISO 639-2t
|
||||
elif lang_iso639.part2t in pytesseract_langs_3:
|
||||
matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part2t)
|
||||
return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
|
||||
|
||||
else:
|
||||
logger.warning(f"{lang} is not a language supported by Tesseract.")
|
||||
return ""
|
||||
logger.warning(f"{lang} is not a language supported by Tesseract.")
|
||||
return ""
|
||||
|
||||
|
||||
def _get_iso639_language_object(lang: str) -> Optional[iso639.Language]:
|
||||
language = _cached_iso639_language_match(lang)
|
||||
if language is not None:
|
||||
return language
|
||||
logger.warning(f"{lang} is not a valid standard language code.")
|
||||
return None
|
||||
|
||||
|
||||
def _get_all_tesseract_langcodes_with_prefix(prefix: str) -> list[str]:
|
||||
"""
|
||||
Get all matching tesseract langcodes with this prefix (may be one or multiple variants).
|
||||
"""
|
||||
return [langcode for langcode in PYTESSERACT_LANG_CODES if langcode.startswith(prefix)]
|
||||
|
||||
|
||||
def detect_languages(
|
||||
text: str,
|
||||
languages: Optional[list[str]] = ["auto"],
|
||||
) -> Optional[list[str]]:
|
||||
"""
|
||||
Detects the list of languages present in the text (in the default "auto" mode),
|
||||
or formats and passes through the user inputted document languages if provided.
|
||||
"""
|
||||
if languages is None:
|
||||
languages = ["auto"]
|
||||
if not isinstance(languages, list):
|
||||
raise TypeError(
|
||||
'The language parameter must be a list of language codes as strings, ex. ["eng"]',
|
||||
)
|
||||
|
||||
# Skip language detection for partitioners that use other partitioners.
|
||||
# For example, partition_msg relies on partition_html and partition_text, but the metadata
|
||||
# gets overwritten after elements have been returned by _html and _text,
|
||||
# so `languages` would be detected twice.
|
||||
# Also return None if there is no text.
|
||||
if languages[0] == "" or text.strip() == "":
|
||||
return None
|
||||
|
||||
# If text contains special characters (like ñ, å, or Korean/Mandarin/etc.) it will NOT default
|
||||
# to English. It will default to English if text is only ascii characters and is short.
|
||||
if _ASCII_RE.match(text) and len(text.split()) < 5:
|
||||
logger.debug(f'short text: "{text}". Defaulting to English.')
|
||||
return ["eng"]
|
||||
|
||||
# set seed for deterministic langdetect outputs
|
||||
DetectorFactory.seed = 0
|
||||
|
||||
doc_languages: list[str] = []
|
||||
|
||||
# user inputted languages:
|
||||
# if "auto" is included in the list of inputs, language detection will be triggered
|
||||
# and the rest of the inputted languages will be ignored
|
||||
if languages and "auto" not in languages:
|
||||
for lang in languages:
|
||||
str_lang = TESSERACT_LANGUAGES_AND_CODES.get(lang.lower(), lang)
|
||||
language = _get_iso639_language_object(str_lang[:3])
|
||||
if language:
|
||||
doc_languages.append(language.part3)
|
||||
|
||||
# language detection:
|
||||
else:
|
||||
# warn if any values other than "auto" were provided
|
||||
if len(languages) > 1:
|
||||
logger.warning(
|
||||
f'Since "auto" is present in the input languages provided ({languages}), '
|
||||
"the language will be auto detected and the rest of the inputted "
|
||||
"languages will be ignored.",
|
||||
)
|
||||
|
||||
try:
|
||||
langdetect_result = detect_langs(text)
|
||||
except lang_detect_exception.LangDetectException as e:
|
||||
logger.warning(e)
|
||||
return None # None as default
|
||||
|
||||
langdetect_langs: list[str] = []
|
||||
|
||||
# NOTE(robinson) - Chinese gets detected with codes zh-cn, zh-tw, zh-hk for various
|
||||
# Chinese variants. We normalizes these because there is a single model for Chinese
|
||||
# machine translation
|
||||
# TODO(shreya): decide how to maintain nonstandard chinese script information
|
||||
for langobj in langdetect_result:
|
||||
lang_val = str(langobj.lang)
|
||||
if lang_val.startswith("zh"): # pyright: ignore
|
||||
langdetect_langs.append("zho")
|
||||
else:
|
||||
language = _get_iso639_language_object(lang_val[:3]) # pyright: ignore
|
||||
if language:
|
||||
langdetect_langs.append(language.part3)
|
||||
|
||||
# remove duplicate chinese (if exists) without modifying order
|
||||
seen = set(doc_languages)
|
||||
for lang in langdetect_langs:
|
||||
if lang not in seen:
|
||||
doc_languages.append(lang)
|
||||
seen.add(lang)
|
||||
|
||||
return doc_languages
|
||||
|
||||
|
||||
def apply_lang_metadata(
|
||||
elements: Iterable[Element],
|
||||
languages: Optional[list[str]],
|
||||
detect_language_per_element: bool = False,
|
||||
) -> Iterator[Element]:
|
||||
"""Detect language and apply it to metadata.languages for each element in `elements`.
|
||||
If languages is None, default to auto detection.
|
||||
If languages is and empty string, skip."""
|
||||
# -- Note this function has a stream interface, but reads the full `elements` stream into memory
|
||||
# -- before emitting the first updated element as output.
|
||||
|
||||
# The auto `partition` function uses `None` as a default because the default for
|
||||
# `partition_pdf` and `partition_img` conflict with the other partitioners that use ["auto"]
|
||||
if languages is None:
|
||||
languages = ["auto"]
|
||||
|
||||
# Skip language detection for partitioners that use other partitioners.
|
||||
# For example, partition_msg relies on partition_html and partition_text, but the metadata
|
||||
# gets overwritten after elements have been returned by _html and _text,
|
||||
# so `languages` would be detected twice.
|
||||
if languages == [""]:
|
||||
yield from elements
|
||||
return
|
||||
|
||||
# Convert elements to a list to get the text, detect the language, and add it to the elements
|
||||
if not isinstance(elements, list):
|
||||
elements = list(elements)
|
||||
|
||||
full_text = " ".join(str(e.text) for e in elements if hasattr(e, "text") and e.text)
|
||||
detected_languages = detect_languages(text=full_text, languages=languages)
|
||||
if (
|
||||
detected_languages is not None
|
||||
and len(detected_languages) == 1
|
||||
and detect_language_per_element is False
|
||||
):
|
||||
# -- apply detected language to each element's metadata --
|
||||
for e in elements:
|
||||
e.metadata.languages = detected_languages
|
||||
yield e
|
||||
else:
|
||||
for e in elements:
|
||||
if hasattr(e, "text"):
|
||||
text_value = str(e.text) if e.text is not None else ""
|
||||
e.metadata.languages = detect_languages(text_value)
|
||||
yield e
|
||||
else:
|
||||
yield e
|
||||
|
||||
|
||||
def _clean_ocr_languages_arg(ocr_languages: list[str] | str) -> str:
|
||||
"""Fix common incorrect definitions for ocr_languages:
|
||||
defining it as a list, adding extra quotation marks, adding brackets.
|
||||
Returns a single string of ocr_languages"""
|
||||
# extract from list
|
||||
if isinstance(ocr_languages, list):
|
||||
ocr_languages = "+".join(ocr_languages)
|
||||
|
||||
# remove extra quotations
|
||||
ocr_languages = re.sub(r"[\"']", "", ocr_languages)
|
||||
# remove brackets
|
||||
ocr_languages = re.sub(r"[\[\]]", "", ocr_languages)
|
||||
|
||||
return ocr_languages
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _cached_iso639_language_match(lang: str) -> Optional[iso639.Language]:
|
||||
try:
|
||||
return iso639.Language.match(lang.lower()) # pyright: ignore[reportUnknownMemberType]
|
||||
except iso639.LanguageNotFoundError:
|
||||
return None
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Helpers used across multiple partitioners to compute metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import datetime as dt
|
||||
import functools
|
||||
import os
|
||||
from typing import Any, Callable, Iterator, Sequence
|
||||
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
from unstructured.documents.elements import Element, ElementMetadata
|
||||
from unstructured.file_utils.model import FileType
|
||||
from unstructured.partition.common.lang import apply_lang_metadata
|
||||
from unstructured.utils import get_call_args_applying_defaults
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
|
||||
def get_last_modified_date(filename: str) -> str | None:
|
||||
"""Modification time of file at path `filename`, if it exists.
|
||||
|
||||
Returns `None` when `filename` is not a path to a file on the local filesystem.
|
||||
|
||||
Otherwise returns date and time in ISO 8601 string format (YYYY-MM-DDTHH:MM:SS) like
|
||||
"2024-03-05T17:02:53".
|
||||
"""
|
||||
if not os.path.isfile(filename):
|
||||
return None
|
||||
|
||||
modify_date = dt.datetime.fromtimestamp(os.path.getmtime(filename))
|
||||
return modify_date.strftime("%Y-%m-%dT%H:%M:%S%z")
|
||||
|
||||
|
||||
HIERARCHY_RULE_SET = {
|
||||
"Title": [
|
||||
"Text",
|
||||
"UncategorizedText",
|
||||
"NarrativeText",
|
||||
"ListItem",
|
||||
"BulletedText",
|
||||
"Table",
|
||||
"FigureCaption",
|
||||
"CheckBox",
|
||||
"Table",
|
||||
],
|
||||
"Header": [
|
||||
"Title",
|
||||
"Text",
|
||||
"UncategorizedText",
|
||||
"NarrativeText",
|
||||
"ListItem",
|
||||
"BulletedText",
|
||||
"Table",
|
||||
"FigureCaption",
|
||||
"CheckBox",
|
||||
"Table",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def set_element_hierarchy(
|
||||
elements: Sequence[Element], ruleset: dict[str, list[str]] = HIERARCHY_RULE_SET
|
||||
) -> list[Element]:
|
||||
"""Sets `.metadata.parent_id` for each element it applies to.
|
||||
|
||||
`parent_id` assignment is based on the element's category and depth. The importance of an
|
||||
element's category is determined by a rule set. The rule set trumps category_depth. That is,
|
||||
category_depth is only relevant when elements are of the same category.
|
||||
"""
|
||||
stack: list[Element] = []
|
||||
for element in elements:
|
||||
if element.metadata.parent_id is not None:
|
||||
continue
|
||||
parent_id = None
|
||||
element_category = getattr(element, "category", None)
|
||||
element_category_depth = getattr(element.metadata, "category_depth", 0) or 0
|
||||
|
||||
# -- skip elements without a category --
|
||||
if not element_category:
|
||||
continue
|
||||
|
||||
while stack:
|
||||
top_element: Element = stack[-1]
|
||||
top_element_category = getattr(top_element, "category")
|
||||
top_element_category_depth = (
|
||||
getattr(
|
||||
top_element.metadata,
|
||||
"category_depth",
|
||||
0,
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
if (
|
||||
top_element_category == element_category
|
||||
and top_element_category_depth < element_category_depth
|
||||
) or (
|
||||
top_element_category != element_category
|
||||
and element_category in ruleset.get(top_element_category, [])
|
||||
):
|
||||
parent_id = top_element.id
|
||||
break
|
||||
|
||||
stack.pop()
|
||||
|
||||
element.metadata.parent_id = parent_id
|
||||
stack.append(element)
|
||||
|
||||
return list(elements)
|
||||
|
||||
|
||||
# ================================================================================================
|
||||
# METADATA POST-PARTITIONING PROCESSING DECORATOR
|
||||
# ================================================================================================
|
||||
|
||||
|
||||
def apply_metadata(
|
||||
file_type: FileType | None = None,
|
||||
) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
|
||||
"""Post-process element-metadata for this document.
|
||||
|
||||
This decorator adds a post-processing step to a partitioner, primarily to apply metadata that
|
||||
is common to all partitioners. It assumes the following responsibilities:
|
||||
|
||||
- Hash element-ids. Computes and applies SHA1 hash element.id when `unique_element_ids`
|
||||
argument is False.
|
||||
|
||||
- Element Hierarchy. Computes and applies `parent_id` metadata based on `category_depth`
|
||||
etc. added by partitioner.
|
||||
|
||||
- Language metadata. Computes and applies `language` metadata based on a language detection
|
||||
model.
|
||||
|
||||
- Apply `filetype` (MIME-type) metadata. There are three cases; first one in this order that
|
||||
applies is used:
|
||||
|
||||
- `metadata_file_type` argument is present in call, use that.
|
||||
- `file_type` decorator argument is populated, use that.
|
||||
- `file_type` decorator argument is omitted or None, don't apply `.metadata.filetype`
|
||||
(assume the partitioner will do that for itself, like `partition_image()`.
|
||||
|
||||
- Replace `filename` with `metadata_filename` when present.
|
||||
|
||||
- Replace `last_modified` with `metadata_last_modified` when present.
|
||||
|
||||
- Apply `url` metadata when present.
|
||||
"""
|
||||
|
||||
def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
|
||||
"""The decorator function itself.
|
||||
|
||||
This function is returned by the `apply_metadata()` function and is the actual decorator.
|
||||
Think of `apply_metadata()` as a factory function that configures this decorator, in
|
||||
particular by setting its `file_type` value.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
|
||||
elements = func(*args, **kwargs)
|
||||
call_args = get_call_args_applying_defaults(func, *args, **kwargs)
|
||||
|
||||
# ------------------------------------------------------------------------------------
|
||||
# unique-ify elements
|
||||
# ------------------------------------------------------------------------------------
|
||||
# Do this first to ensure all following operations behave as expected. It's easy for a
|
||||
# partitioner to re-use an element or metadata instance when its values are common to
|
||||
# multiple elements. This can lead to very hard-to diagnose bugs downstream when
|
||||
# mutating one element unexpectedly also mutates others (because they are the same
|
||||
# instance).
|
||||
# ------------------------------------------------------------------------------------
|
||||
|
||||
elements = _uniqueify_elements_and_metadata(elements)
|
||||
|
||||
# ------------------------------------------------------------------------------------
|
||||
# apply metadata - do this first because it affects the hash computation.
|
||||
# ------------------------------------------------------------------------------------
|
||||
|
||||
# -- `language` - auto-detect language (e.g. eng, spa) --
|
||||
languages = call_args.get("languages")
|
||||
detect_language_per_element = call_args.get("detect_language_per_element", False)
|
||||
elements = list(
|
||||
apply_lang_metadata(
|
||||
elements=elements,
|
||||
languages=languages,
|
||||
detect_language_per_element=detect_language_per_element,
|
||||
)
|
||||
)
|
||||
|
||||
# == apply filetype, filename, last_modified, and url metadata ===================
|
||||
metadata_kwargs: dict[str, Any] = {}
|
||||
|
||||
# -- `filetype` (MIME-type) metadata --
|
||||
metadata_file_type = call_args.get("metadata_file_type") or file_type
|
||||
if metadata_file_type is not None:
|
||||
metadata_kwargs["filetype"] = metadata_file_type.mime_type
|
||||
|
||||
# -- `filename` metadata - override with metadata_filename when it's present --
|
||||
filename = call_args.get("metadata_filename") or call_args.get("filename")
|
||||
if filename:
|
||||
metadata_kwargs["filename"] = filename
|
||||
|
||||
# -- `last_modified` metadata - override with metadata_last_modified when present --
|
||||
metadata_last_modified = call_args.get("metadata_last_modified")
|
||||
if metadata_last_modified:
|
||||
metadata_kwargs["last_modified"] = metadata_last_modified
|
||||
|
||||
# -- `url` metadata - record url when present --
|
||||
url = call_args.get("url")
|
||||
if url:
|
||||
metadata_kwargs["url"] = url
|
||||
|
||||
# -- update element.metadata in single pass --
|
||||
for element in elements:
|
||||
# NOTE(robinson) - Attached files have already run through this logic in their own
|
||||
# partitioning function
|
||||
if element.metadata.attached_to_filename:
|
||||
continue
|
||||
element.metadata.update(ElementMetadata(**metadata_kwargs))
|
||||
|
||||
# ------------------------------------------------------------------------------------
|
||||
# compute hash ids (when so requestsd)
|
||||
# ------------------------------------------------------------------------------------
|
||||
|
||||
# -- Compute and apply hash-ids if the user does not want UUIDs. Note this mutates the
|
||||
# -- elements themselves, not their metadata.
|
||||
unique_element_ids: bool = call_args.get("unique_element_ids", False)
|
||||
if unique_element_ids is False:
|
||||
elements = _assign_hash_ids(elements)
|
||||
|
||||
# ------------------------------------------------------------------------------------
|
||||
# assign parent-id - do this after hash computation so parent-id is stable.
|
||||
# ------------------------------------------------------------------------------------
|
||||
|
||||
# -- `parent_id` - process category-level etc. to assign parent-id --
|
||||
elements = set_element_hierarchy(elements)
|
||||
|
||||
return elements
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def _assign_hash_ids(elements: list[Element]) -> list[Element]:
|
||||
"""Converts `.id` of each element from UUID to hash.
|
||||
|
||||
The hash is based on the `.text` of the element, but also on its page-number and sequence number
|
||||
on that page. This provides for deterministic results even when the document is split into one
|
||||
or more fragments for parallel processing.
|
||||
"""
|
||||
# -- generate sequence number for each element on a page --
|
||||
page_seq_counts = {}
|
||||
for element in elements:
|
||||
page_number = element.metadata.page_number
|
||||
seq_on_page_counter = page_seq_counts.get(page_number, 0)
|
||||
element.id_to_hash(seq_on_page_counter)
|
||||
page_seq_counts[page_number] = seq_on_page_counter + 1
|
||||
|
||||
return elements
|
||||
|
||||
|
||||
def _uniqueify_elements_and_metadata(elements: list[Element]) -> list[Element]:
|
||||
"""Ensure each of `elements` and their metadata are unique instances.
|
||||
|
||||
This prevents hard-to-diagnose bugs downstream when mutating one element unexpectedly also
|
||||
mutates others because they are the same instance.
|
||||
"""
|
||||
|
||||
def iter_unique_elements(elements: list[Element]) -> Iterator[Element]:
|
||||
"""Substitute deep-copies of any non-unique elements or metadata in `elements`."""
|
||||
seen_elements: set[int] = set()
|
||||
seen_metadata: set[int] = set()
|
||||
|
||||
for element in elements:
|
||||
if id(element) in seen_elements:
|
||||
element = copy.deepcopy(element)
|
||||
if id(element.metadata) in seen_metadata:
|
||||
element.metadata = copy.deepcopy(element.metadata)
|
||||
seen_elements.add(id(element))
|
||||
seen_metadata.add(id(element.metadata))
|
||||
yield element
|
||||
|
||||
return list(iter_unique_elements(elements))
|
||||
Reference in New Issue
Block a user