修改为东南天坐标系

backend_service/venv/lib/python3.13/site-packages/unstructured/chunking/__init__.py

@@ -0,0 +1,22 @@
+"""Chunking module initializer.
+
+Publishes the public aspects of the chunking sub-package interface.
+"""
+
+from __future__ import annotations
+
+from unstructured.chunking.base import CHUNK_MAX_CHARS_DEFAULT, CHUNK_MULTI_PAGE_DEFAULT
+from unstructured.chunking.dispatch import (
+    Chunker,
+    add_chunking_strategy,
+    register_chunking_strategy,
+)
+
+__all__ = [
+    "CHUNK_MAX_CHARS_DEFAULT",
+    "CHUNK_MULTI_PAGE_DEFAULT",
+    "add_chunking_strategy",
+    # -- these must be published to allow pluggable chunkers in other code-bases --
+    "Chunker",
+    "register_chunking_strategy",
+]

backend_service/venv/lib/python3.13/site-packages/unstructured/chunking/basic.py

@@ -0,0 +1,91 @@
+"""Implementation of baseline chunking.
+
+This is the "plain-vanilla" chunking strategy. All the fundamental chunking behaviors are present in
+this strategy and also in all other strategies. Those are:
+
+- Maximally fill each chunk with sequential elements.
+- Isolate oversized elements and divide (only) those chunks by text-splitting.
+- Overlap when requested.
+
+"Fancier" strategies add higher-level semantic-unit boundaries to be respected. For example, in the
+by-title strategy, section boundaries are respected, meaning a chunk never contains text from two
+different sections. When a new section is detected the current chunk is closed and a new one
+started.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+from unstructured.chunking.base import ChunkingOptions, PreChunker
+from unstructured.documents.elements import Element
+
+
+def chunk_elements(
+    elements: Iterable[Element],
+    *,
+    include_orig_elements: Optional[bool] = None,
+    max_characters: Optional[int] = None,
+    new_after_n_chars: Optional[int] = None,
+    overlap: Optional[int] = None,
+    overlap_all: Optional[bool] = None,
+) -> list[Element]:
+    """Combine sequential `elements` into chunks, respecting specified text-length limits.
+
+    Produces a sequence of `CompositeElement`, `Table`, and `TableChunk` elements (chunks).
+
+    Parameters
+    ----------
+    elements
+        A list of unstructured elements. Usually the output of a partition function.
+    include_orig_elements
+        When `True` (default), add elements from pre-chunk to the `.metadata.orig_elements` field
+        of the chunk(s) formed from that pre-chunk. Among other things, this allows access to
+        original-element metadata that cannot be consolidated and is dropped in the course of
+        chunking.
+    max_characters
+        Hard maximum chunk length. No chunk will exceed this length. A single element that exceeds
+        this length will be divided into two or more chunks using text-splitting.
+    new_after_n_chars
+        A chunk that of this length or greater is not extended to include the next element, even if
+        that element would fit without exceeding `max_characters`. A "soft max" length that can be
+        used in conjunction with `max_characters` to limit most chunks to a preferred length while
+        still allowing larger elements to be included in a single chunk without resorting to
+        text-splitting. Defaults to `max_characters` when not specified, which effectively disables
+        any soft window. Specifying 0 for this argument causes each element to appear in a chunk by
+        itself (although an element with text longer than `max_characters` will be still be split
+        into two or more chunks).
+    overlap
+        Specifies the length of a string ("tail") to be drawn from each chunk and prefixed to the
+        next chunk as a context-preserving mechanism. By default, this only applies to split-chunks
+        where an oversized element is divided into multiple chunks by text-splitting.
+    overlap_all
+        Default: `False`. When `True`, apply overlap between "normal" chunks formed from whole
+        elements and not subject to text-splitting. Use this with caution as it produces a certain
+        level of "pollution" of otherwise clean semantic chunk boundaries.
+    """
+    # -- raises ValueError on invalid parameters --
+    opts = _BasicChunkingOptions.new(
+        include_orig_elements=include_orig_elements,
+        max_characters=max_characters,
+        new_after_n_chars=new_after_n_chars,
+        overlap=overlap,
+        overlap_all=overlap_all,
+    )
+
+    return _chunk_elements(elements, opts)
+
+
+def _chunk_elements(elements: Iterable[Element], opts: _BasicChunkingOptions) -> list[Element]:
+    """Implementation of actual basic chunking."""
+    # -- Note(scanny): it might seem like over-abstraction for this to be a separate function but
+    # -- it eases overriding or adding individual chunking options when customizing a stock chunker.
+    return [
+        chunk
+        for pre_chunk in PreChunker.iter_pre_chunks(elements, opts)
+        for chunk in pre_chunk.iter_chunks()
+    ]
+
+
+class _BasicChunkingOptions(ChunkingOptions):
+    """Options for `basic` chunking."""

backend_service/venv/lib/python3.13/site-packages/unstructured/chunking/dispatch.py

@@ -0,0 +1,129 @@
+"""Handles dispatch of elements to a chunking-strategy by name.
+
+Also provides the `@add_chunking_strategy` decorator which is the chief current user of "by-name"
+chunking dispatch.
+"""
+
+from __future__ import annotations
+
+import dataclasses as dc
+import functools
+import inspect
+from typing import Any, Callable, Iterable, Optional, Protocol
+
+from typing_extensions import ParamSpec
+
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import Element
+from unstructured.utils import get_call_args_applying_defaults, lazyproperty
+
+_P = ParamSpec("_P")
+
+
+class Chunker(Protocol):
+    """Abstract interface for chunking functions."""
+
+    def __call__(
+        self, elements: Iterable[Element], *, max_characters: Optional[int]
+    ) -> list[Element]:
+        """A chunking function must have this signature.
+
+        In particular it must minimally have an `elements` parameter and all chunkers will have a
+        `max_characters` parameter (doesn't need to follow `elements` directly). All others can
+        vary by chunker.
+        """
+        ...
+
+
+def add_chunking_strategy(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+    """Decorator for chunking text.
+
+    Chunks the element sequence produced by the partitioner it decorates when a `chunking_strategy`
+    argument is present in the partitioner call and it names an available chunking strategy.
+    """
+    # -- Patch the docstring of the decorated function to add chunking strategy and
+    # -- chunking-related argument documentation. This only applies when `chunking_strategy`
+    # -- is an explicit argument of the decorated function and "chunking_strategy" is not
+    # -- already mentioned in the docstring.
+    if func.__doc__ and (
+        "chunking_strategy" in func.__code__.co_varnames and "chunking_strategy" not in func.__doc__
+    ):
+        func.__doc__ += (
+            "\nchunking_strategy"
+            + "\n\tStrategy used for chunking text into larger or smaller elements."
+            + "\n\tDefaults to `None` with optional arg of 'basic' or 'by_title'."
+            + "\n\tAdditional Parameters:"
+            + "\n\t\tmultipage_sections"
+            + "\n\t\t\tIf True, sections can span multiple pages. Defaults to True."
+            + "\n\t\tcombine_text_under_n_chars"
+            + "\n\t\t\tCombines elements (for example a series of titles) until a section"
+            + "\n\t\t\treaches a length of n characters. Only applies to 'by_title' strategy."
+            + "\n\t\tnew_after_n_chars"
+            + "\n\t\t\tCuts off chunks once they reach a length of n characters; a soft max."
+            + "\n\t\tmax_characters"
+            + "\n\t\t\tChunks elements text and text_as_html (if present) into chunks"
+            + "\n\t\t\tof length n characters, a hard max."
+        )
+
+    @functools.wraps(func)
+    def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+        """The decorated function is replaced with this one."""
+
+        # -- call the partitioning function to get the elements --
+        elements = func(*args, **kwargs)
+
+        # -- look for a chunking-strategy argument --
+        call_args = get_call_args_applying_defaults(func, *args, **kwargs)
+        chunking_strategy = call_args.pop("chunking_strategy", None)
+
+        # -- no chunking-strategy means no chunking --
+        if chunking_strategy is None:
+            return elements
+
+        # -- otherwise, chunk away :) --
+        return chunk(elements, chunking_strategy, **call_args)
+
+    return wrapper
+
+
+def chunk(elements: Iterable[Element], chunking_strategy: str, **kwargs: Any) -> list[Element]:
+    """Dispatch chunking of `elements` to the chunking function for `chunking_strategy`."""
+    chunker_spec = _chunker_registry.get(chunking_strategy)
+
+    if chunker_spec is None:
+        raise ValueError(f"unrecognized chunking strategy {repr(chunking_strategy)}")
+
+    # -- `kwargs` will in general be an omnibus dict of all keyword arguments to the partitioner;
+    # -- pick out and use only those supported by this chunker.
+    chunking_kwargs = {k: v for k, v in kwargs.items() if k in chunker_spec.kw_arg_names}
+
+    return chunker_spec.chunker(elements, **chunking_kwargs)
+
+
+def register_chunking_strategy(name: str, chunker: Chunker) -> None:
+    """Make chunker available by using `name` as `chunking_strategy` arg in partitioner call."""
+    _chunker_registry[name] = _ChunkerSpec(chunker)
+
+
+@dc.dataclass(frozen=True)
+class _ChunkerSpec:
+    """A registry entry for a chunker."""
+
+    chunker: Chunker
+    """The "chunk_by_{x}() function that implements this chunking strategy."""
+
+    @lazyproperty
+    def kw_arg_names(self) -> tuple[str, ...]:
+        """Keyword arguments supported by this chunker.
+
+        These are all arguments other than the required `elements: list[Element]` first parameter.
+        """
+        sig = inspect.signature(self.chunker)
+        return tuple(key for key in sig.parameters if key != "elements")
+
+
+_chunker_registry: dict[str, _ChunkerSpec] = {
+    "basic": _ChunkerSpec(chunk_elements),
+    "by_title": _ChunkerSpec(chunk_by_title),
+}

backend_service/venv/lib/python3.13/site-packages/unstructured/chunking/title.py

@@ -0,0 +1,167 @@
+"""Implementation of chunking by title.
+
+Main entry point is the `@add_chunking_strategy()` decorator.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Iterator, Optional
+
+from unstructured.chunking.base import (
+    CHUNK_MULTI_PAGE_DEFAULT,
+    BoundaryPredicate,
+    ChunkingOptions,
+    PreChunkCombiner,
+    PreChunker,
+    is_on_next_page,
+    is_title,
+)
+from unstructured.documents.elements import Element
+from unstructured.utils import lazyproperty
+
+
+def chunk_by_title(
+    elements: Iterable[Element],
+    *,
+    combine_text_under_n_chars: Optional[int] = None,
+    include_orig_elements: Optional[bool] = None,
+    max_characters: Optional[int] = None,
+    multipage_sections: Optional[bool] = None,
+    new_after_n_chars: Optional[int] = None,
+    overlap: Optional[int] = None,
+    overlap_all: Optional[bool] = None,
+) -> list[Element]:
+    """Uses title elements to identify sections within the document for chunking.
+
+    Splits off into a new CompositeElement when a title is detected or if metadata changes, which
+    happens when page numbers or sections change. Cuts off sections once they have exceeded a
+    character length of max_characters.
+
+    Parameters
+    ----------
+    elements
+        A list of unstructured elements. Usually the output of a partition function.
+    combine_text_under_n_chars
+        Combines elements (for example a series of titles) until a section reaches a length of
+        n characters. Defaults to `max_characters` which combines chunks whenever space allows.
+        Specifying 0 for this argument suppresses combining of small chunks. Note this value is
+        "capped" at the `new_after_n_chars` value since a value higher than that would not change
+        this parameter's effect.
+    include_orig_elements
+        When `True` (default), add elements from pre-chunk to the `.metadata.orig_elements` field
+        of the chunk(s) formed from that pre-chunk. Among other things, this allows access to
+        original-element metadata that cannot be consolidated and is dropped in the course of
+        chunking.
+    max_characters
+        Chunks elements text and text_as_html (if present) into chunks of length
+        n characters (hard max)
+    multipage_sections
+        If True, sections can span multiple pages. Defaults to True.
+    new_after_n_chars
+        Cuts off new sections once they reach a length of n characters (soft max). Defaults to
+        `max_characters` when not specified, which effectively disables any soft window.
+        Specifying 0 for this argument causes each element to appear in a chunk by itself (although
+        an element with text longer than `max_characters` will be still be split into two or more
+        chunks).
+    overlap
+        Specifies the length of a string ("tail") to be drawn from each chunk and prefixed to the
+        next chunk as a context-preserving mechanism. By default, this only applies to split-chunks
+        where an oversized element is divided into multiple chunks by text-splitting.
+    overlap_all
+        Default: `False`. When `True`, apply overlap between "normal" chunks formed from whole
+        elements and not subject to text-splitting. Use this with caution as it entails a certain
+        level of "pollution" of otherwise clean semantic chunk boundaries.
+    """
+    opts = _ByTitleChunkingOptions.new(
+        combine_text_under_n_chars=combine_text_under_n_chars,
+        include_orig_elements=include_orig_elements,
+        max_characters=max_characters,
+        multipage_sections=multipage_sections,
+        new_after_n_chars=new_after_n_chars,
+        overlap=overlap,
+        overlap_all=overlap_all,
+    )
+    return _chunk_by_title(elements, opts)
+
+
+def _chunk_by_title(elements: Iterable[Element], opts: _ByTitleChunkingOptions) -> list[Element]:
+    """Implementation of actual "by-title" chunking."""
+    # -- Note(scanny): it might seem like over-abstraction for this to be a separate function but
+    # -- it eases overriding or adding individual chunking options when customizing a stock chunker.
+    pre_chunks = PreChunkCombiner(
+        PreChunker.iter_pre_chunks(elements, opts), opts=opts
+    ).iter_combined_pre_chunks()
+
+    return [chunk for pre_chunk in pre_chunks for chunk in pre_chunk.iter_chunks()]
+
+
+class _ByTitleChunkingOptions(ChunkingOptions):
+    """Adds the by-title-specific chunking options to the base case.
+
+    `by_title`-specific options:
+
+    combine_text_under_n_chars
+        A remedy to over-chunking caused by elements mis-identified as Title elements.
+        Every Title element would start a new chunk and this setting mitigates that, at the
+        expense of sometimes violating legitimate semantic boundaries.
+    multipage_sections
+        Indicates that page-boundaries should not be respected while chunking, i.e. elements
+        appearing on two different pages can appear in the same chunk.
+    """
+
+    @lazyproperty
+    def boundary_predicates(self) -> tuple[BoundaryPredicate, ...]:
+        """The semantic-boundary detectors to be applied to break pre-chunks.
+
+        For the `by_title` strategy these are sections indicated by a title (section-heading), an
+        explicit section metadata item (only present for certain document types), and optionally
+        page boundaries.
+        """
+
+        def iter_boundary_predicates() -> Iterator[BoundaryPredicate]:
+            yield is_title
+            if not self.multipage_sections:
+                yield is_on_next_page()
+
+        return tuple(iter_boundary_predicates())
+
+    @lazyproperty
+    def combine_text_under_n_chars(self) -> int:
+        """Combine consecutive text pre-chunks if former is smaller than this and both will fit.
+
+        - Does not combine text chunks if together they would exceed the chunking window.
+        - Defaults to `max_characters` when not specified.
+        - Is reduced to `new_after_n_chars` when it exceeds that value.
+        """
+        # -- `combine_text_under_n_chars` defaults to `max_characters` when not specified --
+        arg_value = self._kwargs.get("combine_text_under_n_chars")
+        return self.hard_max if arg_value is None else arg_value
+
+    @lazyproperty
+    def multipage_sections(self) -> bool:
+        """When False, break pre-chunks on page-boundaries."""
+        arg_value = self._kwargs.get("multipage_sections")
+        return CHUNK_MULTI_PAGE_DEFAULT if arg_value is None else bool(arg_value)
+
+    def _validate(self) -> None:
+        """Raise ValueError if request option-set is invalid."""
+        # -- start with base-class validations --
+        super()._validate()
+
+        # -- `combine_text_under_n_chars == 0` is valid (suppresses chunk combination)
+        # -- but a negative value is not
+        if self.combine_text_under_n_chars < 0:
+            raise ValueError(
+                f"'combine_text_under_n_chars' argument must be >= 0,"
+                f" got {self.combine_text_under_n_chars}"
+            )
+
+        # -- `combine_text_under_n_chars` > `max_characters` can produce behavior confusing to
+        # -- users. The chunking behavior would be no different than when
+        # -- `combine_text_under_n_chars == max_characters`, but if `max_characters` is left to
+        # -- default (500) then it can look like chunk-combining isn't working.
+        if self.combine_text_under_n_chars > self.hard_max:
+            raise ValueError(
+                f"'combine_text_under_n_chars' argument must not exceed `max_characters`"
+                f" value, got {self.combine_text_under_n_chars} > {self.hard_max}"
+            )