修改为东南天坐标系

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/element_type.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+
+from typing_extensions import TypeAlias
+
+FrequencyDict: TypeAlias = "dict[tuple[str, int | None], int]"
+"""Like:
+    {
+        ("ListItem", 0): 2,
+        ("NarrativeText", None): 2,
+        ("Title", 0): 5,
+        ("UncategorizedText", None): 6,
+    }
+"""
+
+
+def get_element_type_frequency(
+    elements: str,
+) -> FrequencyDict:
+    """
+    Calculate the frequency of Element Types from a list of elements.
+
+    Args:
+        elements (str): String-formatted json of all elements (as a result of elements_to_json).
+    Returns:
+        Element type and its frequency in dictionary format.
+    """
+    frequency: dict[tuple[str, int | None], int] = {}
+    if len(elements) == 0:
+        return frequency
+    for element in json.loads(elements):
+        type = element.get("type")
+        category_depth = element["metadata"].get("category_depth")
+        key = (type, category_depth)
+        if key not in frequency:
+            frequency[key] = 1
+        else:
+            frequency[key] += 1
+    return frequency
+
+
+def calculate_element_type_percent_match(
+    output: FrequencyDict,
+    source: FrequencyDict,
+    category_depth_weight: float = 0.5,
+) -> float:
+    """Calculate the percent match between two frequency dictionary.
+
+    Intended to use with `get_element_type_frequency` function. The function counts the absolute
+    exact match (type and depth), and counts the weighted match (correct type but different depth),
+    then normalized with source's total elements.
+    """
+    if len(output) == 0 or len(source) == 0:
+        return 0.0
+
+    output_copy = output.copy()
+    source_copy = source.copy()
+    total_source_element_count = 0
+    total_match_element_count = 0
+
+    unmatched_depth_output: dict[str, int] = {}
+    unmatched_depth_source: dict[str, int] = {}
+
+    # loop through the output list to find match with source
+    for k, _ in output_copy.items():
+        if k in source_copy:
+            match_count = min(output_copy[k], source_copy[k])
+            total_match_element_count += match_count
+            total_source_element_count += match_count
+
+            # update the dictionary by removing already matched values
+            output_copy[k] -= match_count
+            source_copy[k] -= match_count
+
+        # add unmatched leftovers from output_copy to a new dictionary
+        element_type = k[0]
+        if element_type not in unmatched_depth_output:
+            unmatched_depth_output[element_type] = output_copy[k]
+        else:
+            unmatched_depth_output[element_type] += output_copy[k]
+
+    # add unmatched leftovers from source_copy to a new dictionary
+    unmatched_depth_source = _convert_to_frequency_without_depth(source_copy)
+
+    # loop through the source list to match any existing partial match left
+    for k, _ in unmatched_depth_source.items():
+        total_source_element_count += unmatched_depth_source[k]
+        if k in unmatched_depth_output:
+            match_count = min(unmatched_depth_output[k], unmatched_depth_source[k])
+            total_match_element_count += match_count * category_depth_weight
+
+    return min(max(total_match_element_count / total_source_element_count, 0.0), 1.0)
+
+
+def _convert_to_frequency_without_depth(d: FrequencyDict) -> dict[str, int]:
+    """
+    Takes in element frequency with depth of format (type, depth): value
+    and converts to dictionary without depth of format type: value
+    """
+    res: dict[str, int] = {}
+    for k, v in d.items():
+        element_type = k[0]
+        if element_type not in res:
+            res[element_type] = v
+        else:
+            res[element_type] += v
+    return res

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/evaluate.py

@@ -0,0 +1,897 @@
+#! /usr/bin/env python3
+
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import logging
+import os
+import sys
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Union
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from unstructured.metrics.element_type import (
+    calculate_element_type_percent_match,
+    get_element_type_frequency,
+)
+from unstructured.metrics.object_detection import (
+    ObjectDetectionEvalProcessor,
+)
+from unstructured.metrics.table.table_eval import TableEvalProcessor
+from unstructured.metrics.text_extraction import calculate_accuracy, calculate_percent_missing_text
+from unstructured.metrics.utils import (
+    _count,
+    _display,
+    _format_grouping_output,
+    _mean,
+    _prepare_output_cct,
+    _pstdev,
+    _read_text_file,
+    _rename_aggregated_columns,
+    _stdev,
+    _write_to_file,
+)
+
+logger = logging.getLogger("unstructured.eval")
+handler = logging.StreamHandler()
+handler.name = "eval_log_handler"
+formatter = logging.Formatter("%(asctime)s %(processName)-10s %(levelname)-8s %(message)s")
+handler.setFormatter(formatter)
+
+# Only want to add the handler once
+if "eval_log_handler" not in [h.name for h in logger.handlers]:
+    logger.addHandler(handler)
+
+logger.setLevel(logging.DEBUG)
+
+AGG_HEADERS = ["metric", "average", "sample_sd", "population_sd", "count"]
+AGG_HEADERS_MAPPING = {
+    "index": "metric",
+    "_mean": "average",
+    "_stdev": "sample_sd",
+    "_pstdev": "population_sd",
+    "_count": "count",
+}
+OUTPUT_TYPE_OPTIONS = ["json", "txt"]
+
+
+@dataclass
+class BaseMetricsCalculator(ABC):
+    """Foundation class for specialized metrics calculators.
+
+    It provides a common interface for calculating metrics based on outputs and ground truths.
+    Those can be provided as either directories or lists of files.
+    """
+
+    documents_dir: str | Path
+    ground_truths_dir: str | Path
+
+    def __post_init__(self):
+        """Discover all files in the provided directories."""
+        self.documents_dir = Path(self.documents_dir).resolve()
+        self.ground_truths_dir = Path(self.ground_truths_dir).resolve()
+
+        # -- auto-discover all files in the directories --
+        self._document_paths = [
+            path.relative_to(self.documents_dir)
+            for path in self.documents_dir.glob("*")
+            if path.is_file()
+        ]
+        self._ground_truth_paths = [
+            path.relative_to(self.ground_truths_dir)
+            for path in self.ground_truths_dir.glob("*")
+            if path.is_file()
+        ]
+
+    @property
+    @abstractmethod
+    def default_tsv_name(self):
+        """Default name for the per-document metrics TSV file."""
+
+    @property
+    @abstractmethod
+    def default_agg_tsv_name(self):
+        """Default name for the aggregated metrics TSV file."""
+
+    @abstractmethod
+    def _generate_dataframes(self, rows: list) -> tuple[pd.DataFrame, pd.DataFrame]:
+        """Generates pandas DataFrames from the list of rows.
+
+        The first DF (index 0) is a dataframe containing metrics per file.
+        The second DF (index 1) is a dataframe containing the aggregated
+            metrics.
+        """
+
+    def on_files(
+        self,
+        document_paths: Optional[list[str | Path]] = None,
+        ground_truth_paths: Optional[list[str | Path]] = None,
+    ) -> BaseMetricsCalculator:
+        """Overrides the default list of files to process."""
+        if document_paths:
+            self._document_paths = [Path(p) for p in document_paths]
+
+        if ground_truth_paths:
+            self._ground_truth_paths = [Path(p) for p in ground_truth_paths]
+
+        return self
+
+    def calculate(
+        self,
+        executor: Optional[concurrent.futures.Executor] = None,
+        export_dir: Optional[str | Path] = None,
+        visualize_progress: bool = True,
+        display_agg_df: bool = True,
+    ) -> pd.DataFrame:
+        """Calculates metrics for each document using the provided executor.
+
+        * Optionally, the results can be exported and displayed.
+        * It loops through the list of structured output from all of `documents_dir` or
+        selected files from `document_paths`, and compares them with gold-standard
+        of the same file name under `ground_truths_dir` or selected files from `ground_truth_paths`.
+
+        Args:
+            executor: concurrent.futures.Executor instance
+            export_dir: directory to export the results
+            visualize_progress: whether to display progress bar
+            display_agg_df: whether to display the aggregated results
+
+        Returns:
+            Metrics for each document as a pandas DataFrame
+        """
+        if executor is None:
+            executor = self._default_executor()
+        rows = self._process_all_documents(executor, visualize_progress)
+        df, agg_df = self._generate_dataframes(rows)
+
+        if export_dir is not None:
+            _write_to_file(export_dir, self.default_tsv_name, df)
+            _write_to_file(export_dir, self.default_agg_tsv_name, agg_df)
+
+        if display_agg_df is True:
+            _display(agg_df)
+        return df
+
+    @classmethod
+    def _default_executor(cls):
+        max_processors = int(os.environ.get("MAX_PROCESSES", os.cpu_count()))
+        logger.info(f"Configuring a pool of {max_processors} processors for parallel processing.")
+        return cls._get_executor_class()(max_workers=max_processors)
+
+    @classmethod
+    def _get_executor_class(
+        cls,
+    ) -> type[concurrent.futures.ThreadPoolExecutor] | type[concurrent.futures.ProcessPoolExecutor]:
+        return concurrent.futures.ProcessPoolExecutor
+
+    def _process_all_documents(
+        self, executor: concurrent.futures.Executor, visualize_progress: bool
+    ) -> list:
+        """Triggers processing of all documents using the provided executor.
+
+        Failures are omitted from the returned result.
+        """
+        with executor:
+            return [
+                row
+                for row in tqdm(
+                    executor.map(self._try_process_document, self._document_paths),
+                    total=len(self._document_paths),
+                    leave=False,
+                    disable=not visualize_progress,
+                )
+                if row is not None
+            ]
+
+    def _try_process_document(self, doc: Path) -> Optional[list]:
+        """Safe wrapper around the document processing method."""
+        logger.info(f"Processing {doc}")
+        try:
+            return self._process_document(doc)
+        except Exception as e:
+            logger.error(f"Failed to process document {doc}: {e}")
+            return None
+
+    @abstractmethod
+    def _process_document(self, doc: Path) -> Optional[list]:
+        """Should return all metadata and metrics for a single document."""
+
+
+@dataclass
+class TableStructureMetricsCalculator(BaseMetricsCalculator):
+    """Calculates the following metrics for tables:
+        - tables found accuracy
+        - table-level accuracy
+        - element in column index accuracy
+        - element in row index accuracy
+        - element's column content accuracy
+        - element's row content accuracy
+    It also calculates the aggregated accuracy.
+    """
+
+    cutoff: Optional[float] = None
+    weighted_average: bool = True
+    include_false_positives: bool = True
+
+    def __post_init__(self):
+        super().__post_init__()
+
+    @property
+    def supported_metric_names(self):
+        return [
+            "total_tables",
+            "table_level_acc",
+            "table_detection_recall",
+            "table_detection_precision",
+            "table_detection_f1",
+            "composite_structure_acc",
+            "element_col_level_index_acc",
+            "element_row_level_index_acc",
+            "element_col_level_content_acc",
+            "element_row_level_content_acc",
+        ]
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-table-structure-accuracy.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-table-structure-accuracy.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        doc_path = Path(doc)
+        out_filename = doc_path.stem
+        doctype = Path(out_filename).suffix
+        src_gt_filename = out_filename + ".json"
+        connector = doc_path.parts[-2] if len(doc_path.parts) > 1 else None
+
+        if src_gt_filename in self._ground_truth_paths:  # type: ignore
+            return None
+
+        prediction_file = self.documents_dir / doc
+        if not prediction_file.exists():
+            logger.warning(f"Prediction file {prediction_file} does not exist, skipping")
+            return None
+
+        ground_truth_file = self.ground_truths_dir / src_gt_filename
+        if not ground_truth_file.exists():
+            logger.warning(f"Ground truth file {ground_truth_file} does not exist, skipping")
+            return None
+
+        processor_from_text_as_html = TableEvalProcessor.from_json_files(
+            prediction_file=prediction_file,
+            ground_truth_file=ground_truth_file,
+            cutoff=self.cutoff,
+            source_type="html",
+        )
+        report_from_html = processor_from_text_as_html.process_file()
+        return [
+            out_filename,
+            doctype,
+            connector,
+            report_from_html.total_predicted_tables,
+        ] + [getattr(report_from_html, metric) for metric in self.supported_metric_names]
+
+    def _generate_dataframes(self, rows):
+        headers = [
+            "filename",
+            "doctype",
+            "connector",
+            "total_predicted_tables",
+        ] + self.supported_metric_names
+
+        df = pd.DataFrame(rows, columns=headers)
+        df["_table_weights"] = df["total_tables"]
+
+        if self.include_false_positives:
+            # we give false positive tables a 1 table worth of weight in computing table level acc
+            df["_table_weights"][df.total_tables.eq(0) & df.total_predicted_tables.gt(0)] = 1
+
+        # filter down to only those with actual and/or predicted tables
+        has_tables_df = df[df["_table_weights"] > 0]
+
+        if not self.weighted_average:
+            # for all non zero elements assign them value 1
+            df["_table_weights"] = df["_table_weights"].apply(
+                lambda table_weight: 1 if table_weight != 0 else 0
+            )
+
+        if has_tables_df.empty:
+            agg_df = pd.DataFrame(
+                [[metric, None, None, None, 0] for metric in self.supported_metric_names]
+            ).reset_index()
+        else:
+            element_metrics_results = {}
+            for metric in self.supported_metric_names:
+                metric_df = has_tables_df[has_tables_df[metric].notnull()]
+                agg_metric = metric_df[metric].agg([_stdev, _pstdev, _count]).transpose()
+                if metric.startswith("total_tables"):
+                    agg_metric["_mean"] = metric_df[metric].mean()
+                elif metric.startswith("table_level_acc"):
+                    agg_metric["_mean"] = np.round(
+                        np.average(metric_df[metric], weights=metric_df["_table_weights"]),
+                        3,
+                    )
+                else:
+                    # false positive tables do not contribute to table structure and content
+                    # extraction metrics
+                    agg_metric["_mean"] = np.round(
+                        np.average(metric_df[metric], weights=metric_df["total_tables"]),
+                        3,
+                    )
+                if agg_metric.empty:
+                    element_metrics_results[metric] = pd.Series(
+                        data=[None, None, None, 0], index=["_mean", "_stdev", "_pstdev", "_count"]
+                    )
+                else:
+                    element_metrics_results[metric] = agg_metric
+            agg_df = pd.DataFrame(element_metrics_results).transpose().reset_index()
+        agg_df = agg_df.rename(columns=AGG_HEADERS_MAPPING)
+        return df, agg_df
+
+
+@dataclass
+class TextExtractionMetricsCalculator(BaseMetricsCalculator):
+    """Calculates text accuracy and percent missing between document and ground truth texts.
+
+    It also calculates the aggregated accuracy and percent missing.
+    """
+
+    group_by: Optional[str] = None
+    weights: tuple[int, int, int] = (1, 1, 1)
+    document_type: str = "json"
+
+    def __post_init__(self):
+        super().__post_init__()
+        self._validate_inputs()
+
+    @property
+    def default_tsv_name(self) -> str:
+        return "all-docs-cct.tsv"
+
+    @property
+    def default_agg_tsv_name(self) -> str:
+        return "aggregate-scores-cct.tsv"
+
+    def calculate(
+        self,
+        executor: Optional[concurrent.futures.Executor] = None,
+        export_dir: Optional[str | Path] = None,
+        visualize_progress: bool = True,
+        display_agg_df: bool = True,
+    ) -> pd.DataFrame:
+        """See the parent class for the method's docstring."""
+        df = super().calculate(
+            executor=executor,
+            export_dir=export_dir,
+            visualize_progress=visualize_progress,
+            display_agg_df=display_agg_df,
+        )
+
+        if export_dir is not None and self.group_by:
+            get_mean_grouping(self.group_by, df, export_dir, "text_extraction")
+        return df
+
+    def _validate_inputs(self):
+        if not self._document_paths:
+            logger.info("No output files to calculate to edit distances for, exiting")
+            sys.exit(0)
+        if self.document_type not in OUTPUT_TYPE_OPTIONS:
+            raise ValueError(
+                "Specified file type under `documents_dir` or `output_list` should be one of "
+                f"`json` or `txt`. The given file type is {self.document_type}, exiting."
+            )
+        for path in self._document_paths:
+            try:
+                path.suffixes[-1]
+            except IndexError:
+                logger.error(f"File {path} does not have a suffix, skipping")
+                continue
+            if path.suffixes[-1] != f".{self.document_type}":
+                logger.warning(
+                    "The directory contains file type inconsistent with the given input. "
+                    "Please note that some files will be skipped."
+                )
+        if not all(path.suffixes[-1] == f".{self.document_type}" for path in self._document_paths):
+            logger.warning(
+                "The directory contains file type inconsistent with the given input. "
+                "Please note that some files will be skipped."
+            )
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        filename = doc.stem
+        doctype = doc.suffixes[-2]
+        connector = doc.parts[0] if len(doc.parts) > 1 else None
+
+        output_cct, source_cct = self._get_ccts(doc)
+        # NOTE(amadeusz): Levenshtein distance calculation takes too long
+        # skip it if file sizes differ wildly
+        if 0.5 < len(output_cct.encode()) / len(source_cct.encode()) < 2.0:
+            accuracy = round(calculate_accuracy(output_cct, source_cct, self.weights), 3)
+        else:
+            # 0.01 to distinguish it was set manually
+            accuracy = 0.01
+        percent_missing = round(calculate_percent_missing_text(output_cct, source_cct), 3)
+        return [filename, doctype, connector, accuracy, percent_missing]
+
+    def _get_ccts(self, doc: Path) -> tuple[str, str]:
+        output_cct = _prepare_output_cct(
+            docpath=self.documents_dir / doc, output_type=self.document_type
+        )
+        source_cct = _read_text_file(self.ground_truths_dir / doc.with_suffix(".txt"))
+
+        return output_cct, source_cct
+
+    def _generate_dataframes(self, rows):
+        headers = ["filename", "doctype", "connector", "cct-accuracy", "cct-%missing"]
+        df = pd.DataFrame(rows, columns=headers)
+
+        acc = df[["cct-accuracy"]].agg([_mean, _stdev, _pstdev, _count]).transpose()
+        miss = df[["cct-%missing"]].agg([_mean, _stdev, _pstdev, _count]).transpose()
+        if acc.shape[1] == 0 and miss.shape[1] == 0:
+            agg_df = pd.DataFrame(columns=AGG_HEADERS)
+        else:
+            agg_df = pd.concat((acc, miss)).reset_index()
+            agg_df.columns = AGG_HEADERS
+
+        return df, agg_df
+
+
+@dataclass
+class ElementTypeMetricsCalculator(BaseMetricsCalculator):
+    """
+    Calculates element type frequency accuracy, percent missing and
+    aggregated accuracy between document and ground truth.
+    """
+
+    group_by: Optional[str] = None
+
+    def calculate(
+        self,
+        executor: Optional[concurrent.futures.Executor] = None,
+        export_dir: Optional[str | Path] = None,
+        visualize_progress: bool = True,
+        display_agg_df: bool = False,
+    ) -> pd.DataFrame:
+        """See the parent class for the method's docstring."""
+        df = super().calculate(
+            executor=executor,
+            export_dir=export_dir,
+            visualize_progress=visualize_progress,
+            display_agg_df=display_agg_df,
+        )
+
+        if export_dir is not None and self.group_by:
+            get_mean_grouping(self.group_by, df, export_dir, "element_type")
+        return df
+
+    @property
+    def default_tsv_name(self) -> str:
+        return "all-docs-element-type-frequency.tsv"
+
+    @property
+    def default_agg_tsv_name(self) -> str:
+        return "aggregate-scores-element-type.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        filename = doc.stem
+        doctype = doc.suffixes[-2]
+        connector = doc.parts[0] if len(doc.parts) > 1 else None
+
+        output = get_element_type_frequency(_read_text_file(self.documents_dir / doc))
+        source = get_element_type_frequency(
+            _read_text_file(self.ground_truths_dir / doc.with_suffix(".json"))
+        )
+        accuracy = round(calculate_element_type_percent_match(output, source), 3)
+        return [filename, doctype, connector, accuracy]
+
+    def _generate_dataframes(self, rows):
+        headers = ["filename", "doctype", "connector", "element-type-accuracy"]
+        df = pd.DataFrame(rows, columns=headers)
+        if df.empty:
+            agg_df = pd.DataFrame(["element-type-accuracy", None, None, None, 0]).transpose()
+        else:
+            agg_df = df.agg({"element-type-accuracy": [_mean, _stdev, _pstdev, _count]}).transpose()
+            agg_df = agg_df.reset_index()
+
+        agg_df.columns = AGG_HEADERS
+
+        return df, agg_df
+
+
+def get_mean_grouping(
+    group_by: str,
+    data_input: Union[pd.DataFrame, str],
+    export_dir: str,
+    eval_name: str,
+    agg_name: Optional[str] = None,
+    export_filename: Optional[str] = None,
+) -> None:
+    """Aggregates accuracy and missing metrics by column name 'doctype' or 'connector',
+    or 'all' for all rows. Export to TSV.
+    If `all`, passing export_name is recommended.
+
+    Args:
+        group_by (str): Grouping category ('doctype' or 'connector' or 'all').
+        data_input (Union[pd.DataFrame, str]): DataFrame or path to a CSV/TSV file.
+        export_dir (str): Directory for the exported TSV file.
+        eval_name (str): Evaluated metric ('text_extraction' or 'element_type').
+        agg_name (str, optional): String to use with export filename. Default is `cct` for
+            group_by `text_extraction` and `element-type` for `element_type`
+        export_name (str, optional): Export filename.
+    """
+    if group_by not in ("doctype", "connector") and group_by != "all":
+        raise ValueError("Invalid grouping category. Returning a non-group evaluation.")
+
+    if eval_name == "text_extraction":
+        agg_fields = ["cct-accuracy", "cct-%missing"]
+        agg_name = "cct"
+    elif eval_name == "element_type":
+        agg_fields = ["element-type-accuracy"]
+        agg_name = "element-type"
+    elif eval_name == "object_detection":
+        agg_fields = ["f1_score", "m_ap"]
+        agg_name = "object-detection"
+    else:
+        raise ValueError(
+            f"Unknown metric for eval {eval_name}. "
+            f"Expected `text_extraction` or `element_type` or `table_extraction`."
+        )
+
+    if isinstance(data_input, str):
+        if not os.path.exists(data_input):
+            raise FileNotFoundError(f"File {data_input} not found.")
+        if data_input.endswith(".csv"):
+            df = pd.read_csv(data_input, header=None)
+        elif data_input.endswith(".tsv"):
+            df = pd.read_csv(data_input, sep="\t")
+        elif data_input.endswith(".txt"):
+            df = pd.read_csv(data_input, sep="\t", header=None)
+        else:
+            raise ValueError("Please provide a .csv or .tsv file.")
+    else:
+        df = data_input
+
+    if df.empty:
+        raise SystemExit("Data is empty. Exiting.")
+    elif group_by != "all" and (group_by not in df.columns or df[group_by].isnull().all()):
+        raise SystemExit(
+            f"Data cannot be aggregated by `{group_by}`."
+            f" Check if it's empty or the column is missing/empty."
+        )
+
+    grouped_df = []
+    if group_by and group_by != "all":
+        for field in agg_fields:
+            grouped_df.append(
+                _rename_aggregated_columns(
+                    df.groupby(group_by).agg({field: [_mean, _stdev, _pstdev, _count]})
+                )
+            )
+    if group_by == "all":
+        df["grouping_key"] = 0
+        for field in agg_fields:
+            grouped_df.append(
+                _rename_aggregated_columns(
+                    df.groupby("grouping_key").agg({field: [_mean, _stdev, _pstdev, _count]})
+                )
+            )
+    grouped_df = _format_grouping_output(*grouped_df)
+    if "grouping_key" in grouped_df.columns.get_level_values(0):
+        grouped_df = grouped_df.drop("grouping_key", axis=1, level=0)
+
+    if export_filename:
+        if not export_filename.endswith(".tsv"):
+            export_filename = export_filename + ".tsv"
+        _write_to_file(export_dir, export_filename, grouped_df)
+    else:
+        _write_to_file(export_dir, f"all-{group_by}-agg-{agg_name}.tsv", grouped_df)
+
+
+def filter_metrics(
+    data_input: Union[str, pd.DataFrame],
+    filter_list: Union[str, List[str]],
+    filter_by: str = "filename",
+    export_filename: Optional[str] = None,
+    export_dir: str = "metrics",
+    return_type: str = "file",
+) -> Optional[pd.DataFrame]:
+    """Reads the data_input file and filter only selected row available in filter_list.
+
+    Args:
+        data_input (str, dataframe): the source data, path to file or dataframe
+        filter_list (str, list): the filter, path to file or list of string
+        filter_by (str): data_input's column to filter the filter_list to
+        export_filename (str, optional): export filename. required when return_type is "file"
+        export_dir (str, optional): export directory. default to <current directory>/metrics
+        return_type (str): "file" or "dataframe"
+    """
+    if isinstance(data_input, str):
+        if not os.path.exists(data_input):
+            raise FileNotFoundError(f"File {data_input} not found.")
+        if data_input.endswith(".csv"):
+            df = pd.read_csv(data_input, header=None)
+        elif data_input.endswith(".tsv"):
+            df = pd.read_csv(data_input, sep="\t")
+        elif data_input.endswith(".txt"):
+            df = pd.read_csv(data_input, sep="\t", header=None)
+        else:
+            raise ValueError("Please provide a .csv or .tsv file.")
+    else:
+        df = data_input
+
+    if isinstance(filter_list, str):
+        if not os.path.exists(filter_list):
+            raise FileNotFoundError(f"File {filter_list} not found.")
+        if filter_list.endswith(".csv"):
+            filter_df = pd.read_csv(filter_list, header=None)
+        elif filter_list.endswith(".tsv"):
+            filter_df = pd.read_csv(filter_list, sep="\t")
+        elif filter_list.endswith(".txt"):
+            filter_df = pd.read_csv(filter_list, sep="\t", header=None)
+        else:
+            raise ValueError("Please provide a .csv or .tsv file.")
+        filter_list = filter_df.iloc[:, 0].astype(str).values.tolist()
+    elif not isinstance(filter_list, list):
+        raise ValueError("Please provide a List of strings or path to file.")
+
+    if filter_by not in df.columns:
+        raise ValueError("`filter_by` key does not exists in the data provided.")
+
+    res = df[df[filter_by].isin(filter_list)]
+
+    if res.empty:
+        raise SystemExit("No common file names between data_input and filter_list. Exiting.")
+
+    if return_type == "dataframe":
+        return res
+    elif return_type == "file" and export_filename:
+        _write_to_file(export_dir, export_filename, res)
+    elif return_type == "file" and not export_filename:
+        raise ValueError("Please provide `export_filename`.")
+    else:
+        raise ValueError("Return type must be either `dataframe` or `file`.")
+
+
+@dataclass
+class ObjectDetectionMetricsCalculatorBase(BaseMetricsCalculator, ABC):
+    """
+    Calculates object detection metrics for each document:
+    - f1 score
+    - precision
+    - recall
+    - average precision (mAP)
+    It also calculates aggregated metrics.
+    """
+
+    def __post_init__(self):
+        super().__post_init__()
+        self._document_paths = [
+            path.relative_to(self.documents_dir)
+            for path in self.documents_dir.rglob("analysis/*/layout_dump/object_detection.json")
+            if path.is_file()
+        ]
+
+    @property
+    def supported_metric_names(self):
+        return ["f1_score", "precision", "recall", "m_ap"]
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-object-detection-metrics.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-object-detection-metrics.tsv"
+
+    def _find_file_in_ground_truth(self, file_stem: str) -> Optional[Path]:
+        """Find the file corresponding to OD model dump file among the set of ground truth files
+
+        The files in ground truth paths keep the original extension and have .json suffix added,
+        e.g.:
+        some_document.pdf.json
+        poster.jpg.json
+
+        To compare to `file_stem` we need to take the prefix part of the file, thus double-stem
+        is applied.
+        """
+        for path in self._ground_truth_paths:
+            if Path(path.stem).stem == file_stem:
+                return path
+        return None
+
+    def _get_paths(self, doc: Path) -> tuple(str, Path, Path):
+        """Resolves ground doctype, prediction file path and ground truth path.
+
+        As OD dump directory structure differes from other simple outputs, it needs
+        a specific processing to match the output OD dump file with corresponding
+        OD GT file.
+
+        The outputs are placed in a dicrectory structure:
+
+        analysis
+        |- document_name
+            |- layout_dump
+                |- object_detection.json
+            |- bboxes # not used in this evaluation
+
+        and the GT file is pleced in od_gt directory for given dataset
+
+        dataset_name
+        |- od_gt
+            |- document_name.pdf.json
+
+        Args:
+            doc (Path): path to the OD dump file
+
+        Returns:
+            tuple: doctype, prediction file path, ground truth path
+        """
+        od_dump_path = Path(doc)
+        file_stem = od_dump_path.parts[-3]  # we take the `document_name` - so the filename stem
+
+        src_gt_filename = self._find_file_in_ground_truth(file_stem)
+
+        if src_gt_filename not in self._ground_truth_paths:
+            raise ValueError(f"Ground truth file {src_gt_filename} not found in list of GT files")
+
+        doctype = Path(src_gt_filename.stem).suffix[1:]
+
+        prediction_file = self.documents_dir / doc
+        if not prediction_file.exists():
+            logger.warning(f"Prediction file {prediction_file} does not exist, skipping")
+            raise ValueError(f"Prediction file {prediction_file} does not exist")
+
+        ground_truth_file = self.ground_truths_dir / src_gt_filename
+        if not ground_truth_file.exists():
+            logger.warning(f"Ground truth file {ground_truth_file} does not exist, skipping")
+            raise ValueError(f"Ground truth file {ground_truth_file} does not exist")
+
+        return doctype, prediction_file, ground_truth_file
+
+    def _generate_dataframes(self, rows) -> tuple[pd.DataFrame, pd.DataFrame]:
+        headers = ["filename", "doctype", "connector"] + self.supported_metric_names
+        df = pd.DataFrame(rows, columns=headers)
+
+        if df.empty:
+            agg_df = pd.DataFrame(columns=AGG_HEADERS)
+        else:
+            element_metrics_results = {}
+            for metric in self.supported_metric_names:
+                metric_df = df[df[metric].notnull()]
+                agg_metric = metric_df[metric].agg([_mean, _stdev, _pstdev, _count]).transpose()
+                if agg_metric.empty:
+                    element_metrics_results[metric] = pd.Series(
+                        data=[None, None, None, 0], index=["_mean", "_stdev", "_pstdev", "_count"]
+                    )
+                else:
+                    element_metrics_results[metric] = agg_metric
+            agg_df = pd.DataFrame(element_metrics_results).transpose().reset_index()
+        agg_df.columns = AGG_HEADERS
+
+        return df, agg_df
+
+
+class ObjectDetectionPerClassMetricsCalculator(ObjectDetectionMetricsCalculatorBase):
+    def __post_init__(self):
+        super().__post_init__()
+        self.per_class_metric_names: list[str] | None = None
+        self._set_supported_metrics()
+
+    @property
+    def supported_metric_names(self):
+        if self.per_class_metric_names:
+            return self.per_class_metric_names
+        else:
+            raise ValueError("per_class_metrics not initialized - cannot get class names")
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-object-detection-metrics-per-class.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-object-detection-metrics-per-class.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        """Calculate both class-aggregated and per-class metrics for a single document.
+
+        Args:
+            doc (Path): path to the OD dump file
+
+        Returns:
+            tuple: a tuple of aggregated and per-class metrics for a single document
+        """
+        try:
+            doctype, prediction_file, ground_truth_file = self._get_paths(doc)
+        except ValueError as e:
+            logger.error(f"Failed to process document {doc}: {e}")
+            return None
+
+        processor = ObjectDetectionEvalProcessor.from_json_files(
+            prediction_file_path=prediction_file,
+            ground_truth_file_path=ground_truth_file,
+        )
+        _, per_class_metrics = processor.get_metrics()
+
+        per_class_metrics_row = [
+            ground_truth_file.stem,
+            doctype,
+            None,  # connector
+        ]
+
+        for combined_metric_name in self.supported_metric_names:
+            metric = "_".join(combined_metric_name.split("_")[:-1])
+            class_name = combined_metric_name.split("_")[-1]
+            class_metrics = getattr(per_class_metrics, metric)
+            per_class_metrics_row.append(class_metrics[class_name])
+        return per_class_metrics_row
+
+    def _set_supported_metrics(self):
+        """Sets the supported metrics based on the classes found in the ground truth files.
+        The difference between per class and aggregated calculator is that the list of classes
+        (so the metrics) bases on the contents of the GT / prediction files.
+        """
+        metrics = ["f1_score", "precision", "recall", "m_ap"]
+        classes = set()
+        for gt_file in self._ground_truth_paths:
+            gt_file_path = self.ground_truths_dir / gt_file
+            with open(gt_file_path) as f:
+                gt = json.load(f)
+                gt_classes = gt["object_detection_classes"]
+                classes.update(gt_classes)
+        per_class_metric_names = []
+        for metric in metrics:
+            for class_name in classes:
+                per_class_metric_names.append(f"{metric}_{class_name}")
+        self.per_class_metric_names = sorted(per_class_metric_names)
+
+
+class ObjectDetectionAggregatedMetricsCalculator(ObjectDetectionMetricsCalculatorBase):
+    """Calculates object detection metrics for each document and aggregates by all classes"""
+
+    @property
+    def supported_metric_names(self):
+        return ["f1_score", "precision", "recall", "m_ap"]
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-object-detection-metrics.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-object-detection-metrics.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        """Calculate both class-aggregated and per-class metrics for a single document.
+
+        Args:
+            doc (Path): path to the OD dump file
+
+        Returns:
+            list: a list of aggregated metrics for a single document
+        """
+        try:
+            doctype, prediction_file, ground_truth_file = self._get_paths(doc)
+        except ValueError as e:
+            logger.error(f"Failed to process document {doc}: {e}")
+            return None
+
+        processor = ObjectDetectionEvalProcessor.from_json_files(
+            prediction_file_path=prediction_file,
+            ground_truth_file_path=ground_truth_file,
+        )
+        metrics, _ = processor.get_metrics()
+
+        return [
+            ground_truth_file.stem,
+            doctype,
+            None,  # connector
+        ] + [getattr(metrics, metric) for metric in self.supported_metric_names]

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/object_detection.py

@@ -0,0 +1,719 @@
+"""
+Implements object detection metrics: average precision, precision, recall, and f1 score.
+"""
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import torch
+
+IOU_THRESHOLDS = torch.tensor(
+    [0.5000, 0.5500, 0.6000, 0.6500, 0.7000, 0.7500, 0.8000, 0.8500, 0.9000, 0.9500]
+)
+SCORE_THRESHOLD = 0.1
+RECALL_THRESHOLDS = torch.arange(0, 1.01, 0.01)
+
+
+@dataclass
+class ObjectDetectionAggregatedEvaluation:
+    """Class representing a gathered class-aggregated object detection metrics"""
+
+    f1_score: float
+    precision: float
+    recall: float
+    m_ap: float
+
+
+@dataclass
+class ObjectDetectionPerClassEvaluation:
+    """Class representing a gathered object detection metrics per-class"""
+
+    f1_score: dict[str, float]
+    precision: dict[str, float]
+    recall: dict[str, float]
+    m_ap: dict[str, float]
+
+    @classmethod
+    def from_tensors(cls, ap, precision, recall, f1, class_labels):
+        f1_score = {class_labels[i]: f1[i] for i in range(len(class_labels))}
+        precision = {class_labels[i]: precision[i] for i in range(len(class_labels))}
+        recall = {class_labels[i]: recall[i] for i in range(len(class_labels))}
+        m_ap = {class_labels[i]: ap[i] for i in range(len(class_labels))}
+
+        return cls(f1_score, precision, recall, m_ap)
+
+
+class ObjectDetectionEvalProcessor:
+    iou_thresholds = IOU_THRESHOLDS
+    score_threshold = SCORE_THRESHOLD
+    recall_thresholds = RECALL_THRESHOLDS
+
+    def __init__(
+        self,
+        document_preds: list[torch.Tensor],
+        document_targets: list[torch.Tensor],
+        pages_height: list[int],
+        pages_width: list[int],
+        class_labels: list[str],
+        device: str = "cpu",
+    ):
+        """
+        Initializes the ObjectDetection prediction and ground truth.
+
+        Args:
+            document_preds (list):      list (of length pages of document) of
+                                        Tensors of shape (num_predictions, 6)
+                                        format: (x1, y1, x2, y2, confidence,class_label)
+                                        where x1,y1,x2,y2 are according to image size
+            document_targets (list):    list (of length pages of document) of
+                                        Tensors of shape (num_targets, 6)
+                                        format: (label, x1, y1, x2, y2)
+                                        where x,y,w,h are according to image size
+            pages_height (list):        list of height of each page in the document
+            pages_width (list):         list of width of each page in the document
+            class_labels (list):        list of class labels
+        """
+        self.device = device
+        self.document_preds = [pred.to(device) for pred in document_preds]
+        self.document_targets = [target.to(device) for target in document_targets]
+        self.pages_height = pages_height
+        self.pages_width = pages_width
+        self.class_labels = class_labels
+
+    @classmethod
+    def from_json_files(
+        cls,
+        prediction_file_path: Path,
+        ground_truth_file_path: Path,
+    ) -> "ObjectDetectionEvalProcessor":
+        """
+        Initializes the ObjectDetection prediction and ground truth,
+        and converts the data to the required format.
+
+        Args:
+            prediction_file_path (Path): path to json file with predictions dump from OD model
+            ground_truth_file_path (Path): path to json file with OD ground truth data
+        """
+        # TODO: Test after https://unstructured-ai.atlassian.net/browse/ML-92
+        # is done.
+        with open(prediction_file_path) as f:
+            predictions_data = json.load(f)
+        with open(ground_truth_file_path) as f:
+            ground_truth_data = json.load(f)
+
+        assert sorted(predictions_data["object_detection_classes"]) == sorted(
+            ground_truth_data["object_detection_classes"]
+        ), "Classes in predictions and ground truth do not match."
+        assert len(predictions_data["pages"]) == len(
+            ground_truth_data["pages"]
+        ), "Pages number in predictions and ground truth do not match."
+        for pred_page, gt_page in zip(
+            sorted(predictions_data["pages"], key=lambda p: p["number"]),
+            sorted(ground_truth_data["pages"], key=lambda p: p["number"]),
+        ):
+            assert pred_page["number"] == gt_page["number"], (
+                f"Page numbers in predictions {prediction_file_path.name} "
+                f"({pred_page['number']}) and ground truth {ground_truth_file_path.name} "
+                f"({gt_page['number']}) do not match."
+            )
+            page_num = pred_page["number"]
+
+            # TODO: translate the bboxes instead of raising error
+            assert pred_page["size"] == gt_page["size"], (
+                f"Page sizes in predictions {prediction_file_path.name} "
+                f"({pred_page['size'][0]} x {pred_page['size'][1]}) "
+                f"and ground truth {ground_truth_file_path.name} ({gt_page['size'][0]} x "
+                f"{gt_page['size'][1]}) do not match for page {page_num}."
+            )
+
+        class_labels = predictions_data["object_detection_classes"]
+        document_preds = cls._process_data(predictions_data, class_labels, prediction=True)
+        document_targets = cls._process_data(ground_truth_data, class_labels)
+        pages_height, pages_width = cls._parse_page_dimensions(predictions_data)
+
+        return cls(document_preds, document_targets, pages_height, pages_width, class_labels)
+
+    def get_metrics(
+        self,
+    ) -> tuple[ObjectDetectionAggregatedEvaluation, ObjectDetectionPerClassEvaluation]:
+        """Get per document OD metrics.
+
+        Returns:
+            tuple: Tuple of ObjectDetectionAggregatedEvaluation and
+                ObjectDetectionPerClassEvaluation
+        """
+        document_matchings = []
+        for preds, targets, height, width in zip(
+            self.document_preds, self.document_targets, self.pages_height, self.pages_width
+        ):
+            # iterate over each page
+            page_matching_tensors = self._compute_page_detection_matching(
+                preds=preds,
+                targets=targets,
+                height=height,
+                width=width,
+            )
+            document_matchings.append(page_matching_tensors)
+
+        # compute metrics for all detections and targets
+        mean_ap, mean_precision, mean_recall, mean_f1 = (
+            -1.0,
+            -1.0,
+            -1.0,
+            -1.0,
+        )
+
+        num_cls = len(self.class_labels)
+        mean_ap_per_class = np.full(num_cls, np.nan)
+        mean_precision_per_class = np.full(num_cls, np.nan)
+        mean_recall_per_class = np.full(num_cls, np.nan)
+        mean_f1_per_class = np.full(num_cls, np.nan)
+
+        if len(document_matchings):
+            matching_info_tensors = [torch.cat(x, 0) for x in list(zip(*document_matchings))]
+
+            # shape (n_class, nb_iou_thresh)
+            (
+                ap_per_present_classes,
+                precision_per_present_classes,
+                recall_per_present_classes,
+                f1_per_present_classes,
+                present_classes,
+            ) = self._compute_detection_metrics(
+                *matching_info_tensors,
+            )
+
+            # Precision, recall and f1 are computed for IoU threshold range, averaged over classes
+            # results before version 3.0.4 (Dec 11 2022) were computed only for smallest value
+            # (i.e IoU 0.5 if metric is @0.5:0.95)
+            mean_precision, mean_recall, mean_f1 = (
+                precision_per_present_classes.mean(),
+                recall_per_present_classes.mean(),
+                f1_per_present_classes.mean(),
+            )
+
+            # MaP is averaged over IoU thresholds and over classes
+            mean_ap = ap_per_present_classes.mean()
+
+            # Fill array of per-class AP scores with values for classes that were present in the
+            # dataset
+            ap_per_class = ap_per_present_classes.mean(1)
+            precision_per_class = precision_per_present_classes.mean(1)
+            recall_per_class = recall_per_present_classes.mean(1)
+            f1_per_class = f1_per_present_classes.mean(1)
+            for i, class_index in enumerate(present_classes):
+                mean_ap_per_class[class_index] = float(ap_per_class[i])
+
+                mean_precision_per_class[class_index] = float(precision_per_class[i])
+                mean_recall_per_class[class_index] = float(recall_per_class[i])
+                mean_f1_per_class[class_index] = float(f1_per_class[i])
+
+        od_per_class_evaluation = ObjectDetectionPerClassEvaluation.from_tensors(
+            ap=mean_ap_per_class,
+            precision=mean_precision_per_class,
+            recall=mean_recall_per_class,
+            f1=mean_f1_per_class,
+            class_labels=self.class_labels,
+        )
+
+        od_evaluation = ObjectDetectionAggregatedEvaluation(
+            f1_score=float(mean_f1),
+            precision=float(mean_precision),
+            recall=float(mean_recall),
+            m_ap=float(mean_ap),
+        )
+
+        return od_evaluation, od_per_class_evaluation
+
+    @staticmethod
+    def _parse_page_dimensions(data: dict) -> tuple[list, list]:
+        """
+        Process the page dimensions from the json file to the required format.
+        """
+        pages_height = []
+        pages_width = []
+        for page in data["pages"]:
+            pages_height.append(page["size"]["height"])
+            pages_width.append(page["size"]["width"])
+        return pages_height, pages_width
+
+    @staticmethod
+    def _process_data(data: dict, class_labels, prediction: bool = False) -> list[dict]:
+        """
+        Process the elements from the json file to the required format.
+        """
+        pages_list = []
+        for page in data["pages"]:
+            page_elements = []
+            for element in page["elements"]:
+                # Extract coordinates, confidence, and class label from each prediction
+                class_label = element["type"]
+                class_idx = class_labels.index(class_label)
+                x1, y1, x2, y2 = element["bbox"]
+                if prediction:
+                    confidence = element["prob"]
+                    page_elements.append([x1, y1, x2, y2, confidence, class_idx])
+                else:
+                    page_elements.append([class_idx, x1, y1, x2, y2])
+            page_tensor = torch.tensor(page_elements)
+            pages_list.append(page_tensor)
+
+        return pages_list
+
+    @staticmethod
+    def _get_top_k_idx_per_cls(
+        preds_scores: torch.Tensor, preds_cls: torch.Tensor, top_k: int
+    ) -> torch.Tensor:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Get the indexes of all the top k predictions for every class
+
+        Args:
+            preds_scores:   The confidence scores, vector of shape (n_pred)
+            preds_cls:      The predicted class, vector of shape (n_pred)
+            top_k:          Number of predictions to keep per class, ordered by confidence score
+
+        Returns:
+            top_k_idx:     Indexes of the top k predictions. length <= (k * n_unique_class)
+        """
+        n_unique_cls = torch.max(preds_cls)
+        mask = preds_cls.view(-1, 1) == torch.arange(
+            n_unique_cls + 1, device=preds_scores.device
+        ).view(1, -1)
+        preds_scores_per_cls = preds_scores.view(-1, 1) * mask
+
+        sorted_scores_per_cls, sorting_idx = preds_scores_per_cls.sort(0, descending=True)
+        idx_with_satisfying_scores = sorted_scores_per_cls[:top_k, :].nonzero(as_tuple=False)
+        top_k_idx = sorting_idx[idx_with_satisfying_scores.split(1, dim=1)]
+        return top_k_idx.view(-1)
+
+    @staticmethod
+    def _change_bbox_bounds_for_image_size(
+        boxes: np.ndarray, img_shape: tuple[int, int]
+    ) -> np.ndarray:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Clips bboxes to image boundaries.
+
+        Args:
+            bboxes:         Input bounding boxes in XYXY format of [..., 4] shape
+            img_shape:      Image shape (height, width).
+        Returns:
+            clipped_boxes:  Clipped bboxes in XYXY format of [..., 4] shape
+        """
+        boxes[..., [0, 2]] = boxes[..., [0, 2]].clip(min=0, max=img_shape[1])
+        boxes[..., [1, 3]] = boxes[..., [1, 3]].clip(min=0, max=img_shape[0])
+        return boxes
+
+    @staticmethod
+    def _box_iou(box1: torch.Tensor, box2: torch.Tensor) -> torch.Tensor:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Return intersection-over-union (Jaccard index) of boxes.
+        Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
+
+        Args:
+            box1: Tensor of shape [N, 4]
+            box2: Tensor of shape [M, 4]
+
+        Returns:
+            iou:    Tensor of shape [N, M]: the NxM matrix containing the pairwise IoU values
+                    for every element in boxes1 and boxes2
+        """
+
+        def box_area(box):
+            # box = 4xn
+            return (box[2] - box[0]) * (box[3] - box[1])
+
+        area1 = box_area(box1.T)
+        area2 = box_area(box2.T)
+
+        # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
+        inter = (
+            (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2]))
+            .clamp(0)
+            .prod(2)
+        )
+        return inter / (area1[:, None] + area2 - inter)  # iou = inter / (area1 + area2 - inter)
+
+    def _compute_targets(
+        self,
+        preds_box_xyxy: torch.Tensor,
+        preds_cls: torch.Tensor,
+        targets_box_xyxy: torch.Tensor,
+        targets_cls: torch.Tensor,
+        preds_matched: torch.Tensor,
+        targets_matched: torch.Tensor,
+        preds_idx_to_use: torch.Tensor,
+        iou_thresholds: torch.Tensor,
+    ) -> torch.Tensor:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Computes the matching targets based on IoU for regular scenarios.
+
+        Args:
+            preds_box_xyxy: (torch.Tensor) Predicted bounding boxes in XYXY format.
+            preds_cls: (torch.Tensor) Predicted classes.
+            targets_box_xyxy: (torch.Tensor) Target bounding boxes in XYXY format.
+            targets_cls: (torch.Tensor) Target classes.
+            preds_matched: (torch.Tensor) Tensor indicating which predictions are matched.
+            targets_matched: (torch.Tensor) Tensor indicating which targets are matched.
+            preds_idx_to_use: (torch.Tensor) Indices of predictions to use.
+
+        Returns:
+            targets: Computed matching targets.
+        """
+        # shape = (n_preds x n_targets)
+        iou = self._box_iou(preds_box_xyxy[preds_idx_to_use], targets_box_xyxy)
+
+        # Fill IoU values at index (i, j) with 0 when the prediction (i) and target(j)
+        # are of different class
+        # Filling with 0 is equivalent to ignore these values
+        # since with want IoU > iou_threshold > 0
+        cls_mismatch = preds_cls[preds_idx_to_use].view(-1, 1) != targets_cls.view(1, -1)
+        iou[cls_mismatch] = 0
+
+        # The matching priority is first detection confidence and then IoU value.
+        # The detection is already sorted by confidence in NMS,
+        # so here for each prediction we order the targets by iou.
+        sorted_iou, target_sorted = iou.sort(descending=True, stable=True)
+
+        # Only iterate over IoU values higher than min threshold to speed up the process
+        for pred_selected_i, target_sorted_i in (sorted_iou > iou_thresholds[0]).nonzero(
+            as_tuple=False
+        ):
+            # pred_selected_i and target_sorted_i are relative to filters/sorting,
+            # so we extract their absolute indexes
+            pred_i = preds_idx_to_use[pred_selected_i]
+            target_i = target_sorted[pred_selected_i, target_sorted_i]
+
+            # Vector[j], True when IoU(pred_i, target_i) is above the (j)th threshold
+            is_iou_above_threshold = sorted_iou[pred_selected_i, target_sorted_i] > iou_thresholds
+
+            # Vector[j], True when both pred_i and target_i are not matched yet
+            # for the (j)th threshold
+            are_candidates_free = torch.logical_and(
+                ~preds_matched[pred_i, :], ~targets_matched[target_i, :]
+            )
+
+            # Vector[j], True when (pred_i, target_i) can be matched for the (j)th threshold
+            are_candidates_good = torch.logical_and(is_iou_above_threshold, are_candidates_free)
+
+            # For every threshold (j) where target_i and pred_i can be matched together
+            # ( are_candidates_good[j]==True )
+            # fill the matching placeholders with True
+            targets_matched[target_i, are_candidates_good] = True
+            preds_matched[pred_i, are_candidates_good] = True
+
+            # When all the targets are matched with a prediction for every IoU Threshold, stop.
+            if targets_matched.all():
+                break
+
+        return preds_matched
+
+    def _compute_page_detection_matching(
+        self,
+        preds: torch.Tensor,
+        targets: torch.Tensor,
+        height: int,
+        width: int,
+        top_k: int = 100,
+        return_on_cpu: bool = True,
+    ) -> tuple:
+        # Adapted from: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Match predictions (NMS output) and the targets (ground truth) with respect to metric
+        and confidence score for a given image.
+
+        Args:
+            preds:          Tensor of shape (num_img_predictions, 6)
+                            format: (x1, y1, x2, y2, confidence, class_label)
+                            where x1,y1,x2,y2 are according to image size
+            targets:        targets for this image of shape (num_img_targets, 5)
+                            format:     (label, x1, y1, x2, y2)
+                            where x1,y1,x2,y2 are according to image size
+            height:         dimensions of the image
+            width:          dimensions of the image
+            top_k:          Number of predictions to keep per class, ordered by confidence score
+            return_on_cpu:  If True, the output will be returned on "CPU", otherwise it will be
+                            returned on "device"
+
+        Returns:
+            preds_matched:      Tensor of shape (num_img_predictions, n_thresholds)
+                                True when prediction (i) is matched with a target with respect to
+                                the (j)th threshold
+            preds_to_ignore:    Tensor of shape (num_img_predictions, n_thresholds)
+                                True when prediction (i) is matched with a crowd target with
+                                respect to the (j)th threshold
+            preds_scores:       Tensor of shape (num_img_predictions),
+                                confidence score for every prediction
+            preds_cls:          Tensor of shape (num_img_predictions),
+                                predicted class for every prediction
+            targets_cls:        Tensor of shape (num_img_targets),
+                                ground truth class for every target
+        """
+        thresholds = self.iou_thresholds.to(device=self.device)
+        num_thresholds = len(thresholds)
+
+        if preds is None or len(preds) == 0:
+            preds_matched = torch.zeros((0, num_thresholds), dtype=torch.bool, device=self.device)
+            preds_to_ignore = torch.zeros((0, num_thresholds), dtype=torch.bool, device=self.device)
+            preds_scores = torch.tensor([], dtype=torch.float32, device=self.device)
+            preds_cls = torch.tensor([], dtype=torch.float32, device=self.device)
+            targets_cls = targets[:, 0].to(device=self.device)
+            return preds_matched, preds_to_ignore, preds_scores, preds_cls, targets_cls
+
+        preds_matched = torch.zeros(
+            len(preds), num_thresholds, dtype=torch.bool, device=self.device
+        )
+        targets_matched = torch.zeros(
+            len(targets), num_thresholds, dtype=torch.bool, device=self.device
+        )
+        preds_to_ignore = torch.zeros(
+            len(preds), num_thresholds, dtype=torch.bool, device=self.device
+        )
+
+        preds_cls, preds_box, preds_scores = preds[:, -1], preds[:, 0:4], preds[:, 4]
+        targets_cls, targets_box = targets[:, 0], targets[:, 1:5]
+
+        # Ignore all but the predictions that were top_k for their class
+        preds_idx_to_use = self._get_top_k_idx_per_cls(preds_scores, preds_cls, top_k)
+        preds_to_ignore[:, :] = True
+        preds_to_ignore[preds_idx_to_use] = False
+
+        if len(targets) > 0:  # or len(crowd_targets) > 0:
+            self._change_bbox_bounds_for_image_size(preds, (height, width))
+
+            preds_matched = self._compute_targets(
+                preds_box,
+                preds_cls,
+                targets_box,
+                targets_cls,
+                preds_matched,
+                targets_matched,
+                preds_idx_to_use,
+                thresholds,
+            )
+
+        return preds_matched, preds_to_ignore, preds_scores, preds_cls, targets_cls
+
+    def _compute_detection_metrics(
+        self,
+        preds_matched: torch.Tensor,
+        preds_to_ignore: torch.Tensor,
+        preds_scores: torch.Tensor,
+        preds_cls: torch.Tensor,
+        targets_cls: torch.Tensor,
+    ) -> tuple:
+        # Adapted from: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Compute the list of precision, recall, MaP and f1 for every class.
+
+        Args:
+            preds_matched:      Tensor of shape (num_predictions, n_iou_thresholds)
+                                True when prediction (i) is matched with a target with respect
+                                to the (j)th IoU threshold
+            preds_to_ignore     Tensor of shape (num_predictions, n_iou_thresholds)
+                                True when prediction (i) is matched with a crowd target with
+                                respect to the (j)th IoU threshold
+            preds_scores:       Tensor of shape (num_predictions),
+                                confidence score for every prediction
+            preds_cls:          Tensor of shape (num_predictions),
+                                predicted class for every prediction
+            targets_cls:        Tensor of shape (num_targets),
+                                ground truth class for every target box to be detected
+
+        Returns:
+            ap, precision, recall, f1:  Tensors of shape (n_class, nb_iou_thrs)
+            unique_classes:             Vector with all unique target classes
+        """
+
+        preds_matched, preds_to_ignore = preds_matched.to(self.device), preds_to_ignore.to(
+            self.device
+        )
+        preds_scores, preds_cls, targets_cls = (
+            preds_scores.to(self.device),
+            preds_cls.to(self.device),
+            targets_cls.to(self.device),
+        )
+
+        recall_thresholds = self.recall_thresholds.to(self.device)
+        score_threshold = self.score_threshold
+
+        unique_classes = torch.unique(targets_cls).long()
+
+        n_class, nb_iou_thrs = len(unique_classes), preds_matched.shape[-1]
+
+        ap = torch.zeros((n_class, nb_iou_thrs), device=self.device)
+        precision = torch.zeros((n_class, nb_iou_thrs), device=self.device)
+        recall = torch.zeros((n_class, nb_iou_thrs), device=self.device)
+
+        for cls_i, class_value in enumerate(unique_classes):
+            cls_preds_idx, cls_targets_idx = (preds_cls == class_value), (
+                targets_cls == class_value
+            )
+            (
+                cls_ap,
+                cls_precision,
+                cls_recall,
+            ) = self._compute_detection_metrics_per_cls(
+                preds_matched=preds_matched[cls_preds_idx],
+                preds_to_ignore=preds_to_ignore[cls_preds_idx],
+                preds_scores=preds_scores[cls_preds_idx],
+                n_targets=cls_targets_idx.sum(),
+                recall_thresholds=recall_thresholds,
+                score_threshold=score_threshold,
+            )
+            ap[cls_i, :] = cls_ap
+            precision[cls_i, :] = cls_precision
+            recall[cls_i, :] = cls_recall
+
+        f1 = 2 * precision * recall / (precision + recall + 1e-16)
+        return ap, precision, recall, f1, unique_classes
+
+    def _compute_detection_metrics_per_cls(
+        self,
+        preds_matched: torch.Tensor,
+        preds_to_ignore: torch.Tensor,
+        preds_scores: torch.Tensor,
+        n_targets: int,
+        recall_thresholds: torch.Tensor,
+        score_threshold: float,
+    ):
+        # Adapted from: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Compute the list of precision, recall and MaP of a given class for every recall threshold.
+
+        Args:
+            preds_matched:      Tensor of shape (num_predictions, n_thresholds)
+                                True when prediction (i) is matched with a target
+                                with respect to the(j)th threshold
+            preds_to_ignore     Tensor of shape (num_predictions, n_thresholds)
+                                True when prediction (i) is matched with a crowd target
+                                with respect to the (j)th threshold
+            preds_scores:       Tensor of shape (num_predictions),
+                                confidence score for every prediction
+            n_targets:          Number of target boxes of this class
+            recall_thresholds:  Tensor of shape (max_n_rec_thresh)
+                                list of recall thresholds used to compute MaP
+            score_threshold:    Minimum confidence score to consider a prediction
+                                for the computation of precision and recall (not MaP)
+
+        Returns:
+            ap, precision, recall:     Tensors of shape (nb_thrs)
+        """
+
+        nb_iou_thrs = preds_matched.shape[-1]
+
+        tps = preds_matched
+        fps = torch.logical_and(
+            torch.logical_not(preds_matched), torch.logical_not(preds_to_ignore)
+        )
+
+        if len(tps) == 0:
+            return (
+                torch.zeros(nb_iou_thrs, device=self.device),
+                torch.zeros(nb_iou_thrs, device=self.device),
+                torch.zeros(nb_iou_thrs, device=self.device),
+            )
+
+        # Sort by decreasing score
+        dtype = (
+            torch.uint8
+            if preds_scores.is_cuda and preds_scores.dtype is torch.bool
+            else preds_scores.dtype
+        )
+        sort_ind = torch.argsort(preds_scores.to(dtype), descending=True)
+        tps = tps[sort_ind, :]
+        fps = fps[sort_ind, :]
+        preds_scores = preds_scores[sort_ind].contiguous()
+
+        # Rolling sum over the predictions
+        rolling_tps = torch.cumsum(tps, axis=0, dtype=torch.float)
+        rolling_fps = torch.cumsum(fps, axis=0, dtype=torch.float)
+
+        rolling_recalls = rolling_tps / n_targets
+        rolling_precisions = rolling_tps / (
+            rolling_tps + rolling_fps + torch.finfo(torch.float64).eps
+        )
+
+        # Reversed cummax to only have decreasing values
+        rolling_precisions = rolling_precisions.flip(0).cummax(0).values.flip(0)
+
+        # ==================
+        # RECALL & PRECISION
+
+        # We want the rolling precision/recall at index i so that:
+        # preds_scores[i-1] >= score_threshold > preds_scores[i]
+        # Note: torch.searchsorted works on increasing sequence and preds_scores is decreasing,
+        # so we work with "-"
+        # Note2: right=True due to negation
+        lowest_score_above_threshold = torch.searchsorted(
+            -preds_scores, -score_threshold, right=True
+        )
+
+        if (
+            lowest_score_above_threshold == 0
+        ):  # Here score_threshold > preds_scores[0], so no pred is above the threshold
+            recall = torch.zeros(nb_iou_thrs, device=self.device)
+            precision = torch.zeros(
+                nb_iou_thrs, device=self.device
+            )  # the precision is not really defined when no pred but we need to give it a value
+        else:
+            recall = rolling_recalls[lowest_score_above_threshold - 1]
+            precision = rolling_precisions[lowest_score_above_threshold - 1]
+
+        # ==================
+        # AVERAGE PRECISION
+
+        # shape = (nb_iou_thrs, n_recall_thresholds)
+        recall_thresholds = recall_thresholds.view(1, -1).repeat(nb_iou_thrs, 1)
+
+        # We want the index i so that:
+        # rolling_recalls[i-1] < recall_thresholds[k] <= rolling_recalls[i]
+        # Note:  when recall_thresholds[k] > max(rolling_recalls), i = len(rolling_recalls)
+        # Note2: we work with transpose (.T) to apply torch.searchsorted on first dim
+        # instead of the last one
+        recall_threshold_idx = torch.searchsorted(
+            rolling_recalls.T.contiguous(), recall_thresholds, right=False
+        ).T
+
+        # When recall_thresholds[k] > max(rolling_recalls),
+        # rolling_precisions[i] is not defined, and we want precision = 0
+        rolling_precisions = torch.cat(
+            (rolling_precisions, torch.zeros(1, nb_iou_thrs, device=self.device)), dim=0
+        )
+
+        # shape = (n_recall_thresholds, nb_iou_thrs)
+        sampled_precision_points = torch.gather(
+            input=rolling_precisions, index=recall_threshold_idx, dim=0
+        )
+
+        # Average over the recall_thresholds
+        ap = sampled_precision_points.mean(0)
+
+        return ap, precision, recall
+
+
+if __name__ == "__main__":
+    from dataclasses import asdict
+
+    # Example usage
+    prediction_file_paths = [Path("pths/to/predictions.json"), Path("pths/to/predictions2.json")]
+    ground_truth_file_paths = [
+        Path("pths/to/ground_truth.json"),
+        Path("pths/to/ground_truth2.json"),
+    ]
+
+    for prediction_file_path, ground_truth_file_path in zip(
+        prediction_file_paths, ground_truth_file_paths
+    ):
+        eval_processor = ObjectDetectionEvalProcessor.from_json_files(
+            prediction_file_path, ground_truth_file_path
+        )
+
+        metrics, per_class_metrics = eval_processor.get_metrics()
+        print(f"Metrics for {ground_truth_file_path.name}:\n{asdict(metrics)}")
+        print(f"Per class Metrics for {ground_truth_file_path.name}:\n{asdict(per_class_metrics)}")

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/table/table_alignment.py

@@ -0,0 +1,180 @@
+import difflib
+from typing import Any, Dict, List
+
+import numpy as np
+import pandas as pd
+from unstructured_inference.models.eval import compare_contents_as_df
+
+
+class TableAlignment:
+    def __init__(self, cutoff: float = 0.8):
+        self.cutoff = cutoff
+
+    @staticmethod
+    def get_content_in_tables(table_data: List[List[Dict[str, Any]]]) -> List[str]:
+        # Replace below docstring with google-style docstring
+        """Extracts and concatenates the content of cells from each table in a list of tables.
+
+        Args:
+          table_data: A list of tables, each table being a list of cell data dictionaries.
+
+        Returns:
+          List of strings where each string represents the concatenated content of one table.
+        """
+        return [" ".join([d["content"] for d in td if "content" in d]) for td in table_data]
+
+    @staticmethod
+    def get_table_level_alignment(
+        predicted_table_data: List[List[Dict[str, Any]]],
+        ground_truth_table_data: List[List[Dict[str, Any]]],
+    ) -> List[int]:
+        """Compares predicted table data with ground truth data to find the best
+        matching table index for each predicted table.
+
+        Args:
+          predicted_table_data: A list of predicted tables.
+          ground_truth_table_data: A list of ground truth tables.
+
+        Returns:
+          A list of indices indicating the best match in the ground truth for
+          each predicted table.
+
+        """
+        ground_truth_texts = TableAlignment.get_content_in_tables(ground_truth_table_data)
+        matched_indices = []
+        for td in predicted_table_data:
+            reference = TableAlignment.get_content_in_tables([td])[0]
+            matches = difflib.get_close_matches(reference, ground_truth_texts, cutoff=0.1, n=1)
+            matched_indices.append(ground_truth_texts.index(matches[0]) if matches else -1)
+        return matched_indices
+
+    @staticmethod
+    def _zip_to_dataframe(table_data: List[Dict[str, Any]]) -> pd.DataFrame:
+        df = pd.DataFrame(table_data, columns=["row_index", "col_index", "content"])
+        df = df.set_index("row_index")
+        df["col_index"] = df["col_index"].astype(str)
+        return df
+
+    @staticmethod
+    def get_element_level_alignment(
+        predicted_table_data: List[List[Dict[str, Any]]],
+        ground_truth_table_data: List[List[Dict[str, Any]]],
+        matched_indices: List[int],
+        cutoff: float = 0.8,
+    ) -> Dict[str, float]:
+        """Aligns elements of the predicted tables with the ground truth tables at the cell level.
+
+        Args:
+          predicted_table_data: A list of predicted tables.
+          ground_truth_table_data: A list of ground truth tables.
+          matched_indices: Indices of the best matching ground truth table for each predicted table.
+          cutoff: The cutoff value for the close matches.
+
+        Returns:
+          A dictionary with column and row alignment accuracies.
+
+        """
+        content_diff_cols = []
+        content_diff_rows = []
+        col_index_acc = []
+        row_index_acc = []
+
+        for idx, td in zip(matched_indices, predicted_table_data):
+            if idx == -1:
+                content_diff_cols.append(0)
+                content_diff_rows.append(0)
+                col_index_acc.append(0)
+                row_index_acc.append(0)
+                continue
+            ground_truth_td = ground_truth_table_data[idx]
+
+            # Get row and col content accuracy
+            predict_table_df = TableAlignment._zip_to_dataframe(td)
+            ground_truth_table_df = TableAlignment._zip_to_dataframe(ground_truth_td)
+
+            table_content_diff = compare_contents_as_df(
+                ground_truth_table_df.fillna(""),
+                predict_table_df.fillna(""),
+            )
+            content_diff_cols.append(table_content_diff["by_col_token_ratio"])
+            content_diff_rows.append(table_content_diff["by_row_token_ratio"])
+
+            aligned_element_col_count = 0
+            aligned_element_row_count = 0
+            total_element_count = 0
+            # Get row and col index accuracy
+            ground_truth_td_contents_list = [gtd["content"].lower() for gtd in ground_truth_td]
+            used_indices = set()
+            indices_tuple_pairs = []
+            for td_ele in td:
+                content = td_ele["content"].lower()
+                row_index = td_ele["row_index"]
+                col_idx = td_ele["col_index"]
+
+                matches = difflib.get_close_matches(
+                    content,
+                    ground_truth_td_contents_list,
+                    cutoff=cutoff,
+                    n=1,
+                )
+                # BUG FIX: the previous matched_idx will only output the first matched index if
+                # the match has duplicates in the
+                # ground_truth_td_contents_list, the current fix will output its correspondence idx
+                # once matching is exhausted, it will go back search again the same fashion
+                matching_indices = []
+                if matches != []:
+                    b_indices = [
+                        i
+                        for i, b_string in enumerate(ground_truth_td_contents_list)
+                        if b_string == matches[0] and i not in used_indices
+                    ]
+                    if not b_indices:
+                        # If all indices are used, reset used_indices and use the first index
+                        used_indices.clear()
+                        b_indices = [
+                            i
+                            for i, b_string in enumerate(ground_truth_td_contents_list)
+                            if b_string == matches[0] and i not in used_indices
+                        ]
+                    matching_index = b_indices[0]
+                    matching_indices.append(matching_index)
+                    used_indices.add(matching_index)
+                else:
+                    matching_indices = [-1]
+                matched_idx = matching_indices[0]
+                if matched_idx >= 0:
+                    gt_row_index = ground_truth_td[matched_idx]["row_index"]
+                    gt_col_index = ground_truth_td[matched_idx]["col_index"]
+                    indices_tuple_pairs.append(((row_index, col_idx), (gt_row_index, gt_col_index)))
+
+            for indices_tuple_pair in indices_tuple_pairs:
+                if indices_tuple_pair[0][0] == indices_tuple_pair[1][0]:
+                    aligned_element_row_count += 1
+                if indices_tuple_pair[0][1] == indices_tuple_pair[1][1]:
+                    aligned_element_col_count += 1
+                total_element_count += 1
+
+            table_col_index_acc = 0
+            table_row_index_acc = 0
+            if total_element_count > 0:
+                table_col_index_acc = round(aligned_element_col_count / total_element_count, 2)
+                table_row_index_acc = round(aligned_element_row_count / total_element_count, 2)
+
+            col_index_acc.append(table_col_index_acc)
+            row_index_acc.append(table_row_index_acc)
+
+        not_found_gt_table_indexes = [
+            id for id in range(len(ground_truth_table_data)) if id not in matched_indices
+        ]
+        for _ in not_found_gt_table_indexes:
+            content_diff_cols.append(0)
+            content_diff_rows.append(0)
+            col_index_acc.append(0)
+            row_index_acc.append(0)
+
+        return {
+            "col_index_acc": round(np.mean(col_index_acc), 2),
+            "row_index_acc": round(np.mean(row_index_acc), 2),
+            "col_content_acc": round(np.mean(content_diff_cols) / 100.0, 2),
+            "row_content_acc": round(np.mean(content_diff_rows) / 100.0, 2),
+        }

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/table/table_eval.py

@@ -0,0 +1,342 @@
+"""
+The purpose of this script is to create a comprehensive metric for table evaluation
+1. Verify table identification.
+    a. Concatenate all text in the table and ground truth.
+    b. Calculate the difference to find the closest matches.
+    c. If contents are too different, mark as a failure.
+
+2. For each identified table:
+    a. Align elements at the level of individual elements.
+    b. Match elements by text.
+    c. Determine indexes for both predicted and actual data.
+    d. Compare index tuples at column and row levels to assess content shifts.
+    e. Compare the token orders by flattened along column and row levels
+    f. Note: Imperfect HTML is acceptable unless it impedes parsing,
+       in which case the table is considered failed.
+
+Example
+python table_eval.py  \
+    --prediction_file "model_output.pdf.json" \
+    --ground_truth_file "ground_truth.pdf.json"
+"""
+
+import difflib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import click
+import numpy as np
+
+from unstructured.metrics.table.table_alignment import TableAlignment
+from unstructured.metrics.table.table_extraction import (
+    extract_and_convert_tables_from_ground_truth,
+    extract_and_convert_tables_from_prediction,
+)
+
+
+@dataclass
+class TableEvaluation:
+    """Class representing a gathered table metrics."""
+
+    total_tables: int
+    total_predicted_tables: int
+    table_level_acc: float
+    table_detection_recall: float
+    table_detection_precision: float
+    table_detection_f1: float
+    element_col_level_index_acc: float
+    element_row_level_index_acc: float
+    element_col_level_content_acc: float
+    element_row_level_content_acc: float
+
+    @property
+    def composite_structure_acc(self) -> float:
+        return (
+            self.element_col_level_index_acc
+            + self.element_row_level_index_acc
+            + (self.element_col_level_content_acc + self.element_row_level_content_acc) / 2
+        ) / 3
+
+
+def table_level_acc(predicted_table_data, ground_truth_table_data, matched_indices):
+    """computes for each predicted table its accurary compared to ground truth.
+
+    The accuracy is defined as the SequenceMatcher.ratio() between those two strings. If a
+    prediction does not have a matched ground truth its accuracy is 0
+    """
+    score = np.zeros((len(matched_indices),))
+    ground_truth_text = TableAlignment.get_content_in_tables(ground_truth_table_data)
+    for idx, predicted in enumerate(predicted_table_data):
+        matched_idx = matched_indices[idx]
+        if matched_idx == -1:
+            # false positive; default score 0
+            continue
+        score[idx] = difflib.SequenceMatcher(
+            None,
+            TableAlignment.get_content_in_tables([predicted])[0],
+            ground_truth_text[matched_idx],
+        ).ratio()
+    return score
+
+
+def _count_predicted_tables(matched_indices: List[int]) -> int:
+    """Counts the number of predicted tables that have a corresponding match in the ground truth.
+
+    Args:
+      matched_indices: List of indices indicating matches between predicted
+        and ground truth tables.
+
+    Returns:
+      The count of matched predicted tables.
+
+    """
+    return sum(1 for idx in matched_indices if idx >= 0)
+
+
+def calculate_table_detection_metrics(
+    matched_indices: list[int], ground_truth_tables_number: int
+) -> tuple[float, float, float]:
+    """
+    Calculate the table detection metrics: recall, precision, and f1 score.
+    Args:
+        matched_indices:
+            List of indices indicating matches between predicted and ground truth tables
+            For example: matched_indices[i] = j means that the
+            i-th predicted table is matched with the j-th ground truth table.
+        ground_truth_tables_number: the number of ground truth tables.
+
+    Returns:
+        Tuple of recall, precision, and f1 scores
+    """
+    predicted_tables_number = len(matched_indices)
+
+    matched_set = set(matched_indices)
+    if -1 in matched_set:
+        matched_set.remove(-1)
+
+    true_positive = len(matched_set)
+    false_positive = predicted_tables_number - true_positive
+    positive = ground_truth_tables_number
+
+    recall = true_positive / positive if positive > 0 else 0
+    precision = (
+        true_positive / (true_positive + false_positive)
+        if true_positive + false_positive > 0
+        else 0
+    )
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0
+
+    return recall, precision, f1
+
+
+class TableEvalProcessor:
+    def __init__(
+        self,
+        prediction: List[Dict[str, Any]],
+        ground_truth: List[Dict[str, Any]],
+        cutoff: float = 0.8,
+        source_type: str = "html",
+    ):
+        """
+        Initializes the TableEvalProcessor prediction and ground truth.
+
+        Args:
+            ground_truth: Ground truth table data. The tables text should be in the deckerd format.
+            prediction: Predicted table data.
+            cutoff: The cutoff value for the element level alignment. Default is 0.8.
+
+        Examples:
+            ground_truth: [
+                {
+                    "type": "Table",
+                    "text": [
+                        {
+                            "id": "f4c35dae-105b-46f5-a77a-7fbc199d6aca",
+                            "x": 0,
+                            "y": 0,
+                            "w": 1,
+                            "h": 1,
+                            "content": "Cell text"
+                        },
+                        ...
+                }
+            ]
+            prediction: [
+                {
+                    "element_id": <id_string>,
+                    ...
+                    "metadata": {
+                        ...
+                        "text_as_html": "<table><thead><tr><th rowspan=\"2\">June....
+                                                                </tr></td></table>",
+                        "table_as_cells":
+                        [
+                            {
+                                "x": 0,
+                                "y": 0,
+                                "w": 1,
+                                "h": 2,
+                                "content": "June"
+                            },
+                            ...
+                        ]
+                    }
+                },
+            ]
+
+        """
+        self.prediction = prediction
+        self.ground_truth = ground_truth
+        self.cutoff = cutoff
+        self.source_type = source_type
+
+    @classmethod
+    def from_json_files(
+        cls,
+        prediction_file: Path,
+        ground_truth_file: Path,
+        cutoff: Optional[float] = None,
+        source_type: str = "html",
+    ) -> "TableEvalProcessor":
+        """Factory classmethod to initialize the object with path to json files instead of dicts
+
+        Args:
+          prediction_file: Path to the json file containing the predicted table data.
+          ground_truth_file: Path to the json file containing the ground truth table data.
+          source_type: 'cells' or 'html'. 'cells' refers to reading 'table_as_cells' field while
+            'html' is extracted from 'text_as_html'
+          cutoff: The cutoff value for the element level alignment.
+            If not set, class default value is used (=0.8).
+
+        Returns:
+          TableEvalProcessor: An instance of the class initialized with the provided data.
+        """
+        with open(prediction_file) as f:
+            prediction = json.load(f)
+        with open(ground_truth_file) as f:
+            ground_truth = json.load(f)
+        if cutoff is not None:
+            return cls(
+                prediction=prediction,
+                ground_truth=ground_truth,
+                cutoff=cutoff,
+                source_type=source_type,
+            )
+        else:
+            return cls(prediction=prediction, ground_truth=ground_truth, source_type=source_type)
+
+    def process_file(self) -> TableEvaluation:
+        """Processes the files and computes table-level and element-level accuracy.
+
+        Returns:
+            TableEvaluation: A dataclass object containing the computed metrics.
+        """
+        ground_truth_table_data = extract_and_convert_tables_from_ground_truth(
+            self.ground_truth,
+        )
+
+        predicted_table_data = extract_and_convert_tables_from_prediction(
+            file_elements=self.prediction, source_type=self.source_type
+        )
+        is_table_in_gt = bool(ground_truth_table_data)
+        is_table_predicted = bool(predicted_table_data)
+        if not is_table_in_gt:
+            # There is no table data in ground truth, you either got perfect score or 0
+            score = 0 if is_table_predicted else np.nan
+            table_acc = 1 if not is_table_predicted else 0
+            return TableEvaluation(
+                total_tables=0,
+                total_predicted_tables=len(predicted_table_data),
+                table_level_acc=table_acc,
+                table_detection_recall=score,
+                table_detection_precision=score,
+                table_detection_f1=score,
+                element_col_level_index_acc=score,
+                element_row_level_index_acc=score,
+                element_col_level_content_acc=score,
+                element_row_level_content_acc=score,
+            )
+        if is_table_in_gt and not is_table_predicted:
+            return TableEvaluation(
+                total_tables=len(ground_truth_table_data),
+                total_predicted_tables=0,
+                table_level_acc=0,
+                table_detection_recall=0,
+                table_detection_precision=0,
+                table_detection_f1=0,
+                element_col_level_index_acc=0,
+                element_row_level_index_acc=0,
+                element_col_level_content_acc=0,
+                element_row_level_content_acc=0,
+            )
+        else:
+            # We have both ground truth tables and predicted tables
+            matched_indices = TableAlignment.get_table_level_alignment(
+                predicted_table_data,
+                ground_truth_table_data,
+            )
+            predicted_table_acc = np.mean(
+                table_level_acc(predicted_table_data, ground_truth_table_data, matched_indices)
+            )
+
+            metrics = TableAlignment.get_element_level_alignment(
+                predicted_table_data,
+                ground_truth_table_data,
+                matched_indices,
+                cutoff=self.cutoff,
+            )
+
+            (
+                table_detection_recall,
+                table_detection_precision,
+                table_detection_f1,
+            ) = calculate_table_detection_metrics(
+                matched_indices=matched_indices,
+                ground_truth_tables_number=len(ground_truth_table_data),
+            )
+
+            evaluation = TableEvaluation(
+                total_tables=len(ground_truth_table_data),
+                total_predicted_tables=len(predicted_table_data),
+                table_level_acc=predicted_table_acc,
+                table_detection_recall=table_detection_recall,
+                table_detection_precision=table_detection_precision,
+                table_detection_f1=table_detection_f1,
+                element_col_level_index_acc=metrics.get("col_index_acc", 0),
+                element_row_level_index_acc=metrics.get("row_index_acc", 0),
+                element_col_level_content_acc=metrics.get("col_content_acc", 0),
+                element_row_level_content_acc=metrics.get("row_content_acc", 0),
+            )
+            return evaluation
+
+
+@click.command()
+@click.option(
+    "--prediction_file", help="Path to the model prediction JSON file", type=click.Path(exists=True)
+)
+@click.option(
+    "--ground_truth_file", help="Path to the ground truth JSON file", type=click.Path(exists=True)
+)
+@click.option(
+    "--cutoff",
+    type=float,
+    show_default=True,
+    default=0.8,
+    help="The cutoff value for the element level alignment. \
+        If not set, a default value is used",
+)
+def run(prediction_file: str, ground_truth_file: str, cutoff: Optional[float]):
+    """Runs the table evaluation process and prints the computed metrics."""
+    processor = TableEvalProcessor.from_json_files(
+        Path(prediction_file),
+        Path(ground_truth_file),
+        cutoff=cutoff,
+    )
+    report = processor.process_file()
+    print(report)
+
+
+if __name__ == "__main__":
+    run()

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/table/table_extraction.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from bs4 import BeautifulSoup
+from unstructured_inference.models.tables import cells_to_html
+
+EMPTY_CELL = {
+    "row_index": "",
+    "col_index": "",
+    "content": "",
+}
+
+
+def _move_cells_for_spanned_cells(cells: List[Dict[str, Any]]):
+    """Move cells to the right if spanned cells have an influence on the rendering.
+
+    Args:
+        cells: List of cells in the table in Deckerd format.
+
+    Returns:
+        List of cells in the table in Deckerd format with cells moved to the right if spanned.
+    """
+    sorted_cells = sorted(cells, key=lambda x: (x["y"], x["x"]))
+    cells_occupied_by_spanned = set()
+    for cell in sorted_cells:
+        if cell["w"] > 1 or cell["h"] > 1:
+            for i in range(cell["y"], cell["y"] + cell["h"]):
+                for j in range(cell["x"], cell["x"] + cell["w"]):
+                    if (i, j) != (cell["y"], cell["x"]):
+                        cells_occupied_by_spanned.add((i, j))
+        while (cell["y"], cell["x"]) in cells_occupied_by_spanned:
+            cell_y, cell_x = cell["y"], cell["x"]
+            cells_to_the_right = [c for c in sorted_cells if c["y"] == cell_y and c["x"] >= cell_x]
+            for cell_to_move in cells_to_the_right:
+                cell_to_move["x"] += 1
+            cells_occupied_by_spanned.remove((cell_y, cell_x))
+    return sorted_cells
+
+
+def html_table_to_deckerd(content: str) -> List[Dict[str, Any]]:
+    """Convert html format to Deckerd table structure.
+
+    Args:
+        content: The html content with a table to extract.
+
+    Returns:
+        A list of dictionaries where each dictionary represents a cell in the table.
+    """
+
+    soup = BeautifulSoup(content, "html.parser")
+    table = soup.find("table")
+    rows = table.find_all(["tr"])
+    table_data = []
+
+    for i, row in enumerate(rows):
+        cells = row.find_all(["th", "td"])
+        for j, cell_data in enumerate(cells):
+            cell = {
+                "y": i,
+                "x": j,
+                "w": int(cell_data.attrs.get("colspan", 1)),
+                "h": int(cell_data.attrs.get("rowspan", 1)),
+                "content": cell_data.text,
+            }
+            table_data.append(cell)
+    return _move_cells_for_spanned_cells(table_data)
+
+
+def deckerd_table_to_html(cells: List[Dict[str, Any]]) -> str:
+    """Convert Deckerd table structure to html format.
+
+    Args:
+        cells: List of dictionaries where each dictionary represents a cell in the table.
+
+    Returns:
+        A string with the html content of the table.
+    """
+    transformer_cells = []
+    # determine which cells are in header. Consider row 0 as header
+    # but spans may make it larger
+    first_row_cells = [cell for cell in cells if cell["y"] == 0]
+    header_length = max(cell["w"] for cell in first_row_cells)
+    header_rows = set(range(header_length))
+    for cell in cells:
+        cell_data = {
+            "row_nums": list(range(cell["y"], cell["y"] + cell["h"])),
+            "column_nums": list(range(cell["x"], cell["x"] + cell["w"])),
+            "w": cell["w"],
+            "h": cell["h"],
+            "cell text": cell["content"],
+            "column header": cell["y"] in header_rows,
+        }
+        transformer_cells.append(cell_data)
+    # reuse the existing function to convert to HTML
+    table = cells_to_html(transformer_cells)
+    return table
+
+
+def _convert_table_from_html(content: str) -> List[Dict[str, Any]]:
+    """Convert html format to table structure. As a middle step it converts
+    html to the Deckerd format as it's more convenient to work with.
+
+    Args:
+        content: The html content with a table to extract.
+
+    Returns:
+        A list of dictionaries where each dictionary represents a cell in the table.
+    """
+    deckerd_cells = html_table_to_deckerd(content)
+    return _convert_table_from_deckerd(deckerd_cells)
+
+
+def _convert_table_from_deckerd(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Convert deckerd format to table structure.
+
+    Args:
+      content: The deckerd formatted content with a table to extract.
+
+    Returns:
+      A list of dictionaries where each dictionary represents a cell in the table.
+    """
+    table_data = []
+    for table in content:
+        try:
+            cell_data = {
+                "row_index": table["y"],
+                "col_index": table["x"],
+                "content": table["content"],
+            }
+        except KeyError:
+            cell_data = EMPTY_CELL
+        except TypeError:
+            cell_data = EMPTY_CELL
+        table_data.append(cell_data)
+    return table_data
+
+
+def _sort_table_cells(table_data: List[List[Dict[str, Any]]]) -> List[List[Dict[str, Any]]]:
+    return sorted(table_data, key=lambda cell: (cell["row_index"], cell["col_index"]))
+
+
+def extract_and_convert_tables_from_ground_truth(
+    file_elements: List[Dict[str, Any]],
+) -> List[List[Dict[str, Any]]]:
+    """Extracts and converts tables data to a structured format based on the specified table type.
+
+    Args:
+        file_elements: List of elements from the ground truth file.
+
+    Returns:
+        A list of tables with each table represented as a list of cell data dictionaries.
+
+    """
+    ground_truth_table_data = []
+    for element in file_elements:
+        if "type" in element and element["type"] == "Table" and "text" in element:
+            try:
+                converted_data = _convert_table_from_deckerd(
+                    element["text"],
+                )
+                ground_truth_table_data.append(_sort_table_cells(converted_data))
+            except Exception as e:
+                print(f"Error converting ground truth data: {e}")
+                ground_truth_table_data.append({})
+
+    return ground_truth_table_data
+
+
+def extract_and_convert_tables_from_prediction(
+    file_elements: List[Dict[str, Any]], source_type: str = "html"
+) -> List[List[Dict[str, Any]]]:
+    """Extracts and converts table data to a structured format
+
+    Args:
+      file_elements: List of elements from the file.
+      source_type: 'cells' or 'html'. 'cells' refers to reading 'table_as_cells' field while
+        'html' is extracted from 'text_as_html'
+
+    Returns:
+      A list of tables with each table represented as a list of cell data dictionaries.
+
+    """
+    source_type_to_extraction_strategies = {
+        "html": extract_cells_from_text_as_html,
+        "cells": extract_cells_from_table_as_cells,
+    }
+    if source_type not in source_type_to_extraction_strategies:
+        raise ValueError(
+            f'source_type {source_type} is not valid. Allowed source_types are "html" and "cells"'
+        )
+
+    extract_cells_fn = source_type_to_extraction_strategies[source_type]
+    fallback_extract_cells_fn = (
+        extract_cells_from_table_as_cells
+        if source_type == "cells"
+        else extract_cells_from_text_as_html
+    )
+
+    predicted_table_data = []
+    for element in file_elements:
+        if element.get("type") == "Table":
+            extracted_cells = extract_cells_fn(element)
+            if not extracted_cells:
+                extracted_cells = fallback_extract_cells_fn(element)
+            if extracted_cells:
+                sorted_cells = _sort_table_cells(extracted_cells)
+                predicted_table_data.append(sorted_cells)
+
+    return predicted_table_data
+
+
+def extract_cells_from_text_as_html(element: Dict[str, Any]) -> List[Dict[str, Any]] | None:
+    """Extracts and parse cells from "text_as_html" field in Element structure
+
+    Args:
+        element: Example element:
+        {
+            "type": "Table",
+            "metadata": {
+                "text_as_html": "<table>
+                                    <thead>
+                                        <tr>
+                                            <th>Month A.</th>
+                                        </tr>
+                                    </thead>
+                                    </tbody>
+                                        <tr>
+                                            <td>22</td><
+                                        </tr>
+                                    </tbody>
+                                </table>"
+            }
+        }
+
+    Returns:
+        List of extracted cells in a format:
+        [
+            {
+                "row_index": 0,
+                "col_index": 0,
+                "content": "Month A.",
+            },
+            ...,
+        ]
+    """
+    val = element["metadata"].get("text_as_html")
+    if not val or "<table>" not in val:
+        return None
+
+    predicted_cells = None
+    try:
+        predicted_cells = _convert_table_from_html(val)
+    except Exception as e:
+        print(f"Error converting Unstructured table data: {e}")
+
+    return predicted_cells
+
+
+def extract_cells_from_table_as_cells(element: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extracts and parse cells from "table_as_cells" field in Element structure
+
+    Args:
+        element: Example element:
+        {
+            "type": "Table",
+            "metadata": {
+                "table_as_cells": [{"x": 0, "y": 0, "w": 1, "h": 1, "content": "Month A."},
+                                   {"x": 0, "y": 1, "w": 1, "h": 1, "content": "22"}]
+            }
+        }
+
+    Returns:
+        List of extracted cells in a format:
+        [
+            {
+                "row_index": 0,
+                "col_index": 0,
+                "content": "Month A.",
+            },
+            ...,
+        ]
+    """
+    predicted_cells = element["metadata"].get("table_as_cells")
+    converted_cells = None
+    if predicted_cells:
+        converted_cells = _convert_table_from_deckerd(predicted_cells)
+    return converted_cells

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/table/table_formats.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass
+from typing import Union
+
+
+@dataclass
+class SimpleTableCell:
+    x: int
+    y: int
+    w: int
+    h: int
+    content: str = ""
+
+    def to_dict(self):
+        return {
+            "x": self.x,
+            "y": self.y,
+            "w": self.w,
+            "h": self.h,
+            "content": self.content,
+        }
+
+    @classmethod
+    def from_table_transformer_cell(cls, tatr_table_cell: dict[str, Union[list[int], str]]):
+        """
+        Args:
+            tatr_table_cell (dict):
+                Cell in a format returned by Table Transformer model, for example:
+                    {
+                        "row_nums": [1,2,3],
+                        "column_nums": [2],
+                        "cell text": "Text inside cell"
+                    }
+        """
+
+        row_nums = tatr_table_cell.get("row_nums", [])
+        column_nums = tatr_table_cell.get("column_nums", [])
+
+        if not row_nums:
+            raise ValueError(f'Cell {tatr_table_cell} has missing values under "row_nums" key')
+        if not column_nums:
+            raise ValueError(f'Cell {tatr_table_cell} has missing values under "column_nums" key')
+
+        return cls(
+            x=min(column_nums),
+            y=min(row_nums),
+            w=len(column_nums),
+            h=len(row_nums),
+            content=tatr_table_cell.get("cell text", ""),
+        )

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/table_structure.py

@@ -0,0 +1,48 @@
+import numpy as np
+import pandas as pd
+from PIL import Image
+
+from unstructured.partition.pdf import convert_pdf_to_images
+from unstructured.partition.pdf_image.ocr import get_table_tokens
+from unstructured.partition.utils.ocr_models.ocr_interface import OCRAgent
+from unstructured.utils import requires_dependencies
+
+
+@requires_dependencies("unstructured_inference")
+def image_or_pdf_to_dataframe(filename: str) -> pd.DataFrame:
+    """helper to JUST run table transformer on the input image/pdf file. It assumes the input is
+    JUST a table. This is intended to facilitate metric tracking on table structure detection ALONE
+    without mixing metric of element detection model"""
+    from unstructured_inference.models.tables import load_agent, tables_agent
+
+    load_agent()
+
+    if filename.endswith(".pdf"):
+        image = list(convert_pdf_to_images(filename))[0].convert("RGB")
+    else:
+        image = Image.open(filename).convert("RGB")
+
+    ocr_agent = OCRAgent.get_agent(language="eng")
+
+    return tables_agent.run_prediction(
+        image, ocr_tokens=get_table_tokens(image, ocr_agent), result_format="dataframe"
+    )
+
+
+@requires_dependencies("unstructured_inference")
+def eval_table_transformer_for_file(
+    filename: str,
+    true_table_filename: str,
+    eval_func: str = "token_ratio",
+) -> float:
+    """evaluate the predicted table structure vs. actual table structure by column and row as a
+    number between 0 and 1"""
+    from unstructured_inference.models.eval import compare_contents_as_df
+
+    pred_table = image_or_pdf_to_dataframe(filename).fillna("").replace(np.nan, "")
+    actual_table = pd.read_csv(true_table_filename).astype(str).fillna("").replace(np.nan, "")
+
+    results = np.array(
+        list(compare_contents_as_df(actual_table, pred_table, eval_func=eval_func).values()),
+    )
+    return results.mean() / 100.0

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/text_extraction.py

@@ -0,0 +1,251 @@
+from typing import Dict, Optional, Tuple
+
+from rapidfuzz.distance import Levenshtein
+
+from unstructured.cleaners.core import clean_bullets, remove_sentence_punctuation
+
+
+def calculate_accuracy(
+    output: Optional[str],
+    source: Optional[str],
+    weights: Tuple[int, int, int] = (2, 1, 1),
+) -> float:
+    """
+    Calculates accuracy by calling calculate_edit_distance function using `return_as=score`.
+    The function will return complement of the edit distance instead.
+    """
+    return calculate_edit_distance(output, source, weights, return_as="score")
+
+
+def calculate_edit_distance(
+    output: Optional[str],
+    source: Optional[str],
+    weights: Tuple[int, int, int] = (2, 1, 1),
+    return_as: str = "distance",
+    standardize_whitespaces: bool = True,
+) -> float:
+    """
+    Calculates edit distance using Levenshtein distance between two strings.
+
+    Args:
+        output (str): The target string to be compared.
+        source (str): The reference string against which 'output' is compared.
+        weights (Tuple[int, int, int], optional): A tuple containing weights
+            for insertion, deletion, and substitution operations in the edit
+            distance calculation. Default is (2, 1, 1).
+        return_as (str, optional): The type of result to return, one of
+            ["score", "distance"].
+            Default is "distance".
+
+    Returns:
+        float: The calculated edit distance or similarity score between
+            the 'output' and 'source' strings.
+
+    Raises:
+        ValueError: If 'return_as' is not one of the valid return types
+        ["score", "distance"].
+
+    Note:
+        This function calculates the edit distance (or similarity score) between
+        two strings using the Levenshtein distance algorithm. The 'weights' parameter
+        allows customizing the cost of insertion, deletion, and substitution
+        operations. The 'return_as' parameter determines the type of result to return:
+        - "score": Returns the similarity score, where 1.0 indicates a perfect match.
+        - "distance": Returns the raw edit distance value.
+
+    """
+    return_types = ["score", "distance"]
+    if return_as not in return_types:
+        raise ValueError("Invalid return value type. Expected one of: %s" % return_types)
+    output = standardize_quotes(prepare_str(output, standardize_whitespaces))
+    source = standardize_quotes(prepare_str(source, standardize_whitespaces))
+    distance = Levenshtein.distance(output, source, weights=weights)  # type: ignore
+    # lower bounded the char length for source string at 1.0 because to avoid division by zero
+    # in the case where source string is empty, the distance should be at 100%
+    source_char_len = max(len(source), 1.0)  # type: ignore
+    bounded_percentage_distance = min(max(distance / source_char_len, 0.0), 1.0)
+    if return_as == "score":
+        return 1 - bounded_percentage_distance
+    elif return_as == "distance":
+        return distance
+    return 0.0
+
+
+def bag_of_words(text: str) -> Dict[str, int]:
+    """
+    Outputs the bag of words (BOW) found in the input text and their frequencies.
+
+    Takes "clean, concatenated text" (CCT) from a document as input.
+
+    Removes sentence punctuation, but not punctuation within a word (ex. apostrophes).
+    """
+    bow: Dict[str, int] = {}
+    incorrect_word: str = ""
+    words = clean_bullets(remove_sentence_punctuation(text.lower(), ["-", "'"])).split()
+
+    i = 0
+    while i < len(words):
+        if len(words[i]) > 1:
+            if words[i] in bow:
+                bow[words[i]] += 1
+            else:
+                bow[words[i]] = 1
+            i += 1
+        else:
+            j = i
+            incorrect_word = ""
+
+            while j < len(words) and len(words[j]) == 1:
+                incorrect_word += words[j]
+                j += 1
+
+            if len(incorrect_word) == 1 and words[i].isalnum():
+                if incorrect_word in bow:
+                    bow[incorrect_word] += 1
+                else:
+                    bow[incorrect_word] = 1
+            i = j
+    return bow
+
+
+def calculate_percent_missing_text(
+    output: Optional[str],
+    source: Optional[str],
+) -> float:
+    """
+    Creates the bag of words (BOW) found in each input text and their frequencies, then compares the
+    output BOW against the source BOW to calculate the % of text from the source text missing from
+    the output text.
+
+    Takes "clean, concatenated text" (CCT) from a document output and the ground truth source text
+    as inputs.
+
+    If the output text contains all words from the source text and then some extra, result will be
+    0% missing text - this calculation does not penalize duplication.
+
+    A spaced-out word (ex. h e l l o) is considered missing; individual characters of a word
+    will not be counted as separate words.
+
+    Returns the percentage of missing text represented as a decimal between 0 and 1.
+    """
+    output = prepare_str(output)
+    source = prepare_str(source)
+    output_bow = bag_of_words(output)
+    source_bow = bag_of_words(source)
+
+    # get total words in source bow while counting missing words
+    total_source_word_count = 0
+    total_missing_word_count = 0
+
+    for source_word, source_count in source_bow.items():
+        total_source_word_count += source_count
+        if source_word not in output_bow:
+            # entire count is missing
+            total_missing_word_count += source_count
+        else:
+            output_count = output_bow[source_word]
+            total_missing_word_count += max(source_count - output_count, 0)
+
+    # calculate percent missing text
+    if total_source_word_count == 0:
+        return 0  # nothing missing because nothing in source document
+
+    fraction_missing = round(total_missing_word_count / total_source_word_count, 3)
+    return min(fraction_missing, 1)  # limit to 100%
+
+
+def prepare_str(string: Optional[str], standardize_whitespaces: bool = False) -> str:
+    if not string:
+        return ""
+    if standardize_whitespaces:
+        return " ".join(string.split())
+    return str(string)  # type: ignore
+
+
+def standardize_quotes(text: str) -> str:
+    """
+    Converts all unicode quotes to standard ASCII quotes with comprehensive coverage.
+
+    Args:
+        text (str): The input text to be standardized.
+
+    Returns:
+        str: The text with standardized quotes.
+    """
+    # Double Quotes Dictionary
+    double_quotes = {
+        '"': "U+0022",  # noqa 601 # Standard typewriter/programmer's quote
+        '"': "U+201C",  # noqa 601 # Left double quotation mark
+        '"': "U+201D",  # noqa 601 # Right double quotation mark
+        "„": "U+201E",  # Double low-9 quotation mark
+        "‟": "U+201F",  # Double high-reversed-9 quotation mark
+        "«": "U+00AB",  # Left-pointing double angle quotation mark
+        "»": "U+00BB",  # Right-pointing double angle quotation mark
+        "❝": "U+275D",  # Heavy double turned comma quotation mark ornament
+        "❞": "U+275E",  # Heavy double comma quotation mark ornament
+        "⹂": "U+2E42",  # Double low-reversed-9 quotation mark
+        "🙶": "U+1F676",  # SANS-SERIF HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT
+        "🙷": "U+1F677",  # SANS-SERIF HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT
+        "🙸": "U+1F678",  # SANS-SERIF HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT
+        "⠦": "U+2826",  # Braille double closing quotation mark
+        "⠴": "U+2834",  # Braille double opening quotation mark
+        "〝": "U+301D",  # REVERSED DOUBLE PRIME QUOTATION MARK
+        "〞": "U+301E",  # DOUBLE PRIME QUOTATION MARK
+        "〟": "U+301F",  # LOW DOUBLE PRIME QUOTATION MARK
+        "＂": "U+FF02",  # FULLWIDTH QUOTATION MARK
+        ",,": "U+275E",  # LOW HEAVY DOUBLE COMMA ORNAMENT
+    }
+
+    # Single Quotes Dictionary
+    single_quotes = {
+        "'": "U+0027",  # noqa 601 # Standard typewriter/programmer's quote
+        "'": "U+2018",  # noqa 601 # Left single quotation mark
+        "'": "U+2019",  # noqa 601 # Right single quotation mark # noqa: W605
+        "‚": "U+201A",  # Single low-9 quotation mark
+        "‛": "U+201B",  # Single high-reversed-9 quotation mark
+        "‹": "U+2039",  # Single left-pointing angle quotation mark
+        "›": "U+203A",  # Single right-pointing angle quotation mark
+        "❛": "U+275B",  # Heavy single turned comma quotation mark ornament
+        "❜": "U+275C",  # Heavy single comma quotation mark ornament
+        "「": "U+300C",  # Left corner bracket
+        "」": "U+300D",  # Right corner bracket
+        "『": "U+300E",  # Left white corner bracket
+        "』": "U+300F",  # Right white corner bracket
+        "﹁": "U+FE41",  # PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET
+        "﹂": "U+FE42",  # PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET
+        "﹃": "U+FE43",  # PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET
+        "﹄": "U+FE44",  # PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET
+        "＇": "U+FF07",  # FULLWIDTH APOSTROPHE
+        "｢": "U+FF62",  # HALFWIDTH LEFT CORNER BRACKET
+        "｣": "U+FF63",  # HALFWIDTH RIGHT CORNER BRACKET
+    }
+
+    double_quote_standard = '"'
+    single_quote_standard = "'"
+
+    # Apply double quote replacements
+    for unicode_val in double_quotes.values():
+        unicode_char = unicode_to_char(unicode_val)
+        if unicode_char in text:
+            text = text.replace(unicode_char, double_quote_standard)
+
+    # Apply single quote replacements
+    for unicode_val in single_quotes.values():
+        unicode_char = unicode_to_char(unicode_val)
+        if unicode_char in text:
+            text = text.replace(unicode_char, single_quote_standard)
+
+    return text
+
+
+def unicode_to_char(unicode_val: str) -> str:
+    """
+    Converts a Unicode value to a character.
+
+    Args:
+        unicode_val (str): The Unicode value to convert.
+
+    Returns:
+        str: The character corresponding to the Unicode value.
+    """
+    return chr(int(unicode_val.replace("U+", ""), 16))

backend_service/venv/lib/python3.13/site-packages/unstructured/metrics/utils.py

@@ -0,0 +1,246 @@
+import logging
+import os
+import re
+import statistics
+from pathlib import Path
+from typing import List, Optional, Union
+
+import click
+import pandas as pd
+
+from unstructured.staging.base import elements_from_json, elements_to_text
+
+logger = logging.getLogger("unstructured.eval")
+
+
+def _prepare_output_cct(docpath: str, output_type: str) -> str:
+    """
+    Convert given input document (path) into cct-ready. The function only support conversion
+    from `json` or `txt` file.
+    """
+    try:
+        if output_type == "json":
+            output_cct = elements_to_text(elements_from_json(docpath))
+        elif output_type == "txt":
+            output_cct = _read_text_file(docpath)
+        else:
+            raise ValueError(
+                f"File type not supported. Expects one of `json` or `txt`, \
+                    but received {output_type} instead."
+            )
+    except ValueError as e:
+        logger.error(f"Could not read the file {docpath}")
+        raise e
+    return output_cct
+
+
+def _listdir_recursive(dir: str) -> List[str]:
+    """
+    Recursively lists all files in the given directory and its subdirectories.
+    Returns a list of all files found, with each file's path relative to the
+    initial directory.
+    """
+    listdir = []
+    for dirpath, _, filenames in os.walk(dir):
+        for filename in filenames:
+            # Remove the starting directory from the path to show the relative path
+            relative_path = os.path.relpath(dirpath, dir)
+            if relative_path == ".":
+                listdir.append(filename)
+            else:
+                listdir.append(os.path.join(relative_path, filename))
+    return listdir
+
+
+def _rename_aggregated_columns(df):
+    """
+    Renames aggregated columns in a DataFrame based on a predefined mapping.
+
+    Parameters:
+    df (pandas.DataFrame): The DataFrame with aggregated columns to rename.
+
+    Returns:
+    pandas.DataFrame: A new DataFrame with renamed aggregated columns.
+    """
+    rename_map = {"_mean": "mean", "_stdev": "stdev", "_pstdev": "pstdev", "_count": "count"}
+    return df.rename(columns=rename_map)
+
+
+def _format_grouping_output(*df):
+    """
+    Concatenates multiple pandas DataFrame objects along the columns (side-by-side)
+    and resets the index.
+    """
+    return pd.concat(df, axis=1).reset_index()
+
+
+def _display(df):
+    """
+    Displays the evaluation metrics in a formatted text table.
+    """
+    if len(df) == 0:
+        return
+    headers = df.columns.tolist()
+    col_widths = [
+        max(len(header), max(len(str(item)) for item in df[header])) for header in headers
+    ]
+    click.echo(" ".join(header.ljust(col_widths[i]) for i, header in enumerate(headers)))
+    click.echo("-" * sum(col_widths) + "-" * (len(headers) - 1))
+    for _, row in df.iterrows():
+        formatted_row = []
+        for item in row:
+            if isinstance(item, float):
+                formatted_row.append(f"{item:.3f}")
+            else:
+                formatted_row.append(str(item))
+        click.echo(
+            " ".join(formatted_row[i].ljust(col_widths[i]) for i in range(len(formatted_row))),
+        )
+
+
+def _write_to_file(
+    directory: str, filename: str, df: pd.DataFrame, mode: str = "w", overwrite: bool = True
+):
+    """
+    Save the metrics report to tsv file. The function allows an option 1) to choose `mode`
+    as `w` (write) or `a` (append) and 2) to `overwrite` the file if filename existed or not.
+    """
+    if mode not in ["w", "a"]:
+        raise ValueError("Mode not supported. Mode must be one of [w, a].")
+    if directory:
+        Path(directory).mkdir(exist_ok=True)
+    if "count" in df.columns:
+        df["count"] = df["count"].astype(int)
+    if "filename" in df.columns and "connector" in df.columns:
+        df.sort_values(by=["connector", "filename"], inplace=True)
+    if not overwrite:
+        filename = _get_non_duplicated_filename(directory, filename)
+    df.to_csv(
+        os.path.join(directory, filename), sep="\t", mode=mode, index=False, header=(mode == "w")
+    )
+
+
+def _sorting_key(filename):
+    """
+    A function that defines the sorting method for duplicated file names. For example,
+    with filename.ext filename (1).ext filename (2).ext filename (10).ext - this function
+    extracts the integer in the bracket and sort those numbers ascendingly.
+    """
+    # Regular expression to find the number in the filename
+    numbers = re.findall(r"(\d+)", filename)
+    if numbers:
+        # If there's a number, return it as an integer for sorting
+        return int(numbers[-1])
+    else:
+        # If no number, return 0 so these files come first
+        return 0
+
+
+def _uniquity_file(file_list, target_filename) -> str:
+    """
+    Checks the duplicity of the file name from the list and run the numerical check
+    of the minimum number needed as extension to not overwrite the exising file.
+    Returns a string of file name in the format of `filename (<min number>).ext`.
+    """
+    original_filename, extension = target_filename.rsplit(".", 1)
+    pattern = rf"^{re.escape(original_filename)}(?: \((\d+)\))?\.{re.escape(extension)}$"
+    duplicated_files = sorted([f for f in file_list if re.match(pattern, f)], key=_sorting_key)
+
+    numbers = []
+    for file in duplicated_files:
+        match = re.search(r"\((\d+)\)", file)
+        if match:
+            numbers.append(int(match.group(1)))
+
+    numbers.sort()
+
+    counter = 1
+    for number in numbers:
+        if number == counter:
+            counter += 1
+        else:
+            break
+
+    return original_filename + " (" + str(counter) + ")." + extension
+
+
+def _get_non_duplicated_filename(dir, filename) -> str:
+    """
+    Helper function to calls the `_uniquity_file` function. Takes in directory and file name
+    to check on.
+    """
+    filename = _uniquity_file(os.listdir(dir), filename)
+    return filename
+
+
+def _mean(scores: Union[pd.Series, List[float]], rounding: Optional[int] = 3) -> Union[float, None]:
+    """
+    Find mean from the list. Returns None if no element in the list.
+
+    Args:
+        rounding (int): optional argument that allows user to define decimal points. Default at 3.
+    """
+    if len(scores) == 0:
+        return None
+    mean = statistics.mean(scores)
+    if not rounding:
+        return mean
+    return round(mean, rounding)
+
+
+def _stdev(scores: List[Optional[float]], rounding: Optional[int] = 3) -> Union[float, None]:
+    """
+    Find standard deviation from the list.
+    Returns None if only 0 or 1 element in the list.
+
+    Args:
+        rounding (int): optional argument that allows user to define decimal points. Default at 3.
+    """
+    # Filter out None values
+    scores = [score for score in scores if score is not None]
+    # Proceed only if there are more than one value
+    if len(scores) <= 1:
+        return None
+    if not rounding:
+        return statistics.stdev(scores)
+    return round(statistics.stdev(scores), rounding)
+
+
+def _pstdev(scores: List[Optional[float]], rounding: Optional[int] = 3) -> Union[float, None]:
+    """
+    Find population standard deviation from the list.
+    Returns None if only 0 or 1 element in the list.
+
+    Args:
+        rounding (int): optional argument that allows user to define decimal points. Default at 3.
+    """
+    scores = [score for score in scores if score is not None]
+    if len(scores) <= 1:
+        return None
+    if not rounding:
+        return statistics.pstdev(scores)
+    return round(statistics.pstdev(scores), rounding)
+
+
+def _count(scores: List[Optional[float]]) -> float:
+    """
+    Returns the row count of the list.
+    """
+    return len(scores)
+
+
+def _read_text_file(path):
+    """
+    Reads the contents of a text file and returns it as a string.
+    """
+    # Check if the file exists
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"The file at {path} does not exist.")
+
+    try:
+        with open(path, errors="ignore") as f:
+            text = f.read()
+        return text
+    except OSError as e:
+        # Handle other I/O related errors
+        raise IOError(f"An error occurred when reading the file at {path}: {e}")