Source code for torchocr.metrics.detection

"""Text-detection metrics."""

from collections.abc import Sequence
from dataclasses import dataclass

import numpy as np
import shapely
from torch import Tensor

from .recognition import normalize_text


[docs] @dataclass(frozen=True) class HmeanResult: """Dataset-level detection scores under the ICDAR IoU protocol. ``num_gt`` and ``num_pred`` exclude don't-care regions and invalid polygons; they are the denominators of ``recall`` and ``precision``. """ precision: float recall: float hmean: float num_matched: int num_gt: int num_pred: int
[docs] class DetectionHmean: """ICDAR-2015 text-detection precision / recall / hmean. Reproduces the official RRC "IoU" evaluation (the protocol behind every published ICDAR-2015 detection number, and PaddleOCR's ``DetectionIoUEvaluator``): 1. Polygons that are not valid (e.g. self-intersecting) are dropped. 2. A prediction is *don't care* if more than ``ignore_threshold`` of its own area lies inside a single ignored ground-truth region (``###`` transcriptions in ICDAR files). 3. Remaining pairs are matched greedily and one-to-one in (ground truth, prediction) order whenever IoU > ``iou_threshold``. 4. Matches and care counts are summed over all images before the ratios are taken, so large images are not averaged away. Usage follows the stateful-metric idiom:: metric = DetectionHmean() for preds, target in loader: metric.update(preds, target.polygons, target.ignore) result = metric.compute() # HmeanResult Args: iou_threshold: Minimum IoU (exclusive) for a match. Default 0.5. ignore_threshold: Fraction (exclusive) of a prediction's area that must overlap an ignored region to discard it. Default 0.5. """ def __init__(self, iou_threshold: float = 0.5, ignore_threshold: float = 0.5) -> None: if not 0.0 <= iou_threshold <= 1.0: raise ValueError(f"iou_threshold must lie in [0, 1]; got {iou_threshold}.") if not 0.0 <= ignore_threshold <= 1.0: raise ValueError(f"ignore_threshold must lie in [0, 1]; got {ignore_threshold}.") self.iou_threshold = iou_threshold self.ignore_threshold = ignore_threshold self.reset() def reset(self) -> None: self._matched = 0 self._gt_care = 0 self._pred_care = 0
[docs] def update(self, preds: Tensor, targets: Tensor, ignore: Tensor | None = None) -> None: """Accumulate one image. Args: preds: Predicted regions, either ``(N, P, 2)`` polygons with ``P >= 3`` vertices or ``(N, 4)`` ``xyxy`` boxes. targets: Ground-truth regions in the same formats. ignore: Optional ``(M,)`` bool mask over ``targets`` marking don't-care regions. Default: nothing ignored. """ pairs, gt_care, pred_care = self._match(preds, targets, ignore) self._matched += len(pairs) self._gt_care += gt_care self._pred_care += pred_care
def _match( self, preds: Tensor, targets: Tensor, ignore: Tensor | None ) -> tuple[list[tuple[int, int]], int, int]: """Return matched ``(target_index, pred_index)`` pairs and the care counts.""" pred_polys = _to_polygons(preds, "preds") gt_polys = _to_polygons(targets, "targets") if ignore is None: gt_ignore = np.zeros(len(gt_polys), dtype=bool) else: gt_ignore = ignore.detach().cpu().numpy().astype(bool).reshape(-1) if gt_ignore.shape[0] != len(gt_polys): raise ValueError( f"ignore has {gt_ignore.shape[0]} entries but targets has {len(gt_polys)} regions." ) gt_index = np.flatnonzero(shapely.is_valid(gt_polys)) pred_index = np.flatnonzero(shapely.is_valid(pred_polys)) gt_polys, gt_ignore = gt_polys[gt_index], gt_ignore[gt_index] pred_polys = pred_polys[pred_index] pred_ignore = np.zeros(len(pred_polys), dtype=bool) if gt_ignore.any() and len(pred_polys): dont_care = gt_polys[gt_ignore] overlap = shapely.area(shapely.intersection(pred_polys[:, None], dont_care[None, :])) pred_area = shapely.area(pred_polys)[:, None] with np.errstate(divide="ignore", invalid="ignore"): fraction = np.where(pred_area > 0, overlap / pred_area, 0.0) pred_ignore = (fraction > self.ignore_threshold).any(axis=1) pairs: list[tuple[int, int]] = [] if len(gt_polys) and len(pred_polys): inter = shapely.area(shapely.intersection(gt_polys[:, None], pred_polys[None, :])) union = shapely.area(shapely.union(gt_polys[:, None], pred_polys[None, :])) with np.errstate(divide="ignore", invalid="ignore"): iou = np.where(union > 0, inter / union, 0.0) # Greedy in index order -- intentionally not Hungarian, to stay # bit-compatible with the official script. pred_taken = pred_ignore.copy() for g in np.flatnonzero(~gt_ignore): for p in np.flatnonzero(~pred_taken & (iou[g] > self.iou_threshold)): pred_taken[p] = True pairs.append((int(gt_index[g]), int(pred_index[p]))) break return pairs, int((~gt_ignore).sum()), int((~pred_ignore).sum()) def compute(self) -> HmeanResult: recall = self._matched / self._gt_care if self._gt_care else 0.0 precision = self._matched / self._pred_care if self._pred_care else 0.0 hmean = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 return HmeanResult( precision=precision, recall=recall, hmean=hmean, num_matched=self._matched, num_gt=self._gt_care, num_pred=self._pred_care, )
def _to_polygons(regions: Tensor, name: str) -> np.ndarray: """Convert ``(N, P, 2)`` polygons or ``(N, 4)`` xyxy boxes to shapely polygons.""" coords = regions.detach().cpu().double().numpy() if coords.ndim == 2 and coords.shape[1] == 4: x1, y1, x2, y2 = coords.T coords = np.stack([np.stack([x1, y1], -1), np.stack([x2, y1], -1), np.stack([x2, y2], -1), np.stack([x1, y2], -1)], axis=1) if coords.ndim != 3 or coords.shape[2] != 2 or coords.shape[1] < 3: raise ValueError( f"{name} must be (N, P, 2) polygons with P >= 3 or (N, 4) xyxy boxes; " f"got {tuple(regions.shape)}." ) if coords.shape[0] == 0: return np.empty(0, dtype=object) return shapely.polygons(coords)
[docs] class EndToEndHmean(DetectionHmean): """End-to-end text spotting hmean: a detection counts only if it is also read correctly. Regions are matched exactly as in :class:`DetectionHmean`; a matched pair is a true positive only when the normalized transcriptions are equal. This follows the ICDAR-2015 end-to-end task with a generic vocabulary (no lexicon), simplified: text equality uses :class:`~torchocr.metrics.RecognitionAccuracy`'s normalization instead of the official script's special-character rules. Args: iou_threshold, ignore_threshold: As in :class:`DetectionHmean`. case_sensitive, alphanumeric_only: Text normalization before comparison. Defaults follow the scene-text convention (case-insensitive, letters and digits only). """ def __init__( self, iou_threshold: float = 0.5, ignore_threshold: float = 0.5, case_sensitive: bool = False, alphanumeric_only: bool = True, ) -> None: super().__init__(iou_threshold, ignore_threshold) self.case_sensitive = case_sensitive self.alphanumeric_only = alphanumeric_only
[docs] def update( # type: ignore[override] self, preds: Tensor, targets: Tensor, ignore: Tensor | None, pred_texts: Sequence[str], target_texts: Sequence[str], ) -> None: """Accumulate one image; ``pred_texts`` / ``target_texts`` align with the regions.""" if len(pred_texts) != preds.shape[0]: raise ValueError(f"pred_texts has {len(pred_texts)} entries for {preds.shape[0]} predictions.") if len(target_texts) != targets.shape[0]: raise ValueError(f"target_texts has {len(target_texts)} entries for {targets.shape[0]} targets.") pairs, gt_care, pred_care = self._match(preds, targets, ignore) norm = lambda t: normalize_text(t, self.case_sensitive, self.alphanumeric_only) # noqa: E731 self._matched += sum(norm(pred_texts[p]) == norm(target_texts[g]) for g, p in pairs) self._gt_care += gt_care self._pred_care += pred_care