Source code for torchocr.transforms.v2

"""Polygon-aware augmentation on top of :mod:`torchvision.transforms.v2`.

torchvision's v2 transforms move :class:`~torchvision.tv_tensors.KeyPoints`
of any leading shape, so ``(N, P, 2)`` text polygons follow flips,
rotations, affines, resizes and crops once wrapped. :class:`TextDetectionTarget`
is a pytree node, so its ``texts`` and ``ignore`` ride along untouched.
This module adds what is specific to text:

- :func:`wrap_sample` wraps a dataset sample as tv_tensors;
- :class:`RandomTextCrop` crops without cutting through any text region
  (PaddleOCR's ``EastRandomCropData``);
- :class:`SanitizeTextPolygons` drops regions moved off the canvas and
  marks partly visible ones don't-care, keeping ``texts`` and ``ignore``
  aligned.

Example::

    from torchvision.transforms import v2
    from torchocr.transforms.v2 import RandomTextCrop, SanitizeTextPolygons, wrap_sample

    augment = v2.Compose([
        v2.RandomRotation(10), v2.RandomAffine(0, scale=(0.5, 3.0)),
        RandomTextCrop(), v2.Resize(size=None, max_size=640), SanitizeTextPolygons(),
    ])
    image, target = augment(*wrap_sample(*dataset[0]))
    db_targets = MakeDBTargets()(target.polygons, target.ignore, image.shape[-2:])

Pass images as ``tv_tensors.Image`` (what :func:`wrap_sample` does): a
plain tensor would make torchvision treat the first plain tensor it meets
as the image, which could be the ``ignore`` mask.
"""

from typing import Any

import torch
from torch import Tensor
from torch.utils._pytree import tree_flatten, tree_unflatten
from torchvision import tv_tensors
from torchvision.transforms import v2
from torchvision.transforms.v2 import functional as F
from torchvision.transforms.v2._utils import query_size

from ..core.structures import TextDetectionTarget

__all__ = ["RandomTextCrop", "SanitizeTextPolygons", "wrap_sample"]


[docs] def wrap_sample(image: Tensor, target: TextDetectionTarget) -> tuple[tv_tensors.Image, TextDetectionTarget]: """Wrap a ``(C, H, W)`` image and its target as tv_tensors for :mod:`torchvision.transforms.v2`.""" if image.ndim != 3: raise ValueError(f"Expected a (C, H, W) image; got {tuple(image.shape)}.") polygons = tv_tensors.KeyPoints(target.polygons.float(), canvas_size=tuple(image.shape[-2:])) return tv_tensors.Image(image), TextDetectionTarget(polygons, list(target.texts), target.ignore)
[docs] class RandomTextCrop(v2.Transform): """Random crop whose borders never cut through a text polygon. Crop edges are drawn from rows and columns that no polygon's axis-aligned extent covers, so every region ends up fully inside or fully outside the crop on each axis, and the crop holds at least one region. After ``max_tries`` misses (e.g. text everywhere) the image is returned whole. Every polygon in the sample counts, don't-care ones included. Args: min_crop_side_ratio: Minimum crop side, as a fraction of the image side. max_tries: Attempts before giving up and keeping the whole image. """ def __init__(self, min_crop_side_ratio: float = 0.1, max_tries: int = 50) -> None: super().__init__() if not 0 < min_crop_side_ratio <= 1: raise ValueError(f"min_crop_side_ratio must be in (0, 1]; got {min_crop_side_ratio}.") self.min_crop_side_ratio = min_crop_side_ratio self.max_tries = max_tries
[docs] def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: height, width = query_size(flat_inputs) whole = {"top": 0, "left": 0, "height": height, "width": width} polys = [_as_polygons(i) for i in flat_inputs if isinstance(i, tv_tensors.KeyPoints)] polys = torch.cat(polys) if polys else torch.zeros(0, 1, 2) rows = torch.zeros(height, dtype=torch.bool) cols = torch.zeros(width, dtype=torch.bool) for poly in polys.round().long(): x0, x1 = poly[:, 0].min().clamp(0, width), poly[:, 0].max().clamp(0, width) y0, y1 = poly[:, 1].min().clamp(0, height), poly[:, 1].max().clamp(0, height) cols[x0:x1] = True rows[y0:y1] = True free_rows, free_cols = (~rows).nonzero().flatten(), (~cols).nonzero().flatten() if len(polys) == 0 or len(free_rows) == 0 or len(free_cols) == 0: return whole for _ in range(self.max_tries): x0, x1 = _pick_span(free_cols) y0, y1 = _pick_span(free_rows) if x1 - x0 < self.min_crop_side_ratio * width or y1 - y0 < self.min_crop_side_ratio * height: continue if _overlaps(polys, x0, y0, x1, y1).any(): return {"top": y0, "left": x0, "height": y1 - y0, "width": x1 - x0} return whole
[docs] def transform(self, inpt: Any, params: dict[str, Any]) -> Any: return self._call_kernel(F.crop, inpt, **params)
[docs] class SanitizeTextPolygons(v2.Transform): """Drop text regions outside the canvas; mark partly visible ones don't-care. Run it last, after the geometric transforms. A region is dropped when its axis-aligned extent misses the canvas (or when it has fewer than ``min_size`` pixels of extent along either axis); a region that sticks out of the canvas has a clipped word whose transcription no longer matches the pixels, so its ``ignore`` flag is set (with ``partial="ignore"``) or it is dropped (``partial="drop"``). Vertices are not clamped: :class:`~torchocr.transforms.MakeDBTargets` rasterizes polygons that overhang the canvas correctly. Args: partial: ``"ignore"`` (default), ``"drop"`` or ``"keep"`` for regions partly outside the canvas. min_size: Minimum width and height, in pixels, of a kept region. """ def __init__(self, partial: str = "ignore", min_size: float = 1.0) -> None: super().__init__() if partial not in ("ignore", "drop", "keep"): raise ValueError(f'partial must be "ignore", "drop" or "keep"; got {partial!r}.') self.partial = partial self.min_size = min_size
[docs] def forward(self, *inputs: Any) -> Any: # Like v2.SanitizeBoundingBoxes: filtering changes N, so it works on whole # targets rather than leaf by leaf. flat, spec = tree_flatten(inputs if len(inputs) > 1 else inputs[0], is_leaf=lambda x: isinstance(x, TextDetectionTarget)) return tree_unflatten([self._sanitize(x) if isinstance(x, TextDetectionTarget) else x for x in flat], spec)
def _sanitize(self, target: TextDetectionTarget) -> TextDetectionTarget: polygons = target.polygons if not isinstance(polygons, tv_tensors.KeyPoints): raise TypeError("SanitizeTextPolygons needs tv_tensors.KeyPoints polygons; wrap samples with wrap_sample().") if len(polygons) == 0: return target height, width = polygons.canvas_size x, y = polygons[..., 0].as_subclass(Tensor), polygons[..., 1].as_subclass(Tensor) x0, x1, y0, y1 = x.min(1).values, x.max(1).values, y.min(1).values, y.max(1).values visible = (x1 > 0) & (x0 < width) & (y1 > 0) & (y0 < height) big_enough = (x1.clamp(0, width) - x0.clamp(0, width) >= self.min_size) & ( y1.clamp(0, height) - y0.clamp(0, height) >= self.min_size ) inside = (x0 >= 0) & (x1 <= width) & (y0 >= 0) & (y1 <= height) keep = visible & big_enough ignore = target.ignore.clone() if self.partial == "drop": keep &= inside elif self.partial == "ignore": ignore |= ~inside texts = [text for text, k in zip(target.texts, keep.tolist()) if k] kept = tv_tensors.wrap(polygons.as_subclass(Tensor)[keep], like=polygons) # indexing unwraps tv_tensors return TextDetectionTarget(kept, texts, ignore[keep])
def _as_polygons(points: tv_tensors.KeyPoints) -> Tensor: """``(N, P, 2)`` view of keypoints of any leading shape (a lone point is a 1-vertex polygon).""" points = points.as_subclass(Tensor) return points.reshape(-1, 1, 2) if points.ndim == 2 else points.reshape(-1, points.shape[-2], 2) def _pick_span(free: Tensor) -> tuple[int, int]: """Two positions among sorted free indices, from two (possibly equal) runs so a crop can span text.""" runs = torch.tensor_split(free, ((free.diff() != 1).nonzero().flatten() + 1).tolist()) if len(runs) > 1: picks = [run[torch.randint(len(run), ())] for run in (runs[i] for i in torch.randint(len(runs), (2,)))] else: picks = list(free[torch.randint(len(free), (2,))]) return int(min(picks)), int(max(picks)) def _overlaps(polys: Tensor, x0: int, y0: int, x1: int, y1: int) -> Tensor: """``(N,)`` mask of polygons whose extent overlaps the inside of the ``[x0, x1] x [y0, y1]`` rectangle.""" x, y = polys[..., 0], polys[..., 1] return (x.max(1).values > x0) & (x.min(1).values < x1) & (y.max(1).values > y0) & (y.min(1).values < y1)