Source code for torchocr.ops.crop

"""Perspective rectification of text quadrilaterals: the OCR analog of ``roi_align``."""

import torch
from torch import Tensor
from torch.nn import functional as F


[docs] def crop_quads( images: Tensor, quads: Tensor, batch_idx: Tensor, height: int, max_width: int, fill: float = 0.0, vertical_ratio: float = 1.5, ) -> tuple[Tensor, Tensor]: """Rectify text quadrilaterals into fixed-height, left-aligned crops. Each quad is mapped onto a ``height x w_k`` rectangle by the homography sending its corners (in reading order: top-left, top-right, bottom-right, bottom-left) to the rectangle's corners, and sampled bilinearly with :func:`torch.nn.functional.grid_sample`. ``w_k`` keeps the quad's aspect ratio (clamped to ``max_width``); columns past ``w_k`` are set to ``fill``. Quads whose height is at least ``vertical_ratio`` times their width are treated as vertical text and rotated 90 degrees counter-clockwise, as PaddleOCR does. Everything runs on the input device, is batched across all quads of a page, and is differentiable with respect to ``images``. Because per-pixel normalization commutes with bilinear sampling, normalizing the whole page once and then cropping equals cropping then normalizing each crop. Args: images: ``(B, C, H, W)`` float pages. quads: ``(K, 4, 2)`` vertices in continuous pixel coordinates (pixel ``j`` spans ``[j, j + 1)``), in reading order -- use :func:`order_quad_vertices` for detector output. batch_idx: ``(K,)`` page index of each quad. height: Output height (e.g. 32 for CRNN). max_width: Output width; every crop is padded to it. fill: Value of the padding columns. vertical_ratio: Height / width ratio from which a quad is rotated. Returns: ``(crops, widths)``: ``(K, C, height, max_width)`` crops and the ``(K,)`` number of valid (non-padding) columns of each. """ if images.ndim != 4: raise ValueError(f"Expected images of shape (B, C, H, W); got {tuple(images.shape)}.") if quads.ndim != 3 or quads.shape[1:] != (4, 2): raise ValueError(f"Expected quads of shape (K, 4, 2); got {tuple(quads.shape)}.") if batch_idx.shape != (quads.shape[0],): raise ValueError( f"batch_idx must have shape ({quads.shape[0]},) to match quads; got {tuple(batch_idx.shape)}." ) num_quads = quads.shape[0] channels, page_h, page_w = images.shape[1:] crops = images.new_full((num_quads, channels, height, max_width), fill) widths = torch.zeros(num_quads, dtype=torch.long, device=images.device) if num_quads == 0: return crops, widths quads = quads.to(device=images.device, dtype=torch.float64) edge = lambda a, b: (quads[:, a] - quads[:, b]).norm(dim=-1) # noqa: E731 quad_w = torch.maximum(edge(0, 1), edge(3, 2)) quad_h = torch.maximum(edge(0, 3), edge(1, 2)) vertical = quad_h >= vertical_ratio * quad_w # Rotating the crop 90 degrees CCW makes the old top-right the new top-left. quads = torch.where(vertical[:, None, None], quads.roll(-1, dims=1), quads) quad_w, quad_h = torch.where(vertical, quad_h, quad_w), torch.where(vertical, quad_w, quad_h) widths = (height * quad_w / quad_h.clamp_min(1e-6)).round().clamp(1, max_width).long() homographies = _unit_square_to_quads(quads) # (K, 3, 3) # Pixel-centre sampling positions, in unit-square coordinates per crop. cols = torch.arange(max_width, device=images.device, dtype=torch.float64) + 0.5 rows = torch.arange(height, device=images.device, dtype=torch.float64) + 0.5 u = cols[None, None, :] / widths[:, None, None] v = (rows / height)[None, :, None].expand(num_quads, height, max_width) uv1 = torch.stack([u.expand_as(v), v, torch.ones_like(v)], dim=-1) # (K, h, W, 3) xyw = torch.einsum("kij,khwj->khwi", homographies, uv1) xy = xyw[..., :2] / xyw[..., 2:] # align_corners=False: continuous coordinate x maps to 2x/W - 1. scale = torch.tensor([2.0 / page_w, 2.0 / page_h], device=images.device, dtype=torch.float64) grid = (xy * scale - 1).to(images.dtype) valid = (cols[None, :] < widths[:, None])[:, None, None, :] # (K, 1, 1, W) for page in batch_idx.unique().tolist(): members = (batch_idx == page).nonzero().squeeze(1) # One grid_sample per page: stack its crops' grids vertically. page_grid = grid[members].reshape(1, -1, max_width, 2) sampled = F.grid_sample( images[page : page + 1], page_grid, mode="bilinear", padding_mode="border", align_corners=False ) sampled = sampled.reshape(channels, len(members), height, max_width).transpose(0, 1) crops[members] = torch.where(valid[members], sampled, crops[members]) return crops, widths
def _unit_square_to_quads(quads: Tensor) -> Tensor: """Homographies mapping (0,0), (1,0), (1,1), (0,1) to each quad's corners.""" u = quads.new_tensor([0.0, 1.0, 1.0, 0.0]).expand(quads.shape[0], 4) v = quads.new_tensor([0.0, 0.0, 1.0, 1.0]).expand(quads.shape[0], 4) x, y = quads[..., 0], quads[..., 1] zeros, ones = torch.zeros_like(u), torch.ones_like(u) rows_x = torch.stack([u, v, ones, zeros, zeros, zeros, -u * x, -v * x], dim=-1) rows_y = torch.stack([zeros, zeros, zeros, u, v, ones, -u * y, -v * y], dim=-1) system = torch.cat([rows_x, rows_y], dim=1) # (K, 8, 8) target = torch.cat([x, y], dim=1) # (K, 8) # pinv rather than solve: degenerate (zero-area) quads must not raise. h = (torch.linalg.pinv(system) @ target.unsqueeze(-1)).squeeze(-1) return torch.cat([h, torch.ones_like(h[:, :1])], dim=1).reshape(-1, 3, 3)