"""High-level orchestration of detection, recognition, and decoding."""
from typing import Callable
import torch
from torch import Tensor, nn
from .charsets import load_charset
from .core.structures import DocumentTensor
from .decoders import build_decoder
from .models import CRNN, CRNN_ResNet34_VD_Weights, DBNet, DBNet_MobileNetV3_Large_05_Weights, WeightsEnum, get_weight
from .models.detection import DBNetOutput
from .ops import crop_quads, quad_to_box
from .orientation import PageOrientation
from .postprocess import DBPostProcessor
from .transforms import RecognitionPreset
_PostProcessor = Callable[[DBNetOutput], Tensor]
_Decoder = Callable[[Tensor], list[str]]
[docs]
class OCRPipeline:
"""End-to-end OCR pipeline: image -> populated :class:`DocumentTensor`.
Composes a detector, post-processor, recognizer, and decoder:
0. ``orientation`` (if given) sets a page scanned sideways or upside
down upright; every later step sees the upright page.
1. ``detector_transforms`` (if given) resizes/normalizes the image for
the detector, e.g. ``detector_weights.transforms()``.
2. The post-processor yields text regions. Rotated quads are used when it
offers ``quadrilaterals()`` (as :class:`DBPostProcessor` does);
otherwise its ``(K, 5) [batch_idx, x1, y1, x2, y2]`` boxes are used.
Regions are mapped back to input-image coordinates.
3. ``recognizer_transforms.normalize`` (if given) normalizes the
*original-resolution* image, and every region is rectified out of it
with :func:`torchocr.ops.crop_quads` -- aspect-preserving, padded,
one batched call on the model's device.
4. The recognizer and decoder turn crops into strings.
Use :meth:`from_pretrained` to assemble all four from published weights.
Args:
detector: Module returning a :class:`DBNetOutput`.
recognizer: Module mapping ``(K, C, H, W)`` crops to
``(T, K, num_classes)`` logits.
post_processor: Callable mapping a :class:`DBNetOutput` to boxes
(see above).
decoder: Callable mapping ``(T, K, num_classes)`` logits to a
list of K strings.
crop_size: ``(height, max_width)`` of recognizer crops. Default:
from ``recognizer_transforms`` when given, else ``(32, 128)``.
detector_transforms: Optional preset applied to the input image
before the detector (e.g. :class:`~torchocr.transforms.DetectionPreset`).
recognizer_transforms: Optional :class:`~torchocr.transforms.RecognitionPreset`
whose ``normalize`` is applied to the page before cropping.
recognizer_batch_size: Words cropped and recognized per forward
pass, bounding memory on dense documents. Default 64.
min_text_score: Drop regions whose recognition confidence
(``DocumentTensor.scores``) is below this value. ``None`` (default)
keeps every region. 0.8 maximizes ICDAR-2015 *training*-split
end-to-end hmean for the published scene-text models.
orientation: Optional :class:`~torchocr.PageOrientation` run before
detection. Default ``None`` (pages are read as given).
Without presets the input is fed to both models as-is, so it must
already be normalized and sized for the detector (the v0.1 behavior).
"""
def __init__(
self,
detector: nn.Module,
recognizer: nn.Module,
post_processor: _PostProcessor,
decoder: _Decoder,
*,
crop_size: tuple[int, int] | None = None,
detector_transforms: Callable[[Tensor], Tensor] | None = None,
recognizer_transforms: RecognitionPreset | None = None,
recognizer_batch_size: int = 64,
min_text_score: float | None = None,
orientation: PageOrientation | None = None,
) -> None:
if recognizer_batch_size < 1:
raise ValueError(f"recognizer_batch_size must be >= 1; got {recognizer_batch_size}.")
if crop_size is None:
crop_size = (
(recognizer_transforms.height, recognizer_transforms.max_width)
if recognizer_transforms is not None
else (32, 128)
)
if crop_size[0] != 32:
raise ValueError(
f"crop_size height must equal 32 to match the CRNN contract; got {crop_size[0]}."
)
if crop_size[1] <= 0 or crop_size[1] % 4:
raise ValueError(
f"crop_size width must be a positive multiple of 4; got {crop_size[1]}."
)
self.detector = detector
self.recognizer = recognizer
self.post_processor = post_processor
self.decoder = decoder
self.crop_size = crop_size
self.detector_transforms = detector_transforms
self.recognizer_transforms = recognizer_transforms
self.recognizer_batch_size = recognizer_batch_size
self.min_text_score = min_text_score
self.orientation = orientation
[docs]
@classmethod
def from_pretrained(
cls,
detector_weights: WeightsEnum | str | None = None,
recognizer_weights: WeightsEnum | str | None = None,
device: torch.device | str | None = None,
decoder: str = "auto",
min_text_score: float | None = None,
correct_orientation: bool = False,
) -> "OCRPipeline":
"""Assemble a pipeline from published weights, in eval mode.
Each stage gets its own preprocessing (``weights.transforms()``); the
post-processor uses ``detector_weights.meta["postprocess"]`` and the
decoder the charset named in ``recognizer_weights.meta``.
Args:
detector_weights: DBNet weights enum member or ``"Cls.NAME"``.
Default ``DBNet_MobileNetV3_Large_05_Weights.DEFAULT``.
recognizer_weights: CRNN weights enum member or ``"Cls.NAME"``.
Default ``CRNN_ResNet34_VD_Weights.DEFAULT``.
device: Where to place both models. Default: CPU.
decoder: ``"auto"`` uses the decoder declared in
``recognizer_weights.meta["decoder"]`` (e.g. beam search with a
language model), else greedy; ``"greedy"`` or ``"beam"`` force one.
min_text_score: See :class:`OCRPipeline`. Default ``None`` (keep all).
correct_orientation: Set pages upright first with
``PageOrientation.from_pretrained()``. Default ``False``.
"""
det_weights = _resolve(detector_weights, DBNet_MobileNetV3_Large_05_Weights)
rec_weights = _resolve(recognizer_weights, CRNN_ResNet34_VD_Weights)
detector = DBNet(weights=det_weights).eval().to(device)
recognizer = CRNN(weights=rec_weights).eval().to(device)
charset = load_charset(rec_weights.meta["charset"], rec_weights.meta["num_classes"])
return cls(
detector,
recognizer,
DBPostProcessor(**det_weights.meta.get("postprocess", {})),
build_decoder(rec_weights, charset, kind=decoder),
detector_transforms=det_weights.transforms(),
recognizer_transforms=rec_weights.transforms(),
min_text_score=min_text_score,
orientation=PageOrientation.from_pretrained(device=device) if correct_orientation else None,
)
def __call__(self, image: Tensor) -> DocumentTensor:
"""Run the full pipeline on a single ``(3, H, W)`` image.
With presets, ``image`` is a ``uint8`` RGB tensor (as returned by
:func:`torchocr.load_image`). It is returned in
``DocumentTensor.pixels`` -- unmodified, or set upright when
``orientation`` turned it (``DocumentTensor.rotation``); regions are
in the pixel coordinates of ``pixels``.
"""
if image.ndim != 3 or image.shape[0] != 3:
raise ValueError(f"Expected (3, H, W) image; got {tuple(image.shape)}.")
device = next(self.detector.parameters(), image).device
rotation = 0
if self.orientation is not None:
image, rotation = self.orientation.upright(image)
with torch.inference_mode():
pixels = image.to(device)
detector_input = (self.detector_transforms(pixels) if self.detector_transforms else pixels)[None]
detection = self.detector(detector_input)
quads = self._regions(detection).to(device=device, dtype=torch.float32)
if quads.shape[0] == 0:
return DocumentTensor(pixels=image, rotation=rotation)
scale = torch.tensor(
[pixels.shape[-1] / detector_input.shape[-1], pixels.shape[-2] / detector_input.shape[-2]],
device=device,
)
quads = quads * scale
page = self.recognizer_transforms.normalize(pixels) if self.recognizer_transforms else pixels
texts: list[str] = []
scores: list[Tensor] = []
# Bounded chunks: a dense form can hold hundreds of words.
for chunk in quads.split(self.recognizer_batch_size):
crops, _ = crop_quads(
page[None].float(),
chunk,
torch.zeros(chunk.shape[0], dtype=torch.long, device=device),
height=self.crop_size[0],
max_width=self.crop_size[1],
)
logits = self.recognizer(crops)
texts += self.decoder(logits)
scores.append(_text_scores(logits))
text_scores = torch.cat(scores)
if self.min_text_score is not None:
keep = text_scores >= self.min_text_score
quads, text_scores = quads[keep], text_scores[keep]
texts = [t for t, k in zip(texts, keep.tolist()) if k]
return DocumentTensor(
pixels=image,
text=texts,
bounding_boxes=quad_to_box(quads),
polygons=quads,
scores=text_scores,
rotation=rotation,
)
def _regions(self, detection: DBNetOutput) -> Tensor:
"""``(K, 4, 2)`` quads for the single image of ``detection``."""
if hasattr(self.post_processor, "quadrilaterals"):
return self.post_processor.quadrilaterals(detection)[0]
boxes = self.post_processor(detection)[:, 1:]
x1, y1, x2, y2 = boxes.unbind(-1)
return torch.stack([torch.stack(p, -1) for p in ((x1, y1), (x2, y1), (x2, y2), (x1, y2))], dim=1)
def _text_scores(logits: Tensor) -> Tensor:
"""``(K,)`` mean max-probability over the frames whose argmax is not the blank (0 if none)."""
best = logits.float().log_softmax(-1).max(-1) # (T, K)
emitted = best.indices != 0
total = torch.where(emitted, best.values.exp(), torch.zeros_like(best.values)).sum(0)
return total / emitted.sum(0).clamp(min=1)
def _resolve(weights: WeightsEnum | str | None, default: type[WeightsEnum]) -> WeightsEnum:
if weights is None:
return default.DEFAULT
if isinstance(weights, str) and "." in weights:
return get_weight(weights)
if isinstance(weights, WeightsEnum):
return weights
return default.verify(weights)