"""Text detection model definitions."""
from dataclasses import dataclass
from functools import partial
from typing import Literal
import torch
from torch import Tensor, nn
from torch.nn import functional as F
from torchvision.models import ResNet18_Weights, resnet18
from torchvision.models.feature_extraction import create_feature_extractor
from ..transforms import DetectionPreset
from .backbones import MobileNetV3, ResNetVd
from .backbones.mobilenet_v3 import _SEModule
from .hub import BASE_URL, Weights, WeightsEnum, get_weight, load_weights, register_model
_BACKBONE_TAPS = {"layer1": "c2", "layer2": "c3", "layer3": "c4", "layer4": "c5"}
_BACKBONE_CHANNELS = (64, 128, 256, 512)
_BACKBONE_KEYS = ("c2", "c3", "c4", "c5")
[docs]
@dataclass
class DBNetOutput:
"""Probability and threshold maps from a DBNet forward pass."""
probability: Tensor
threshold: Tensor
# Lets torch.export flatten and rebuild the typed output (it is a pytree node).
torch.export.register_dataclass(DBNetOutput, serialized_type_name="torchocr.models.detection.DBNetOutput")
@torch.fx.wrap
def _check_dbnet_input(images: Tensor) -> None:
"""Raise ``ValueError`` unless ``images`` is ``(B, 3, H, W)`` with ``H, W`` multiples of 32.
A module-level function marked ``torch.fx.wrap``: FX records it as an opaque
call instead of tracing its Python ``if``s on proxies, so the model stays
FX-traceable while every run (eager, traced, exported) still checks shapes.
"""
if images.ndim != 4 or images.shape[1] != 3:
raise ValueError(f"Expected (B, 3, H, W) images; got {tuple(images.shape)}.")
height, width = images.shape[-2:]
if height % 32 or width % 32:
raise ValueError(f"Height and width must be divisible by 32; got {height}x{width}.")
class _FPN(nn.Module):
"""Top-down Feature Pyramid Network with lateral connections.
This is the original torchocr FPN, paired with the torchvision
``resnet18`` backbone path. The PaddleOCR-compatible
:class:`_DBFPN` is used with the ``resnet18_vd`` path instead.
"""
def __init__(self, in_channels: tuple[int, ...], out_channels: int) -> None:
super().__init__()
self.lateral = nn.ModuleList(
nn.Conv2d(c, out_channels, kernel_size=1, bias=False) for c in in_channels
)
self.smooth = nn.ModuleList(
nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False)
for _ in in_channels
)
def forward(self, features: list[Tensor]) -> list[Tensor]:
laterals = [conv(f) for conv, f in zip(self.lateral, features)]
for i in range(len(laterals) - 1, 0, -1):
laterals[i - 1] = laterals[i - 1] + F.interpolate(
laterals[i], scale_factor=2.0, mode="nearest"
)
return [smooth(lat) for smooth, lat in zip(self.smooth, laterals)]
class _DBFPN(nn.Module):
"""PaddleOCR-compatible DBFPN with cascaded top-down adds.
Produces a single fused tensor at 1/4 resolution shaped
``(B, out_channels, H/4, W/4)``. Each pyramid level is reduced to
``out_channels // 4`` and concatenated to recover the full
``out_channels`` count -- this matches PaddleOCR's
``ppocr.modeling.necks.db_fpn.DBFPN`` exactly so weights port over
with a small mechanical name remap.
Submodule names (``in2_conv``, ``p2_conv``, ...) match PaddleOCR.
"""
def __init__(self, in_channels: tuple[int, ...], out_channels: int) -> None:
super().__init__()
if len(in_channels) != 4:
raise ValueError(f"DBFPN expects 4 input scales; got {len(in_channels)}.")
c2, c3, c4, c5 = in_channels
self.in2_conv = nn.Conv2d(c2, out_channels, kernel_size=1, bias=False)
self.in3_conv = nn.Conv2d(c3, out_channels, kernel_size=1, bias=False)
self.in4_conv = nn.Conv2d(c4, out_channels, kernel_size=1, bias=False)
self.in5_conv = nn.Conv2d(c5, out_channels, kernel_size=1, bias=False)
smooth = out_channels // 4
self.p2_conv = nn.Conv2d(out_channels, smooth, kernel_size=3, padding=1, bias=False)
self.p3_conv = nn.Conv2d(out_channels, smooth, kernel_size=3, padding=1, bias=False)
self.p4_conv = nn.Conv2d(out_channels, smooth, kernel_size=3, padding=1, bias=False)
self.p5_conv = nn.Conv2d(out_channels, smooth, kernel_size=3, padding=1, bias=False)
def forward(self, features: dict[str, Tensor]) -> Tensor:
c2, c3, c4, c5 = (features[name] for name in _BACKBONE_KEYS)
in5 = self.in5_conv(c5)
in4 = self.in4_conv(c4)
in3 = self.in3_conv(c3)
in2 = self.in2_conv(c2)
out4 = in4 + F.interpolate(in5, scale_factor=2.0, mode="nearest")
out3 = in3 + F.interpolate(out4, scale_factor=2.0, mode="nearest")
out2 = in2 + F.interpolate(out3, scale_factor=2.0, mode="nearest")
p5 = F.interpolate(self.p5_conv(in5), scale_factor=8.0, mode="nearest")
p4 = F.interpolate(self.p4_conv(out4), scale_factor=4.0, mode="nearest")
p3 = F.interpolate(self.p3_conv(out3), scale_factor=2.0, mode="nearest")
p2 = self.p2_conv(out2)
return torch.cat([p5, p4, p3, p2], dim=1)
class _RSELayer(nn.Module):
"""1x1 (or 3x3) conv -> SE block, with optional shortcut.
Used by :class:`_RSEFPN` to inject channel attention at every FPN
convolution. Submodule names match PaddleOCR (``in_conv``,
``se_block``).
"""
def __init__(self, in_channels: int, out_channels: int, kernel_size: int, shortcut: bool = True) -> None:
super().__init__()
self.in_conv = nn.Conv2d(
in_channels,
out_channels,
kernel_size=kernel_size,
padding=kernel_size // 2,
bias=False,
)
self.se_block = _SEModule(out_channels)
self.shortcut = shortcut
def forward(self, x: Tensor) -> Tensor:
x = self.in_conv(x)
return x + self.se_block(x) if self.shortcut else self.se_block(x)
class _RSEFPN(nn.Module):
"""PP-OCRv3 RSEFPN: DBFPN cascade with SE attention at every conv.
Same top-down + concat structure as :class:`_DBFPN` but every
1x1 lateral and 3x3 smooth conv is replaced by an :class:`_RSELayer`.
Default ``out_channels`` is 96, the value PP-OCRv3 ships with.
"""
def __init__(self, in_channels: tuple[int, ...], out_channels: int = 96, shortcut: bool = True) -> None:
super().__init__()
if len(in_channels) != 4:
raise ValueError(f"RSEFPN expects 4 input scales; got {len(in_channels)}.")
smooth = out_channels // 4
# PaddleOCR stores the 4 scales as ModuleLists indexed c2..c5 (i=0..3).
self.ins_conv = nn.ModuleList(
_RSELayer(in_channels[i], out_channels, kernel_size=1, shortcut=shortcut)
for i in range(4)
)
self.inp_conv = nn.ModuleList(
_RSELayer(out_channels, smooth, kernel_size=3, shortcut=shortcut)
for _ in range(4)
)
def forward(self, features: dict[str, Tensor]) -> Tensor:
c2, c3, c4, c5 = (features[name] for name in _BACKBONE_KEYS)
in5 = self.ins_conv[3](c5)
in4 = self.ins_conv[2](c4)
in3 = self.ins_conv[1](c3)
in2 = self.ins_conv[0](c2)
out4 = in4 + F.interpolate(in5, scale_factor=2.0, mode="nearest")
out3 = in3 + F.interpolate(out4, scale_factor=2.0, mode="nearest")
out2 = in2 + F.interpolate(out3, scale_factor=2.0, mode="nearest")
p5 = F.interpolate(self.inp_conv[3](in5), scale_factor=8.0, mode="nearest")
p4 = F.interpolate(self.inp_conv[2](out4), scale_factor=4.0, mode="nearest")
p3 = F.interpolate(self.inp_conv[1](out3), scale_factor=2.0, mode="nearest")
p2 = self.inp_conv[0](out2)
return torch.cat([p5, p4, p3, p2], dim=1)
class _DBHead(nn.Module):
"""One head (binarize or threshold) of a DBNet.
Module layout matches PaddleOCR's ``ppocr.modeling.heads.det_db_head.Head``:
``conv1`` -> ``conv_bn1`` -> ReLU -> ``conv2`` (transpose, x2) ->
``conv_bn2`` -> ReLU -> ``conv3`` (transpose, x2) -> sigmoid.
Sigmoid is applied in forward and is not a registered module, so
the state_dict has exactly the same parameter names as PaddleOCR's
head once the converter strips its ``head.binarize.`` /
``head.thresh.`` prefixes.
"""
def __init__(self, in_channels: int) -> None:
super().__init__()
inner = in_channels // 4
self.conv1 = nn.Conv2d(in_channels, inner, kernel_size=3, padding=1, bias=False)
self.conv_bn1 = nn.BatchNorm2d(inner)
self.conv2 = nn.ConvTranspose2d(inner, inner, kernel_size=2, stride=2)
self.conv_bn2 = nn.BatchNorm2d(inner)
self.conv3 = nn.ConvTranspose2d(inner, 1, kernel_size=2, stride=2)
def forward(self, x: Tensor) -> Tensor:
x = F.relu(self.conv_bn1(self.conv1(x)), inplace=True)
x = F.relu(self.conv_bn2(self.conv2(x)), inplace=True)
return torch.sigmoid(self.conv3(x))
BackboneName = Literal["resnet18", "resnet18_vd", "mobilenet_v3_large_05"]
# PaddleOCR detectors saw cv2-decoded BGR pixels normalized with RGB-ordered
# ImageNet statistics; on ICDAR-2015 "bgr" beats "rgb" for every checkpoint.
_PADDLE_DET_PRESET = partial(DetectionPreset, max_side=960, channel_order="bgr")
# PaddleOCR's DB post-processing defaults, under which the converted checkpoints were scored.
_PADDLE_DB_POSTPROCESS = {"threshold": 0.3, "box_thresh": 0.6, "unclip_ratio": 1.5}
_ICDAR2015_PROTOCOL = (
"ICDAR-2015 test (500 images, 2077 care words), official IoU protocol on rotated quads; "
"transforms() preset, DBPostProcessor(**meta['postprocess']). "
"Reproduce with references/detection/evaluate.py --weights <this enum>."
)
_LINE_LEVEL_CAVEAT = (
"Trained by PaddleOCR on line-level annotations of mostly Chinese/English document and "
"scene text, then converted to PyTorch. ICDAR-2015 scores *word* boxes in low-resolution "
"street scenes, so adjacent words merged into one line count as misses; use this number "
"to compare checkpoints, not as the accuracy you will see on documents."
)
[docs]
class DBNet_ResNet18_VD_Weights(WeightsEnum):
PPOCR_SERVER_V2 = Weights(
url=f"{BASE_URL}/dbnet_resnet18_vd_ppocr_server_v2-59d99b11.pth",
transforms=_PADDLE_DET_PRESET,
meta={
"task": "detection",
"backbone": "resnet18_vd",
"num_params": 12_364_386,
"source": "PaddleOCR ch_ppocr_server_v2.0_det_train, via scripts/convert_paddle_dbnet.py",
"license": "Apache-2.0",
"languages": ["ch", "en"],
"postprocess": _PADDLE_DB_POSTPROCESS,
"_metrics": {"ICDAR2015-test": {"precision": 0.5814, "recall": 0.3216, "hmean": 0.4141}},
"_docs": f"{_LINE_LEVEL_CAVEAT} {_ICDAR2015_PROTOCOL}",
},
)
ICDAR2015 = Weights(
url=f"{BASE_URL}/dbnet_resnet18_vd_ic15-d803cec9.pth",
# Trained and scored at PaddleOCR's ICDAR-2015 test size (736 x 1280).
transforms=partial(DetectionPreset, max_side=1280, channel_order="bgr"),
meta={
"task": "detection",
"backbone": "resnet18_vd",
"num_params": 12_364_386,
"source": (
"torchocr: PPOCR_SERVER_V2 fine-tuned 300 epochs on the 1000 ICDAR-2015 training images "
"with references/detection/train.py"
),
"license": "Apache-2.0 (base weights); fine-tuned on ICDAR 2015, released for research use",
"languages": ["en"],
"postprocess": {"threshold": 0.3, "box_thresh": 0.5, "unclip_ratio": 1.5},
"_metrics": {"ICDAR2015-test": {"precision": 0.8485, "recall": 0.7737, "hmean": 0.8094}},
"_docs": (
"Word-level detector for incidental scene text (street-level photos, Latin script). "
"box_thresh=0.5 maximizes hmean on the *training* split "
"(references/detection/calibrate.py) and was applied once to the test split; with "
"PaddleOCR's box_thresh=0.6 this checkpoint scores 0.794. Weights are the last "
"epoch -- no checkpoint was selected on the test set. Expect lower recall on dense "
f"documents and on non-Latin scripts. {_ICDAR2015_PROTOCOL}"
),
},
)
DEFAULT = PPOCR_SERVER_V2
[docs]
class DBNet_MobileNetV3_Large_05_Weights(WeightsEnum):
PPOCR_V3_CH = Weights(
url=f"{BASE_URL}/dbnet_mobilenet_v3_large_05_ppocr_v3_ch-fc000d1e.pth",
transforms=_PADDLE_DET_PRESET,
meta={
"task": "detection",
"backbone": "mobilenet_v3_large_05",
"num_params": 603_418,
"source": "PaddleOCR ch_PP-OCRv3_det_distill_train (Student), via scripts/convert_paddle_dbnet_v3.py",
"license": "Apache-2.0",
"languages": ["ch", "en"],
"postprocess": _PADDLE_DB_POSTPROCESS,
"_metrics": {"ICDAR2015-test": {"precision": 0.5641, "recall": 0.3370, "hmean": 0.4219}},
"_docs": f"{_LINE_LEVEL_CAVEAT} {_ICDAR2015_PROTOCOL}",
},
)
PPOCR_V3_EN = Weights(
url=f"{BASE_URL}/dbnet_mobilenet_v3_large_05_ppocr_v3_en-9bfd3b59.pth",
transforms=_PADDLE_DET_PRESET,
meta={
"task": "detection",
"backbone": "mobilenet_v3_large_05",
"num_params": 603_418,
"source": "PaddleOCR en_PP-OCRv3_det_distill_train (Student), via scripts/convert_paddle_dbnet_v3.py",
"license": "Apache-2.0",
"languages": ["en"],
"postprocess": _PADDLE_DB_POSTPROCESS,
"_metrics": {"ICDAR2015-test": {"precision": 0.5342, "recall": 0.3755, "hmean": 0.4411}},
"_docs": f"{_LINE_LEVEL_CAVEAT} {_ICDAR2015_PROTOCOL}",
},
)
ICDAR2015 = Weights(
url=f"{BASE_URL}/dbnet_mobilenet_v3_large_05_ic15-d9d17ae7.pth",
# Trained and scored at PaddleOCR's ICDAR-2015 test size (736 x 1280).
transforms=partial(DetectionPreset, max_side=1280, channel_order="bgr"),
meta={
"task": "detection",
"backbone": "mobilenet_v3_large_05",
"num_params": 603_418,
"source": (
"torchocr: PPOCR_V3_EN fine-tuned 300 epochs on the 1000 ICDAR-2015 training images "
"with references/detection/train.py"
),
"license": "Apache-2.0 (base weights); fine-tuned on ICDAR 2015, released for research use",
"languages": ["en"],
"postprocess": {"threshold": 0.3, "box_thresh": 0.45, "unclip_ratio": 1.5},
"_metrics": {"ICDAR2015-test": {"precision": 0.7845, "recall": 0.6890, "hmean": 0.7337}},
"_docs": (
"Word-level detector for incidental scene text (street-level photos, Latin script). "
"box_thresh=0.45 maximizes hmean on the *training* split "
"(references/detection/calibrate.py) and was applied once to the test split; with "
"PaddleOCR's box_thresh=0.6 this checkpoint scores 0.688. Weights are the last "
"epoch -- no checkpoint was selected on the test set. Expect lower recall on dense "
f"documents and on non-Latin scripts. {_ICDAR2015_PROTOCOL}"
),
},
)
DEFAULT = PPOCR_V3_CH
_WEIGHTS_BY_BACKBONE: dict[str, type[WeightsEnum]] = {
"resnet18_vd": DBNet_ResNet18_VD_Weights,
"mobilenet_v3_large_05": DBNet_MobileNetV3_Large_05_Weights,
}
[docs]
class DBNet(nn.Module):
"""DBNet text detector.
The model produces a probability map and a threshold map at the
input resolution. The differentiable-binarization combination of
the two maps is a training-time loss helper and lives outside this
module.
Args:
backbone: Which backbone to instantiate. ``"resnet18"`` uses
torchvision's ResNet-18 with its 7x7 stem and is what
torchocr trains from scratch (no published OCR weights).
``"resnet18_vd"`` and ``"mobilenet_v3_large_05"`` mirror
PaddleOCR's detectors so converted checkpoints load as-is.
Default ``None``: taken from ``weights`` when it is an enum
member, else ``"resnet18"``.
weights: Pretrained OCR weights -- a member of
:class:`DBNet_ResNet18_VD_Weights` or
:class:`DBNet_MobileNetV3_Large_05_Weights`, or a member name
such as ``"DEFAULT"`` resolved against ``backbone``. If the
download fails a ``UserWarning`` is emitted and the model
keeps its random initialization. Pair with
``weights.transforms()`` for the matching preprocessing.
Default ``None`` (random init).
``pretrained_backbone`` is ignored for the PaddleOCR-style
backbones since no canonical ImageNet weights ship with
torchvision for them.
pretrained_backbone: If True and ``backbone="resnet18"``, load
ImageNet weights for the torchvision ResNet-18 backbone.
Default False to keep instantiation offline-safe.
fpn_out_channels: Channels per FPN level. Default 256.
head_inner_channels: Inner channels of the probability and
threshold heads. Used only on the ``resnet18`` path; the
``resnet18_vd`` path uses ``fpn_out_channels // 4`` to
match PaddleOCR.
Note:
The ``resnet18`` backbone is wrapped via
``create_feature_extractor`` (FX-traced). Custom backbones with
data-dependent control flow may fail to trace. The
``resnet18_vd`` path bypasses FX tracing.
"""
def __init__(
self,
backbone: BackboneName | None = None,
weights: WeightsEnum | str | None = None,
pretrained_backbone: bool = False,
fpn_out_channels: int = 256,
head_inner_channels: int = 64,
) -> None:
super().__init__()
if isinstance(weights, str) and "." in weights:
weights = get_weight(weights) # "Cls.NAME" names the architecture too
if backbone is None:
backbone = weights.meta["backbone"] if isinstance(weights, WeightsEnum) else "resnet18"
if weights is not None:
if backbone not in _WEIGHTS_BY_BACKBONE:
raise ValueError(
f"No pretrained weights are published for backbone '{backbone}'. "
f"Backbones with weights: {sorted(_WEIGHTS_BY_BACKBONE)}."
)
weights = _WEIGHTS_BY_BACKBONE[backbone].verify(weights)
self.backbone_name = backbone
if backbone == "resnet18":
backbone_weights = ResNet18_Weights.DEFAULT if pretrained_backbone else None
torchvision_backbone = resnet18(weights=backbone_weights)
self.backbone = create_feature_extractor(
torchvision_backbone, return_nodes=_BACKBONE_TAPS
)
self.fpn = _FPN(_BACKBONE_CHANNELS, fpn_out_channels)
self.fuse = nn.Sequential(
nn.Conv2d(
fpn_out_channels * 4,
fpn_out_channels,
kernel_size=3,
padding=1,
bias=False,
),
nn.BatchNorm2d(fpn_out_channels),
nn.ReLU(inplace=True),
)
self.probability_head = self._make_legacy_head(
fpn_out_channels, head_inner_channels
)
self.threshold_head = self._make_legacy_head(
fpn_out_channels, head_inner_channels
)
elif backbone == "resnet18_vd":
self.backbone = ResNetVd(depth=18)
self.fpn = _DBFPN(self.backbone.out_channels, fpn_out_channels)
self.binarize = _DBHead(fpn_out_channels)
self.thresh = _DBHead(fpn_out_channels)
elif backbone == "mobilenet_v3_large_05":
# PP-OCRv3 detector: MobileNetV3-large scale=0.5 with SE *disabled*
# in the backbone (SE only lives in the RSEFPN). Default neck width
# is 96 for v3 instead of the 256 used by the ResNet-VD detector.
self.backbone = MobileNetV3(model_name="large", scale=0.5, disable_se=True)
v3_fpn_channels = 96 if fpn_out_channels == 256 else fpn_out_channels
self.fpn = _RSEFPN(self.backbone.out_channels, out_channels=v3_fpn_channels)
self.binarize = _DBHead(v3_fpn_channels)
self.thresh = _DBHead(v3_fpn_channels)
else:
raise ValueError(
f"Unknown backbone '{backbone}'. "
"Known: 'resnet18', 'resnet18_vd', 'mobilenet_v3_large_05'."
)
if weights is not None:
load_weights(self, weights)
@staticmethod
def _make_legacy_head(in_channels: int, inner_channels: int) -> nn.Sequential:
return nn.Sequential(
nn.Conv2d(in_channels, inner_channels, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(inner_channels),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(inner_channels, inner_channels, kernel_size=2, stride=2),
nn.BatchNorm2d(inner_channels),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(inner_channels, 1, kernel_size=2, stride=2),
nn.Sigmoid(),
)
[docs]
def forward(self, images: Tensor) -> DBNetOutput:
_check_dbnet_input(images)
feature_maps = self.backbone(images)
if self.backbone_name == "resnet18":
pyramid = self.fpn([feature_maps[name] for name in _BACKBONE_KEYS])
target_size = pyramid[0].shape[-2:]
upsampled = [pyramid[0]] + [
F.interpolate(level, size=target_size, mode="nearest") for level in pyramid[1:]
]
fused = self.fuse(torch.cat(upsampled, dim=1))
return DBNetOutput(
probability=self.probability_head(fused),
threshold=self.threshold_head(fused),
)
# ResNet-VD and MobileNetV3 paths share the same forward shape:
# backbone -> FPN/RSEFPN (single fused tensor) -> two DBHead modules.
fused = self.fpn(feature_maps)
return DBNetOutput(
probability=self.binarize(fused),
threshold=self.thresh(fused),
)
[docs]
@register_model("dbnet_resnet18")
def dbnet_resnet18(**kwargs: object) -> DBNet:
"""DBNet with torchvision's ResNet-18 backbone (no published OCR weights)."""
return DBNet(backbone="resnet18", **kwargs)
[docs]
@register_model("dbnet_resnet18_vd", weights=DBNet_ResNet18_VD_Weights)
def dbnet_resnet18_vd(*, weights: DBNet_ResNet18_VD_Weights | str | None = None, **kwargs: object) -> DBNet:
"""DBNet with PaddleOCR's ResNet-18-VD backbone; see :class:`DBNet_ResNet18_VD_Weights`."""
return DBNet(backbone="resnet18_vd", weights=weights, **kwargs)
[docs]
@register_model("dbnet_mobilenet_v3_large_05", weights=DBNet_MobileNetV3_Large_05_Weights)
def dbnet_mobilenet_v3_large_05(
*, weights: DBNet_MobileNetV3_Large_05_Weights | str | None = None, **kwargs: object
) -> DBNet:
"""PP-OCRv3 detector (MobileNetV3-large x0.5 + RSEFPN); see :class:`DBNet_MobileNetV3_Large_05_Weights`."""
return DBNet(backbone="mobilenet_v3_large_05", weights=weights, **kwargs)