Source code for torchocr.datasets.lmdb_words
"""Word crops stored in the deep-text-recognition-benchmark LMDB layout."""
from collections.abc import Callable
from pathlib import Path
import torch
from torch import Tensor
from torch.utils.data import Dataset
from torchvision.io import ImageReadMode, decode_image
[docs]
class LMDBWords(Dataset[tuple[Tensor, str]]):
"""Cropped word images in the LMDB layout of clovaai's
deep-text-recognition-benchmark.
This is how the standard recognition corpora circulate (MJSynth,
SynthText, IIIT5K, SVT, ICDAR 2013/2015 crops, CUTE80, MLT word crops):
``num-samples`` holds the count and samples ``i = 1..N`` are stored under
``image-%09d`` (encoded image bytes) and ``label-%09d`` (UTF-8 text).
Requires the optional ``lmdb`` package (``pip install torchocr[lmdb]``).
Args:
root: Directory containing ``data.mdb``.
label_filter: Optional predicate on the label; only matching samples
are kept. Evaluated once on all labels at construction (images are
not decoded), e.g. to drop words outside a model's charset.
transform: Optional callable applied to the ``uint8`` image.
Returns:
``(image, text)``: a ``(3, H, W)`` ``uint8`` RGB crop and its label.
"""
def __init__(
self,
root: str | Path,
label_filter: Callable[[str], bool] | None = None,
transform: Callable[[Tensor], Tensor] | None = None,
) -> None:
self.root = Path(root)
if not (self.root / "data.mdb").is_file():
raise FileNotFoundError(f"No LMDB database (data.mdb) under {self.root}.")
self.transform = transform
self._env = None
with self._open().begin() as txn:
count = int(txn.get(b"num-samples"))
labels = [txn.get(f"label-{i:09d}".encode()).decode("utf-8") for i in range(1, count + 1)]
keep = [i for i, label in enumerate(labels, start=1) if label_filter is None or label_filter(label)]
self.indices = keep
self.labels = [labels[i - 1] for i in keep]
# LMDB environments must not cross fork(): DataLoader workers reopen lazily.
self._env.close()
self._env = None
def _open(self):
try:
import lmdb
except ImportError as exc:
raise ImportError("LMDBWords needs the lmdb package: pip install torchocr[lmdb]") from exc
if self._env is None:
self._env = lmdb.open(str(self.root), readonly=True, lock=False, readahead=False, meminit=False)
return self._env
def __len__(self) -> int:
return len(self.indices)
def __getitem__(self, index: int) -> tuple[Tensor, str]:
with self._open().begin() as txn:
data = txn.get(f"image-{self.indices[index]:09d}".encode())
image = decode_image(torch.frombuffer(bytearray(data), dtype=torch.uint8), mode=ImageReadMode.RGB)
if self.transform is not None:
image = self.transform(image)
return image, self.labels[index]