Instructions to use Yesianrohn/openocrv2 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Yesianrohn/openocrv2 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Yesianrohn/openocrv2")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("Yesianrohn/openocrv2", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Yesianrohn/openocrv2 with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Yesianrohn/openocrv2" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Yesianrohn/openocrv2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/Yesianrohn/openocrv2
- SGLang
How to use Yesianrohn/openocrv2 with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Yesianrohn/openocrv2" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Yesianrohn/openocrv2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Yesianrohn/openocrv2" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Yesianrohn/openocrv2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use Yesianrohn/openocrv2 with Docker Model Runner:
docker model run hf.co/Yesianrohn/openocrv2
Download modeling_openocrv2_det.py from Yesianrohn/openocrv2: direct link, hf CLI and curl.
- Browser
- Download file 35.7 kB
-
https://huggingface.co/Yesianrohn/openocrv2/resolve/main/modeling_openocrv2_det.py
- Command line
-
hf download hf://Yesianrohn/openocrv2/modeling_openocrv2_det.py
-
curl -L -o modeling_openocrv2_det.py https://huggingface.co/Yesianrohn/openocrv2/resolve/main/modeling_openocrv2_det.py
35.7 kB
| import numpy as np | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from transformers import PretrainedConfig | |
| from transformers import PreTrainedModel | |
| from transformers.modeling_outputs import ModelOutput | |
| from PIL import Image | |
| from dataclasses import dataclass | |
| class DCNv2(nn.Module): | |
| """Modulated deformable 3x3 convolution with learned offsets and masks. | |
| Parameter names match the frozen checkpoint (``weight``, ``conv_offset.*``). | |
| ``forward`` dispatches to the portable path so this class is importable | |
| without torchvision. | |
| """ | |
| def __init__(self, channels, stride=1, deform_groups=2): | |
| super().__init__() | |
| self.in_channels = channels | |
| self.out_channels = channels | |
| self.stride = (stride, stride) | |
| self.padding = (1, 1) | |
| self.dilation = (1, 1) | |
| self.deform_groups = deform_groups | |
| # Registered the same way nn.Conv2d would, so key names line up. | |
| self.weight = nn.Parameter(torch.empty(channels, channels, 3, 3)) | |
| self.bias = None | |
| self.conv_offset = nn.Conv2d( | |
| channels, deform_groups * 27, 3, stride=stride, padding=1 | |
| ) | |
| def forward(self, x): | |
| offset_y, offset_x, mask = self.conv_offset(x).chunk(3, dim=1) | |
| return deform_conv2d_portable( | |
| x, | |
| torch.cat((offset_y, offset_x), dim=1), | |
| self.weight, | |
| self.bias, | |
| self.stride, | |
| self.padding, | |
| self.dilation, | |
| mask=mask.sigmoid(), | |
| groups=self.deform_groups, | |
| ) | |
| def deform_conv2d_portable( | |
| x, | |
| offset, | |
| weight, | |
| bias=None, | |
| stride=(1, 1), | |
| padding=(1, 1), | |
| dilation=(1, 1), | |
| mask=None, | |
| groups=1, | |
| ): | |
| """Modulated deformable convolution via ``grid_sample``. | |
| ``offset`` has ``2 * groups * 9`` channels ordered as | |
| ``[group][ky][kx][dy, dx]`` — the layout ``torchvision`` expects and the one | |
| the trained ``conv_offset`` produces. | |
| """ | |
| if isinstance(stride, int): | |
| stride = (stride, stride) | |
| if isinstance(padding, int): | |
| padding = (padding, padding) | |
| if isinstance(dilation, int): | |
| dilation = (dilation, dilation) | |
| if (stride, padding, dilation) != ((1, 1), (1, 1), (1, 1)): | |
| raise ValueError( | |
| "Portable DCN covers the frozen stride-1, pad-1, dilation-1 adapters only" | |
| ) | |
| batch, channels, height, width = x.shape | |
| if offset.shape[1] != groups * 18: | |
| raise ValueError( | |
| f"offset has {offset.shape[1]} channels; expected {groups * 18} " | |
| f"for {groups} deform groups" | |
| ) | |
| dtype = x.dtype | |
| device = x.device | |
| delta = offset.reshape(batch, groups, 9, 2, height, width) | |
| yy, xx = torch.meshgrid( | |
| torch.arange(height, device=device, dtype=torch.float32), | |
| torch.arange(width, device=device, dtype=torch.float32), | |
| indexing="ij", | |
| ) | |
| ky, kx = torch.meshgrid( | |
| torch.arange(3, device=device, dtype=torch.float32), | |
| torch.arange(3, device=device, dtype=torch.float32), | |
| indexing="ij", | |
| ) | |
| # Coordinates are expressed in the *unpadded* input frame; padding is handled | |
| # by ``padding_mode="zeros"``. Adding a padding offset here would shift every | |
| # sample by ``padding`` pixels (verified against torchvision: it does not). | |
| y = yy[None, None, None] + (ky.reshape(1, 1, 9, 1, 1) - 1) * dilation[0] + delta[:, :, :, 0] | |
| xcoord = ( | |
| xx[None, None, None] | |
| + (kx.reshape(1, 1, 9, 1, 1) - 1) * dilation[1] | |
| + delta[:, :, :, 1] | |
| ) | |
| grid = torch.stack( | |
| (((xcoord + 0.5) * 2 / width - 1), ((y + 0.5) * 2 / height - 1)), dim=-1 | |
| ) | |
| grid = grid.permute(0, 1, 3, 4, 2, 5).reshape(batch * groups, height, width * 9, 2) | |
| sampled = F.grid_sample( | |
| x.reshape(batch * groups, channels // groups, height, width), | |
| grid.to(dtype), | |
| mode="bilinear", | |
| padding_mode="zeros", | |
| align_corners=False, | |
| ) | |
| sampled = sampled.reshape(batch, groups, channels // groups, height, width, 9) | |
| sampled = sampled.permute(0, 1, 2, 5, 3, 4) | |
| if mask is not None: | |
| sampled = sampled * mask.reshape(batch, groups, 1, 9, height, width).to(dtype) | |
| columns = sampled.reshape(batch, channels * 9, height, width) | |
| # The column tensor is already grouped channel-major, so the 1x1 convolution | |
| # runs with groups=1 over the flattened (channel, kernel-position) axis. | |
| return F.conv2d( | |
| columns, | |
| weight.reshape(weight.shape[0], weight.shape[1] * 9, 1, 1), | |
| bias, | |
| ) | |
| def box_iou(boxes, box): | |
| """IoU of every row in ``boxes`` [N,4] against one ``box`` [4] (xyxy).""" | |
| lower = torch.maximum(boxes[:, 0:2], box[0:2]) | |
| upper = torch.minimum(boxes[:, 2:4], box[2:4]) | |
| wh = (upper - lower).clamp(min=0) | |
| inter = wh[:, 0] * wh[:, 1] | |
| area_boxes = (boxes[:, 2:4] - boxes[:, 0:2]).clamp(min=0).prod(dim=1) | |
| area_box = (box[2:4] - box[0:2]).clamp(min=0).prod() | |
| union = area_boxes + area_box - inter | |
| return torch.where(union > 0, inter / union, torch.zeros_like(inter)) | |
| def nms(boxes, scores, iou_threshold): | |
| """Greedy NMS matching ``torchvision.ops.nms`` ordering. | |
| torchvision sorts by descending score and keeps the first box when IoU is | |
| strictly greater than the threshold; equal scores keep their original order. | |
| """ | |
| if boxes.numel() == 0: | |
| return torch.empty((0,), dtype=torch.long, device=boxes.device) | |
| order = torch.argsort(scores, descending=True, stable=True) | |
| keep = [] | |
| while order.numel(): | |
| index = order[0] | |
| keep.append(index) | |
| if order.numel() == 1: | |
| break | |
| iou = box_iou(boxes[order[1:]], boxes[index]) | |
| order = order[1:][iou <= iou_threshold] | |
| return torch.stack(keep) | |
| def _same_padding(kernel_size, stride=1): | |
| return ((stride - 1) + kernel_size - 1) // 2 | |
| class ConvBNAct(nn.Module): | |
| """Conv + BatchNorm + optional ReLU, the HGNetv2 basic block.""" | |
| def __init__(self, in_chs, out_chs, kernel_size, stride=1, groups=1, use_act=True): | |
| super().__init__() | |
| self.conv = nn.Conv2d( | |
| in_chs, out_chs, kernel_size, stride=stride, | |
| padding=_same_padding(kernel_size, stride), groups=groups, bias=False, | |
| ) | |
| self.bn = nn.BatchNorm2d(out_chs) | |
| self.act = nn.ReLU() if use_act else nn.Identity() | |
| def forward(self, x): | |
| return self.act(self.bn(self.conv(x))) | |
| class LightConvBNAct(nn.Module): | |
| """1x1 pointwise conv followed by a depthwise conv.""" | |
| def __init__(self, in_chs, out_chs, kernel_size): | |
| super().__init__() | |
| self.conv1 = ConvBNAct(in_chs, out_chs, kernel_size=1, use_act=False) | |
| self.conv2 = ConvBNAct(out_chs, out_chs, kernel_size=kernel_size, groups=out_chs) | |
| def forward(self, x): | |
| return self.conv2(self.conv1(x)) | |
| class Stem(nn.Module): | |
| """PP-HGNetv2 stem: two stride-2 stages reaching stride 4.""" | |
| def __init__(self, in_chs=3, mid_chs=32, out_chs=48): | |
| super().__init__() | |
| self.stem1 = ConvBNAct(in_chs, mid_chs, kernel_size=3, stride=2) | |
| self.stem2a = ConvBNAct(mid_chs, mid_chs // 2, kernel_size=2, stride=1) | |
| self.stem2b = ConvBNAct(mid_chs // 2, mid_chs, kernel_size=2, stride=1) | |
| self.stem3 = ConvBNAct(mid_chs * 2, mid_chs, kernel_size=3, stride=2) | |
| self.stem4 = ConvBNAct(mid_chs, out_chs, kernel_size=1, stride=1) | |
| self.pool = nn.MaxPool2d(kernel_size=2, stride=1, ceil_mode=True) | |
| def forward(self, x): | |
| x = self.stem1(x) | |
| x = F.pad(x, (0, 1, 0, 1)) | |
| # Two separate pads: the pooled branch sees the first one only, the | |
| # convolution branch both. Collapsing them changes the spatial size. | |
| branch = F.pad(self.stem2a(x), (0, 1, 0, 1)) | |
| branch = self.stem2b(branch) | |
| pooled = self.pool(x) | |
| return self.stem4(self.stem3(torch.cat([pooled, branch], dim=1))) | |
| class DenseBlock(nn.Module): | |
| """Every layer's output is concatenated, then squeezed by two 1x1 convs.""" | |
| def __init__(self, in_chs, mid_chs, out_chs, layer_num, kernel_size=3, | |
| residual=False, light_block=False): | |
| super().__init__() | |
| self.residual = residual | |
| self.layers = nn.ModuleList( | |
| (LightConvBNAct if light_block else ConvBNAct)( | |
| in_chs if index == 0 else mid_chs, mid_chs, kernel_size=kernel_size | |
| ) | |
| for index in range(layer_num) | |
| ) | |
| total_chs = in_chs + layer_num * mid_chs | |
| self.aggregation = nn.Sequential( | |
| ConvBNAct(total_chs, out_chs // 2, kernel_size=1), | |
| ConvBNAct(out_chs // 2, out_chs, kernel_size=1), | |
| ) | |
| def forward(self, x): | |
| identity = x | |
| collected = [x] | |
| for layer in self.layers: | |
| x = layer(x) | |
| collected.append(x) | |
| x = self.aggregation(torch.cat(collected, dim=1)) | |
| return x + identity if self.residual else x | |
| class DenseStage(nn.Module): | |
| def __init__(self, in_chs, mid_chs, out_chs, block_num, layer_num, | |
| downsample=True, stride=2, light_block=False, kernel_size=3): | |
| super().__init__() | |
| self.downsample = ( | |
| ConvBNAct(in_chs, in_chs, kernel_size=3, stride=stride, | |
| groups=in_chs, use_act=False) | |
| if downsample else nn.Identity() | |
| ) | |
| self.blocks = nn.Sequential(*[ | |
| DenseBlock( | |
| in_chs if index == 0 else out_chs, | |
| mid_chs, | |
| out_chs, | |
| layer_num, | |
| residual=index != 0, | |
| kernel_size=kernel_size, | |
| light_block=light_block, | |
| ) | |
| for index in range(block_num) | |
| ]) | |
| def forward(self, x): | |
| return self.blocks(self.downsample(x)) | |
| HGNETV2_B4_STAGES = ( | |
| (48, 48, 128, 1, False, False, 3, 6), | |
| (128, 96, 512, 1, True, False, 3, 6), | |
| (512, 192, 1024, 3, True, True, 5, 6), | |
| ) | |
| class HGNetv2B4Features(nn.Module): | |
| """HGNetv2-b4 truncated after stage 2: stride 16, 1024 channels. | |
| Stages are plain attributes named ``stages_0`` ... because that is what the | |
| checkpoint calls them (timm's ``flatten_sequential`` renames ``stages.0``). | |
| """ | |
| out_channels = 1024 | |
| def __init__(self): | |
| super().__init__() | |
| self.stem = Stem(in_chs=3, mid_chs=32, out_chs=48) | |
| for index, stage in enumerate(HGNETV2_B4_STAGES): | |
| setattr( | |
| self, | |
| f"stages_{index}", | |
| DenseStage( | |
| in_chs=stage[0], | |
| mid_chs=stage[1], | |
| out_chs=stage[2], | |
| block_num=stage[3], | |
| layer_num=stage[7], | |
| downsample=stage[4], | |
| light_block=stage[5], | |
| kernel_size=stage[6], | |
| ), | |
| ) | |
| self.stage_names = tuple(f"stages_{index}" for index in range(len(HGNETV2_B4_STAGES))) | |
| class DeformableAdapter(nn.Module): | |
| """Identity-initialised bottleneck with one DCNv2 at its centre.""" | |
| def __init__(self, channels, bottleneck_channels): | |
| super().__init__() | |
| self.reduce = nn.Conv2d(channels, bottleneck_channels, 1, bias=False) | |
| self.bn1 = nn.BatchNorm2d(bottleneck_channels) | |
| self.dcn = DCNv2(bottleneck_channels, deform_groups=2) | |
| self.bn2 = nn.BatchNorm2d(bottleneck_channels) | |
| self.expand = nn.Conv2d(bottleneck_channels, channels, 1, bias=False) | |
| self.bn3 = nn.BatchNorm2d(channels) | |
| self.relu = nn.ReLU(inplace=True) | |
| def forward(self, x): | |
| residual = self.relu(self.bn1(self.reduce(x))) | |
| residual = self.relu(self.bn2(self.dcn(residual))) | |
| return x + self.bn3(self.expand(residual)) | |
| class HGNetv2B4Backbone(nn.Module): | |
| """HGNetv2-b4 plus the two deformable adapters.""" | |
| def __init__(self, deformable=True): | |
| super().__init__() | |
| self.deformable = deformable | |
| self.features = HGNetv2B4Features() | |
| self.out_channels = HGNetv2B4Features.out_channels | |
| self.dcn_adapters = nn.ModuleDict() | |
| if deformable: | |
| self.dcn_adapters["stages_1"] = DeformableAdapter(512, 128) | |
| self.dcn_adapters["stages_2"] = DeformableAdapter(1024, 256) | |
| def forward(self, images): | |
| if not self.deformable: | |
| return self.features(images) | |
| x = self.features.stem(images) | |
| for name in self.features.stage_names: | |
| x = getattr(self.features, name)(x) | |
| if name in self.dcn_adapters: | |
| x = self.dcn_adapters[name](x) | |
| return x | |
| class ConvAct(nn.Module): | |
| def __init__(self, in_channels, out_channels, kernel_size=3, activation=True): | |
| super().__init__() | |
| self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, | |
| padding=kernel_size // 2) | |
| self.activate = nn.ReLU(inplace=True) if activation else nn.Identity() | |
| def forward(self, x): | |
| return self.activate(self.conv(x)) | |
| class NeckProjection(nn.Module): | |
| """Lateral conv then output conv (the only active FPN legs in the checkpoint).""" | |
| def __init__(self, in_channels, out_channels=256): | |
| super().__init__() | |
| self.lateral_convs = nn.ModuleList([ConvAct(in_channels, out_channels, 1, False)]) | |
| self.fpn_convs = nn.ModuleList([ConvAct(out_channels, out_channels, 3, False)]) | |
| def forward(self, x): | |
| return self.fpn_convs[0](self.lateral_convs[0](x)) | |
| class PolygonHead(nn.Module): | |
| """Dense/sparse class logits plus 14 low-rank polygon coefficients.""" | |
| def __init__(self, in_channels=256, num_coefficients=14, upsample=4): | |
| super().__init__() | |
| self.upsample = upsample | |
| self.cls_convs = nn.Sequential(ConvAct(in_channels, 32), ConvAct(32, 32)) | |
| self.out_conv_cls_dense = nn.Conv2d(32, 1, 3, padding=1) | |
| self.out_conv_reg_dense = nn.Conv2d(32, num_coefficients, 3, padding=1) | |
| self.out_conv_cls_sparse = nn.Conv2d(32, 1, 3, padding=1) | |
| self.out_conv_reg_sparse = nn.Conv2d(32, num_coefficients, 3, padding=1) | |
| def forward(self, feature): | |
| feature = F.interpolate(feature, scale_factor=self.upsample, | |
| mode="bilinear", align_corners=False) | |
| feature = self.cls_convs(feature) | |
| # ``out_conv_reg_dense`` is kept for checkpoint compatibility but never | |
| # read: decoding uses only the sparse branch. At 1600x1600 input this is | |
| # a 400x400x14 convolution that was pure waste. | |
| return { | |
| "dense_logits": self.out_conv_cls_dense(feature), | |
| "sparse_logits": self.out_conv_cls_sparse(feature), | |
| "sparse_coefficients": self.out_conv_reg_sparse(feature), | |
| } | |
| class DetectorNet(nn.Module): | |
| """Backbone, projection and head, in the order the frozen detector runs them.""" | |
| def __init__(self, num_coefficients=14, deformable=True, upsample=4): | |
| super().__init__() | |
| if upsample not in (1, 2, 4): | |
| raise ValueError("upsample must be 1, 2 or 4") | |
| self.output_stride = 16 // upsample | |
| self.backbone = HGNetv2B4Backbone(deformable=deformable) | |
| self.neck = NeckProjection(self.backbone.out_channels) | |
| self.bbox_head = PolygonHead(num_coefficients=num_coefficients, upsample=upsample) | |
| def forward(self, images): | |
| if images.ndim != 4 or any(size % 32 for size in images.shape[-2:]): | |
| raise ValueError("Expected BCHW images padded to multiples of 32") | |
| return self.bbox_head(self.neck(self.backbone(images))) | |
| def polygon_points(coefficients, basis, stride): | |
| """``[B,K,H,W]`` coefficients -> ``[B,H*W,P,2]`` pixel coords. | |
| """ | |
| batch, _, height, width = coefficients.shape | |
| basis = torch.as_tensor(basis, dtype=torch.float32, device=coefficients.device) | |
| points = coefficients.permute(0, 2, 3, 1).reshape(batch, height * width, -1) @ basis | |
| yy, xx = torch.meshgrid( | |
| torch.arange(height, device=coefficients.device, dtype=torch.float32), | |
| torch.arange(width, device=coefficients.device, dtype=torch.float32), | |
| indexing="ij", | |
| ) | |
| locations = torch.stack((xx.reshape(-1), yy.reshape(-1)), dim=1) | |
| locations = locations.repeat(1, basis.shape[1] // 2) | |
| return (points + locations) * stride | |
| def _drop_degenerate(polygons, content_shape, config): | |
| """Mask out boxes too small to hold readable text. | |
| The optional area cap is disabled by default -- see the comment below for | |
| why "covers half the image" is not a reliable signal. | |
| """ | |
| flat = polygons.reshape(len(polygons), -1, 2) | |
| wh = (flat.amax(dim=1) - flat.amin(dim=1)).clamp(min=0) | |
| min_side = float(config.get("min_box_side", 4.0)) | |
| keep = (wh[:, 0] >= min_side) & (wh[:, 1] >= min_side) | |
| # Area cap is OFF by default: a single-line crop legitimately fills most of | |
| # its own image, so "covers half the page" is not evidence of a bad box. | |
| # The few page-sized regions it would catch are not worth deleting whole | |
| # lines from cropped inputs. | |
| fraction = float(config.get("max_box_area_fraction", 1.0)) | |
| if content_shape is not None and fraction < 1.0: | |
| height, width = content_shape | |
| keep = keep & ((wh[:, 0] * wh[:, 1]) <= fraction * float(height * width)) | |
| return keep | |
| def _suppress_contained(polygons, scores, contain=0.95, ratio=0.35): | |
| """Drop boxes nested inside a larger box. | |
| Measured on 3544 pages: 54% of images contain nested boxes and 40% of them | |
| repeat text already returned by the enclosing line. Containment (not IoU) | |
| is the right test -- a small box inside a large one has a low IoU, so an | |
| IoU filter never fires. | |
| The ratio guard keeps legitimate nesting: a heading inside a paragraph block | |
| is usually more than a third of its area, so it survives. | |
| """ | |
| n = len(polygons) | |
| if n < 2: | |
| return polygons, scores | |
| flat = polygons.reshape(n, -1, 2) | |
| lo, hi = flat.amin(dim=1), flat.amax(dim=1) | |
| areas = (hi - lo).clamp(min=0).prod(dim=1) | |
| order = torch.argsort(areas, descending=True) | |
| lo, hi, areas = lo[order], hi[order], areas[order] | |
| alive = torch.ones(n, dtype=torch.bool, device=areas.device) | |
| for pos in range(n - 1): | |
| if not bool(alive[pos]): | |
| continue | |
| candidate = alive[pos + 1:] | |
| if not bool(candidate.any()): | |
| continue | |
| c_lo, c_hi, c_area = lo[pos + 1:], hi[pos + 1:], areas[pos + 1:] | |
| ix0 = torch.maximum(lo[pos][0], c_lo[:, 0]) | |
| iy0 = torch.maximum(lo[pos][1], c_lo[:, 1]) | |
| ix1 = torch.minimum(hi[pos][0], c_hi[:, 0]) | |
| iy1 = torch.minimum(hi[pos][1], c_hi[:, 1]) | |
| inter = (ix1 - ix0).clamp(min=0) * (iy1 - iy0).clamp(min=0) | |
| nested = (inter / c_area.clamp(min=1e-6) >= contain) & ( | |
| c_area / areas[pos].clamp(min=1e-6) <= ratio | |
| ) | |
| alive[pos + 1:] = candidate & ~nested | |
| keep = torch.zeros(n, dtype=torch.bool, device=areas.device) | |
| keep[order] = alive | |
| return polygons[keep], scores[keep] | |
| def decode_detection(predictions, basis, config, content_shape=None, scale=None): | |
| """Scores plus polygons to accepted boxes, mirroring the frozen decoder. | |
| ``config`` needs ``stride``, ``score_threshold``, ``nms_threshold`` and | |
| ``max_candidates``. ``content_shape`` (h, w pre-padding) masks padding | |
| cells; ``scale`` divides polygons back to original image coordinates. | |
| """ | |
| stride = config["stride"] | |
| points = polygon_points( | |
| predictions["sparse_coefficients"].float(), basis, stride | |
| ) | |
| confidence = ( | |
| predictions["dense_logits"].sigmoid() | |
| * predictions["sparse_logits"].sigmoid() | |
| )[:, 0] | |
| if content_shape is not None: | |
| height, width = content_shape | |
| confidence = confidence.clone() | |
| confidence[:, (height + stride - 1) // stride :] = 0 | |
| confidence[:, :, (width + stride - 1) // stride :] = 0 | |
| results = [] | |
| for polygons, scores in zip(points, confidence.flatten(1)): | |
| selected = (scores > config["score_threshold"]).nonzero(as_tuple=True)[0] | |
| if selected.numel() > config["max_candidates"]: | |
| selected = selected[ | |
| scores[selected].topk(config["max_candidates"]).indices | |
| ] | |
| polygons, scores = polygons[selected], scores[selected] | |
| if polygons.numel(): | |
| flat = polygons.reshape(len(polygons), -1, 2) | |
| boxes = torch.cat((flat.amin(dim=1), flat.amax(dim=1)), dim=1) | |
| keep = nms(boxes, scores, config["nms_threshold"]) | |
| polygons, scores = polygons[keep], scores[keep] | |
| # Added post-processing, all geometry-only so it runs before recognition. | |
| keep = _drop_degenerate(polygons, content_shape, config) | |
| polygons, scores = polygons[keep], scores[keep] | |
| if polygons.numel() and config.get("deduplicate_nested", False): | |
| polygons, scores = _suppress_contained( | |
| polygons, scores, | |
| float(config.get("contain_threshold", 0.95)), | |
| float(config.get("contain_area_ratio", 0.35)), | |
| ) | |
| if content_shape is not None and polygons.numel(): | |
| # Clamp to the content area, not the padded canvas: a polygon | |
| # that runs into the padding would otherwise be cropped with | |
| # replicated border pixels. | |
| content_h, content_w = content_shape | |
| view = polygons.reshape(len(polygons), -1, 2) | |
| view[..., 0].clamp_(0, content_w - 1) | |
| view[..., 1].clamp_(0, content_h - 1) | |
| if scale is not None: | |
| polygons = polygons / torch.as_tensor( | |
| scale, dtype=polygons.dtype, device=polygons.device | |
| ).repeat(polygons.shape[-1] // 2) | |
| results.append( | |
| {"polygons": polygons.cpu().numpy(), "scores": scores.cpu().numpy()} | |
| ) | |
| return results | |
| MEAN = np.array([123.675, 116.28, 103.53], dtype=np.float32) | |
| STD = np.array([58.395, 57.12, 57.375], dtype=np.float32) | |
| def detection_tensor(rgb, size=1600): | |
| """Letterbox one RGB uint8 image to ``size`` and normalise. | |
| Returns ``(tensor [1,3,size,size] float32, scale [2], content (h, w))``. | |
| """ | |
| height, width = rgb.shape[:2] | |
| scale = size / max(height, width) | |
| new_w = max(1, round(width * scale)) | |
| new_h = max(1, round(height * scale)) | |
| resized = np.asarray(Image.fromarray(rgb).resize((new_w, new_h), Image.BILINEAR)) | |
| canvas = np.zeros((size, size, 3), np.float32) | |
| canvas[:new_h, :new_w] = (resized.astype(np.float32) - MEAN) / STD | |
| tensor = np.ascontiguousarray(canvas.transpose(2, 0, 1)[None]) | |
| return tensor, np.array([new_w / width, new_h / height], np.float32), (new_h, new_w) | |
| def _require_cv2(): | |
| """Import OpenCV on demand. | |
| ``transformers`` statically scans a ``trust_remote_code`` module for imports | |
| and refuses to load it when any of them is missing, so a top-level | |
| ``import cv2`` would make the detector unloadable on a machine that only | |
| wants ``detect()``. The import sits inside a ``try`` block because the | |
| scanner does not descend into those. | |
| """ | |
| try: | |
| import cv2 | |
| except ImportError as exc: # pragma: no cover - depends on the environment | |
| raise ImportError( | |
| "OpenCV is required for polygon rectification (det.ocr / the crop " | |
| "step). Install it with `pip install opencv-python-headless` -- the " | |
| "transformers hint `pip install cv2` refers to a package that does " | |
| "not exist. Detection alone (det.detect) works without it." | |
| ) from exc | |
| return cv2 | |
| def polygon_to_quad(polygon, image_shape): | |
| """Return TL/TR/BR/BL float32 corners compatible with OpenOCR's cropper. | |
| Uses OpenOCR's minAreaRect corner convention (sort by x, then split into | |
| left/right pairs), without the int16 coordinate limit. | |
| """ | |
| points = np.asarray(polygon, dtype=np.float32).reshape(-1, 2).copy() | |
| height, width = image_shape[:2] | |
| if len(points) < 3 or not np.isfinite(points).all() or min(height, width) < 2: | |
| return None | |
| cv2 = _require_cv2() | |
| points[:, 0] = np.clip(points[:, 0], 0, width - 1) | |
| points[:, 1] = np.clip(points[:, 1], 0, height - 1) | |
| rect = cv2.minAreaRect(points) | |
| if min(rect[1]) < 2: | |
| return None | |
| corners = cv2.boxPoints(rect) | |
| corners = corners[np.argsort(corners[:, 0], kind="stable")] | |
| left = corners[:2][np.argsort(corners[:2, 1], kind="stable")] | |
| right = corners[2:][np.argsort(corners[2:, 1], kind="stable")] | |
| quad = np.asarray([left[0], right[0], right[1], left[1]], dtype=np.float32) | |
| quad[:, 0] = np.clip(quad[:, 0], 0, width - 1) | |
| quad[:, 1] = np.clip(quad[:, 1], 0, height - 1) | |
| crop_width = max( | |
| np.linalg.norm(quad[0] - quad[1]), np.linalg.norm(quad[2] - quad[3]) | |
| ) | |
| crop_height = max( | |
| np.linalg.norm(quad[0] - quad[3]), np.linalg.norm(quad[1] - quad[2]) | |
| ) | |
| if min(crop_width, crop_height) < 2 or abs(cv2.contourArea(quad)) < 1: | |
| return None | |
| return quad | |
| def get_rotate_crop_image(img, points): | |
| """Four-point perspective crop, identical to the frozen OpenOCR cropper. | |
| Tall crops (height / width >= 1.5) are rotated 90 degrees so vertical text | |
| is fed horizontally; dropping that step costs accuracy on vertical samples. | |
| """ | |
| points = np.asarray(points, dtype=np.float32).reshape(4, 2) | |
| cv2 = _require_cv2() | |
| img_crop_width = int( | |
| max(np.linalg.norm(points[0] - points[1]), np.linalg.norm(points[2] - points[3])) | |
| ) | |
| img_crop_height = int( | |
| max(np.linalg.norm(points[0] - points[3]), np.linalg.norm(points[1] - points[2])) | |
| ) | |
| pts_std = np.float32( | |
| [ | |
| [0, 0], | |
| [img_crop_width, 0], | |
| [img_crop_width, img_crop_height], | |
| [0, img_crop_height], | |
| ] | |
| ) | |
| matrix = cv2.getPerspectiveTransform(points, pts_std) | |
| cropped = cv2.warpPerspective( | |
| img, | |
| matrix, | |
| (img_crop_width, img_crop_height), | |
| borderMode=cv2.BORDER_REPLICATE, | |
| flags=cv2.INTER_CUBIC, | |
| ) | |
| height, width = cropped.shape[0:2] | |
| if width and height * 1.0 / width >= 1.5: | |
| cropped = np.rot90(cropped) | |
| return cropped | |
| class OCRPipeline: | |
| """Detector plus one recogniser, in the frozen pipeline's order. | |
| Detection runs per image, polygons become upright quads, crops are sorted | |
| and bucketed by recognition shape, and regions come back in reading order | |
| (top-to-bottom, then left-to-right of each quad's top-left corner). | |
| """ | |
| def __init__(self, det_model, rec_model, batch_size=32, drop_score=0.0): | |
| self.det_model = det_model | |
| self.rec_model = rec_model | |
| self.batch_size = batch_size | |
| self.drop_score = drop_score | |
| def __call__(self, image, color="rgb"): | |
| """``image`` is a path, bytes, PIL image or HxWx3 array. | |
| Returns ``{"text": [str], "regions": [{text, score, quad, ...}]}``. | |
| """ | |
| return self.run(self._read(image, color)) | |
| def run(self, rgb): | |
| regions = self.detect(rgb) | |
| if not regions: | |
| return {"text": [], "regions": []} | |
| # ``recognition_batches`` expects BGR crops and converts them to RGB | |
| # itself, so the crop stays in BGR rather than being flipped back here. | |
| bgr = np.ascontiguousarray(rgb[:, :, ::-1]) | |
| crops = [get_rotate_crop_image(bgr, region["quad"]) for region in regions] | |
| recognised = self.recognize(crops) | |
| keep = [] | |
| for region, (text, score) in zip(regions, recognised): | |
| if score is not None and not np.isnan(score) and score < self.drop_score: | |
| continue | |
| region = dict(region) | |
| region["text"] = text | |
| region["rec_score"] = None if score is None else float(score) | |
| keep.append(region) | |
| return {"text": [region["text"] for region in keep], "regions": keep} | |
| def detect(self, rgb): | |
| raw = self.det_model.detect([rgb], batch_size=1)[0] | |
| regions = [] | |
| for polygon, score in zip(raw["polygons"], raw["scores"]): | |
| quad = polygon_to_quad(polygon, rgb.shape) | |
| if quad is None: | |
| continue | |
| regions.append( | |
| {"polygon": np.asarray(polygon), "quad": quad, "det_score": float(score)} | |
| ) | |
| return sorted(regions, key=lambda row: (row["quad"][0, 1], row["quad"][0, 0])) | |
| def recognize(self, crops): | |
| return self.rec_model.recognize(crops, batch_size=self.batch_size) | |
| def _read(image, color): | |
| from PIL import Image, ImageOps | |
| def _to_rgb(im): | |
| """Composite alpha over white, apply EXIF orientation, return RGB.""" | |
| im = ImageOps.exif_transpose(im) | |
| if im.mode in ("RGBA", "LA") or "A" in im.mode: | |
| bg = Image.new("RGBA", im.size, (255, 255, 255, 255)) | |
| im = Image.alpha_composite(bg, im.convert("RGBA")).convert("RGB") | |
| elif im.mode == "P": | |
| im = im.convert("RGBA") | |
| bg = Image.new("RGBA", im.size, (255, 255, 255, 255)) | |
| im = Image.alpha_composite(bg, im).convert("RGB") | |
| else: | |
| im = im.convert("RGB") | |
| return im | |
| if isinstance(image, np.ndarray): | |
| if image.ndim == 2: | |
| image = np.stack([image] * 3, axis=-1) | |
| if color == "bgr": | |
| return np.ascontiguousarray(image[:, :, ::-1]) | |
| return np.ascontiguousarray(image) | |
| if isinstance(image, (bytes, bytearray)): | |
| import io | |
| with Image.open(io.BytesIO(image)) as im: | |
| return np.asarray(_to_rgb(im)) | |
| if isinstance(image, (str, bytes)) or hasattr(image, "__fspath__"): | |
| with Image.open(image) as im: | |
| return np.asarray(_to_rgb(im)) | |
| return np.asarray(_to_rgb(image)) | |
| class OpenOCRv2DetConfig(PretrainedConfig): | |
| """Configuration for the OpenOCRv2 detector (``OpenOCRv2_Det``). | |
| The defaults are the frozen post-processing constants from | |
| ``models/manifest.json`` (``models.det``); overriding them changes which | |
| boxes survive, not just the scores. | |
| """ | |
| model_type = "openocrv2_det" | |
| def __init__( | |
| self, | |
| image_size=1600, | |
| stride=4, | |
| score_threshold=0.2, | |
| nms_threshold=0.5, | |
| max_candidates=2000, | |
| num_coefficients=14, | |
| upsample=4, | |
| deformable=True, | |
| architecture="hgnetv2_b4", | |
| **kwargs, | |
| ): | |
| super().__init__(**kwargs) | |
| self.image_size = image_size | |
| self.stride = stride | |
| self.score_threshold = score_threshold | |
| self.nms_threshold = nms_threshold | |
| self.max_candidates = max_candidates | |
| self.num_coefficients = num_coefficients | |
| self.upsample = upsample | |
| self.deformable = deformable | |
| self.architecture = architecture | |
| def decode_config(self): | |
| return { | |
| "stride": self.stride, | |
| "score_threshold": self.score_threshold, | |
| "nms_threshold": self.nms_threshold, | |
| "max_candidates": self.max_candidates, | |
| # Nested-box suppression is OFF by default. It removes text that a | |
| # downstream metric may expect, and ANLS never rewards removing it | |
| # (measured: -0.43 points on a 54-image ablation). Turn it on for | |
| # pseudo-labelling, where the goal is clean boxes for annotators. | |
| "deduplicate_nested": getattr(self, "deduplicate_nested", False), | |
| "contain_threshold": getattr(self, "contain_threshold", 0.95), | |
| "contain_area_ratio": getattr(self, "contain_area_ratio", 0.35), | |
| "min_box_side": getattr(self, "min_box_side", 4.0), | |
| } | |
| class OpenOCRv2DetOutput(ModelOutput): | |
| """Raw detection head outputs plus absolute pixel polygons.""" | |
| dense_logits: torch.FloatTensor | None = None | |
| sparse_logits: torch.FloatTensor | None = None | |
| sparse_coefficients: torch.FloatTensor | None = None | |
| class OpenOCRv2DetModel(PreTrainedModel): | |
| """OpenOCRv2 detector: dense/sparse logits plus 14 polygon coefficients.""" | |
| config_class = OpenOCRv2DetConfig | |
| main_input_name = "pixel_values" | |
| def __init__(self, config): | |
| super().__init__(config) | |
| self.model = DetectorNet( | |
| num_coefficients=config.num_coefficients, | |
| deformable=config.deformable, | |
| upsample=config.upsample, | |
| ) | |
| # Polygon basis (14 x 28). A buffer, so it travels with the checkpoint. | |
| self.register_buffer( | |
| "basis", torch.zeros(config.num_coefficients, config.num_coefficients * 2) | |
| ) | |
| self.post_init() | |
| def forward(self, pixel_values, **kwargs): | |
| predictions = self.model(pixel_values) | |
| return OpenOCRv2DetOutput( | |
| dense_logits=predictions["dense_logits"], | |
| sparse_logits=predictions["sparse_logits"], | |
| sparse_coefficients=predictions["sparse_coefficients"], | |
| ) | |
| def detect(self, images, batch_size=1): | |
| """Run detection on a list of RGB uint8 arrays. | |
| Returns one dict per image with ``polygons``, ``quads``, ``scores`` and the | |
| polygons expressed in original-image pixels. | |
| """ | |
| self.eval() | |
| results = [] | |
| device = next(self.parameters()).device | |
| for start in range(0, len(images), batch_size): | |
| chunk = images[start : start + batch_size] | |
| tensors, scales, contents = [], [], [] | |
| for rgb in chunk: | |
| tensor, scale, content = detection_tensor( | |
| rgb, self.config.image_size | |
| ) | |
| tensors.append(tensor[0]) | |
| scales.append(scale) | |
| contents.append(content) | |
| pixel_values = torch.from_numpy(np.stack(tensors)).to(device) | |
| with torch.no_grad(): | |
| outputs = self(pixel_values=pixel_values) | |
| for index, (scale, content) in enumerate(zip(scales, contents)): | |
| single = { | |
| "dense_logits": outputs.dense_logits[index : index + 1], | |
| "sparse_logits": outputs.sparse_logits[index : index + 1], | |
| "sparse_coefficients": outputs.sparse_coefficients[index : index + 1], | |
| } | |
| decoded = decode_detection( | |
| single, | |
| self.basis, | |
| self.config.decode_config, | |
| content_shape=content, | |
| scale=scale, | |
| )[0] | |
| results.append(decoded) | |
| return results | |
| def ocr(self, image, rec_model, batch_size=32, drop_score=0.0): | |
| """End-to-end OCR on one page; ``rec_model`` is one of the recognizers. | |
| Detect, rectify every polygon into an upright crop, recognise, and return | |
| the regions in reading order. This is the same sequence as | |
| ``OCRPipeline`` -- it just saves the caller from tying the two halves | |
| together (and from re-implementing the cropper, which is where accuracy | |
| is most easily lost). | |
| ``image`` is a path, bytes, PIL image or HxWx3 uint8 array. | |
| Returns ``{"text": [str], "regions": [{text, rec_score, det_score, quad}]}``. | |
| """ | |
| return OCRPipeline( | |
| self, rec_model, batch_size=batch_size, drop_score=drop_score | |
| )(image) | |