Remidesbois's picture
Retrain and validate production PPOCR pipeline on RunPod (2026-11-03)
0f3cc90 verified
Raw History Blame Contribute Delete
4.26 kB
"""Evaluate production-prepared inputs; save every prediction, including failures."""
import argparse
import hashlib
import json
import sys
from pathlib import Path
import numpy as np
import torch
from PIL import Image
from tqdm import tqdm
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'finetune_paddleocr_line_rec'))
from train_ppocrv6_rec import decode_ctc, load_transformers_components, pairs_metric, read_label_file
def prepared_tensor(image_path):
with Image.open(image_path) as image:
rgb = np.asarray(image.convert('RGB'), dtype=np.uint8)
if rgb.shape[0] != 48 or not 960 <= rgb.shape[1] <= 3200:
raise ValueError(f'Unexpected input shape: {image_path}: {rgb.shape}')
return (torch.from_numpy(np.ascontiguousarray(rgb[..., ::-1].transpose(2, 0, 1))).float() / 255 - .5).unsqueeze(0) / .5
def evaluate(model_source, dataset, output, split='val', limit=0, onnx_path=None):
output.mkdir(parents=True, exist_ok=True)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.set_num_threads(4)
_, processor, model = load_transformers_components(str(model_source))
model.eval().to(device)
session = None
if onnx_path:
import onnxruntime as ort
options = ort.SessionOptions(); options.intra_op_num_threads = 4
session = ort.InferenceSession(str(onnx_path), options, providers=['CPUExecutionProvider'])
labels = read_label_file(dataset, split)
if limit:
labels = labels[:limit]
rows = []
# Always start a new evaluation rather than resuming results from other weights.
prediction_path = output / f'{split}_predictions.jsonl'
with prediction_path.open('w', encoding='utf-8') as handle, torch.inference_mode():
for image_path, reference, relative in tqdm(labels, desc=f'evaluate {split}'):
inputs = prepared_tensor(image_path)
probabilities = model(pixel_values=inputs.to(device)).last_hidden_state.float().cpu()
raw = decode_ctc(probabilities, processor.character_list)[0]
row = dict(image=relative, reference=reference, raw=raw, timesteps=probabilities.shape[1], width=inputs.shape[-1])
if session:
ort_probs = session.run(None, {session.get_inputs()[0].name: inputs.numpy()})[0]
row.update(onnx_raw=decode_ctc(torch.from_numpy(ort_probs), processor.character_list)[0], max_abs_diff=float(np.abs(probabilities.numpy() - ort_probs).max()))
rows.append(row)
handle.write(json.dumps(row, ensure_ascii=False) + '\n')
predictions, references = [r['raw'] for r in rows], [r['reference'] for r in rows]
metrics = pairs_metric(predictions, references)
metrics.update(model=str(model_source), split=split, device=str(device), empty_predictions=sum(not r['raw'] for r in rows),
label_sha256=hashlib.sha256((dataset / f'rec_gt_{split}.txt').read_bytes()).hexdigest())
metrics['length_groups'] = {}
for name, lower, upper in [('short_1_12', 0, 12), ('medium_13_40', 12, 40), ('long_over_40', 40, float('inf'))]:
subset = [r for r in rows if lower < len(r['reference']) <= upper]
metrics['length_groups'][name] = pairs_metric([r['raw'] for r in subset], [r['reference'] for r in subset])
if session:
metrics['onnx_parity'] = dict(samples=len(rows), text_matches=sum(r['raw'] == r['onnx_raw'] for r in rows), max_abs_diff=max((r['max_abs_diff'] for r in rows), default=0))
(output / f'{split}_metrics.json').write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(metrics, ensure_ascii=False), flush=True)
return metrics
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--model', required=True, type=Path)
parser.add_argument('--dataset', required=True, type=Path)
parser.add_argument('--output', required=True, type=Path)
parser.add_argument('--split', choices=['train', 'val', 'test'], default='val')
parser.add_argument('--limit', type=int, default=0)
parser.add_argument('--onnx', type=Path)
args = parser.parse_args()
evaluate(args.model, args.dataset, args.output, args.split, args.limit, args.onnx)