Poneglyph-ReaderNet / training_source /docker_scripts /retrain_ppocr_pipeline /evaluate_recognizer.py
Download training_source/docker_scripts/retrain_ppocr_pipeline/evaluate_recognizer.py from Remidesbois/Poneglyph-ReaderNet: direct link, hf CLI and curl.
- Browser
- Download file 4.26 kB
-
https://huggingface.co/Remidesbois/Poneglyph-ReaderNet/resolve/main/training_source/docker_scripts/retrain_ppocr_pipeline/evaluate_recognizer.py
- Command line
-
hf download hf://Remidesbois/Poneglyph-ReaderNet/training_source/docker_scripts/retrain_ppocr_pipeline/evaluate_recognizer.py
-
curl -L -o evaluate_recognizer.py https://huggingface.co/Remidesbois/Poneglyph-ReaderNet/resolve/main/training_source/docker_scripts/retrain_ppocr_pipeline/evaluate_recognizer.py
4.26 kB
| """Evaluate production-prepared inputs; save every prediction, including failures.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from tqdm import tqdm | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'finetune_paddleocr_line_rec')) | |
| from train_ppocrv6_rec import decode_ctc, load_transformers_components, pairs_metric, read_label_file | |
| def prepared_tensor(image_path): | |
| with Image.open(image_path) as image: | |
| rgb = np.asarray(image.convert('RGB'), dtype=np.uint8) | |
| if rgb.shape[0] != 48 or not 960 <= rgb.shape[1] <= 3200: | |
| raise ValueError(f'Unexpected input shape: {image_path}: {rgb.shape}') | |
| return (torch.from_numpy(np.ascontiguousarray(rgb[..., ::-1].transpose(2, 0, 1))).float() / 255 - .5).unsqueeze(0) / .5 | |
| def evaluate(model_source, dataset, output, split='val', limit=0, onnx_path=None): | |
| output.mkdir(parents=True, exist_ok=True) | |
| device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') | |
| torch.set_num_threads(4) | |
| _, processor, model = load_transformers_components(str(model_source)) | |
| model.eval().to(device) | |
| session = None | |
| if onnx_path: | |
| import onnxruntime as ort | |
| options = ort.SessionOptions(); options.intra_op_num_threads = 4 | |
| session = ort.InferenceSession(str(onnx_path), options, providers=['CPUExecutionProvider']) | |
| labels = read_label_file(dataset, split) | |
| if limit: | |
| labels = labels[:limit] | |
| rows = [] | |
| # Always start a new evaluation rather than resuming results from other weights. | |
| prediction_path = output / f'{split}_predictions.jsonl' | |
| with prediction_path.open('w', encoding='utf-8') as handle, torch.inference_mode(): | |
| for image_path, reference, relative in tqdm(labels, desc=f'evaluate {split}'): | |
| inputs = prepared_tensor(image_path) | |
| probabilities = model(pixel_values=inputs.to(device)).last_hidden_state.float().cpu() | |
| raw = decode_ctc(probabilities, processor.character_list)[0] | |
| row = dict(image=relative, reference=reference, raw=raw, timesteps=probabilities.shape[1], width=inputs.shape[-1]) | |
| if session: | |
| ort_probs = session.run(None, {session.get_inputs()[0].name: inputs.numpy()})[0] | |
| row.update(onnx_raw=decode_ctc(torch.from_numpy(ort_probs), processor.character_list)[0], max_abs_diff=float(np.abs(probabilities.numpy() - ort_probs).max())) | |
| rows.append(row) | |
| handle.write(json.dumps(row, ensure_ascii=False) + '\n') | |
| predictions, references = [r['raw'] for r in rows], [r['reference'] for r in rows] | |
| metrics = pairs_metric(predictions, references) | |
| metrics.update(model=str(model_source), split=split, device=str(device), empty_predictions=sum(not r['raw'] for r in rows), | |
| label_sha256=hashlib.sha256((dataset / f'rec_gt_{split}.txt').read_bytes()).hexdigest()) | |
| metrics['length_groups'] = {} | |
| for name, lower, upper in [('short_1_12', 0, 12), ('medium_13_40', 12, 40), ('long_over_40', 40, float('inf'))]: | |
| subset = [r for r in rows if lower < len(r['reference']) <= upper] | |
| metrics['length_groups'][name] = pairs_metric([r['raw'] for r in subset], [r['reference'] for r in subset]) | |
| if session: | |
| metrics['onnx_parity'] = dict(samples=len(rows), text_matches=sum(r['raw'] == r['onnx_raw'] for r in rows), max_abs_diff=max((r['max_abs_diff'] for r in rows), default=0)) | |
| (output / f'{split}_metrics.json').write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding='utf-8') | |
| print(json.dumps(metrics, ensure_ascii=False), flush=True) | |
| return metrics | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--model', required=True, type=Path) | |
| parser.add_argument('--dataset', required=True, type=Path) | |
| parser.add_argument('--output', required=True, type=Path) | |
| parser.add_argument('--split', choices=['train', 'val', 'test'], default='val') | |
| parser.add_argument('--limit', type=int, default=0) | |
| parser.add_argument('--onnx', type=Path) | |
| args = parser.parse_args() | |
| evaluate(args.model, args.dataset, args.output, args.split, args.limit, args.onnx) | |