Remidesbois's picture
Retrain and validate production PPOCR pipeline on RunPod (2026-11-03)
0f3cc90 verified
Raw History Blame Contribute Delete
6.42 kB
"""Retrain the page bubble detector with the preserved OCR page splits."""
import argparse
import hashlib
import json
import shutil
import time
from pathlib import Path
import torch
import yaml
from ultralytics import YOLO
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--batch', type=int, default=16)
parser.add_argument('--patience', type=int, default=20)
parser.add_argument('--model', default='yolo26n.pt')
parser.add_argument('--learning-rate', type=float, default=.001)
parser.add_argument('--mosaic', type=float, default=.5)
parser.add_argument('--scale', type=float, default=.25)
parser.add_argument('--translate', type=float, default=.05)
parser.add_argument('--close-mosaic', type=int, default=10)
parser.add_argument('--warmup-epochs', type=float, default=3.)
parser.add_argument('--warmup-bias-lr', type=float, default=.1)
parser.add_argument('--freeze-batchnorm', action='store_true')
parser.add_argument('--initial-eval', action='store_true')
parser.add_argument('--no-amp', action='store_true')
parser.add_argument('--freeze-backbone', action='store_true')
parser.add_argument('--image-size', type=int, default=800)
args = parser.parse_args()
if not torch.cuda.is_available():
raise RuntimeError('Refusing full training without CUDA')
summary = json.loads((args.dataset / 'dataset_summary.json').read_text())
if summary['smoke'] or summary['limited_recognition']:
raise ValueError('A complete production dataset is required')
pages = json.loads((args.dataset / 'detector/pages.json').read_text())
ids = [p['id'] for p in pages]
if len(ids) != len(set(ids)):
raise ValueError('Page leakage in detector dataset')
args.output.mkdir(parents=True, exist_ok=True)
config = dict(path=str((args.dataset / 'detector').resolve()), train='train/images', val='val/images', test='test/images', names={0: 'bubble'})
yaml_path = args.output / 'data.yaml'
yaml_path.write_text(yaml.safe_dump(config), encoding='utf-8')
model = YOLO(args.model)
if args.freeze_batchnorm:
def freeze_batchnorm(trainer):
for module in trainer.model.modules():
if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):
module.eval()
model.add_callback('on_train_batch_start', freeze_batchnorm)
# First stage starts from generic weights because the previous published
# detector splits are not fully recoverable. A polish stage uses its best.pt.
base_path = Path(model.ckpt_path)
provenance = dict(base_model=str(base_path), base_sha256=hashlib.sha256(base_path.read_bytes()).hexdigest(),
dataset_plan_sha256=hashlib.sha256((args.dataset / 'plan.json').read_bytes()).hexdigest(),
gpu=torch.cuda.get_device_name(0), seed=42, image_size=args.image_size,
annotation_note='All known annotations validated; completeness unproven. Historical holdouts were already inspected.')
(args.output / 'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
started = time.time()
initial_validation = None
if args.initial_eval:
# Ultralytics standalone validation fuses Conv+BN in place. Evaluate a
# separate instance so training still loads every BN parameter/buffer.
initial_validation = YOLO(args.model).val(data=str(yaml_path), split='val', imgsz=args.image_size, device=0, plots=False).results_dict
(args.output / 'initial_validation.json').write_text(json.dumps(initial_validation, indent=2), encoding='utf-8')
expected_state = {k: v.detach().cpu().clone() for k, v in model.model.state_dict().items()}
def verify_loaded_state(trainer):
loaded = trainer.model.state_dict()
changed = [k for k, value in expected_state.items() if k not in loaded or not torch.equal(value, loaded[k].detach().cpu())]
if changed:
raise ValueError(f'Training did not preserve the recovered initial state: {changed[:12]}')
print(json.dumps({'verified_initial_tensors': len(expected_state)}), flush=True)
model.add_callback('on_pretrain_routine_end', verify_loaded_state)
result = model.train(data=str(yaml_path), epochs=args.epochs, imgsz=args.image_size, batch=args.batch, patience=args.patience,
device=0, workers=4, seed=42, deterministic=True, amp=not args.no_amp,
project=str(args.output), name='run', exist_ok=False,
freeze=len(model.model.model)-1 if args.freeze_backbone else None,
optimizer='AdamW', lr0=args.learning_rate, lrf=.01, cos_lr=True,
warmup_epochs=args.warmup_epochs, warmup_bias_lr=args.warmup_bias_lr,
fliplr=0., flipud=0., degrees=0., hsv_h=0., hsv_s=0., hsv_v=.1,
translate=args.translate, scale=args.scale, mosaic=args.mosaic, close_mosaic=args.close_mosaic, cache=False, plots=False)
best = Path(result.save_dir) / 'weights/best.pt'
best_model = YOLO(best)
val = best_model.val(data=str(yaml_path), split='val', imgsz=args.image_size, device=0, plots=False)
selected_initial = bool(initial_validation and val.results_dict['metrics/mAP50-95(B)'] <= initial_validation['metrics/mAP50-95(B)'])
if selected_initial:
shutil.copy2(base_path, args.output / 'selected.pt')
best_model = YOLO(args.output / 'selected.pt')
onnx_path = best_model.export(format='onnx', imgsz=args.image_size, batch=1, dynamic=False, simplify=False, opset=12, nms=True, device='cpu')
report = dict(best_checkpoint=str(best), onnx=str(onnx_path), seconds=time.time()-started, validation=val.results_dict,
initial_validation=initial_validation, selected_initial=selected_initial,
evaluation_note='Ultralytics validation; production preprocessing and threshold must be tested separately.')
(args.output / 'result.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
print(json.dumps(report), flush=True)
if __name__ == '__main__':
main()