File size: 6,423 Bytes
0f3cc90
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
"""Retrain the page bubble detector with the preserved OCR page splits."""
import argparse
import hashlib
import json
import shutil
import time
from pathlib import Path

import torch
import yaml
from ultralytics import YOLO


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', type=Path, required=True)
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--epochs', type=int, default=100)
    parser.add_argument('--batch', type=int, default=16)
    parser.add_argument('--patience', type=int, default=20)
    parser.add_argument('--model', default='yolo26n.pt')
    parser.add_argument('--learning-rate', type=float, default=.001)
    parser.add_argument('--mosaic', type=float, default=.5)
    parser.add_argument('--scale', type=float, default=.25)
    parser.add_argument('--translate', type=float, default=.05)
    parser.add_argument('--close-mosaic', type=int, default=10)
    parser.add_argument('--warmup-epochs', type=float, default=3.)
    parser.add_argument('--warmup-bias-lr', type=float, default=.1)
    parser.add_argument('--freeze-batchnorm', action='store_true')
    parser.add_argument('--initial-eval', action='store_true')
    parser.add_argument('--no-amp', action='store_true')
    parser.add_argument('--freeze-backbone', action='store_true')
    parser.add_argument('--image-size', type=int, default=800)
    args = parser.parse_args()
    if not torch.cuda.is_available():
        raise RuntimeError('Refusing full training without CUDA')
    summary = json.loads((args.dataset / 'dataset_summary.json').read_text())
    if summary['smoke'] or summary['limited_recognition']:
        raise ValueError('A complete production dataset is required')
    pages = json.loads((args.dataset / 'detector/pages.json').read_text())
    ids = [p['id'] for p in pages]
    if len(ids) != len(set(ids)):
        raise ValueError('Page leakage in detector dataset')
    args.output.mkdir(parents=True, exist_ok=True)
    config = dict(path=str((args.dataset / 'detector').resolve()), train='train/images', val='val/images', test='test/images', names={0: 'bubble'})
    yaml_path = args.output / 'data.yaml'
    yaml_path.write_text(yaml.safe_dump(config), encoding='utf-8')
    model = YOLO(args.model)
    if args.freeze_batchnorm:
        def freeze_batchnorm(trainer):
            for module in trainer.model.modules():
                if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):
                    module.eval()
        model.add_callback('on_train_batch_start', freeze_batchnorm)
    # First stage starts from generic weights because the previous published
    # detector splits are not fully recoverable. A polish stage uses its best.pt.
    base_path = Path(model.ckpt_path)
    provenance = dict(base_model=str(base_path), base_sha256=hashlib.sha256(base_path.read_bytes()).hexdigest(),
                      dataset_plan_sha256=hashlib.sha256((args.dataset / 'plan.json').read_bytes()).hexdigest(),
                      gpu=torch.cuda.get_device_name(0), seed=42, image_size=args.image_size,
                      annotation_note='All known annotations validated; completeness unproven. Historical holdouts were already inspected.')
    (args.output / 'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
    started = time.time()
    initial_validation = None
    if args.initial_eval:
        # Ultralytics standalone validation fuses Conv+BN in place. Evaluate a
        # separate instance so training still loads every BN parameter/buffer.
        initial_validation = YOLO(args.model).val(data=str(yaml_path), split='val', imgsz=args.image_size, device=0, plots=False).results_dict
        (args.output / 'initial_validation.json').write_text(json.dumps(initial_validation, indent=2), encoding='utf-8')
        expected_state = {k: v.detach().cpu().clone() for k, v in model.model.state_dict().items()}
        def verify_loaded_state(trainer):
            loaded = trainer.model.state_dict()
            changed = [k for k, value in expected_state.items() if k not in loaded or not torch.equal(value, loaded[k].detach().cpu())]
            if changed:
                raise ValueError(f'Training did not preserve the recovered initial state: {changed[:12]}')
            print(json.dumps({'verified_initial_tensors': len(expected_state)}), flush=True)
        model.add_callback('on_pretrain_routine_end', verify_loaded_state)
    result = model.train(data=str(yaml_path), epochs=args.epochs, imgsz=args.image_size, batch=args.batch, patience=args.patience,
                         device=0, workers=4, seed=42, deterministic=True, amp=not args.no_amp,
                         project=str(args.output), name='run', exist_ok=False,
                         freeze=len(model.model.model)-1 if args.freeze_backbone else None,
                         optimizer='AdamW', lr0=args.learning_rate, lrf=.01, cos_lr=True,
                         warmup_epochs=args.warmup_epochs, warmup_bias_lr=args.warmup_bias_lr,
                         fliplr=0., flipud=0., degrees=0., hsv_h=0., hsv_s=0., hsv_v=.1,
                         translate=args.translate, scale=args.scale, mosaic=args.mosaic, close_mosaic=args.close_mosaic, cache=False, plots=False)
    best = Path(result.save_dir) / 'weights/best.pt'
    best_model = YOLO(best)
    val = best_model.val(data=str(yaml_path), split='val', imgsz=args.image_size, device=0, plots=False)
    selected_initial = bool(initial_validation and val.results_dict['metrics/mAP50-95(B)'] <= initial_validation['metrics/mAP50-95(B)'])
    if selected_initial:
        shutil.copy2(base_path, args.output / 'selected.pt')
        best_model = YOLO(args.output / 'selected.pt')
    onnx_path = best_model.export(format='onnx', imgsz=args.image_size, batch=1, dynamic=False, simplify=False, opset=12, nms=True, device='cpu')
    report = dict(best_checkpoint=str(best), onnx=str(onnx_path), seconds=time.time()-started, validation=val.results_dict,
                  initial_validation=initial_validation, selected_initial=selected_initial,
                  evaluation_note='Ultralytics validation; production preprocessing and threshold must be tested separately.')
    (args.output / 'result.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
    print(json.dumps(report), flush=True)


if __name__ == '__main__':
    main()