File size: 6,514 Bytes
d0f2b17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
"""Evaluate a RAQEEB checkpoint with per-image label-set filtering; on val, select per-class thresholds.

  python evaluate_raqeeb.py --weights ..\\runs\\raqeeb_v1\\weights\\best.pt --split val    # writes thresholds
  python evaluate_raqeeb.py --weights ..\\runs\\raqeeb_v1\\weights\\best.pt --split test   # final held-out report

Each image is scored only on the classes its source exhaustively annotates (see dataset/label_sets.json).
Runs on physical GPU 1 (headless) by default.
"""
import argparse
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path

RAQ = Path(r'D:\CCTV_AI_FLOORPLAN_data\RAQEEB CCTV')
NAMES = ['wall', 'door', 'window', 'open_passage', 'column', 'stair', 'elevator', 'escalator', 'exit_marker',
         'gate', 'space_region']
GPU1_UUID = 'GPU-f4eadf9c-df66-df50-7149-fe323b79330e'
MIN_VAL_INSTANCES = 30  # below this, a class threshold is not considered validated


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--weights', required=True, type=Path)
    ap.add_argument('--split', choices=['val', 'test'], default='val')
    ap.add_argument('--out', type=Path)
    ap.add_argument('--gpu', default=GPU1_UUID, help='CUDA_VISIBLE_DEVICES value (default: physical GPU 1)')
    ap.add_argument('--imgsz', type=int, default=1024)
    ap.add_argument('--batch', type=int, default=8)
    a = ap.parse_args()
    os.environ['CUDA_VISIBLE_DEVICES'] = a.gpu
    os.environ.setdefault('YOLO_CONFIG_DIR', str(RAQ / 'ultralytics_settings'))
    os.environ.setdefault('YOLO_OFFLINE', 'true')
    sys.path.insert(0, str(Path(__file__).resolve().parent))
    import numpy as np
    from ultralytics.utils import colorstr
    from raqeeb_trainer import RaqeebDataset, RaqeebSegValidator

    class EvalValidator(RaqeebSegValidator):
        def build_dataset(self, img_path, mode='val', batch=None):
            return RaqeebDataset(img_path=img_path, imgsz=self.args.imgsz, batch_size=batch, augment=False,
                                 hyp=self.args, rect=True, cache=None, single_cls=False, stride=self.stride, pad=0.5,
                                 prefix=colorstr(f'{mode}: '), task='segment', classes=None, data=self.data,
                                 fraction=1.0)

    out = a.out or RAQ / 'evaluation' / f'{a.weights.parent.parent.name}_{a.split}'
    out.mkdir(parents=True, exist_ok=True)
    args = dict(model=str(a.weights), data=str(RAQ / 'dataset' / 'data.yaml'), split=a.split, imgsz=a.imgsz,
                batch=a.batch, conf=0.001, iou=0.7, max_det=1500, plots=True, task='segment', mode='val', device='0',
                project=str(out.parent), name=out.name, exist_ok=True, half=True, rect=True, workers=8,
                save_json=False)
    v = EvalValidator(args=args)
    v()
    seg, box = v.metrics.seg, v.metrics.box
    nt = v.metrics.nt_per_class  # instances per class (after label-set filtering)
    ni = v.metrics.nt_per_image
    per_class, thresholds = {}, {}
    for k, c in enumerate(seg.ap_class_index):
        c = int(c)
        p, r, ap50, ap = seg.class_result(k)
        bp, br, bap50, bap = box.class_result(k)
        f1 = seg.f1_curve[k]
        best = int(np.argmax(f1))
        thr = float(np.clip(seg.px[best], 0.05, 0.9))
        per_class[NAMES[c]] = dict(id=c, instances=int(nt[c]), images=int(ni[c]), mask_precision=round(float(p), 4),
                                   mask_recall=round(float(r), 4), mask_mAP50=round(float(ap50), 4),
                                   mask_mAP50_95=round(float(ap), 4), box_mAP50=round(float(bap50), 4),
                                   box_mAP50_95=round(float(bap), 4), max_f1=round(float(f1[best]), 4),
                                   max_f1_confidence=round(thr, 4))
        thresholds[str(c)] = round(thr, 4)
    absent = [NAMES[c] for c in range(len(NAMES)) if NAMES[c] not in per_class]
    result = dict(
        weights=str(a.weights.resolve()), split=a.split, created_utc=datetime.now(timezone.utc).isoformat(),
        images=int(v.seen), imgsz=a.imgsz,
        scoring='per-image label-set filtering: each image scored only on the classes its source exhaustively annotates',
        overall=dict(mask_mAP50=round(float(seg.map50), 4), mask_mAP50_95=round(float(seg.map), 4),
                     box_mAP50=round(float(box.map50), 4), box_mAP50_95=round(float(box.map), 4),
                     mask_precision=round(float(seg.mp), 4), mask_recall=round(float(seg.mr), 4)),
        per_class=per_class, classes_without_instances_in_split=absent)
    (out / 'metrics.json').write_text(json.dumps(result, indent=2))
    lines = [f"RAQEEB {a.split} evaluation — {a.weights}", f"images: {v.seen}",
             f"overall mask mAP50 {seg.map50:.3f}  mask mAP50-95 {seg.map:.3f}  box mAP50-95 {box.map:.3f}", '',
             f"{'class':<14}{'inst':>8}{'imgs':>7}{'P':>8}{'R':>8}{'mAP50':>8}{'mAP50-95':>10}{'F1*':>7}{'conf*':>7}"]
    for n, d in per_class.items():
        lines.append(f"{n:<14}{d['instances']:>8}{d['images']:>7}{d['mask_precision']:>8.3f}{d['mask_recall']:>8.3f}"
                     f"{d['mask_mAP50']:>8.3f}{d['mask_mAP50_95']:>10.3f}{d['max_f1']:>7.3f}{d['max_f1_confidence']:>7.3f}")
    if absent:
        lines.append(f"no instances in this split: {', '.join(absent)}")
    lines.append('P/R are at the max-F1 confidence of all classes combined (Ultralytics convention); '
                 'F1*/conf* are each class\'s own best F1 and its confidence.')
    (out / 'METRICS.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8')
    print('\n'.join(lines))
    if a.split == 'val':
        unvalidated = sorted({c for c in range(len(NAMES))
                              if NAMES[c] not in per_class or per_class[NAMES[c]]['instances'] < MIN_VAL_INSTANCES})
        for c in range(len(NAMES)):
            thresholds.setdefault(str(c), 0.5)
        cfg = dict(class_thresholds=thresholds, candidate_confidence_floor=0.05, review_candidate_floor=0.1,
                   review_confidence=0.5, unvalidated_class_ids=unvalidated,
                   selection=f'per-class max mask-F1 confidence on the RAQEEB val split ({v.seen} images), clipped '
                             f'to [0.05, 0.9]; classes with < {MIN_VAL_INSTANCES} val instances are flagged unvalidated',
                   weights=str(a.weights.resolve()))
        (out / 'thresholds.json').write_text(json.dumps(cfg, indent=2))
        print('thresholds ->', out / 'thresholds.json')


if __name__ == '__main__':
    main()