"""Evaluate a RAQEEB checkpoint with per-image label-set filtering; on val, select per-class thresholds. python evaluate_raqeeb.py --weights ..\\runs\\raqeeb_v1\\weights\\best.pt --split val # writes thresholds python evaluate_raqeeb.py --weights ..\\runs\\raqeeb_v1\\weights\\best.pt --split test # final held-out report Each image is scored only on the classes its source exhaustively annotates (see dataset/label_sets.json). Runs on physical GPU 1 (headless) by default. """ import argparse import json import os import sys from datetime import datetime, timezone from pathlib import Path RAQ = Path(r'D:\CCTV_AI_FLOORPLAN_data\RAQEEB CCTV') NAMES = ['wall', 'door', 'window', 'open_passage', 'column', 'stair', 'elevator', 'escalator', 'exit_marker', 'gate', 'space_region'] GPU1_UUID = 'GPU-f4eadf9c-df66-df50-7149-fe323b79330e' MIN_VAL_INSTANCES = 30 # below this, a class threshold is not considered validated def main(): ap = argparse.ArgumentParser() ap.add_argument('--weights', required=True, type=Path) ap.add_argument('--split', choices=['val', 'test'], default='val') ap.add_argument('--out', type=Path) ap.add_argument('--gpu', default=GPU1_UUID, help='CUDA_VISIBLE_DEVICES value (default: physical GPU 1)') ap.add_argument('--imgsz', type=int, default=1024) ap.add_argument('--batch', type=int, default=8) a = ap.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = a.gpu os.environ.setdefault('YOLO_CONFIG_DIR', str(RAQ / 'ultralytics_settings')) os.environ.setdefault('YOLO_OFFLINE', 'true') sys.path.insert(0, str(Path(__file__).resolve().parent)) import numpy as np from ultralytics.utils import colorstr from raqeeb_trainer import RaqeebDataset, RaqeebSegValidator class EvalValidator(RaqeebSegValidator): def build_dataset(self, img_path, mode='val', batch=None): return RaqeebDataset(img_path=img_path, imgsz=self.args.imgsz, batch_size=batch, augment=False, hyp=self.args, rect=True, cache=None, single_cls=False, stride=self.stride, pad=0.5, prefix=colorstr(f'{mode}: '), task='segment', classes=None, data=self.data, fraction=1.0) out = a.out or RAQ / 'evaluation' / f'{a.weights.parent.parent.name}_{a.split}' out.mkdir(parents=True, exist_ok=True) args = dict(model=str(a.weights), data=str(RAQ / 'dataset' / 'data.yaml'), split=a.split, imgsz=a.imgsz, batch=a.batch, conf=0.001, iou=0.7, max_det=1500, plots=True, task='segment', mode='val', device='0', project=str(out.parent), name=out.name, exist_ok=True, half=True, rect=True, workers=8, save_json=False) v = EvalValidator(args=args) v() seg, box = v.metrics.seg, v.metrics.box nt = v.metrics.nt_per_class # instances per class (after label-set filtering) ni = v.metrics.nt_per_image per_class, thresholds = {}, {} for k, c in enumerate(seg.ap_class_index): c = int(c) p, r, ap50, ap = seg.class_result(k) bp, br, bap50, bap = box.class_result(k) f1 = seg.f1_curve[k] best = int(np.argmax(f1)) thr = float(np.clip(seg.px[best], 0.05, 0.9)) per_class[NAMES[c]] = dict(id=c, instances=int(nt[c]), images=int(ni[c]), mask_precision=round(float(p), 4), mask_recall=round(float(r), 4), mask_mAP50=round(float(ap50), 4), mask_mAP50_95=round(float(ap), 4), box_mAP50=round(float(bap50), 4), box_mAP50_95=round(float(bap), 4), max_f1=round(float(f1[best]), 4), max_f1_confidence=round(thr, 4)) thresholds[str(c)] = round(thr, 4) absent = [NAMES[c] for c in range(len(NAMES)) if NAMES[c] not in per_class] result = dict( weights=str(a.weights.resolve()), split=a.split, created_utc=datetime.now(timezone.utc).isoformat(), images=int(v.seen), imgsz=a.imgsz, scoring='per-image label-set filtering: each image scored only on the classes its source exhaustively annotates', overall=dict(mask_mAP50=round(float(seg.map50), 4), mask_mAP50_95=round(float(seg.map), 4), box_mAP50=round(float(box.map50), 4), box_mAP50_95=round(float(box.map), 4), mask_precision=round(float(seg.mp), 4), mask_recall=round(float(seg.mr), 4)), per_class=per_class, classes_without_instances_in_split=absent) (out / 'metrics.json').write_text(json.dumps(result, indent=2)) lines = [f"RAQEEB {a.split} evaluation — {a.weights}", f"images: {v.seen}", f"overall mask mAP50 {seg.map50:.3f} mask mAP50-95 {seg.map:.3f} box mAP50-95 {box.map:.3f}", '', f"{'class':<14}{'inst':>8}{'imgs':>7}{'P':>8}{'R':>8}{'mAP50':>8}{'mAP50-95':>10}{'F1*':>7}{'conf*':>7}"] for n, d in per_class.items(): lines.append(f"{n:<14}{d['instances']:>8}{d['images']:>7}{d['mask_precision']:>8.3f}{d['mask_recall']:>8.3f}" f"{d['mask_mAP50']:>8.3f}{d['mask_mAP50_95']:>10.3f}{d['max_f1']:>7.3f}{d['max_f1_confidence']:>7.3f}") if absent: lines.append(f"no instances in this split: {', '.join(absent)}") lines.append('P/R are at the max-F1 confidence of all classes combined (Ultralytics convention); ' 'F1*/conf* are each class\'s own best F1 and its confidence.') (out / 'METRICS.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8') print('\n'.join(lines)) if a.split == 'val': unvalidated = sorted({c for c in range(len(NAMES)) if NAMES[c] not in per_class or per_class[NAMES[c]]['instances'] < MIN_VAL_INSTANCES}) for c in range(len(NAMES)): thresholds.setdefault(str(c), 0.5) cfg = dict(class_thresholds=thresholds, candidate_confidence_floor=0.05, review_candidate_floor=0.1, review_confidence=0.5, unvalidated_class_ids=unvalidated, selection=f'per-class max mask-F1 confidence on the RAQEEB val split ({v.seen} images), clipped ' f'to [0.05, 0.9]; classes with < {MIN_VAL_INSTANCES} val instances are flagged unvalidated', weights=str(a.weights.resolve())) (out / 'thresholds.json').write_text(json.dumps(cfg, indent=2)) print('thresholds ->', out / 'thresholds.json') if __name__ == '__main__': main()