RAQEEB-CCTV / code /infer_raqeeb.py
swedishtech's picture
RAQEEB-CCTV v1.0.0
d0f2b17 verified
Raw History Blame Contribute Delete
14.3 kB
"""RAQEEB CCTV inference: local PNG/JPEG/PDF segmentation with traceable page/tile coordinates.
Thresholds must come from the packaged validation-selected operating point.
No emergency-exit or interior/exterior role is inferred without evidence.
"""
import argparse
import hashlib
import json
import math
from pathlib import Path
import os
_RAQ = Path(__file__).resolve().parents[1]
os.environ.setdefault('YOLO_CONFIG_DIR', str(_RAQ / 'ultralytics_settings'))
os.environ.setdefault('YOLO_OFFLINE', 'true')
import cv2
import numpy as np
from PIL import Image, ImageDraw, ImageOps
from shapely import affinity, make_valid
from shapely.geometry import Polygon, mapping, shape
from shapely.ops import unary_union
from cubicasa_geometry import polygons
from ultralytics import YOLO
NAMES=['wall','door','window','open_passage','column','stair','elevator','escalator','exit_marker','gate','space_region']
COLORS=[(235,60,60),(20,190,60),(40,100,240),(255,140,30),(130,60,220),(30,190,190),(190,90,120),(190,160,20),(240,30,190),(80,120,40),(110,110,210)]
def sha256(path):
h=hashlib.sha256()
with open(path,'rb') as f:
for b in iter(lambda:f.read(4*1024*1024),b''):h.update(b)
return h.hexdigest()
def mask_geometry(mask):
contours,hierarchy=cv2.findContours(mask.astype(np.uint8),cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
if hierarchy is None:return Polygon()
parts=[];tree=hierarchy[0]
for i,c in enumerate(contours):
if tree[i][3]!=-1 or len(c)<3:continue
shell=c[:,0,:].astype(float)
holes=[];j=tree[i][2]
while j!=-1:
if len(contours[j])>=3:holes.append(contours[j][:,0,:].astype(float))
j=tree[j][0]
p=make_valid(Polygon(shell,holes))
parts.extend(x for x in polygons(p) if x.area>=1)
return unary_union(parts)
def origins(length,size,overlap):
if length<=size:return [0]
vals=list(range(0,length-size+1,size-overlap))
if vals[-1]!=length-size:vals.append(length-size)
return vals
def axis(poly):
c=list(poly.minimum_rotated_rectangle.exterior.coords)
v=max((np.array(c[i+1])-np.array(c[i]) for i in range(4)),key=lambda a:np.linalg.norm(a))
return v/max(np.linalg.norm(v),1e-8)
def merge_instances(items,cell=512):
# Greedy union is recorded and uncertain seam matches remain review items.
# A per-class grid over group bounds limits comparisons to groups that can intersect (exact: inter>0 is
# required for a merge), so large tiled CAD sheets with thousands of detections stay fast.
groups=[];grid={}
def keys(cid,b):
return [(cid,i,j) for i in range(int(b[0]//cell),int(b[2]//cell)+1) for j in range(int(b[1]//cell),int(b[3]//cell)+1)]
def register(g):
for k in keys(g['class_id'],g['geometry'].bounds):grid.setdefault(k,{})[g['_order']]=g
for item in sorted(items,key=lambda x:x['confidence'],reverse=True):
best=None; score=0
cand={}
for k in keys(item['class_id'],item['geometry'].bounds):cand.update(grid.get(k,{}))
for _,g in sorted(cand.items()):
if set(g['tiles']) & set(item['tiles']):continue
inter=g['geometry'].intersection(item['geometry']).area
if inter<=0:continue
small=inter/min(g['geometry'].area,item['geometry'].area)
iou=inter/(g['geometry'].area+item['geometry'].area-inter)
seam=(item['class_id']==0 and item['tile_edge'] and g['tile_edge'] and small>.15 and abs(float(axis(g['geometry'])@axis(item['geometry'])))>.985)
if (iou>.5 or small>.8 or seam) and small>score:best=g;score=small
if best is None:
item['_order']=len(groups);groups.append(item);register(item)
else:
best['geometry']=unary_union([best['geometry'],item['geometry']])
best['tiles']+=item['tiles'];best['tile_edge']|=item['tile_edge']
best['merged_instances']+=1
best['seam_merge']|=(score<=.8)
register(best)
for g in groups:g.pop('_order',None)
return groups
def image_pages(path,dpi,pages):
if path.suffix.lower()!='.pdf':
with Image.open(path) as raw:
w,h=raw.size;orientation=int(raw.getexif().get(274,1));im=ImageOps.exif_transpose(raw).convert('RGB')
transforms={1:[1,0,0,1,0,0],2:[-1,0,0,1,w-1,0],3:[-1,0,0,-1,w-1,h-1],4:[1,0,0,-1,0,h-1],5:[0,1,1,0,0,0],6:[0,1,-1,0,0,h-1],7:[0,-1,-1,0,w-1,h-1],8:[0,-1,1,0,w-1,0]}
yield im,{'page_number':1,'input_type':path.suffix.lower(),'coordinate_units':'original_image_pixels','coordinate_origin':'top_left','raster_to_page_affine':transforms.get(orientation,transforms[1]),'raw_image_size':[w,h],'exif_orientation':orientation,'text_evidence':[],'vector_evidence':[]}
return
import pypdfium2 as pdfium
doc=pdfium.PdfDocument(str(path))
try:
for number in (pages or list(range(1,len(doc)+1))):
if number<1 or number>len(doc):raise ValueError(f'PDF page {number} outside 1..{len(doc)}')
page=doc[number-1];scale=dpi/72
if page.get_width()*page.get_height()*scale*scale>80_000_000:
scale=math.sqrt(80_000_000/(page.get_width()*page.get_height()))
bitmap=page.render(scale=scale);conv=bitmap.get_posconv(page)
p0=np.array(conv.to_page(0,0));px=np.array(conv.to_page(1,0))-p0;py=np.array(conv.to_page(0,1))-p0
transform=[float(px[0]),float(py[0]),float(px[1]),float(py[1]),float(p0[0]),float(p0[1])]
im=bitmap.to_pil().convert('RGB').copy()
evidence=[];tp=page.get_textpage();count=tp.count_chars()
for i in range(min(count,100000)):
text=tp.get_text_range(i,1)
if text.strip():evidence.append({'text':text,'bbox_pdf_points':list(tp.get_charbox(i))})
vectors=[]
for obj in page.get_objects():
try:vectors.append({'object_type':obj.type,'bbox_pdf_points':list(obj.get_bounds())})
except (AttributeError,RuntimeError):pass
meta={'page_number':number,'input_type':'pdf','coordinate_units':'pdf_points','coordinate_origin':'PDF_page_user_space_bottom_left','raster_to_page_affine':transform,'requested_dpi':dpi,'rendered_dpi':scale*72,'pdf_page_size_points':list(page.get_size()),'pdf_rotation':page.get_rotation(),'text_evidence':evidence,'text_characters':count,'text_truncated':count>100000,'vector_evidence':vectors,'vector_support':'Original PDF preserved by reference; object bounds and text boxes retained, no semantic vector classification.'}
tp.close();bitmap.close();page.close()
yield im,meta
finally:doc.close()
def infer_page(model,im,config,args):
w,h=im.size;views=[]
tile=args.tile or max(w,h) # --tile 0 (default): whole page in one view, the scale the model was trained at
for y in origins(h,tile,args.overlap):
for x in origins(w,tile,args.overlap):
views.append((f'tile_{x}_{y}',im.crop((x,y,min(x+tile,w),min(y+tile,h))),[1,0,0,1,x,y],x,y))
if max(w,h)>tile:
context=im.copy();context.thumbnail((args.imgsz,args.imgsz))
views.insert(0,('full_context',context,[w/context.width,0,0,h/context.height,0,0],0,0))
threshold=config['class_thresholds'];floor=float(config.get('candidate_confidence_floor',.01))
items=[];transforms=[]
for name,crop,transform,x,y in views:
result=model.predict(crop,imgsz=args.imgsz,conf=floor,iou=.7,device=args.device,retina_masks=True,max_det=1500,verbose=False)[0]
view_meta={'tile_id':name,'size':list(crop.size),'tile_to_raster_affine':transform}
transforms.append(view_meta)
if name=='full_context':view_meta['context_predictions_for_review']=[]
if result.masks is None:continue
for mask,cls,confidence in zip(result.masks.data.cpu().numpy(),result.boxes.cls.cpu().numpy(),result.boxes.conf.cpu().numpy()):
cid=int(cls);conf=float(confidence)
if str(cid) not in threshold or conf<floor:continue
geom=mask_geometry(mask>.5)
if geom.is_empty:continue
if tuple(mask.shape)!=(crop.height,crop.width):
geom=affinity.scale(geom,crop.width/mask.shape[1],crop.height/mask.shape[0],origin=(0,0))
x0,y0,x1,y1=geom.bounds
edge=name!='full_context' and ((x>0 and x0<3) or (y>0 and y0<3) or (x+crop.width<w and x1>crop.width-4) or (y+crop.height<h and y1>crop.height-4))
geom=affinity.affine_transform(geom,transform)
if name=='full_context':
view_meta['context_predictions_for_review'].append({'class_id':cid,'confidence':conf,'geometry_raster':mapping(geom)})
continue # Coarse context unions can fill fine portal openings.
items.append({'class_id':cid,'confidence':conf,'geometry':geom,'tiles':[name],'tile_edge':bool(edge),'merged_instances':1,'seam_merge':False})
merged=merge_instances(items)
low=[item for item in merged if float(config.get('review_candidate_floor',.05))<=item['confidence']<float(threshold[str(item['class_id'])])]
if low:
transforms.append({'tile_id':'merged_review_candidates','below_operating_threshold_for_review':[{'class_id':item['class_id'],'confidence':item['confidence'],'geometry_raster':mapping(item['geometry']),'tiles':item['tiles']} for item in low[:100]],'truncated':len(low)>100,'candidate_count':len(low)})
return [item for item in merged if item['confidence']>=float(threshold[str(item['class_id'])])],transforms
def main():
p=argparse.ArgumentParser();p.add_argument('input',type=Path);p.add_argument('--weights',type=Path,required=True);p.add_argument('--thresholds',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--device',default='0');p.add_argument('--imgsz',type=int,default=1024);p.add_argument('--tile',type=int,default=0,help='0 = whole page (default; RAQEEB was trained on whole plans). Use e.g. 2048 only for very large sheets');p.add_argument('--overlap',type=int,default=256);p.add_argument('--dpi',type=float,default=300);p.add_argument('--pages',help='Comma-separated one-based PDF page numbers');args=p.parse_args()
if args.tile and not 0<=args.overlap<args.tile:raise ValueError('overlap must be in [0,tile)')
if args.input.suffix.lower() not in {'.png','.jpg','.jpeg','.pdf'}:raise ValueError('Convert this input to PNG, JPEG or PDF first')
args.output.mkdir(parents=True,exist_ok=True)
config=json.loads(args.thresholds.read_text());model=YOLO(str(args.weights))
for i,name in model.names.items():
if NAMES[int(i)]!=name:raise ValueError('Checkpoint canonical class order mismatch')
pages=[int(n) for n in args.pages.split(',')] if args.pages else None
report={'input':str(args.input.resolve()),'input_sha256':sha256(args.input),'weights':str(args.weights.resolve()),'weights_sha256':sha256(args.weights),'model_classes':model.names,'reserved_canonical_labels':dict(enumerate(NAMES)),'operating_point':config,'attributes_supported':'Unknown codes only; PDF text/object evidence retained for review. No automated interior/exterior, entrance/exit or emergency-exit designation.','pages':[]}
for im,meta in image_pages(args.input,args.dpi,pages):
detections,transforms=infer_page(model,im,config,args)
meta.update({'raster_size':list(im.size),'tiles':transforms,'instances':[],'review_queue':[]})
if config.get('release_certification')=='WITHHELD':
meta['review_queue'].append({'scope':'page','reasons':['experimental_model_style_requires_human_review']})
for view in transforms:
for candidate in view.get('below_operating_threshold_for_review',[]):
meta['review_queue'].append({'scope':'candidate','class_id':candidate['class_id'],'label':NAMES[candidate['class_id']],'confidence':candidate['confidence'],'geometry_page':mapping(affinity.affine_transform(shape(candidate['geometry_raster']),meta['raster_to_page_affine'])),'tiles':candidate['tiles'],'reasons':['below_operating_threshold']})
preview=im.convert('RGBA');layer=Image.new('RGBA',im.size);draw=ImageDraw.Draw(layer)
for i,item in enumerate(detections):
cid=item['class_id'];g=item['geometry'];c=COLORS[cid]
for poly in polygons(g):
draw.polygon(list(poly.exterior.coords),fill=(*c,65),outline=(*c,255),width=2)
for ring in poly.interiors:draw.polygon(list(ring.coords),fill=(0,0,0,0))
instance={'id':f'p{meta["page_number"]}_i{i}','class_id':cid,'label':NAMES[cid],'confidence':item['confidence'],'geometry_page':mapping(affinity.affine_transform(g,meta['raster_to_page_affine'])),'geometry_raster':mapping(g),'tiles':item['tiles'],'merged_instances':item['merged_instances'],'attributes':{'wall_side':'WS0','portal_side':'PS0','portal_role':['PR0'],'marker_link':'unresolved','space_use':'unknown'}}
reasons=[]
if cid in config.get('unvalidated_class_ids',[]):reasons.append('unvalidated_class_requires_review')
if item['confidence']<config.get('review_confidence',.5):reasons.append('low_confidence')
if item['tile_edge']:reasons.append('touches_internal_tile_boundary')
if item['seam_merge']:reasons.append('partial_mask_seam_merge')
instance['review_reasons']=reasons;meta['instances'].append(instance)
if reasons:meta['review_queue'].append({'instance_id':instance['id'],'reasons':reasons})
meta['style_review']='Unfamiliar styles require human review; no calibrated style novelty detector is provided.'
overlay=Image.alpha_composite(preview,layer).convert('RGB');overlay.save(args.output/f'page_{meta["page_number"]:04}_overlay.jpg',quality=94)
report['pages'].append(meta)
(args.output/'predictions.json').write_text(json.dumps(report,indent=2),encoding='utf-8')
print(f'Page {meta["page_number"]}: {len(detections)} instances, {len(meta["review_queue"])} queued for review',flush=True)
if __name__=='__main__':main()