"""Build the RAQEEB merged 11-class YOLO-seg dataset from every labelled source in datasets.txt. Stages (each cached, re-runnable): collect -> D:/.../RAQEEB CCTV/work/records/.jsonl.gz (canonical polygons per image) split -> D:/.../RAQEEB CCTV/work/splits.json (group-aware, leak-free train/val/test) write -> D:/.../RAQEEB CCTV/dataset/{images,labels}/{train,val,test}/.jpg|.txt finalize -> data.yaml, label_sets.json, train list with rare-class repeats, manifest, summary Partial labels: most public sources annotate only some of the 11 classes (e.g. RF-07 = doors only). `label_sets.json` records, per source, which classes are exhaustively annotated ("full") and which are positive-only ("partial"). The RAQEEB trainer uses this to ignore negatives for unannotated classes, so an unlabelled wall in a doors-only image is never taught as background. """ import argparse, gzip, io, json, math, os, random, re, sys, time, zipfile, hashlib from collections import Counter, defaultdict from multiprocessing import Pool from pathlib import Path import numpy as np from PIL import Image Image.MAX_IMAGE_PIXELS = None ROOT = Path(r'G:\CCTV_AI_FLOORPLAN') RAQ = Path(r'D:\CCTV_AI_FLOORPLAN_data\RAQEEB CCTV') WORK, DS = RAQ / 'work', RAQ / 'dataset' NAMES = ['wall', 'door', 'window', 'open_passage', 'column', 'stair', 'elevator', 'escalator', 'exit_marker', 'gate', 'space_region'] MAX_SIDE = 1536 ALL = list(range(11)) # source -> zip, {exported COCO name: canonical id}, fully-annotated ids, positive-only ids RF = { 'RF-01': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-01\Training AI.v1i.coco-segmentation.zip', {'wall': 0, 'door': 1, 'window': 2}, [0, 1, 2], []), 'RF-02': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-02\Floor Plan.v81i.coco-segmentation.zip', {'external-walls': 0, 'internal-walls': 0, 'door': 1, 'cav-sliders': 1, 'windows': 2, 'stairs': 5}, [0, 1, 2, 5], []), 'RF-03': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-03\Floorplan-Segmentation.v4-version-3.coco-segmentation.zip', {'wall': 0, 'door': 1, 'window': 2, 'room': 10}, [0, 1, 2], [10]), 'RF-04': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-04\floor-plan.v1i.coco-segmentation.zip', {'wall': 0, 'door': 1, 'window': 2}, [0, 1, 2], []), 'RF-05': (r'D:\CCTV_AI_FLOORPLAN_data\raw\RF-05\Segmenting a Floor Plan.v1i.coco-segmentation.zip', {'door': 1, 'window': 2, 'kitchen': 10, 'bathroom': 10, 'en_suite': 10}, [1, 2], [10]), 'RF-06': (r'D:\CCTV_AI_FLOORPLAN_data\raw\RF-06\Walls.v1i.coco-segmentation.zip', {'wall': 0}, [0], []), 'RF-07': (r'D:\CCTV_AI_FLOORPLAN_data\raw\RF-07\doors detection2.v1i.coco-segmentation.zip', {'doors - v1 2024-12-19 12-26am': 1}, [1], []), 'RF-08': (r'D:\CCTV_AI_FLOORPLAN_data\raw\RF-08\doors detection.v1i.coco-segmentation.zip', {'Windows - v1 2023-05-19 12-58pm': 2}, [2], []), 'RF-09': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-09\floor-spac.v1i.coco-segmentation.zip', {'space': 10, 'stair': 5, 'elevator': 6}, [5, 6, 10], []), 'RF-11': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-11\ArchitecturalBlueprints.v3i.coco-segmentation.zip', {'walls': 0, 'columns': 4}, [4], [0]), 'RF-12': (r'G:\CCTV_AI_FLOORPLAN\raw\RF-12\Evacuation Plan.v2i.coco.zip', {'exit': 8, 'stairs': 5}, [8], [5]), 'RF-13': (r'D:\CCTV_AI_FLOORPLAN_data\raw\RF-13\Floor Plan Walls.v5i.coco.zip', {'wall': 0}, [0], []), 'RF-14': (r'D:\CCTV_AI_FLOORPLAN_data\raw\RF-14\Floor plan.v1i.coco.zip', {'door': 1, 'window': 2, 'bedroom': 10, 'kitchen': 10, 'bathroom': 10, 'living-room': 10}, [1, 2], [10]), } EXCLUDED_SOURCES = { 'RF-10': 'Wine-rack photographs, not floor plans (visually confirmed); "column" labels would teach the wrong object.', } CC_ZIP = r'D:\CCTV_AI_FLOORPLAN_data\research\RES-01\cubicasa5k.zip' CC_SETS = ([0, 1, 2, 3, 4, 5, 10], []) PRV_DIR = Path(r'D:\CCTV_AI_FLOORPLAN_data\prepared\expanded11_v4r1') FPCAD_DIR = RAQ / 'staging' / 'RES-02' GROUPS_FILE = ROOT / 'audit' / 'cross_source_duplicates_v2' / 'all_images.jsonl' VAL_CAP, TEST_CAP, BIG_GROUP = 300, 250, 30 def log(*a): print(time.strftime('%H:%M:%S'), *a, flush=True) def sanitize(s): return re.sub(r'[^A-Za-z0-9._-]+', '_', s) def poly_rows(cls, flat, w, h): """COCO flat pixel polygon -> list with one YOLO row string (or empty).""" a = np.asarray(flat, dtype=np.float64).reshape(-1, 2) a[:, 0] = np.clip(a[:, 0], 0, w) a[:, 1] = np.clip(a[:, 1], 0, h) keep = np.ones(len(a), bool) keep[1:] = np.any(np.abs(np.diff(a, axis=0)) > 1e-6, axis=1) a = a[keep] if len(a) >= 2 and np.allclose(a[0], a[-1]): a = a[:-1] if len(a) < 3: return [] x, y = a[:, 0], a[:, 1] area = 0.5 * abs(np.dot(x, np.roll(y, -1)) - np.dot(y, np.roll(x, -1))) if area < 1.0: return [] n = a / np.array([w, h]) return [str(cls) + ' ' + ' '.join(f'{v:.6f}' for v in n.ravel())] def write_records(src, recs): WORK.joinpath('records').mkdir(parents=True, exist_ok=True) with gzip.open(WORK / 'records' / f'{src}.jsonl.gz', 'wt', encoding='utf-8') as f: for r in recs: f.write(json.dumps(r) + '\n') def read_records(src): p = WORK / 'records' / f'{src}.jsonl.gz' with gzip.open(p, 'rt', encoding='utf-8') as f: return [json.loads(l) for l in f] # ---------------------------------------------------------------- collect: Roboflow def collect_rf(src): zpath, mapping, full, partial = RF[src] z = zipfile.ZipFile(zpath) recs, dropped = [], Counter() for j in [n for n in z.namelist() if n.endswith('_annotations.coco.json')]: split = j.split('/')[0] split = {'valid': 'val'}.get(split, split) d = json.loads(z.read(j)) cats = {c['id']: c['name'] for c in d['categories']} by = defaultdict(list) for a in d['annotations']: by[a['image_id']].append(a) prefix = j.rsplit('/', 1)[0] for im in d['images']: w, h = im['width'], im['height'] rows, counts = [], Counter() for a in by.get(im['id'], []): name = cats[a['category_id']] if name not in mapping: dropped[name] += 1 continue c = mapping[name] seg = a.get('segmentation') polys = [s for s in seg if isinstance(s, list) and len(s) >= 6] if isinstance(seg, list) else [] if not polys: # box-only annotation -> rectangle polygon x, y, bw, bh = a['bbox'] polys = [[x, y, x + bw, y, x + bw, y + bh, x, y + bh]] for s in polys: r = poly_rows(c, s, w, h) rows += r counts[c] += len(r) member = f"{prefix}/{im['file_name']}" stem = sanitize(member.rsplit('.', 1)[0]) recs.append(dict(id=f'{src}__{stem}', src=src, member=member, zip=zpath, src_split=split, w=w, h=h, rows=rows, counts={str(k): v for k, v in counts.items()}, raw_annotations=len(by.get(im['id'], [])))) write_records(src, recs) return src, len(recs), dict(dropped) # ---------------------------------------------------------------- collect: CubiCasa5K _CCZ = None def _cc_zip(): global _CCZ if _CCZ is None: _CCZ = zipfile.ZipFile(CC_ZIP) return _CCZ def cc_parse(folder): """folder like 'cubicasa5k/high_quality/1245/'. Returns record dict.""" sys.path.insert(0, str(Path(__file__).resolve().parent)) from cubicasa_geometry import parse_core, matrix, polygons as shp_polys from defusedxml import ElementTree as ET from shapely.geometry import Polygon, shape, box from shapely.ops import unary_union z = _cc_zip() png = z.read(folder + 'F1_scaled.png') with Image.open(io.BytesIO(png)) as im: W, H = im.size svg = z.read(folder + 'model.svg') rows, counts, issues = [], Counter(), [] frame = box(0, 0, W, H) def emit(c, geom): for p in shp_polys(geom.intersection(frame)): if p.area < 1.0: continue ext = np.asarray(p.exterior.coords)[:-1] r = poly_rows(c, ext.ravel(), W, H) rows.extend(r) counts[c] += len(r) try: objs, rep = parse_core(svg, W, H) for o in objs: emit(o['class_id'], o['geometry']) for cand in rep.get('leafless_opening_candidates', []): emit(3, shape(cand['geometry'])) issues += rep.get('issues', []) except Exception as e: # keep going with the other classes issues.append(f'core_failed:{e}') # Columns (4), stairs (5), spaces (10) root = ET.fromstring(svg) def pts_of(el, T): tag = el.tag.split('}')[-1] if tag == 'polygon': nums = [float(v) for v in re.split(r'[\s,]+', el.get('points', '').strip()) if v] if len(nums) < 6 or len(nums) % 2: return None P = np.c_[np.array(nums).reshape(-1, 2), np.ones(len(nums) // 2)] elif tag == 'circle': cx, cy, r = (float(el.get(k, 0)) for k in ('cx', 'cy', 'r')) t = np.linspace(0, 2 * np.pi, 24, endpoint=False) P = np.c_[cx + r * np.cos(t), cy + r * np.sin(t), np.ones(24)] elif tag == 'rect': x, y, w_, h_ = (float(el.get(k, 0)) for k in ('x', 'y', 'width', 'height')) P = np.array([[x, y, 1], [x + w_, y, 1], [x + w_, y + h_, 1], [x, y + h_, 1]]) else: return None xy = (T @ matrix(el.get('transform', '')) @ P.T).T[:, :2] try: g = Polygon(xy).buffer(0) except Exception: return None return g if g.area > 0 else None def direct_shapes(e, T): out = [] for ch in e: g = pts_of(ch, T) if g is not None: out.append(g) return out def walk(e, parent): T = parent @ matrix(e.get('transform', '')) cls = (e.get('class') or '').split() eid = e.get('id') if cls[:1] == ['Column'] or eid == 'Column': for g in direct_shapes(e, T): emit(4, g) return if cls[:1] == ['Stairs']: parts = [] def sw(x, TT): T2 = TT @ matrix(x.get('transform', '')) if (x.get('class') or '').split()[:1] in (['Flight'], ['Landing'], ['Winding']): parts.extend(direct_shapes(x, T2)) for ch in x: if ch.tag.split('}')[-1] == 'g': sw(ch, T2) for ch in e: if ch.tag.split('}')[-1] == 'g': sw(ch, T) if not parts: parts = direct_shapes(e, T) if parts: emit(5, unary_union(parts)) return if cls[:1] == ['Space']: sh = direct_shapes(e, T) if sh: emit(10, sh[0]) # spaces never nest objects we need; still walk for safety for ch in e: if ch.tag.split('}')[-1] == 'g': walk(ch, T) try: walk(root, np.eye(3)) except Exception as e: issues.append(f'extra_failed:{e}') style, num = folder.strip('/').split('/')[1:3] return dict(id=f'RES-01__{style}_{num}', src='RES-01', member=folder + 'F1_scaled.png', zip=CC_ZIP, w=W, h=H, rows=rows, counts={str(k): v for k, v in counts.items()}, issues=issues[:20]) def collect_cc(procs): z = zipfile.ZipFile(CC_ZIP) split_of = {} for t in ('train', 'val', 'test'): for line in z.read(f'cubicasa5k/{t}.txt').decode().split(): split_of['cubicasa5k' + line.strip()] = t folders = sorted({n.rsplit('/', 1)[0] + '/' for n in z.namelist() if n.endswith('model.svg')}) log(f'CubiCasa: parsing {len(folders)} SVGs with {procs} processes') with Pool(procs) as pool: recs = [] for i, r in enumerate(pool.imap_unordered(cc_parse, folders, chunksize=8)): r['src_split'] = split_of.get(r['member'].rsplit('/', 1)[0] + '/', 'train') recs.append(r) if (i + 1) % 1000 == 0: log(f' CubiCasa {i + 1}/{len(folders)}') recs.sort(key=lambda r: r['id']) write_records('RES-01', recs) return len(recs) # ---------------------------------------------------------------- collect: previous reviewed set, FloorPlanCAD def collect_prv(): recs = [] for split in ('train', 'val', 'test'): for img in sorted((PRV_DIR / 'images' / split).glob('*.png')): lab = PRV_DIR / 'labels' / split / (img.stem + '.txt') rows = [l.strip() for l in lab.read_text().splitlines() if l.strip()] if lab.exists() else [] with Image.open(img) as im: W, H = im.size counts = Counter(int(r.split()[0]) for r in rows) recs.append(dict(id=f'PRV__{split}_{sanitize(img.stem)}', src='PRV', member=str(img), zip=None, src_split=split, w=W, h=H, rows=rows, counts={str(k): v for k, v in counts.items()})) write_records('PRV', recs) return len(recs) def collect_fpcad(): man = FPCAD_DIR / 'manifest.jsonl' if not man.exists(): log('FloorPlanCAD staging not ready; skipping') return 0 recs = [] for l in man.read_text(encoding='utf-8').splitlines(): m = json.loads(l) pid = m['page_id'] img, lab = FPCAD_DIR / 'images' / f'{pid}.jpg', FPCAD_DIR / 'labels' / f'{pid}.txt' if not img.exists() or not lab.exists(): continue rows = [x.strip() for x in lab.read_text().splitlines() if x.strip()] counts = Counter(int(r.split()[0]) for r in rows) # pages carrying FloorPlanCAD "opening symbol" (id 12) marks: about half are casement windows labelled # only as id 12, so windows on those pages are not exhaustively annotated demote = [2] if (FPCAD_DIR.parent / 'RES-02_work' / 'opening_symbol_labels' / f'{pid}.txt').exists() else [] recs.append(dict(id=f'RES-02__{sanitize(pid)}', src='RES-02', member=str(img), zip=None, src_split=m.get('source_split', 'train'), w=m.get('width'), h=m.get('height'), rows=rows, counts={str(k): v for k, v in counts.items()}, fp_group=m.get('group'), demote=demote)) write_records('RES-02', recs) return len(recs) # ---------------------------------------------------------------- label sets def label_sets(): sets = {s: dict(full=v[2], partial=v[3]) for s, v in RF.items()} sets['RES-01'] = dict(full=CC_SETS[0], partial=CC_SETS[1]) sets['PRV'] = dict(full=ALL, partial=[]) fp = FPCAD_DIR / 'annotated_classes.json' if fp.exists(): v = json.loads(fp.read_text()) sets['RES-02'] = dict(full=sorted(v if isinstance(v, list) else v.get('annotated', [])), partial=[]) sets['UNLABELLED'] = dict(full=[], partial=[]) return sets # ---------------------------------------------------------------- split def assign_splits(seed=20261111): srcs = sorted(p.name.split('.')[0] for p in (WORK / 'records').glob('*.jsonl.gz')) recs = [r for s in srcs for r in read_records(s)] log(f'split: {len(recs)} records from {srcs}') gmap = {} for l in GROUPS_FILE.open(encoding='utf-8'): d = json.loads(l) gmap[(d['source'], d['member'])] = d['global_drawing_group'] cc_group = {} for (s, m), g in gmap.items(): if s == 'RES-01': _, style, num, _ = m.split('/') cc_group[(style, num)] = g for r in recs: g = None if r['src'] == 'RES-01': g = gmap.get(('RES-01', r['member'])) elif r['src'].startswith('RF-'): g = gmap.get((r['src'], r['member'])) elif r['src'] == 'PRV': m = re.match(r'cc5k_(colorful|high_quality_architectural|high_quality)_(\d+)__', Path(r['member']).name) if m: g = cc_group.get((m.group(1), m.group(2))) if g is None: g = 'prv_' + Path(r['member']).name.split('__')[0] elif r['src'] == 'RES-02': g = 'fpcad_' + str(r.get('fp_group') or r['id']) r['group'] = g or ('solo_' + r['id']) r['unlabelled'] = (r['src'] in ('RF-02', 'RF-06') and r.get('raw_annotations', 1) == 0) groups = defaultdict(list) for i, r in enumerate(recs): groups[r['group']].append(i) split = {} rng = random.Random(seed) big = {g for g, m in groups.items() if len(m) > BIG_GROUP} # over-chained hash clusters -> train only by_src = defaultdict(list) for i, r in enumerate(recs): by_src[r['src']].append(i) # Budget per source and held-out split; every member of an accepted group counts against its own source. budget = {} for src, idx in by_src.items(): has = Counter(recs[i]['src_split'] for i in idx) for target, cap in (('val', VAL_CAP), ('test', TEST_CAP)): n_src = has.get(target, 0) if src == 'RES-02' and target == 'val': n_src = has.get('test', 0) if n_src == 0: # train-only sources (e.g. RF-02): carve a modest held-out set n_src = cap budget[(src, target)] = min(cap, n_src) used = Counter() def try_take(g, target): mem = groups[g] if g in big or any(j in split for j in mem): return False need = Counter(recs[j]['src'] for j in mem) if any(used[(s, target)] + n > budget.get((s, target), 0) for s, n in need.items()): return False for j in mem: split[j] = target used.update({(s, target): n for s, n in need.items()}) return True # incremental: keep every existing assignment (images are already written to their split folders) prev_path = WORK / 'splits.json' if prev_path.exists(): prev = json.loads(prev_path.read_text()) for i, r in enumerate(recs): if r['id'] in prev: split[i] = prev[r['id']]['split'] if split[i] != 'train': used[(r['src'], split[i])] += 1 done_srcs = {r['src'] for r in recs if r['id'] in prev} log(f'incremental split: keeping {len(prev)} existing assignments from {sorted(done_srcs)}') # previous reviewed held-out pages keep their split when their group is small enough for i, r in enumerate(recs): if r['src'] == 'PRV' and r['src_split'] in ('val', 'test') and i not in split: g = r['group'] if g not in big: for j in groups[g]: split[j] = r['src_split'] used[(recs[j]['src'], r['src_split'])] += 1 for src in sorted(by_src, key=lambda s: len(by_src[s])): # small sources first if src == 'PRV': continue idx = by_src[src] has = Counter(recs[i]['src_split'] for i in idx) for target in ('val', 'test'): want_from = target if has.get(target) else 'train' if src == 'RES-02' and target == 'val': want_from = 'test' cand = sorted({recs[i]['group'] for i in idx if recs[i]['src_split'] == want_from and i not in split and not recs[i]['unlabelled']}) rng.shuffle(cand) for g in cand: if used[(src, target)] >= budget[(src, target)]: break try_take(g, target) for i in range(len(recs)): split.setdefault(i, 'train') out = {r['id']: dict(split=split[i], group=r['group'], src=r['src'], src_split=r['src_split'], unlabelled=r['unlabelled']) for i, r in enumerate(recs)} (WORK / 'splits.json').write_text(json.dumps(out)) tab = defaultdict(Counter) for v in out.values(): tab[v['src']][v['split'] + ('_unlabelled' if v['unlabelled'] else '')] += 1 for s in sorted(tab): log(f' {s}: {dict(tab[s])}') return out # ---------------------------------------------------------------- write _ZIPS = {} def _zip(p): if p not in _ZIPS: _ZIPS[p] = zipfile.ZipFile(p) return _ZIPS[p] def write_one(job): r, split = job img_out = DS / 'images' / split / f"{r['id']}.jpg" lab_out = DS / 'labels' / split / f"{r['id']}.txt" if img_out.exists() and lab_out.exists(): return r['id'], 'skip', None try: if r['zip']: data = _zip(r['zip']).read(r['member']) im = Image.open(io.BytesIO(data)) else: im = Image.open(r['member']) im.load() if im.mode in ('RGBA', 'LA', 'P'): bg = Image.new('RGB', im.size, 'white') im = im.convert('RGBA') bg.paste(im, mask=im.split()[-1]) im = bg elif im.mode != 'RGB': im = im.convert('RGB') if (im.size[0], im.size[1]) != (r['w'], r['h']) and r['w'] and r['h']: # labels are normalised; record mismatch for audit only pass s = MAX_SIDE / max(im.size) if s < 1: im = im.resize((max(1, round(im.size[0] * s)), max(1, round(im.size[1] * s))), Image.LANCZOS) tmp = img_out.with_suffix('.tmp.jpg') im.save(tmp, quality=95) os.replace(tmp, img_out) lab_out.write_text('\n'.join(r['rows']) + ('\n' if r['rows'] else '')) return r['id'], 'ok', im.size except Exception as e: return r['id'], 'error', str(e) def write_all(procs): splits = json.loads((WORK / 'splits.json').read_text()) for sp in ('train', 'val', 'test'): (DS / 'images' / sp).mkdir(parents=True, exist_ok=True) (DS / 'labels' / sp).mkdir(parents=True, exist_ok=True) srcs = sorted(p.name.split('.')[0] for p in (WORK / 'records').glob('*.jsonl.gz')) errors, sizes = [], {} for s in srcs: jobs = [(r, splits[r['id']]['split']) for r in read_records(s) if r['id'] in splits and not splits[r['id']]['unlabelled']] t = time.time() with Pool(procs) as pool: for rid, st, info in pool.imap_unordered(write_one, jobs, chunksize=16): if st == 'error': errors.append((rid, info)) elif st == 'ok': sizes[rid] = info log(f'write {s}: {len(jobs)} images in {time.time() - t:.0f}s, errors so far {len(errors)}') (WORK / 'write_errors.json').write_text(json.dumps(errors, indent=1)) return errors # ---------------------------------------------------------------- finalize def finalize(rfs_threshold=0.08, max_repeat=4): splits = json.loads((WORK / 'splits.json').read_text()) sets = label_sets() srcs = sorted(p.name.split('.')[0] for p in (WORK / 'records').glob('*.jsonl.gz')) manifest, lists = [], defaultdict(list) inst = defaultdict(Counter) img_has = defaultdict(Counter) per_src = defaultdict(Counter) train_recs = [] overrides, demoted = {}, Counter() for s in srcs: for r in read_records(s): sp = splits.get(r['id']) if sp is None: continue if not r['rows'] and s.startswith('RF-') and r.get('raw_annotations', 0) == 0: sp['unlabelled'] = True # a Roboflow image with no annotations at all is unlabelled, not a negative full = set(sets[s]['full']) has = {int(k) for k, v in r['counts'].items() if v} if s != 'PRV' and r['rows']: # per-image completeness: an image with walls but no door (or window) labels is treated as # not door- (window-) annotated, and vice versa; rooms likewise in room-annotating sources if 0 in has and not (has & {1}) and 1 in full: full.discard(1); demoted['door'] += 1 if 0 in has and not (has & {2}) and 2 in full: full.discard(2); demoted['window'] += 1 if (has & {1, 2}) and 0 not in has and 0 in full: full.discard(0); demoted['wall'] += 1 if 10 in full and 10 not in has: full.discard(10); demoted['space_region'] += 1 for c in r.get('demote', []): if c in full: full.discard(c); demoted[f'{NAMES[c]}_source_flag'] += 1 if full != set(sets[s]['full']): overrides[r['id']] = sorted(full) if sp['unlabelled']: manifest.append(dict(id=r['id'], src=s, member=r['member'], split='excluded_unlabelled', group=sp['group'])) per_src[s]['excluded_unlabelled'] += 1 continue img = DS / 'images' / sp['split'] / f"{r['id']}.jpg" if not img.exists(): per_src[s]['missing'] += 1 continue split = sp['split'] lists[split].append(str(img)) per_src[s][split] += 1 for k, v in r['counts'].items(): inst[split][int(k)] += v for k in r['counts']: img_has[split][int(k)] += 1 if split == 'train': train_recs.append((str(img), {int(k) for k in r['counts']})) manifest.append(dict(id=r['id'], src=s, member=r['member'], src_split=r['src_split'], split=split, group=sp['group'], counts=r['counts'], full=sorted(full), partial=sets[s]['partial'])) # repeat-factor sampling (LVIS style) for rare classes in train n_train = len(train_recs) freq = {c: img_has['train'][c] / max(n_train, 1) for c in ALL} rf = {c: (max(1.0, math.sqrt(rfs_threshold / freq[c])) if freq[c] > 0 else 1.0) for c in ALL} train_list, repeats = [], Counter() rng = random.Random(7) for path, cl in train_recs: r = max([rf[c] for c in cl] + [1.0]) r = min(r, max_repeat) k = int(r) + (1 if rng.random() < r - int(r) else 0) train_list += [path] * k repeats[k] += 1 rng.shuffle(train_list) (DS / 'train.txt').write_text('\n'.join(train_list) + '\n') (DS / 'train_unique.txt').write_text('\n'.join(sorted(lists['train'])) + '\n') for sp in ('val', 'test'): (DS / f'{sp}.txt').write_text('\n'.join(sorted(lists[sp])) + '\n') (DS / 'label_sets.json').write_text(json.dumps(dict( names=NAMES, sources=sets, image_full_overrides=overrides, rule='id prefix before "__" selects the source; "full" classes are exhaustively annotated (negatives count), ' '"partial" classes contribute positives only, other classes are ignored. image_full_overrides replaces ' 'the source "full" list for individual images (keyed by file stem) whose labels are visibly incomplete.'), indent=2)) yaml = [f"path: {DS.as_posix()}", "train: train.txt", "val: val.txt", "test: test.txt", "names:"] yaml += [f" {i}: {n}" for i, n in enumerate(NAMES)] (DS / 'data.yaml').write_text('\n'.join(yaml) + '\n') with (DS / 'manifest.jsonl').open('w', encoding='utf-8') as f: for m in manifest: f.write(json.dumps(m) + '\n') summary = dict( created=time.strftime('%Y-%m-%d %H:%M:%S'), names=NAMES, images_by_split={k: len(v) for k, v in lists.items()}, train_entries_with_repeats=len(train_list), repeat_histogram=dict(repeats), repeat_factor_threshold=rfs_threshold, class_repeat_factor={NAMES[c]: round(rf[c], 2) for c in ALL}, instances_by_split={sp: {NAMES[c]: inst[sp][c] for c in ALL} for sp in inst}, images_with_class_by_split={sp: {NAMES[c]: img_has[sp][c] for c in ALL} for sp in img_has}, per_source=per_src, excluded_sources=EXCLUDED_SOURCES, label_sets=sets, max_side=MAX_SIDE, per_image_demotions=dict(demoted), per_image_overrides=len(overrides)) (DS / 'summary.json').write_text(json.dumps(summary, indent=2)) log(json.dumps(summary['images_by_split']), 'train entries', len(train_list)) return summary def main(): ap = argparse.ArgumentParser() ap.add_argument('stages', nargs='+', choices=['collect', 'collect_fpcad', 'split', 'write', 'finalize']) ap.add_argument('--procs', type=int, default=28) ap.add_argument('--only', nargs='*') a = ap.parse_args() RAQ.mkdir(parents=True, exist_ok=True) if 'collect' in a.stages: todo = a.only or (list(RF) + ['RES-01', 'PRV']) rf = [s for s in todo if s in RF] if rf: with Pool(min(len(rf), 8)) as pool: for src, n, dropped in pool.imap_unordered(collect_rf, rf): log(f'collect {src}: {n} images; dropped source classes {dropped}') if 'PRV' in todo: log(f'collect PRV: {collect_prv()}') if 'RES-01' in todo: log(f'collect RES-01: {collect_cc(a.procs)}') if 'collect_fpcad' in a.stages: log(f'collect RES-02: {collect_fpcad()}') if 'split' in a.stages: assign_splits() if 'write' in a.stages: errs = write_all(a.procs) log(f'write errors: {len(errs)}') if 'finalize' in a.stages: finalize() if __name__ == '__main__': main()