Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified Download eval_twin.py from constructelligence/painting-vision-robotics-kit: direct link, hf CLI and curl.
- Browser
- Download file 11.8 kB
-
https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/eval_twin.py
- Command line
-
hf download hf://constructelligence/painting-vision-robotics-kit/eval_twin.py
-
curl -L -o eval_twin.py https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/eval_twin.py
11.8 kB
| #!/usr/bin/env python3 | |
| """Does the digital twin finish a facade better than flying the plan blind, and does its belief track a truth it cannot see? | |
| Simulated job, truth hidden from the twin: each pass is flown with a logged sideways error plus a hidden one; paint is | |
| whatever the truth film says (>= 0.5 of target = painted). Three strategies, same facade, same errors, 30 jobs each: | |
| blind fly the plan once, call it done (what a plan-only system does) | |
| twin fly, log telemetry, camera-observe, fly the gaps the belief shows, repeat until the belief says done | |
| observe as 'twin' but belief from camera only (flight log ignored) | |
| Scored against truth: painted share of the target, false-complete rate (a **complete** verdict while truth is under 97 %), | |
| the share that stops with a hand-finish strip (residual work correctly left to a person rather than claimed done), | |
| flights, paint flown. | |
| ``--truth timing`` (default) deposits paint from the timed path - each pass at its own speed and standoff, with a hidden | |
| speed error per pass - and the twin does the same. ``--truth legacy`` re-derives the numbers published before the timing | |
| model. ``--truth both`` runs both on the *same* seeds and the *same* hidden errors (paired), so the difference is the | |
| model's and not sampling noise. | |
| python eval_twin.py [--jobs 30] [--truth timing|legacy|both] [--drift-mm 20] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import numpy as np | |
| import application as ap | |
| import drone_areas as da | |
| import drone_path as dp | |
| import drone_twin as dt | |
| MM = 25.0 | |
| LOG_SIGMA, HIDDEN_SIGMA, POS = 0.02, 0.012, 0.02 | |
| def facade(): | |
| c = np.full((300, 480), 2, np.uint8) | |
| for r in (40, 160): | |
| for x in (50, 160, 270, 380): | |
| c[r:r + 70, x:x + 60] = 8 | |
| c[220:300, 200:260] = 7 | |
| return c | |
| def spray(): | |
| tip = ap.SprayTip.parse("517") | |
| return ap.spray_settings(tip.flow_lps(2000), tip.fan_angle_deg, 300, 38, 0.38) | |
| def true_share(film, target): | |
| return float(((film >= dt.PAINTED_FILM) & target).sum() / max(int(target.sum()), 1)) | |
| def stroke_len(strokes): | |
| return sum(np.hypot(s[-1][0] - s[0][0], s[-1][1] - s[0][1]) for s in strokes) * MM / 1000.0 | |
| def corrupt(seen, rng, shift_px=0.0, edge_px=0.0): | |
| """A camera read as it really arrives: shifted (registration / parallax) and with wandering edges (segmentation). | |
| ``shift_px`` is the sigma of a whole-read translation; ``edge_px`` the scale of local edge error: the read is blurred | |
| at that scale, noise added and re-thresholded, which moves each edge by about +-edge_px with no net bias. Pixels | |
| shifted in from outside the frame are returned as not observed. | |
| """ | |
| from scipy import ndimage | |
| out, valid = seen.astype(np.float32), np.ones(seen.shape, bool) | |
| if edge_px > 0: | |
| soft = ndimage.gaussian_filter(out, edge_px) | |
| noise = ndimage.gaussian_filter(rng.normal(0, 1, seen.shape), edge_px) | |
| noise *= 0.25 / max(noise.std(), 1e-6) | |
| out = (soft + noise >= 0.5).astype(np.float32) | |
| if shift_px > 0: | |
| dy, dx = (int(round(v)) for v in rng.normal(0.0, shift_px, 2)) | |
| out = np.roll(out, (dy, dx), axis=(0, 1)) | |
| if dy > 0: valid[:dy] = False | |
| elif dy < 0: valid[dy:] = False | |
| if dx > 0: valid[:, :dx] = False | |
| elif dx < 0: valid[:, dx:] = False | |
| return out >= 0.5, valid | |
| GATES = {"none": {}, "innov": {"resolution_px": 6}, "edge": {"edge_px": 3}, "edge+innov": {"edge_px": 3, "resolution_px": 6}} | |
| def job(strategy, seed, max_rounds=4, drift_mm=0.0, model="timing", shift_px=0.0, edge_px=0.0, gate="none"): | |
| rng = np.random.default_rng(seed) | |
| sp, classes = spray(), facade() | |
| twin = dt.Twin() | |
| f = twin.add_facade("south", classes, MM, masked_openings=True) | |
| plan = da.plan_passes(classes, MM, sp, positioning_m=POS, masked_openings=True) | |
| strokes = plan["main_strokes_px"] + plan["edge_strokes_px"] | |
| truth = np.zeros(classes.shape, np.float32) | |
| flown_m, flights, declared, stopped, verdict, waste_l, audits = 0.0, 0, False, False, "none", 0.0, 0 | |
| for rnd in range(max_rounds): | |
| flown = dt.fly(strokes, MM, POS, rng) # logged positions | |
| # Drawn once and given to whichever truth model runs, so a legacy/timing comparison is paired. | |
| hidden = rng.normal(0.0, HIDDEN_SIGMA * 1000.0 / MM, len(strokes)) | |
| jitter = 1.0 + rng.normal(0.0, 0.10, len(strokes)) | |
| truth = dt.truth_film(flown, classes.shape, MM, sp["fan_width_mm"], sp["pass_spacing_mm"], HIDDEN_SIGMA, rng, | |
| truth, unlogged_drift_mm=drift_mm, model=model, spray=sp, standoff_mm=sp["standoff_mm"], | |
| hidden_offsets_px=hidden, speed_jitter=jitter) | |
| flown_m += stroke_len(strokes); flights += 1 | |
| if strategy == "blind": | |
| verdict, declared, stopped = "declared done", True, True | |
| break | |
| if strategy == "twin": | |
| timeline = (dp.from_strokes(flown, MM, classes.shape, sp, standoff_mm=sp["standoff_mm"]) | |
| if model == "timing" else None) | |
| audit = twin.apply_sortie("south", flown, sp["fan_width_mm"], sp["pass_spacing_mm"], telemetry_sigma_m=LOG_SIGMA, | |
| timeline=timeline) | |
| if audit: | |
| waste_l += audit.get("waste_l", 0.0) | |
| audits += 1 | |
| seen = dt.detector((truth >= dt.PAINTED_FILM) & f.target | ((truth >= dt.PAINTED_FILM) & ~f.target), rng) | |
| seen, valid = corrupt(seen, rng, shift_px, edge_px) | |
| twin.observe("south", seen, valid=valid, **GATES[gate]) | |
| st = twin.status("south", sp["fan_width_mm"]) | |
| verdict = st["verdict"] | |
| if verdict != "another_sortie": # finished, or the last strips are for a hand finish: no more drone flights | |
| stopped = True | |
| declared = verdict == "complete" # only a "complete" verdict is a claim that the wall is done | |
| break | |
| nxt = twin.next_sortie("south", sp["fan_width_mm"]) | |
| strokes = nxt["plan"]["strokes"] | |
| tg = f.target & (truth < dt.PAINTED_FILM) | |
| known = twin.gaps("south") if strategy != "blind" else np.zeros_like(tg) | |
| return {"true": true_share(truth, f.target), "declared": declared, "stopped": stopped, "verdict": verdict, | |
| "flights": flights, "paint_m": flown_m, | |
| "belief": twin.coverage("south")["believed_painted"] if strategy != "blind" else None, | |
| "gap_recall": float((tg & known).sum() / max(tg.sum(), 1)), "gap_precision": float((tg & known).sum() / max(known.sum(), 1)) if known.any() else float("nan"), | |
| "left_m2": float(tg.sum()) * (MM / 1000) ** 2, | |
| "waste_l": waste_l if audits else None} | |
| def summarise(rows): | |
| return {"true": np.mean([r["true"] for r in rows]), | |
| "false_complete": np.mean([r["declared"] and r["true"] < 0.97 for r in rows]), | |
| "hand_finish": np.mean([r["verdict"] in ("hand_finish", "needs_look") for r in rows]), | |
| "claims_done": np.mean([r["verdict"] in ("complete", "declared done") for r in rows]), | |
| "flights": np.mean([r["flights"] for r in rows]), "paint_m": np.mean([r["paint_m"] for r in rows]), | |
| "belief_err": np.mean([abs(r["belief"] - r["true"]) for r in rows]) if rows[0]["belief"] is not None else float("nan"), | |
| "gap_recall": np.mean([r["gap_recall"] for r in rows]), | |
| "gap_precision": (lambda p: float(np.mean(p)) if p else float("nan"))([r["gap_precision"] for r in rows if np.isfinite(r["gap_precision"])]), | |
| "waste_l": np.mean([r["waste_l"] for r in rows if r["waste_l"] is not None]) if any(r["waste_l"] is not None for r in rows) else float("nan")} | |
| # Read error levels on the twin's 480-px-wide facade, from real measurements (2026-10-24): | |
| # alignment (drone_align on real facades, synthetic move): median 0.004 % of width -> ~0 px, so no shift term from it; | |
| # segmentation edges (C2-cal, 87 real facade frames): median 0.49 % of width = 2.3 px; p90 image 0.79 % = 3.8 px. | |
| # "+shift" adds a 2 px whole-read shift as a stand-in for parallax, which no real pair has measured yet. | |
| READS = {"perfect": (0.0, 0.0), "real-median": (0.0, 2.3), "real-p90": (0.0, 3.8), "real-p90+shift": (2.0, 3.8)} | |
| def gate_study(jobs, out_path=None): | |
| import json | |
| res = [] | |
| print(f"{'drift mm':>8s} {'read':16s} {'gate':11s} {'true painted':>13s} {'false complete':>15s} {'hand finish':>12s} " | |
| f"{'flights':>8s} {'|belief-truth|':>15s} {'knows gaps':>11s} {'precision':>10s}") | |
| for drift in (0.0, 150.0): | |
| for read, (shift, edge) in READS.items(): | |
| for gate in GATES: | |
| s = summarise([job("twin", seed, drift_mm=drift, shift_px=shift, edge_px=edge, gate=gate) for seed in range(jobs)]) | |
| res.append({"drift_mm": drift, "read": read, "gate": gate, **{k: float(v) for k, v in s.items()}}) | |
| print(f"{drift:8.0f} {read:16s} {gate:11s} {s['true']:13.1%} {s['false_complete']:15.0%} {s['hand_finish']:12.0%} " | |
| f"{s['flights']:8.2f} {s['belief_err']:15.3f} {s['gap_recall']:11.0%} {s['gap_precision']:10.0%}", flush=True) | |
| if out_path: | |
| with open(out_path, "w") as stream: | |
| json.dump({"jobs": jobs, "reads": READS, "gates": GATES, "rows": res}, stream, indent=1) | |
| return res | |
| def main(): | |
| ap_ = argparse.ArgumentParser() | |
| ap_.add_argument("--jobs", type=int, default=30) | |
| ap_.add_argument("--drift-mm", type=float, default=0.0, help="spray carried sideways by wind, NOT in the flight log") | |
| ap_.add_argument("--truth", choices=("timing", "legacy", "both"), default="timing", | |
| help="timing deposits from the timed path; legacy is the model published before it; both pairs them") | |
| ap_.add_argument("--gate-study", action="store_true", | |
| help="twin strategy only: observation gating (none / innov / edge / both) against wind drift and REAL " | |
| "read error levels (benchmark/records/edge_error_real.json, align_real.json), paired seeds") | |
| ap_.add_argument("--out", help="write the gate study as JSON") | |
| a = ap_.parse_args() | |
| if a.gate_study: | |
| gate_study(a.jobs, a.out) | |
| return | |
| models = ("timing", "legacy") if a.truth == "both" else (a.truth,) | |
| print(f"{'truth':8s} {'strategy':10s} {'true painted':>13s} {'claims done':>12s} {'false complete':>15s} {'hand finish':>12s} " | |
| f"{'flights':>8s} {'paint flown m':>14s} {'|belief-truth|':>15s} {'knows gaps':>11s} {'precision':>10s} {'waste L':>8s}") | |
| out = {} | |
| for model in models: | |
| for strat in ("blind", "observe", "twin"): | |
| s = summarise([job(strat, seed, drift_mm=a.drift_mm, model=model) for seed in range(a.jobs)]) | |
| out[(model, strat)] = s | |
| done = s["claims_done"] | |
| print(f"{model:8s} {strat:10s} {s['true']:13.1%} {done:12.0%} {s['false_complete']:15.0%} {s['hand_finish']:12.0%} " | |
| f"{s['flights']:8.2f} {s['paint_m']:14.0f} {s['belief_err']:15.3f} {s['gap_recall']:11.0%} " | |
| f"{s['gap_precision']:10.0%} {s['waste_l']:8.2f}") | |
| if len(models) == 2: | |
| print("\npaired difference (timing - legacy), same seeds and hidden errors:") | |
| for strat in ("blind", "observe", "twin"): | |
| t, l = out[("timing", strat)], out[("legacy", strat)] | |
| print(f" {strat:8s} true painted {t['true'] - l['true']:+.1%} " | |
| f"false complete {t['false_complete'] - l['false_complete']:+.0%} " | |
| f"flights {t['flights'] - l['flights']:+.2f} |belief-truth| {t['belief_err'] - l['belief_err']:+.3f}") | |
| if __name__ == "__main__": | |
| main() | |