#!/usr/bin/env python3 """Does the digital twin finish a facade better than flying the plan blind, and does its belief track a truth it cannot see? Simulated job, truth hidden from the twin: each pass is flown with a logged sideways error plus a hidden one; paint is whatever the truth film says (>= 0.5 of target = painted). Three strategies, same facade, same errors, 30 jobs each: blind fly the plan once, call it done (what a plan-only system does) twin fly, log telemetry, camera-observe, fly the gaps the belief shows, repeat until the belief says done observe as 'twin' but belief from camera only (flight log ignored) Scored against truth: painted share of the target, false-complete rate (a **complete** verdict while truth is under 97 %), the share that stops with a hand-finish strip (residual work correctly left to a person rather than claimed done), flights, paint flown. ``--truth timing`` (default) deposits paint from the timed path - each pass at its own speed and standoff, with a hidden speed error per pass - and the twin does the same. ``--truth legacy`` re-derives the numbers published before the timing model. ``--truth both`` runs both on the *same* seeds and the *same* hidden errors (paired), so the difference is the model's and not sampling noise. python eval_twin.py [--jobs 30] [--truth timing|legacy|both] [--drift-mm 20] """ from __future__ import annotations import argparse import numpy as np import application as ap import drone_areas as da import drone_path as dp import drone_twin as dt MM = 25.0 LOG_SIGMA, HIDDEN_SIGMA, POS = 0.02, 0.012, 0.02 def facade(): c = np.full((300, 480), 2, np.uint8) for r in (40, 160): for x in (50, 160, 270, 380): c[r:r + 70, x:x + 60] = 8 c[220:300, 200:260] = 7 return c def spray(): tip = ap.SprayTip.parse("517") return ap.spray_settings(tip.flow_lps(2000), tip.fan_angle_deg, 300, 38, 0.38) def true_share(film, target): return float(((film >= dt.PAINTED_FILM) & target).sum() / max(int(target.sum()), 1)) def stroke_len(strokes): return sum(np.hypot(s[-1][0] - s[0][0], s[-1][1] - s[0][1]) for s in strokes) * MM / 1000.0 def corrupt(seen, rng, shift_px=0.0, edge_px=0.0): """A camera read as it really arrives: shifted (registration / parallax) and with wandering edges (segmentation). ``shift_px`` is the sigma of a whole-read translation; ``edge_px`` the scale of local edge error: the read is blurred at that scale, noise added and re-thresholded, which moves each edge by about +-edge_px with no net bias. Pixels shifted in from outside the frame are returned as not observed. """ from scipy import ndimage out, valid = seen.astype(np.float32), np.ones(seen.shape, bool) if edge_px > 0: soft = ndimage.gaussian_filter(out, edge_px) noise = ndimage.gaussian_filter(rng.normal(0, 1, seen.shape), edge_px) noise *= 0.25 / max(noise.std(), 1e-6) out = (soft + noise >= 0.5).astype(np.float32) if shift_px > 0: dy, dx = (int(round(v)) for v in rng.normal(0.0, shift_px, 2)) out = np.roll(out, (dy, dx), axis=(0, 1)) if dy > 0: valid[:dy] = False elif dy < 0: valid[dy:] = False if dx > 0: valid[:, :dx] = False elif dx < 0: valid[:, dx:] = False return out >= 0.5, valid GATES = {"none": {}, "innov": {"resolution_px": 6}, "edge": {"edge_px": 3}, "edge+innov": {"edge_px": 3, "resolution_px": 6}} def job(strategy, seed, max_rounds=4, drift_mm=0.0, model="timing", shift_px=0.0, edge_px=0.0, gate="none"): rng = np.random.default_rng(seed) sp, classes = spray(), facade() twin = dt.Twin() f = twin.add_facade("south", classes, MM, masked_openings=True) plan = da.plan_passes(classes, MM, sp, positioning_m=POS, masked_openings=True) strokes = plan["main_strokes_px"] + plan["edge_strokes_px"] truth = np.zeros(classes.shape, np.float32) flown_m, flights, declared, stopped, verdict, waste_l, audits = 0.0, 0, False, False, "none", 0.0, 0 for rnd in range(max_rounds): flown = dt.fly(strokes, MM, POS, rng) # logged positions # Drawn once and given to whichever truth model runs, so a legacy/timing comparison is paired. hidden = rng.normal(0.0, HIDDEN_SIGMA * 1000.0 / MM, len(strokes)) jitter = 1.0 + rng.normal(0.0, 0.10, len(strokes)) truth = dt.truth_film(flown, classes.shape, MM, sp["fan_width_mm"], sp["pass_spacing_mm"], HIDDEN_SIGMA, rng, truth, unlogged_drift_mm=drift_mm, model=model, spray=sp, standoff_mm=sp["standoff_mm"], hidden_offsets_px=hidden, speed_jitter=jitter) flown_m += stroke_len(strokes); flights += 1 if strategy == "blind": verdict, declared, stopped = "declared done", True, True break if strategy == "twin": timeline = (dp.from_strokes(flown, MM, classes.shape, sp, standoff_mm=sp["standoff_mm"]) if model == "timing" else None) audit = twin.apply_sortie("south", flown, sp["fan_width_mm"], sp["pass_spacing_mm"], telemetry_sigma_m=LOG_SIGMA, timeline=timeline) if audit: waste_l += audit.get("waste_l", 0.0) audits += 1 seen = dt.detector((truth >= dt.PAINTED_FILM) & f.target | ((truth >= dt.PAINTED_FILM) & ~f.target), rng) seen, valid = corrupt(seen, rng, shift_px, edge_px) twin.observe("south", seen, valid=valid, **GATES[gate]) st = twin.status("south", sp["fan_width_mm"]) verdict = st["verdict"] if verdict != "another_sortie": # finished, or the last strips are for a hand finish: no more drone flights stopped = True declared = verdict == "complete" # only a "complete" verdict is a claim that the wall is done break nxt = twin.next_sortie("south", sp["fan_width_mm"]) strokes = nxt["plan"]["strokes"] tg = f.target & (truth < dt.PAINTED_FILM) known = twin.gaps("south") if strategy != "blind" else np.zeros_like(tg) return {"true": true_share(truth, f.target), "declared": declared, "stopped": stopped, "verdict": verdict, "flights": flights, "paint_m": flown_m, "belief": twin.coverage("south")["believed_painted"] if strategy != "blind" else None, "gap_recall": float((tg & known).sum() / max(tg.sum(), 1)), "gap_precision": float((tg & known).sum() / max(known.sum(), 1)) if known.any() else float("nan"), "left_m2": float(tg.sum()) * (MM / 1000) ** 2, "waste_l": waste_l if audits else None} def summarise(rows): return {"true": np.mean([r["true"] for r in rows]), "false_complete": np.mean([r["declared"] and r["true"] < 0.97 for r in rows]), "hand_finish": np.mean([r["verdict"] in ("hand_finish", "needs_look") for r in rows]), "claims_done": np.mean([r["verdict"] in ("complete", "declared done") for r in rows]), "flights": np.mean([r["flights"] for r in rows]), "paint_m": np.mean([r["paint_m"] for r in rows]), "belief_err": np.mean([abs(r["belief"] - r["true"]) for r in rows]) if rows[0]["belief"] is not None else float("nan"), "gap_recall": np.mean([r["gap_recall"] for r in rows]), "gap_precision": (lambda p: float(np.mean(p)) if p else float("nan"))([r["gap_precision"] for r in rows if np.isfinite(r["gap_precision"])]), "waste_l": np.mean([r["waste_l"] for r in rows if r["waste_l"] is not None]) if any(r["waste_l"] is not None for r in rows) else float("nan")} # Read error levels on the twin's 480-px-wide facade, from real measurements (2026-10-24): # alignment (drone_align on real facades, synthetic move): median 0.004 % of width -> ~0 px, so no shift term from it; # segmentation edges (C2-cal, 87 real facade frames): median 0.49 % of width = 2.3 px; p90 image 0.79 % = 3.8 px. # "+shift" adds a 2 px whole-read shift as a stand-in for parallax, which no real pair has measured yet. READS = {"perfect": (0.0, 0.0), "real-median": (0.0, 2.3), "real-p90": (0.0, 3.8), "real-p90+shift": (2.0, 3.8)} def gate_study(jobs, out_path=None): import json res = [] print(f"{'drift mm':>8s} {'read':16s} {'gate':11s} {'true painted':>13s} {'false complete':>15s} {'hand finish':>12s} " f"{'flights':>8s} {'|belief-truth|':>15s} {'knows gaps':>11s} {'precision':>10s}") for drift in (0.0, 150.0): for read, (shift, edge) in READS.items(): for gate in GATES: s = summarise([job("twin", seed, drift_mm=drift, shift_px=shift, edge_px=edge, gate=gate) for seed in range(jobs)]) res.append({"drift_mm": drift, "read": read, "gate": gate, **{k: float(v) for k, v in s.items()}}) print(f"{drift:8.0f} {read:16s} {gate:11s} {s['true']:13.1%} {s['false_complete']:15.0%} {s['hand_finish']:12.0%} " f"{s['flights']:8.2f} {s['belief_err']:15.3f} {s['gap_recall']:11.0%} {s['gap_precision']:10.0%}", flush=True) if out_path: with open(out_path, "w") as stream: json.dump({"jobs": jobs, "reads": READS, "gates": GATES, "rows": res}, stream, indent=1) return res def main(): ap_ = argparse.ArgumentParser() ap_.add_argument("--jobs", type=int, default=30) ap_.add_argument("--drift-mm", type=float, default=0.0, help="spray carried sideways by wind, NOT in the flight log") ap_.add_argument("--truth", choices=("timing", "legacy", "both"), default="timing", help="timing deposits from the timed path; legacy is the model published before it; both pairs them") ap_.add_argument("--gate-study", action="store_true", help="twin strategy only: observation gating (none / innov / edge / both) against wind drift and REAL " "read error levels (benchmark/records/edge_error_real.json, align_real.json), paired seeds") ap_.add_argument("--out", help="write the gate study as JSON") a = ap_.parse_args() if a.gate_study: gate_study(a.jobs, a.out) return models = ("timing", "legacy") if a.truth == "both" else (a.truth,) print(f"{'truth':8s} {'strategy':10s} {'true painted':>13s} {'claims done':>12s} {'false complete':>15s} {'hand finish':>12s} " f"{'flights':>8s} {'paint flown m':>14s} {'|belief-truth|':>15s} {'knows gaps':>11s} {'precision':>10s} {'waste L':>8s}") out = {} for model in models: for strat in ("blind", "observe", "twin"): s = summarise([job(strat, seed, drift_mm=a.drift_mm, model=model) for seed in range(a.jobs)]) out[(model, strat)] = s done = s["claims_done"] print(f"{model:8s} {strat:10s} {s['true']:13.1%} {done:12.0%} {s['false_complete']:15.0%} {s['hand_finish']:12.0%} " f"{s['flights']:8.2f} {s['paint_m']:14.0f} {s['belief_err']:15.3f} {s['gap_recall']:11.0%} " f"{s['gap_precision']:10.0%} {s['waste_l']:8.2f}") if len(models) == 2: print("\npaired difference (timing - legacy), same seeds and hidden errors:") for strat in ("blind", "observe", "twin"): t, l = out[("timing", strat)], out[("legacy", strat)] print(f" {strat:8s} true painted {t['true'] - l['true']:+.1%} " f"false complete {t['false_complete'] - l['false_complete']:+.0%} " f"flights {t['flights'] - l['flights']:+.2f} |belief-truth| {t['belief_err'] - l['belief_err']:+.3f}") if __name__ == "__main__": main()