RoboRACE-Net / racenet_analysis.py
Nabidnur's picture
RACE-Net weights, card and code
6adf6f7 verified
Raw History Blame Contribute Delete
4.43 kB
#!/usr/bin/env python3
"""RACE-Net scaling study on an independent synthetic generator (CPU, about a minute).
The generator is `trackA.synthetic` (beta-distributed scores, independent errors), which is NOT the
distribution RACE-Net was trained on (`racenet.sample_batch`). Three sweeps, 20 seeds each:
liar fraction J = 5 and J = 16 agents, window T = 100
window length J = 5, 3 liars, T = 8 … 256 (how many unlabelled episodes each method needs)
swarm size J = 3 … 24, 60% liars (training used J <= 11 columns)
receiver skill one near-perfect peer and a receiver between 55% and 90% accuracy
Writes results/racenet/analysis.json.
"""
import json
import sys
import time
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from robotics.bench import trackA # noqa: E402
from robotics.swarm import attacks, fusion, racenet # noqa: E402
METHODS = {
"receiver_only": lambda P: fusion.single(P, P, 0),
"majority": lambda P: fusion.majority(P, P),
"dawid_skene": lambda P: fusion.dawid_skene(P, P),
"race_online": lambda P: fusion.race(P, P, 0),
"race_s_online": lambda P: fusion.race_s(P, P, 0)[:2],
"race_net": lambda P: racenet.race_net(P, P, 0),
}
V1 = Path(__file__).resolve().parents[1] / "models/racenet/candidates/v1_local"
if racenet.available(V1): # the first release (generator v1), shown next to the released model
METHODS["race_net_v1"] = lambda P: racenet.race_net(P, P, 0, path=V1)
METHODS["race_pool"] = lambda P: fusion.race_pool(P, P, 0)
ATTACKS = ("invert", "collude", "mix", "noise")
SEEDS = 20
def cell(J, T, m, seed, accs=None):
"""One window: receiver accuracy 0.80, peers 0.65-0.85, m lying peers, averaged over the attack list."""
rng = np.random.default_rng(1_000_003 * seed + 1009 * J + 17 * T + m)
accs = list(accs) if accs else [0.80] + list(rng.uniform(0.65, 0.85, J - 1))
_, (P, y, _) = trackA.synthetic(n=T, accs=accs, seed=int(rng.integers(1 << 30)))
liars = list(1 + rng.choice(J - 1, m, replace=False)) if m else []
out, sec = {k: [] for k in METHODS}, {k: 0.0 for k in METHODS}
for a in (ATTACKS if m else ("none",)):
Pa = attacks.apply(P, liars, a, "eval", np.random.default_rng(seed)) if m else P
for k, f in METHODS.items():
t = time.perf_counter()
s, thr = f(Pa)
sec[k] += time.perf_counter() - t
out[k].append(float(np.mean((np.asarray(s) > thr) == y)))
n = len(ATTACKS) if m else 1
return {k: float(np.mean(v)) for k, v in out.items()}, {k: v / n for k, v in sec.items()}
def sweep(points):
rows = []
for label, J, T, m, *fixed in points:
accs, secs = zip(*(cell(J, T, m, s, *fixed) for s in range(SEEDS)))
rows.append({"x": label, "J": J, "T": T, "liars": m,
"acc": {k: float(np.mean([a[k] for a in accs])) for k in METHODS},
"se": {k: float(np.std([a[k] for a in accs]) / np.sqrt(SEEDS)) for k in METHODS},
"ms": {k: 1000 * float(np.median([s[k] for s in secs])) for k in METHODS}})
print(label, J, T, m, {k: round(100 * v, 1) for k, v in rows[-1]["acc"].items()}, flush=True)
return rows
def main():
assert racenet.available(), "train RACE-Net first (scripts/train_racenet.py)"
out = {"generator": "trackA.synthetic (independent of the training generator)", "seeds": SEEDS, "attacks": list(ATTACKS),
"liar_fraction_J5": sweep([(m / 5, 5, 100, m) for m in range(0, 5)]),
"liar_fraction_J16": sweep([(m / 16, 16, 100, m) for m in (0, 3, 6, 8, 10, 12, 14, 15)]),
"window": sweep([(T, 5, T, 3) for T in (8, 16, 32, 64, 128, 256)]),
"swarm_size": sweep([(J, J, 100, int(round(0.6 * J))) for J in (3, 5, 7, 9, 12, 16, 24)]),
# heterogeneous skill: one near-perfect peer, one strong, two weak; the receiver's own accuracy varies
"receiver_skill_clean": sweep([(a, 5, 100, 0, (a, 0.97, 0.9, 0.7, 0.6)) for a in (0.55, 0.6, 0.7, 0.8, 0.9)]),
"receiver_skill_2liars": sweep([(a, 5, 100, 2, (a, 0.97, 0.9, 0.7, 0.6)) for a in (0.55, 0.6, 0.7, 0.8, 0.9)])}
p = Path("results/racenet")
p.mkdir(parents=True, exist_ok=True)
(p / "analysis.json").write_text(json.dumps(out, indent=1))
print("wrote", p / "analysis.json")
if __name__ == "__main__":
main()