NOEON / scripts /run_alignment_experiment.py
openhands
openhands
Build NOEON unified brain: memory, alignment, planning, evaluation
c2f861b
Raw History Blame Contribute Delete
4.7 kB
#!/usr/bin/env python3
"""Run NOEON's real multimodal alignment experiment and record the results.
Two experiments are run, both with genuine source embeddings read from the Hub:
* **Real pairs.** The same token id is looked up in each slot's real
``embed_tokens`` matrix and the cross-slot cosine and retrieval accuracy are
measured *before* and *after* training a contrastive projection.
* **Control.** The target slot's rows are deliberately scrambled (a fixed random
permutation of the batch), so the two sides are *known* to be misaligned. The
projection must recover the alignment. This proves the trainer and the metric
actually work — without a control, "after > before" could be an artefact.
The results are written to ``reports/alignment.json``. Nothing is asserted about
behaviour that was not measured.
Usage:
python scripts/run_alignment_experiment.py --tokens 256 --steps 300
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch
from noeon.alignment import align_pair, save_projection
from noeon.alignment.extract import extract_all
ROOT = Path(__file__).resolve().parent.parent
def run(tokens: int, steps: int, seed: int) -> dict:
ids = list(range(1000, 1000 + tokens))
emb = extract_all(ids)
rows = {k: e.rows for k, e in emb.items()}
slots = list(rows)
pairs = [("vision", "computer_use"), ("vision", "generation"),
("computer_use", "generation")]
real = []
for a, b in pairs:
proj, res = align_pair(rows[a], rows[b], (a, b), steps=steps, seed=seed)
real.append(res.to_dict())
save_projection(proj, ROOT / "checkpoints" / "alignment" / f"{a}__{b}",
{"pair": [a, b], "steps": steps, "seed": seed,
"result": res.to_dict()})
print(f"real {a:12s}->{b:12s} cos {res.before_cosine:.4f} -> "
f"{res.after_cosine:.4f} top1 {res.before_top1:.3f} -> "
f"{res.after_top1:.3f}")
# Control: scramble the target so it is genuinely misaligned.
torch.manual_seed(seed)
perm = torch.randperm(tokens)
ctrl = []
a, b = "vision", "generation"
proj, res = align_pair(rows[a], rows[b][perm], (f"{a}", f"{b}(scrambled)"),
steps=steps, seed=seed)
ctrl.append(res.to_dict())
print(f"ctrl {a:12s}->{b}(scrambled) cos {res.before_cosine:.4f} -> "
f"{res.after_cosine:.4f} top1 {res.before_top1:.3f} -> "
f"{res.after_top1:.3f}")
return {
"tokens": tokens, "steps": steps, "seed": seed,
"source": "weights/upstream/*/embed_tokens (real, via Hub range reads)",
"probe": "token-embedding rows for identical token ids across slots",
"real_pairs": real,
"control_scrambled": ctrl,
"state": "measured",
"conclusion": _conclusion(real, ctrl),
}
def _conclusion(real: list[dict], ctrl: list[dict]) -> str:
real_aligned = all(r["before_cosine"] > 0.9 for r in real)
real_top1 = all(r["before_top1"] > 0.99 for r in real)
ctrl_recovers = ctrl[0]["after_top1"] > ctrl[0]["before_top1"] + 0.5
parts = []
if real_aligned and real_top1:
parts.append(
"The three slots are ALREADY strongly aligned at the token-embedding "
"level (cross-slot cosine > 0.9 and same-token retrieval at "
"ceiling), so a projection is not required to make their embedding "
"spaces comparable — the shared vocabulary and shared pretraining "
"lineage already do that.")
else:
parts.append("The real slots are NOT fully aligned; a learned projection "
"is required.")
if ctrl_recovers:
parts.append(
"On the deliberately scrambled control the same trainer recovers "
"alignment from near-chance, which shows the method and metric are "
"real and not an artefact.")
return " ".join(parts)
def main(argv: list[str]) -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--tokens", type=int, default=256)
ap.add_argument("--steps", type=int, default=300)
ap.add_argument("--seed", type=int, default=0)
args = ap.parse_args(argv[1:])
result = run(args.tokens, args.steps, args.seed)
out = ROOT / "reports" / "alignment.json"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(result, indent=2))
print(f"\nwrote {out}")
print("conclusion:", result["conclusion"])
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv))