#!/usr/bin/env python3 """Run NOEON's real multimodal alignment experiment and record the results. Two experiments are run, both with genuine source embeddings read from the Hub: * **Real pairs.** The same token id is looked up in each slot's real ``embed_tokens`` matrix and the cross-slot cosine and retrieval accuracy are measured *before* and *after* training a contrastive projection. * **Control.** The target slot's rows are deliberately scrambled (a fixed random permutation of the batch), so the two sides are *known* to be misaligned. The projection must recover the alignment. This proves the trainer and the metric actually work — without a control, "after > before" could be an artefact. The results are written to ``reports/alignment.json``. Nothing is asserted about behaviour that was not measured. Usage: python scripts/run_alignment_experiment.py --tokens 256 --steps 300 """ from __future__ import annotations import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import torch from noeon.alignment import align_pair, save_projection from noeon.alignment.extract import extract_all ROOT = Path(__file__).resolve().parent.parent def run(tokens: int, steps: int, seed: int) -> dict: ids = list(range(1000, 1000 + tokens)) emb = extract_all(ids) rows = {k: e.rows for k, e in emb.items()} slots = list(rows) pairs = [("vision", "computer_use"), ("vision", "generation"), ("computer_use", "generation")] real = [] for a, b in pairs: proj, res = align_pair(rows[a], rows[b], (a, b), steps=steps, seed=seed) real.append(res.to_dict()) save_projection(proj, ROOT / "checkpoints" / "alignment" / f"{a}__{b}", {"pair": [a, b], "steps": steps, "seed": seed, "result": res.to_dict()}) print(f"real {a:12s}->{b:12s} cos {res.before_cosine:.4f} -> " f"{res.after_cosine:.4f} top1 {res.before_top1:.3f} -> " f"{res.after_top1:.3f}") # Control: scramble the target so it is genuinely misaligned. torch.manual_seed(seed) perm = torch.randperm(tokens) ctrl = [] a, b = "vision", "generation" proj, res = align_pair(rows[a], rows[b][perm], (f"{a}", f"{b}(scrambled)"), steps=steps, seed=seed) ctrl.append(res.to_dict()) print(f"ctrl {a:12s}->{b}(scrambled) cos {res.before_cosine:.4f} -> " f"{res.after_cosine:.4f} top1 {res.before_top1:.3f} -> " f"{res.after_top1:.3f}") return { "tokens": tokens, "steps": steps, "seed": seed, "source": "weights/upstream/*/embed_tokens (real, via Hub range reads)", "probe": "token-embedding rows for identical token ids across slots", "real_pairs": real, "control_scrambled": ctrl, "state": "measured", "conclusion": _conclusion(real, ctrl), } def _conclusion(real: list[dict], ctrl: list[dict]) -> str: real_aligned = all(r["before_cosine"] > 0.9 for r in real) real_top1 = all(r["before_top1"] > 0.99 for r in real) ctrl_recovers = ctrl[0]["after_top1"] > ctrl[0]["before_top1"] + 0.5 parts = [] if real_aligned and real_top1: parts.append( "The three slots are ALREADY strongly aligned at the token-embedding " "level (cross-slot cosine > 0.9 and same-token retrieval at " "ceiling), so a projection is not required to make their embedding " "spaces comparable — the shared vocabulary and shared pretraining " "lineage already do that.") else: parts.append("The real slots are NOT fully aligned; a learned projection " "is required.") if ctrl_recovers: parts.append( "On the deliberately scrambled control the same trainer recovers " "alignment from near-chance, which shows the method and metric are " "real and not an artefact.") return " ".join(parts) def main(argv: list[str]) -> int: ap = argparse.ArgumentParser() ap.add_argument("--tokens", type=int, default=256) ap.add_argument("--steps", type=int, default=300) ap.add_argument("--seed", type=int, default=0) args = ap.parse_args(argv[1:]) result = run(args.tokens, args.steps, args.seed) out = ROOT / "reports" / "alignment.json" out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(result, indent=2)) print(f"\nwrote {out}") print("conclusion:", result["conclusion"]) return 0 if __name__ == "__main__": raise SystemExit(main(sys.argv))