Download scripts/run_alignment_experiment.py from Frankenstein-Labs/NOEON: direct link, hf CLI and curl.
- Browser
- Download file 4.7 kB
-
https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/run_alignment_experiment.py
- Command line
-
hf download hf://Frankenstein-Labs/NOEON/scripts/run_alignment_experiment.py
-
curl -L -o run_alignment_experiment.py https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/run_alignment_experiment.py
4.7 kB
| #!/usr/bin/env python3 | |
| """Run NOEON's real multimodal alignment experiment and record the results. | |
| Two experiments are run, both with genuine source embeddings read from the Hub: | |
| * **Real pairs.** The same token id is looked up in each slot's real | |
| ``embed_tokens`` matrix and the cross-slot cosine and retrieval accuracy are | |
| measured *before* and *after* training a contrastive projection. | |
| * **Control.** The target slot's rows are deliberately scrambled (a fixed random | |
| permutation of the batch), so the two sides are *known* to be misaligned. The | |
| projection must recover the alignment. This proves the trainer and the metric | |
| actually work — without a control, "after > before" could be an artefact. | |
| The results are written to ``reports/alignment.json``. Nothing is asserted about | |
| behaviour that was not measured. | |
| Usage: | |
| python scripts/run_alignment_experiment.py --tokens 256 --steps 300 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| import torch | |
| from noeon.alignment import align_pair, save_projection | |
| from noeon.alignment.extract import extract_all | |
| ROOT = Path(__file__).resolve().parent.parent | |
| def run(tokens: int, steps: int, seed: int) -> dict: | |
| ids = list(range(1000, 1000 + tokens)) | |
| emb = extract_all(ids) | |
| rows = {k: e.rows for k, e in emb.items()} | |
| slots = list(rows) | |
| pairs = [("vision", "computer_use"), ("vision", "generation"), | |
| ("computer_use", "generation")] | |
| real = [] | |
| for a, b in pairs: | |
| proj, res = align_pair(rows[a], rows[b], (a, b), steps=steps, seed=seed) | |
| real.append(res.to_dict()) | |
| save_projection(proj, ROOT / "checkpoints" / "alignment" / f"{a}__{b}", | |
| {"pair": [a, b], "steps": steps, "seed": seed, | |
| "result": res.to_dict()}) | |
| print(f"real {a:12s}->{b:12s} cos {res.before_cosine:.4f} -> " | |
| f"{res.after_cosine:.4f} top1 {res.before_top1:.3f} -> " | |
| f"{res.after_top1:.3f}") | |
| # Control: scramble the target so it is genuinely misaligned. | |
| torch.manual_seed(seed) | |
| perm = torch.randperm(tokens) | |
| ctrl = [] | |
| a, b = "vision", "generation" | |
| proj, res = align_pair(rows[a], rows[b][perm], (f"{a}", f"{b}(scrambled)"), | |
| steps=steps, seed=seed) | |
| ctrl.append(res.to_dict()) | |
| print(f"ctrl {a:12s}->{b}(scrambled) cos {res.before_cosine:.4f} -> " | |
| f"{res.after_cosine:.4f} top1 {res.before_top1:.3f} -> " | |
| f"{res.after_top1:.3f}") | |
| return { | |
| "tokens": tokens, "steps": steps, "seed": seed, | |
| "source": "weights/upstream/*/embed_tokens (real, via Hub range reads)", | |
| "probe": "token-embedding rows for identical token ids across slots", | |
| "real_pairs": real, | |
| "control_scrambled": ctrl, | |
| "state": "measured", | |
| "conclusion": _conclusion(real, ctrl), | |
| } | |
| def _conclusion(real: list[dict], ctrl: list[dict]) -> str: | |
| real_aligned = all(r["before_cosine"] > 0.9 for r in real) | |
| real_top1 = all(r["before_top1"] > 0.99 for r in real) | |
| ctrl_recovers = ctrl[0]["after_top1"] > ctrl[0]["before_top1"] + 0.5 | |
| parts = [] | |
| if real_aligned and real_top1: | |
| parts.append( | |
| "The three slots are ALREADY strongly aligned at the token-embedding " | |
| "level (cross-slot cosine > 0.9 and same-token retrieval at " | |
| "ceiling), so a projection is not required to make their embedding " | |
| "spaces comparable — the shared vocabulary and shared pretraining " | |
| "lineage already do that.") | |
| else: | |
| parts.append("The real slots are NOT fully aligned; a learned projection " | |
| "is required.") | |
| if ctrl_recovers: | |
| parts.append( | |
| "On the deliberately scrambled control the same trainer recovers " | |
| "alignment from near-chance, which shows the method and metric are " | |
| "real and not an artefact.") | |
| return " ".join(parts) | |
| def main(argv: list[str]) -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--tokens", type=int, default=256) | |
| ap.add_argument("--steps", type=int, default=300) | |
| ap.add_argument("--seed", type=int, default=0) | |
| args = ap.parse_args(argv[1:]) | |
| result = run(args.tokens, args.steps, args.seed) | |
| out = ROOT / "reports" / "alignment.json" | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(result, indent=2)) | |
| print(f"\nwrote {out}") | |
| print("conclusion:", result["conclusion"]) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main(sys.argv)) | |