File size: 4,695 Bytes
c2f861b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
#!/usr/bin/env python3
"""Run NOEON's real multimodal alignment experiment and record the results.

Two experiments are run, both with genuine source embeddings read from the Hub:

* **Real pairs.** The same token id is looked up in each slot's real
  ``embed_tokens`` matrix and the cross-slot cosine and retrieval accuracy are
  measured *before* and *after* training a contrastive projection.
* **Control.** The target slot's rows are deliberately scrambled (a fixed random
  permutation of the batch), so the two sides are *known* to be misaligned. The
  projection must recover the alignment. This proves the trainer and the metric
  actually work — without a control, "after > before" could be an artefact.

The results are written to ``reports/alignment.json``. Nothing is asserted about
behaviour that was not measured.

Usage:
    python scripts/run_alignment_experiment.py --tokens 256 --steps 300
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

import torch

from noeon.alignment import align_pair, save_projection
from noeon.alignment.extract import extract_all

ROOT = Path(__file__).resolve().parent.parent


def run(tokens: int, steps: int, seed: int) -> dict:
    ids = list(range(1000, 1000 + tokens))
    emb = extract_all(ids)
    rows = {k: e.rows for k, e in emb.items()}
    slots = list(rows)

    pairs = [("vision", "computer_use"), ("vision", "generation"),
             ("computer_use", "generation")]
    real = []
    for a, b in pairs:
        proj, res = align_pair(rows[a], rows[b], (a, b), steps=steps, seed=seed)
        real.append(res.to_dict())
        save_projection(proj, ROOT / "checkpoints" / "alignment" / f"{a}__{b}",
                        {"pair": [a, b], "steps": steps, "seed": seed,
                         "result": res.to_dict()})
        print(f"real  {a:12s}->{b:12s}  cos {res.before_cosine:.4f} -> "
              f"{res.after_cosine:.4f}  top1 {res.before_top1:.3f} -> "
              f"{res.after_top1:.3f}")

    # Control: scramble the target so it is genuinely misaligned.
    torch.manual_seed(seed)
    perm = torch.randperm(tokens)
    ctrl = []
    a, b = "vision", "generation"
    proj, res = align_pair(rows[a], rows[b][perm], (f"{a}", f"{b}(scrambled)"),
                           steps=steps, seed=seed)
    ctrl.append(res.to_dict())
    print(f"ctrl  {a:12s}->{b}(scrambled)  cos {res.before_cosine:.4f} -> "
          f"{res.after_cosine:.4f}  top1 {res.before_top1:.3f} -> "
          f"{res.after_top1:.3f}")

    return {
        "tokens": tokens, "steps": steps, "seed": seed,
        "source": "weights/upstream/*/embed_tokens (real, via Hub range reads)",
        "probe": "token-embedding rows for identical token ids across slots",
        "real_pairs": real,
        "control_scrambled": ctrl,
        "state": "measured",
        "conclusion": _conclusion(real, ctrl),
    }


def _conclusion(real: list[dict], ctrl: list[dict]) -> str:
    real_aligned = all(r["before_cosine"] > 0.9 for r in real)
    real_top1 = all(r["before_top1"] > 0.99 for r in real)
    ctrl_recovers = ctrl[0]["after_top1"] > ctrl[0]["before_top1"] + 0.5
    parts = []
    if real_aligned and real_top1:
        parts.append(
            "The three slots are ALREADY strongly aligned at the token-embedding "
            "level (cross-slot cosine > 0.9 and same-token retrieval at "
            "ceiling), so a projection is not required to make their embedding "
            "spaces comparable — the shared vocabulary and shared pretraining "
            "lineage already do that.")
    else:
        parts.append("The real slots are NOT fully aligned; a learned projection "
                     "is required.")
    if ctrl_recovers:
        parts.append(
            "On the deliberately scrambled control the same trainer recovers "
            "alignment from near-chance, which shows the method and metric are "
            "real and not an artefact.")
    return " ".join(parts)


def main(argv: list[str]) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--tokens", type=int, default=256)
    ap.add_argument("--steps", type=int, default=300)
    ap.add_argument("--seed", type=int, default=0)
    args = ap.parse_args(argv[1:])

    result = run(args.tokens, args.steps, args.seed)
    out = ROOT / "reports" / "alignment.json"
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps(result, indent=2))
    print(f"\nwrote {out}")
    print("conclusion:", result["conclusion"])
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv))