File size: 6,938 Bytes
9c6799a
 
2481b65
 
9c6799a
 
 
 
 
 
 
 
 
 
 
 
 
2481b65
9c6799a
2ea7404
9c6799a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2481b65
9c6799a
2ea7404
 
 
 
9c6799a
 
 
 
 
 
 
 
 
 
 
 
 
2ea7404
9c6799a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e631c4e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9c6799a
 
2481b65
 
 
9c6799a
 
e631c4e
 
9c6799a
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
"""Tables for the probe and representation results -> report/tables_extra.tex.

TableDomainsTwo: per-domain validation loss after 2B tokens (report/probe2.json, plus the DROID control
from report/probe3.json), mean over seeds.
TableRank: effective rank of final-layer features and dormant MLP units (report/rank.json, rank2.json).
"""

import json
import re
from collections import defaultdict
from pathlib import Path

import numpy as np

REPORT = Path(__file__).resolve().parents[1] / "report"
DOMAINS = [("ssv2", "SSv2 action templates"), ("ssv2_abs", "same templates, abstract nouns"),
           ("abs_tmpl", "other templates, abstract nouns"), ("droid", "DROID robot instructions"),
           ("droid_abs", "same instructions, abstract nouns"),
           ("piqa", "PIQA"), ("fineweb", "FineWeb-Edu"), ("wiki", "WikiText-103"), ("gsm8k", "GSM8K")]
MEDIUM = [("scratch", r"(?:^|-)scratch-none-lr2e-3"), ("pretrained", r"-ivgpt-none-lr3e-3"), ("S\\&P", r"-sp0\.4-none")]
SMALL = [("scratch", r"^act-sm-scratch"), ("shuffled", r"^act-base-shuffled"), ("pretrained", r"^act-base-ivgpt")]


def group(probe, pattern, budget):
    rows = [v for k, v in probe.items() if re.search(pattern, k) and k.split("-")[-1 if "-s" not in k[-3:] else -2] == budget]
    return rows


def domain_table(probe):
    cols = [(lab, group(probe, p, "2000M")) for lab, p in MEDIUM] + [(lab, group(probe, p, "2000M")) for lab, p in SMALL]
    for lab, rs in cols:
        assert rs, lab
    lines = [r"\newcommand{\TableDomainsTwo}{", r"\begin{tabular}{lcccccc}", r"\toprule",
             r"& \multicolumn{3}{c}{medium (400M backbone)} & \multicolumn{3}{c}{small (113M backbone)} \\",
             r"\cmidrule(lr){2-4}\cmidrule(lr){5-7}",
             "text & " + " & ".join(lab for lab, _ in cols) + r" \\", r"\midrule"]
    for key, name in DOMAINS:
        vals = [float(np.mean([r[key] for r in rs])) for _, rs in cols]
        cells = []
        for block in (vals[:3], vals[3:]):
            lo = min(block)
            cells += [rf"\textbf{{{v:.3f}}}" if v == lo else f"{v:.3f}" for v in block]
        lines.append(f"{name} & " + " & ".join(cells) + r" \\")
        if key in ("abs_tmpl", "droid_abs"):
            lines.append(r"\midrule")
    lines.append(r"\midrule")
    for key, name in [("copy_second", "copying, second copy"), ("icl_score", "in-context score")]:
        vals = [float(np.mean([r[key] for r in rs])) for _, rs in cols]
        lines.append(f"{name} & " + " & ".join(f"{v:.2f}" if key == "copy_second" else f"{v:.3f}" for v in vals) + r" \\")
    lines += [r"\bottomrule", r"\end{tabular}}"]
    print("seeds per column:", [(lab, len(rs)) for lab, rs in cols])
    return lines


def rank_table(rank):
    def mean_of(prefixes):
        rs = [v for k, v in rank.items() if any(k == p or (p.startswith("trained/") and k.startswith(p)) for p in prefixes)]
        return (float(np.mean([r["eff_rank"] for r in rs])), float(np.mean([r["dormant"] for r in rs])),
                float(np.mean([np.mean(r["per_layer_dormant"][:2]) for r in rs])), len(rs))

    rows = [
        ("medium", "scratch", ["init/m_scratch"], ["trained/10006a-scratch", "trained/050000-scratch"]),
        ("", "pretrained", ["init/m_ivgpt"], ["trained/10007-ivgpt", "trained/050000-ivgpt", "trained/z0723-ivgpt"]),
        ("", "shuffled", ["init/m_shuffled"], ["trained/z0700-shuffled"]),
        ("", "rescaled", ["init/m_rescaled"], []),
        ("", "S\\&P", ["init/m_sp"], ["trained/0600-sp0.4", "trained/0551-sp0.4"]),
        ("", "shuffled S\\&P", ["init/m_spshuf"], ["trained/z0702-spshuf"]),
        ("small", "scratch", ["init/s_scratch"], ["trained/act-sm-scratch"]),
        ("", "pretrained", ["init/s_base"], ["trained/act-base-ivgpt"]),
        ("", "shuffled", ["init/s_shuffled"], ["trained/act-base-shuffled"]),
        ("", "S\\&P", ["init/s_sp"], []),
    ]
    lines = [r"\newcommand{\TableRank}{", r"\begin{tabular}{llcccccc}", r"\toprule",
             r"& & \multicolumn{3}{c}{at initialization} & \multicolumn{3}{c}{after 500M tokens} \\",
             r"\cmidrule(lr){3-5}\cmidrule(lr){6-8}",
             r"model & initialization & eff.\ rank & dormant & dormant, layers 1--2 & eff.\ rank & dormant & dormant, layers 1--2 \\",
             r"\midrule"]
    for size, lab, init, trained in rows:
        if size == "small":
            lines.append(r"\midrule")
        a = mean_of(init)
        cells = [f"{a[0]:.0f}", f"{100 * a[1]:.1f}\\%", f"{100 * a[2]:.1f}\\%"]
        if trained:
            b = mean_of(trained)
            cells += [f"{b[0]:.0f}", f"{100 * b[1]:.1f}\\%", f"{100 * b[2]:.1f}\\%"]
            print(size, lab, "trained seeds", b[3])
        else:
            cells += ["--", "--", "--"]
        lines.append(f"{size} & {lab} & " + " & ".join(cells) + r" \\")
    lines += [r"\bottomrule", r"\end{tabular}}"]
    return lines


def position_table(pos):
    """Loss by position in the 1024-token window after 2B tokens, mean over seeds."""
    bins = list(next(iter(pos.values()))["ssv2"].keys())
    groups = [("medium", "scratch", r"^045\d+-scratch"), ("", "pretrained", r"^045\d+-ivgpt"),
              ("small", "scratch", r"^act-sm-scratch"), ("", "shuffled", r"^act-base-shuffled"),
              ("", "pretrained", r"^act-base-ivgpt")]
    lines = [r"\newcommand{\TablePosition}{", r"\begin{tabular}{lll" + "c" * len(bins) + "}", r"\toprule",
             r"& & & \multicolumn{" + str(len(bins)) + r"}{c}{token position in the window} \\",
             r"\cmidrule(lr){4-" + str(3 + len(bins)) + "}",
             "text & model & initialization & " + " & ".join(b.replace("-", "--") for b in bins) + r" \\", r"\midrule"]
    for key, name in [("ssv2", "action templates"), ("fineweb", "FineWeb-Edu")]:
        for i, (size, lab, pat) in enumerate(groups):
            rs = [v for k, v in pos.items() if re.search(pat, k) and "2000M" in k]
            vals = [float(np.mean([r[key][b] for r in rs])) for b in bins]
            first = name if i == 0 else ""
            lines.append(f"{first} & {size} & {lab} & " + " & ".join(f"{v:.3f}" for v in vals) + r" \\")
        if key == "ssv2":
            lines.append(r"\midrule")
    lines += [r"\bottomrule", r"\end{tabular}}"]
    return lines


def main():
    probe = json.loads((REPORT / "probe2.json").read_text())
    for run, v in json.loads((REPORT / "probe3.json").read_text()).items():  # adds the DROID noun-swap control
        if run in probe:
            probe[run]["droid_abs"] = v["droid_abs"]
    rank = json.loads((REPORT / "rank.json").read_text()) | json.loads((REPORT / "rank2.json").read_text())
    out = domain_table(probe) + rank_table(rank)
    if (REPORT / "position.json").exists():
        out += position_table(json.loads((REPORT / "position.json").read_text()))
    (REPORT / "tables_extra.tex").write_text("\n".join(out) + "\n")
    print("wrote", REPORT / "tables_extra.tex")


if __name__ == "__main__":
    main()