Babak Barazandeh
Clean upload: all experiment code and logs, PII and benchmark data scrubbed
58c26c2
Raw History Blame Contribute Delete
14.8 kB
#!/usr/bin/env python3
"""R2 β€” Which checkpoint is which?
Stage 1: weights only, no passes, no tokenizer. 2 runs per pair.
P1: lora1_merged vs base (architecture-matched, blind)
P2: decision-2 vs base (architecture differs, sanity)
"""
import json
import subprocess
import sys
from pathlib import Path
from textwrap import dedent
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness.orchestrator import Orchestrator
DGX = "<user>@<GPU_HOST>"
DGX_PASS = "<GPU_PASS>"
LOG_DIR = Path(__file__).resolve().parent / "logs"
# ---- Load answer key ----
def get_answer_key():
cmd = (
f"sshpass -p {DGX_PASS} ssh -o StrictHostKeyChecking=no {DGX} "
f"'cat /home/<user>/wsp_v2/eval/answer_key.json'"
)
result = subprocess.run(cmd, shell=True, capture_output=True, timeout=15)
return json.loads(result.stdout)
AK = get_answer_key()
P1 = AK["R2_P1"]
P1_ID_A = P1["id_A"]
P1_ID_B = P1["id_B"]
P2 = AK["R2_P2"]
P2_ID_A = P2["id_A"]
P2_ID_B = P2["id_B"]
# ---- Forward code for P1: both are standard transformers ----
FORWARD_CODE_P1 = dedent(f"""\
# Forward pass pseudocode β€” applies to BOTH checkpoints ({P1_ID_A} and {P1_ID_B})
#
# Architecture: 24 transformer layers, hidden dimension 1024, vocabulary 248320.
# - 18 layers use linear attention (selective state-space style with conv1d, gated projections,
# A_log/dt_bias parameters, and a group norm over 16 heads of dimension 128).
# - 6 layers at positions [3, 7, 11, 15, 19, 23] use standard grouped-query self-attention
# (16 query heads, 4 KV heads, head dim 128, with QK norms).
# - Every layer has an RMSNorm before attention and before the MLP.
# - MLP: SwiGLU with intermediate dimension 3584 (gate_proj, up_proj, down_proj).
# - Final RMSNorm on the output hidden states.
#
# Input: token IDs -> embed_tokens [248320, 1024] -> hidden states [batch, seq, 1024].
# Each layer: h = h + attn(layernorm(h)); h = h + mlp(layernorm(h)).
# Output: lm_head [248320, 1024] applied to the final hidden states -> logits over vocabulary.
#
# Both checkpoints share this exact architecture. They differ only in weight values.
""")
# ---- Forward codes for P2: base (standard) vs fine-tune (base + gates + scorer) ----
FORWARD_CODE_P2_STANDARD = dedent(f"""\
# Forward pass pseudocode for checkpoint {P2_ID_A}
#
# Architecture: 24 transformer layers, hidden dimension 1024, vocabulary 248320.
# - 18 layers use linear attention (selective state-space style with conv1d, gated projections,
# A_log/dt_bias parameters, and a group norm over 16 heads of dimension 128).
# - 6 layers at positions [3, 7, 11, 15, 19, 23] use standard grouped-query self-attention
# (16 query heads, 4 KV heads, head dim 128, with QK norms).
# - Every layer has an RMSNorm before attention and before the MLP.
# - MLP: SwiGLU with intermediate dimension 3584 (gate_proj, up_proj, down_proj).
# - Final RMSNorm on the output hidden states.
#
# Input: token IDs -> embed_tokens [248320, 1024] -> hidden states [batch, seq, 1024].
# Each layer: h = h + attn(layernorm(h)); h = h + mlp(layernorm(h)).
# Output: final hidden states after the last layer norm.
""")
FORWARD_CODE_P2_FINETUNE = dedent(f"""\
# Forward pass pseudocode for checkpoint {P2_ID_B}
#
# Architecture: same 24-layer backbone as {P2_ID_A} (see that description), plus two additions:
#
# ADDITION 1 β€” Bidirectional scan gates.
# On 6 of the 18 linear-attention layers (at positions [2, 6, 10, 14, 18, 22]), the linear
# attention module is wrapped: the input is processed both left-to-right (normal) and
# right-to-left (reversed). The two outputs are combined as:
# output = forward_output + tanh(gate) * reversed_output
# where gate is a single learned scalar per layer (stored as reverse_gate.N).
#
# ADDITION 2 β€” Bilinear position scorer (the output head).
# Instead of a language-model head, this checkpoint scores a set of marked positions in the
# input against a single readout position:
#
# Given hidden states h of shape [batch, seq, 1024]:
# - One position in the sequence is designated the readout position (index r).
# - K other positions are designated scored positions (indices p_1 ... p_K).
# - q = Q @ h[r] # Q is a learned matrix [256, 1024] -> q is [256]
# - k_i = K_proj @ h[p_i] # K_proj is a learned matrix [256, 1024] -> k_i is [256]
# - score_i = dot(q, k_i) / 256
# - probabilities = softmax(scores)
#
# The readout and scored positions are determined at inference time by special token IDs
# embedded in the input sequence.
#
# Extra tensors in this checkpoint (not present in {P2_ID_A}):
# query [256, 1024], key [256, 1024], reverse_gate.{{2,6,10,14,18,22}} [1] each.
""")
# Swap forward codes if d2 is A (the forward code labels must match the hex IDs)
if P2["label_A"] == "decision2":
FORWARD_CODE_P2_STANDARD, FORWARD_CODE_P2_FINETUNE = FORWARD_CODE_P2_FINETUNE, FORWARD_CODE_P2_STANDARD
# Re-generate with swapped IDs
FORWARD_CODE_P2_STANDARD = dedent(f"""\
# Forward pass pseudocode for checkpoint {P2_ID_B}
#
# Architecture: 24 transformer layers, hidden dimension 1024, vocabulary 248320.
# - 18 layers use linear attention (selective state-space style with conv1d, gated projections,
# A_log/dt_bias parameters, and a group norm over 16 heads of dimension 128).
# - 6 layers at positions [3, 7, 11, 15, 19, 23] use standard grouped-query self-attention
# (16 query heads, 4 KV heads, head dim 128, with QK norms).
# - Every layer has an RMSNorm before attention and before the MLP.
# - MLP: SwiGLU with intermediate dimension 3584 (gate_proj, up_proj, down_proj).
# - Final RMSNorm on the output hidden states.
#
# Input: token IDs -> embed_tokens [248320, 1024] -> hidden states [batch, seq, 1024].
# Each layer: h = h + attn(layernorm(h)); h = h + mlp(layernorm(h)).
# Output: final hidden states after the last layer norm.
""")
FORWARD_CODE_P2_FINETUNE = dedent(f"""\
# Forward pass pseudocode for checkpoint {P2_ID_A}
#
# Architecture: same 24-layer backbone as {P2_ID_B} (see that description), plus two additions:
#
# ADDITION 1 β€” Bidirectional scan gates.
# On 6 of the 18 linear-attention layers (at positions [2, 6, 10, 14, 18, 22]), the linear
# attention module is wrapped: the input is processed both left-to-right (normal) and
# right-to-left (reversed). The two outputs are combined as:
# output = forward_output + tanh(gate) * reversed_output
# where gate is a single learned scalar per layer (stored as reverse_gate.N).
#
# ADDITION 2 β€” Bilinear position scorer (the output head).
# Instead of a language-model head, this checkpoint scores a set of marked positions in the
# input against a single readout position:
#
# Given hidden states h of shape [batch, seq, 1024]:
# - One position in the sequence is designated the readout position (index r).
# - K other positions are designated scored positions (indices p_1 ... p_K).
# - q = Q @ h[r] # Q is a learned matrix [256, 1024] -> q is [256]
# - k_i = K_proj @ h[p_i] # K_proj is a learned matrix [256, 1024] -> k_i is [256]
# - score_i = dot(q, k_i) / 256
# - probabilities = softmax(scores)
#
# The readout and scored positions are determined at inference time by special token IDs
# embedded in the input sequence.
#
# Extra tensors in this checkpoint (not present in {P2_ID_B}):
# query [256, 1024], key [256, 1024], reverse_gate.{{2,6,10,14,18,22}} [1] each.
""")
# ---- Sandbox preludes ----
def make_prelude(run_id, id_A, id_B):
return dedent(f"""\
import os, json, numpy as np
from pathlib import Path
WORK = "/home/<user>/wsp_v2/work/{run_id}"
_CHECKPOINT_DIRS = {{
"{id_A}": os.path.join(WORK, "{id_A}"),
"{id_B}": os.path.join(WORK, "{id_B}"),
}}
def load(checkpoint_id, tensor_name):
\"\"\"Load a tensor as float32 numpy array from the named checkpoint.\"\"\"
d = _CHECKPOINT_DIRS.get(checkpoint_id)
if d is None:
raise ValueError(f"Unknown checkpoint: {{checkpoint_id}}. Available: {{list(_CHECKPOINT_DIRS)}}")
path = os.path.join(d, tensor_name + ".npy")
if not os.path.exists(path):
available = sorted(f[:-4] for f in os.listdir(d) if f.endswith(".npy"))
raise FileNotFoundError(f"No tensor '{{tensor_name}}' in {{checkpoint_id}}. "
f"Available ({{len(available)}}): {{available[:20]}}...")
return np.load(path)
def list_tensors(checkpoint_id):
\"\"\"List all available tensor names in a checkpoint.\"\"\"
d = _CHECKPOINT_DIRS.get(checkpoint_id)
if d is None:
raise ValueError(f"Unknown checkpoint: {{checkpoint_id}}")
return sorted(f[:-4] for f in os.listdir(d) if f.endswith(".npy"))
print(f"Workspace: {{WORK}}")
for cid, d in _CHECKPOINT_DIRS.items():
n = len([f for f in os.listdir(d) if f.endswith(".npy")])
print(f"Checkpoint {{cid}}: {{n}} tensors")
""")
# ---- P1 prompt (architecture-matched, blind) ----
P1_PROMPT = dedent(f"""\
You have a Python sandbox running on a GPU server. Write ```python code blocks and they will
be executed; you will see the printed output. You have numpy, scipy, sklearn, torch, and pandas.
Two neural network checkpoints are mounted as float32 .npy files.
**Checkpoint {P1_ID_A}** and **Checkpoint {P1_ID_B}** β€” both share the same architecture.
One is a base pretrained language model. The other is a fine-tune of that base, specialized
for a particular domain or task. They have identical tensor names and shapes; only values differ.
{FORWARD_CODE_P1}
Pre-defined functions in your sandbox:
- `load(checkpoint_id, tensor_name)` β€” returns a float32 numpy array
- `list_tensors(checkpoint_id)` β€” returns all tensor names in a checkpoint
**Rules:**
- You may NOT run the network or any layer on any input, real or constructed.
Linear algebra on the weight matrices (SVD, norms, projections, diffs) is allowed.
- No forward passes, no simulating inference, no constructing inputs and propagating them
through any weight matrix as if it were a layer processing data.
- Max 100 code turns.
Start by listing the tensors in both checkpoints, then investigate. When you are ready,
give your final answer (with no code block) addressing these questions:
1. Which checkpoint is the domain-specialized fine-tune, and which is the base? Confidence 0–100.
2. What domain or task is the fine-tune specialized for? Confidence 0–100.
3. Where does the difference live? Which layers and tensor groups show the largest changes,
and what do those changes suggest about what was learned?
""")
# ---- P2 prompt (architecture differs, sanity) ----
P2_PROMPT = dedent(f"""\
You have a Python sandbox running on a GPU server. Write ```python code blocks and they will
be executed; you will see the printed output. You have numpy, scipy, sklearn, torch, and pandas.
Two neural network checkpoints are mounted as float32 .npy files.
**Checkpoint {P2_ID_A}** and **Checkpoint {P2_ID_B}** β€” one of these is a fine-tune of the other.
They share the same backbone but one has additional architectural components. You may diff
them freely.
{FORWARD_CODE_P2_STANDARD}
{FORWARD_CODE_P2_FINETUNE}
Pre-defined functions in your sandbox:
- `load(checkpoint_id, tensor_name)` β€” returns a float32 numpy array
- `list_tensors(checkpoint_id)` β€” returns all tensor names in a checkpoint
**Rules:**
- You may NOT run the network or any layer on any input, real or constructed.
Linear algebra on the weight matrices (SVD, norms, projections, diffs) is allowed.
- No forward passes, no simulating inference, no constructing inputs and propagating them
through any weight matrix as if it were a layer processing data.
- Max 100 code turns.
Start by listing the tensors in both checkpoints, then investigate. When you are ready,
give your final answer (with no code block) addressing these questions:
1. Which checkpoint is the fine-tune, and which is the base? Confidence 0–100.
2. What domain or task is the fine-tune specialized for? Confidence 0–100.
3. Where does the difference live? Which layers and tensor groups show the largest changes,
and what do those changes suggest about what was learned?
""")
def run_single(run_id: str, prompt: str, condition: str, id_A: str, id_B: str):
print(f"\n{'#'*60}")
print(f"STARTING {run_id} ({condition})")
print(f"{'#'*60}")
prelude = make_prelude(run_id, id_A, id_B)
orch = Orchestrator(
run_id=run_id,
log_dir=str(LOG_DIR),
sandbox_prelude=prelude,
effort="xhigh",
max_turns=100,
)
return orch.run(prompt)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--run", choices=[
"R2_P1_r0", "R2_P1_r1", "R2_P2_r0", "R2_P2_r1",
"P1", "P2", "all"
], default="all")
args = parser.parse_args()
runs = {
"R2_P1_r0": (P1_PROMPT, "P1 blind, attempt 0", P1_ID_A, P1_ID_B),
"R2_P1_r1": (P1_PROMPT, "P1 blind, attempt 1", P1_ID_A, P1_ID_B),
"R2_P2_r0": (P2_PROMPT, "P2 sanity, attempt 0", P2_ID_A, P2_ID_B),
"R2_P2_r1": (P2_PROMPT, "P2 sanity, attempt 1", P2_ID_A, P2_ID_B),
}
if args.run == "all":
for run_id, (prompt, condition, idA, idB) in runs.items():
run_single(run_id, prompt, condition, idA, idB)
elif args.run == "P1":
for run_id in ["R2_P1_r0", "R2_P1_r1"]:
prompt, condition, idA, idB = runs[run_id]
run_single(run_id, prompt, condition, idA, idB)
elif args.run == "P2":
for run_id in ["R2_P2_r0", "R2_P2_r1"]:
prompt, condition, idA, idB = runs[run_id]
run_single(run_id, prompt, condition, idA, idB)
else:
prompt, condition, idA, idB = runs[args.run]
run_single(args.run, prompt, condition, idA, idB)
if __name__ == "__main__":
main()