Babak Barazandeh
Clean upload: all experiment code and logs, PII and benchmark data scrubbed
58c26c2 Download experiments/R2/run_R2.py from cribl-ai/Weight-Space-Programming: direct link, hf CLI and curl.
- Browser
- Download file 14.8 kB
-
https://huggingface.co/cribl-ai/Weight-Space-Programming/resolve/main/experiments/R2/run_R2.py
- Command line
-
hf download hf://cribl-ai/Weight-Space-Programming/experiments/R2/run_R2.py
-
curl -L -o run_R2.py https://huggingface.co/cribl-ai/Weight-Space-Programming/resolve/main/experiments/R2/run_R2.py
14.8 kB
| #!/usr/bin/env python3 | |
| """R2 β Which checkpoint is which? | |
| Stage 1: weights only, no passes, no tokenizer. 2 runs per pair. | |
| P1: lora1_merged vs base (architecture-matched, blind) | |
| P2: decision-2 vs base (architecture differs, sanity) | |
| """ | |
| import json | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| from textwrap import dedent | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from harness.orchestrator import Orchestrator | |
| DGX = "<user>@<GPU_HOST>" | |
| DGX_PASS = "<GPU_PASS>" | |
| LOG_DIR = Path(__file__).resolve().parent / "logs" | |
| # ---- Load answer key ---- | |
| def get_answer_key(): | |
| cmd = ( | |
| f"sshpass -p {DGX_PASS} ssh -o StrictHostKeyChecking=no {DGX} " | |
| f"'cat /home/<user>/wsp_v2/eval/answer_key.json'" | |
| ) | |
| result = subprocess.run(cmd, shell=True, capture_output=True, timeout=15) | |
| return json.loads(result.stdout) | |
| AK = get_answer_key() | |
| P1 = AK["R2_P1"] | |
| P1_ID_A = P1["id_A"] | |
| P1_ID_B = P1["id_B"] | |
| P2 = AK["R2_P2"] | |
| P2_ID_A = P2["id_A"] | |
| P2_ID_B = P2["id_B"] | |
| # ---- Forward code for P1: both are standard transformers ---- | |
| FORWARD_CODE_P1 = dedent(f"""\ | |
| # Forward pass pseudocode β applies to BOTH checkpoints ({P1_ID_A} and {P1_ID_B}) | |
| # | |
| # Architecture: 24 transformer layers, hidden dimension 1024, vocabulary 248320. | |
| # - 18 layers use linear attention (selective state-space style with conv1d, gated projections, | |
| # A_log/dt_bias parameters, and a group norm over 16 heads of dimension 128). | |
| # - 6 layers at positions [3, 7, 11, 15, 19, 23] use standard grouped-query self-attention | |
| # (16 query heads, 4 KV heads, head dim 128, with QK norms). | |
| # - Every layer has an RMSNorm before attention and before the MLP. | |
| # - MLP: SwiGLU with intermediate dimension 3584 (gate_proj, up_proj, down_proj). | |
| # - Final RMSNorm on the output hidden states. | |
| # | |
| # Input: token IDs -> embed_tokens [248320, 1024] -> hidden states [batch, seq, 1024]. | |
| # Each layer: h = h + attn(layernorm(h)); h = h + mlp(layernorm(h)). | |
| # Output: lm_head [248320, 1024] applied to the final hidden states -> logits over vocabulary. | |
| # | |
| # Both checkpoints share this exact architecture. They differ only in weight values. | |
| """) | |
| # ---- Forward codes for P2: base (standard) vs fine-tune (base + gates + scorer) ---- | |
| FORWARD_CODE_P2_STANDARD = dedent(f"""\ | |
| # Forward pass pseudocode for checkpoint {P2_ID_A} | |
| # | |
| # Architecture: 24 transformer layers, hidden dimension 1024, vocabulary 248320. | |
| # - 18 layers use linear attention (selective state-space style with conv1d, gated projections, | |
| # A_log/dt_bias parameters, and a group norm over 16 heads of dimension 128). | |
| # - 6 layers at positions [3, 7, 11, 15, 19, 23] use standard grouped-query self-attention | |
| # (16 query heads, 4 KV heads, head dim 128, with QK norms). | |
| # - Every layer has an RMSNorm before attention and before the MLP. | |
| # - MLP: SwiGLU with intermediate dimension 3584 (gate_proj, up_proj, down_proj). | |
| # - Final RMSNorm on the output hidden states. | |
| # | |
| # Input: token IDs -> embed_tokens [248320, 1024] -> hidden states [batch, seq, 1024]. | |
| # Each layer: h = h + attn(layernorm(h)); h = h + mlp(layernorm(h)). | |
| # Output: final hidden states after the last layer norm. | |
| """) | |
| FORWARD_CODE_P2_FINETUNE = dedent(f"""\ | |
| # Forward pass pseudocode for checkpoint {P2_ID_B} | |
| # | |
| # Architecture: same 24-layer backbone as {P2_ID_A} (see that description), plus two additions: | |
| # | |
| # ADDITION 1 β Bidirectional scan gates. | |
| # On 6 of the 18 linear-attention layers (at positions [2, 6, 10, 14, 18, 22]), the linear | |
| # attention module is wrapped: the input is processed both left-to-right (normal) and | |
| # right-to-left (reversed). The two outputs are combined as: | |
| # output = forward_output + tanh(gate) * reversed_output | |
| # where gate is a single learned scalar per layer (stored as reverse_gate.N). | |
| # | |
| # ADDITION 2 β Bilinear position scorer (the output head). | |
| # Instead of a language-model head, this checkpoint scores a set of marked positions in the | |
| # input against a single readout position: | |
| # | |
| # Given hidden states h of shape [batch, seq, 1024]: | |
| # - One position in the sequence is designated the readout position (index r). | |
| # - K other positions are designated scored positions (indices p_1 ... p_K). | |
| # - q = Q @ h[r] # Q is a learned matrix [256, 1024] -> q is [256] | |
| # - k_i = K_proj @ h[p_i] # K_proj is a learned matrix [256, 1024] -> k_i is [256] | |
| # - score_i = dot(q, k_i) / 256 | |
| # - probabilities = softmax(scores) | |
| # | |
| # The readout and scored positions are determined at inference time by special token IDs | |
| # embedded in the input sequence. | |
| # | |
| # Extra tensors in this checkpoint (not present in {P2_ID_A}): | |
| # query [256, 1024], key [256, 1024], reverse_gate.{{2,6,10,14,18,22}} [1] each. | |
| """) | |
| # Swap forward codes if d2 is A (the forward code labels must match the hex IDs) | |
| if P2["label_A"] == "decision2": | |
| FORWARD_CODE_P2_STANDARD, FORWARD_CODE_P2_FINETUNE = FORWARD_CODE_P2_FINETUNE, FORWARD_CODE_P2_STANDARD | |
| # Re-generate with swapped IDs | |
| FORWARD_CODE_P2_STANDARD = dedent(f"""\ | |
| # Forward pass pseudocode for checkpoint {P2_ID_B} | |
| # | |
| # Architecture: 24 transformer layers, hidden dimension 1024, vocabulary 248320. | |
| # - 18 layers use linear attention (selective state-space style with conv1d, gated projections, | |
| # A_log/dt_bias parameters, and a group norm over 16 heads of dimension 128). | |
| # - 6 layers at positions [3, 7, 11, 15, 19, 23] use standard grouped-query self-attention | |
| # (16 query heads, 4 KV heads, head dim 128, with QK norms). | |
| # - Every layer has an RMSNorm before attention and before the MLP. | |
| # - MLP: SwiGLU with intermediate dimension 3584 (gate_proj, up_proj, down_proj). | |
| # - Final RMSNorm on the output hidden states. | |
| # | |
| # Input: token IDs -> embed_tokens [248320, 1024] -> hidden states [batch, seq, 1024]. | |
| # Each layer: h = h + attn(layernorm(h)); h = h + mlp(layernorm(h)). | |
| # Output: final hidden states after the last layer norm. | |
| """) | |
| FORWARD_CODE_P2_FINETUNE = dedent(f"""\ | |
| # Forward pass pseudocode for checkpoint {P2_ID_A} | |
| # | |
| # Architecture: same 24-layer backbone as {P2_ID_B} (see that description), plus two additions: | |
| # | |
| # ADDITION 1 β Bidirectional scan gates. | |
| # On 6 of the 18 linear-attention layers (at positions [2, 6, 10, 14, 18, 22]), the linear | |
| # attention module is wrapped: the input is processed both left-to-right (normal) and | |
| # right-to-left (reversed). The two outputs are combined as: | |
| # output = forward_output + tanh(gate) * reversed_output | |
| # where gate is a single learned scalar per layer (stored as reverse_gate.N). | |
| # | |
| # ADDITION 2 β Bilinear position scorer (the output head). | |
| # Instead of a language-model head, this checkpoint scores a set of marked positions in the | |
| # input against a single readout position: | |
| # | |
| # Given hidden states h of shape [batch, seq, 1024]: | |
| # - One position in the sequence is designated the readout position (index r). | |
| # - K other positions are designated scored positions (indices p_1 ... p_K). | |
| # - q = Q @ h[r] # Q is a learned matrix [256, 1024] -> q is [256] | |
| # - k_i = K_proj @ h[p_i] # K_proj is a learned matrix [256, 1024] -> k_i is [256] | |
| # - score_i = dot(q, k_i) / 256 | |
| # - probabilities = softmax(scores) | |
| # | |
| # The readout and scored positions are determined at inference time by special token IDs | |
| # embedded in the input sequence. | |
| # | |
| # Extra tensors in this checkpoint (not present in {P2_ID_B}): | |
| # query [256, 1024], key [256, 1024], reverse_gate.{{2,6,10,14,18,22}} [1] each. | |
| """) | |
| # ---- Sandbox preludes ---- | |
| def make_prelude(run_id, id_A, id_B): | |
| return dedent(f"""\ | |
| import os, json, numpy as np | |
| from pathlib import Path | |
| WORK = "/home/<user>/wsp_v2/work/{run_id}" | |
| _CHECKPOINT_DIRS = {{ | |
| "{id_A}": os.path.join(WORK, "{id_A}"), | |
| "{id_B}": os.path.join(WORK, "{id_B}"), | |
| }} | |
| def load(checkpoint_id, tensor_name): | |
| \"\"\"Load a tensor as float32 numpy array from the named checkpoint.\"\"\" | |
| d = _CHECKPOINT_DIRS.get(checkpoint_id) | |
| if d is None: | |
| raise ValueError(f"Unknown checkpoint: {{checkpoint_id}}. Available: {{list(_CHECKPOINT_DIRS)}}") | |
| path = os.path.join(d, tensor_name + ".npy") | |
| if not os.path.exists(path): | |
| available = sorted(f[:-4] for f in os.listdir(d) if f.endswith(".npy")) | |
| raise FileNotFoundError(f"No tensor '{{tensor_name}}' in {{checkpoint_id}}. " | |
| f"Available ({{len(available)}}): {{available[:20]}}...") | |
| return np.load(path) | |
| def list_tensors(checkpoint_id): | |
| \"\"\"List all available tensor names in a checkpoint.\"\"\" | |
| d = _CHECKPOINT_DIRS.get(checkpoint_id) | |
| if d is None: | |
| raise ValueError(f"Unknown checkpoint: {{checkpoint_id}}") | |
| return sorted(f[:-4] for f in os.listdir(d) if f.endswith(".npy")) | |
| print(f"Workspace: {{WORK}}") | |
| for cid, d in _CHECKPOINT_DIRS.items(): | |
| n = len([f for f in os.listdir(d) if f.endswith(".npy")]) | |
| print(f"Checkpoint {{cid}}: {{n}} tensors") | |
| """) | |
| # ---- P1 prompt (architecture-matched, blind) ---- | |
| P1_PROMPT = dedent(f"""\ | |
| You have a Python sandbox running on a GPU server. Write ```python code blocks and they will | |
| be executed; you will see the printed output. You have numpy, scipy, sklearn, torch, and pandas. | |
| Two neural network checkpoints are mounted as float32 .npy files. | |
| **Checkpoint {P1_ID_A}** and **Checkpoint {P1_ID_B}** β both share the same architecture. | |
| One is a base pretrained language model. The other is a fine-tune of that base, specialized | |
| for a particular domain or task. They have identical tensor names and shapes; only values differ. | |
| {FORWARD_CODE_P1} | |
| Pre-defined functions in your sandbox: | |
| - `load(checkpoint_id, tensor_name)` β returns a float32 numpy array | |
| - `list_tensors(checkpoint_id)` β returns all tensor names in a checkpoint | |
| **Rules:** | |
| - You may NOT run the network or any layer on any input, real or constructed. | |
| Linear algebra on the weight matrices (SVD, norms, projections, diffs) is allowed. | |
| - No forward passes, no simulating inference, no constructing inputs and propagating them | |
| through any weight matrix as if it were a layer processing data. | |
| - Max 100 code turns. | |
| Start by listing the tensors in both checkpoints, then investigate. When you are ready, | |
| give your final answer (with no code block) addressing these questions: | |
| 1. Which checkpoint is the domain-specialized fine-tune, and which is the base? Confidence 0β100. | |
| 2. What domain or task is the fine-tune specialized for? Confidence 0β100. | |
| 3. Where does the difference live? Which layers and tensor groups show the largest changes, | |
| and what do those changes suggest about what was learned? | |
| """) | |
| # ---- P2 prompt (architecture differs, sanity) ---- | |
| P2_PROMPT = dedent(f"""\ | |
| You have a Python sandbox running on a GPU server. Write ```python code blocks and they will | |
| be executed; you will see the printed output. You have numpy, scipy, sklearn, torch, and pandas. | |
| Two neural network checkpoints are mounted as float32 .npy files. | |
| **Checkpoint {P2_ID_A}** and **Checkpoint {P2_ID_B}** β one of these is a fine-tune of the other. | |
| They share the same backbone but one has additional architectural components. You may diff | |
| them freely. | |
| {FORWARD_CODE_P2_STANDARD} | |
| {FORWARD_CODE_P2_FINETUNE} | |
| Pre-defined functions in your sandbox: | |
| - `load(checkpoint_id, tensor_name)` β returns a float32 numpy array | |
| - `list_tensors(checkpoint_id)` β returns all tensor names in a checkpoint | |
| **Rules:** | |
| - You may NOT run the network or any layer on any input, real or constructed. | |
| Linear algebra on the weight matrices (SVD, norms, projections, diffs) is allowed. | |
| - No forward passes, no simulating inference, no constructing inputs and propagating them | |
| through any weight matrix as if it were a layer processing data. | |
| - Max 100 code turns. | |
| Start by listing the tensors in both checkpoints, then investigate. When you are ready, | |
| give your final answer (with no code block) addressing these questions: | |
| 1. Which checkpoint is the fine-tune, and which is the base? Confidence 0β100. | |
| 2. What domain or task is the fine-tune specialized for? Confidence 0β100. | |
| 3. Where does the difference live? Which layers and tensor groups show the largest changes, | |
| and what do those changes suggest about what was learned? | |
| """) | |
| def run_single(run_id: str, prompt: str, condition: str, id_A: str, id_B: str): | |
| print(f"\n{'#'*60}") | |
| print(f"STARTING {run_id} ({condition})") | |
| print(f"{'#'*60}") | |
| prelude = make_prelude(run_id, id_A, id_B) | |
| orch = Orchestrator( | |
| run_id=run_id, | |
| log_dir=str(LOG_DIR), | |
| sandbox_prelude=prelude, | |
| effort="xhigh", | |
| max_turns=100, | |
| ) | |
| return orch.run(prompt) | |
| def main(): | |
| import argparse | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--run", choices=[ | |
| "R2_P1_r0", "R2_P1_r1", "R2_P2_r0", "R2_P2_r1", | |
| "P1", "P2", "all" | |
| ], default="all") | |
| args = parser.parse_args() | |
| runs = { | |
| "R2_P1_r0": (P1_PROMPT, "P1 blind, attempt 0", P1_ID_A, P1_ID_B), | |
| "R2_P1_r1": (P1_PROMPT, "P1 blind, attempt 1", P1_ID_A, P1_ID_B), | |
| "R2_P2_r0": (P2_PROMPT, "P2 sanity, attempt 0", P2_ID_A, P2_ID_B), | |
| "R2_P2_r1": (P2_PROMPT, "P2 sanity, attempt 1", P2_ID_A, P2_ID_B), | |
| } | |
| if args.run == "all": | |
| for run_id, (prompt, condition, idA, idB) in runs.items(): | |
| run_single(run_id, prompt, condition, idA, idB) | |
| elif args.run == "P1": | |
| for run_id in ["R2_P1_r0", "R2_P1_r1"]: | |
| prompt, condition, idA, idB = runs[run_id] | |
| run_single(run_id, prompt, condition, idA, idB) | |
| elif args.run == "P2": | |
| for run_id in ["R2_P2_r0", "R2_P2_r1"]: | |
| prompt, condition, idA, idB = runs[run_id] | |
| run_single(run_id, prompt, condition, idA, idB) | |
| else: | |
| prompt, condition, idA, idB = runs[args.run] | |
| run_single(args.run, prompt, condition, idA, idB) | |
| if __name__ == "__main__": | |
| main() | |