Archive full MsTok step 17167 and matched FP64 generation evaluation

iclr-debug-decoder-135b-20260916/decoder-mstok/checkpoint-17167-README.md

@@ -0,0 +1,12 @@
+# Decoder-aligned joint MsTok: step 17,167
+
+Intermediate checkpoint from the original full `iclr-debug-decoder-135b-20260916` run (135B learning-rate horizon).
+
+- 17,167 optimizer updates; 13,500,678,144 raw input positions including padding.
+- Full resumable checkpoint, including model components, optimizer, scheduler, configuration, and eight per-rank RNG states.
+- Fresh evaluation: 640 samples across seeds 0–4, supplied true level-zero code, untruncated temperature-1 sampling. Generator uses float32; reference GPT-2 Large uses FP64 with eager attention. TF32 disabled.
+- Generation PPL: 109.261 +/- 2.332 standard error across seeds. Mean empirical token entropy: 4.364 nats.
+- Filtering the existing samples for entropy >= 4.5 retains 109/640 (17.03%), with pooled token-weighted PPL 201.717.
+- Replacement characters appear in 66.56% of all samples. Low generation PPL alone does not establish generation quality.
+
+Evaluation artifacts are in `evaluation/step-17167-fp64-rerun-20260919`. Raw generations, per-sample scores, accepted samples, evaluator scripts, and configuration/provenance are included. This is a fresh matched evaluation; any previously archived step-17,167 results used different generation settings and are preserved separately. Absolute paths describe the original environment. Export component weights using `utils.iclr_training.export_training_checkpoint` from the matching source checkout.

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/controller.py

@@ -0,0 +1,54 @@
+import datetime
+import json
+import math
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+REPO = Path('/home/ubuntu/mstok-runs/iclr-debug-joint-20260916')
+STAGE = Path('/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok')
+OUT = STAGE / 'evaluation/step-17167-fp64-rerun-20260919'
+EXPORT = STAGE / 'exports/step-17167'
+ENV = dict(os.environ, CUDA_VISIBLE_DEVICES='0', OMP_NUM_THREADS='1',
+           TOKENIZERS_PARALLELISM='false', PYTORCH_ALLOC_CONF='expandable_segments:True')
+
+def status(state, **extra):
+    payload = dict(state=state, updated_utc=datetime.datetime.now(datetime.timezone.utc).isoformat(),
+                   controller_pid=os.getpid(), **extra)
+    temporary = OUT / 'manual-eval-status.tmp'
+    temporary.write_text(json.dumps(payload, indent=2) + '\n')
+    temporary.replace(OUT / 'manual-eval-status.json')
+    print(json.dumps(payload), flush=True)
+
+try:
+    for seed in range(5):
+        result = OUT / f'random-seed-{seed}.json'
+        if not result.exists():
+            status('evaluating', seed=seed)
+            code = "import runpy,torch; torch.cuda.set_per_process_memory_fraction(0.15); runpy.run_path('/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/evaluator.py',run_name='__main__')"
+            command = [sys.executable, '-u', '-c', code, '--ckpt', str(EXPORT/'ncp.pt'),
+                       '--codec', str(EXPORT/'vqvae.pt'), '--validation-source',
+                       '/home/ubuntu/data/full_owt/valid_gpt2.bin', '--ref-model', 'gpt2-large',
+                       '--ref-dtype', 'float64', '--seed', str(seed), '--n-samples', '128',
+                       '--batch-size', '4', '--temperature', '1', '--top-k', '0', '--top-p', '1',
+                       '--save-all-generations', '--out', str(result)]
+            with result.with_suffix('.log').open('a') as log:
+                subprocess.run(command, cwd=REPO, env=ENV, stdout=log, stderr=subprocess.STDOUT,
+                               check=True, timeout=1800)
+        row = json.loads(result.read_text())
+        assert row['checkpoint_step'] == 17167 and row['seed'] == seed
+        assert row['reference_dtype'] == 'float64' and not row['tf32_allowed']
+        assert row['protocol']['n_levels'] == 16 and row['protocol']['n_provided_levels'] == 1
+        assert row['requested_samples'] == row['scored_samples'] == 128
+        assert row['scored_tokens'] > 0
+        assert math.isfinite(row['mean_ppl']) and math.isfinite(row['mean_entropy_nats'])
+        status('seed-complete', seed=seed, mean_ppl=row['mean_ppl'], entropy=row['mean_entropy_nats'])
+    with (OUT/'summarize.log').open('w') as log:
+        subprocess.run([sys.executable, 'evaluation/summarize_gen_ppl.py', '--input-glob',
+                        str(OUT/'random-seed-[0-4].json'), '--out', str(OUT/'summary.json')],
+                       cwd=REPO, env=ENV, stdout=log, stderr=subprocess.STDOUT, check=True)
+    status('complete', summary=str(OUT/'summary.json'))
+except BaseException as exc:
+    status('failed', error=str(exc))
+    raise

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/evaluator.py

@@ -0,0 +1,187 @@
+"""Evaluate random-sampling generation PPL from a completed NCP checkpoint."""
+
+import argparse
+import json
+import math
+import sys
+from collections import Counter
+from pathlib import Path
+
+REPO_ROOT = Path("/home/ubuntu/mstok-runs/iclr-debug-joint-20260916")
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+import hydra
+import torch
+from omegaconf import OmegaConf
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from data.serialized_dataset import SerializedDataset
+from evaluation.gen_ppl import generation_perplexity
+from utils.misc import set_manual_seed
+from utils.registry import trainers
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ckpt", required=True)
+    parser.add_argument("--codec")
+    parser.add_argument("--validation-source")
+    parser.add_argument("--ref-model", default="gpt2-large")
+    parser.add_argument("--ref-dtype", choices=("bfloat16", "float32", "float64"), default="bfloat16",
+                        help="Legacy default retained; new ICLR runs explicitly use float32")
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--n-samples", type=int, default=128)
+    parser.add_argument("--batch-size", type=int, default=8)
+    parser.add_argument("--temperature", type=float, default=1.0)
+    parser.add_argument("--top-k", type=int, default=0)
+    parser.add_argument("--top-p", type=float, default=1.0)
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--save-all-generations", action="store_true")
+    args = parser.parse_args()
+
+    device = "cuda"
+    checkpoint = torch.load(args.ckpt, map_location="cpu", weights_only=False)
+    cfg = checkpoint["args"]
+    OmegaConf.set_struct(cfg, False)
+    if args.codec:
+        cfg.tokenizer.vqvae.checkpoint_path = args.codec
+    if args.validation_source:
+        cfg.dataset.validate_source = args.validation_source
+    cfg.optimization.token_corruption_prob = 0.0
+    if getattr(cfg.optimization, "stochastic_encode", None) is not None:
+        cfg.optimization.stochastic_encode.enabled = False
+    else:
+        cfg.optimization.stochastic_encode = {"enabled": False}
+
+    set_manual_seed(args.seed)
+    # Seed setup enables TF32 by default, so enforce evaluation precision after it.
+    if args.ref_dtype in ("float32", "float64"):
+        torch.backends.cuda.matmul.allow_tf32 = False
+        torch.backends.cudnn.allow_tf32 = False
+    Trainer = hydra.utils.get_class(trainers[cfg.task])
+    trainer = Trainer(cfg, device)
+    trainer.model.load_state_dict(checkpoint["model"])
+    model = trainer.model.to(device).eval()
+    tokenizer = trainer.tokenizer
+    levels = list(model.levels)
+    prefix_len = int(cfg.model.prefix_len)
+
+    dataset = SerializedDataset(
+        cfg.dataset.validate_source,
+        int(cfg.model.context_length),
+        cfg.task,
+        tokenizer.pad_token_id,
+    )
+    full_sequences, _ = dataset.get_rand_batch(args.n_samples)
+    prefix_sequences = full_sequences[:, :prefix_len]
+    target_sequences = full_sequences[:, prefix_len:]
+
+    prompts = []
+    generations = []
+    with torch.no_grad():
+        for start in range(0, args.n_samples, args.batch_size):
+            end = min(start + args.batch_size, args.n_samples)
+            prefix = prefix_sequences[start:end].to(device)
+            target_idx = tokenizer.encode_pre_tokenized_idx(
+                target_sequences[start:end].to(device)
+            )
+            level0 = target_idx[:, : levels[0]]
+            generated_idx = model.generate(
+                level0,
+                prefix,
+                temperature=args.temperature,
+                top_k=args.top_k,
+                top_p=args.top_p,
+            )
+            reconstructions = tokenizer.decode_multiscales(
+                generated_idx, skip_special_tokens=True
+            )
+            generations.extend(
+                reconstructions[row][-1] for row in range(len(reconstructions))
+            )
+            prompts.extend(
+                tokenizer.pre_tokenizer.decode_batch(
+                    prefix.cpu().tolist(), skip_special_tokens=True
+                )
+            )
+
+    if args.save_all_generations:
+        # Preserve every generated sample even if scoring fails or some are empty.
+        samples_path = Path(args.out).with_suffix(".samples.json")
+        samples_path.parent.mkdir(parents=True, exist_ok=True)
+        samples_path.write_text(json.dumps(dict(seed=args.seed, checkpoint=args.ckpt,
+            prompts=prompts, generations=generations), indent=2) + "\n")
+    kept = [(prompt, text) for prompt, text in zip(prompts, generations) if text.strip()]
+    prompts = [prompt for prompt, _ in kept]
+    generations = [text for _, text in kept]
+
+    entropies = []
+    for text in generations:
+        ids = tokenizer.pre_tokenizer.encode(text)
+        if ids:
+            counts = Counter(ids)
+            entropies.append(
+                -sum((count / len(ids)) * math.log(count / len(ids)) for count in counts.values())
+            )
+
+    del model, trainer, tokenizer
+    torch.cuda.empty_cache()
+
+    ref_tokenizer = AutoTokenizer.from_pretrained(args.ref_model)
+    ref_model = AutoModelForCausalLM.from_pretrained(
+        args.ref_model, torch_dtype=getattr(torch, args.ref_dtype),
+        **({"attn_implementation": "eager"} if args.ref_dtype == "float64" else {})
+    ).to(device).eval()
+    assert all(p.dtype == getattr(torch, args.ref_dtype) for p in ref_model.parameters())
+    result = generation_perplexity(
+        ref_model,
+        ref_tokenizer,
+        prompts,
+        generations,
+        score_only_generated=True,
+        batch_size=args.batch_size,
+        device=device,
+    )
+
+    assert result.nll_sum.dtype == getattr(torch, args.ref_dtype)
+    assert bool(torch.isfinite(result.ppl).all()) and bool((result.token_count > 0).all())
+    payload = {
+        "checkpoint": str(Path(args.ckpt).resolve()),
+        "checkpoint_step": int(checkpoint["step"]),
+        "codec": str(Path(cfg.tokenizer.vqvae.checkpoint_path).resolve()),
+        "reference_model": args.ref_model,
+        "reference_dtype": args.ref_dtype,
+        "reference_attention_implementation": ref_model.config._attn_implementation,
+        "per_sample_nll_sum": result.nll_sum.tolist(),
+        "per_sample_token_count": result.token_count.tolist(),
+        "per_sample_ppl": result.ppl.tolist(),
+        "per_sample_entropy_nats": entropies,
+        "tf32_allowed": torch.backends.cuda.matmul.allow_tf32,
+        "protocol": {
+            "sampling": "random" if args.top_k == 0 and args.top_p == 1.0 else "truncated",
+            "temperature": args.temperature,
+            "top_k": args.top_k,
+            "top_p": args.top_p,
+            "n_provided_levels": 1,
+            "n_levels": len(levels),
+            "document_aware_validation_sampling": dataset.doc_offsets is not None,
+        },
+        "seed": args.seed,
+        "requested_samples": args.n_samples,
+        "scored_samples": len(generations),
+        "skipped_empty_samples": args.n_samples - len(generations),
+        "scored_tokens": int(result.token_count.sum()),
+        "mean_ppl": float(result.mean_ppl),
+        "median_ppl": float(result.ppl.median()),
+        "mean_entropy_nats": sum(entropies) / len(entropies),
+        "sample_generations": generations if args.save_all_generations else generations[:8],
+    }
+    output_path = Path(args.out)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(payload, indent=2) + "\n")
+    print(json.dumps({key: value for key, value in payload.items() if key != "sample_generations"}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/manifest.json

@@ -0,0 +1,26 @@
+{
+  "checkpoint_step": 17167,
+  "input_positions": 13500678144,
+  "source_checkpoint": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/milestone-iter-17167.pt",
+  "checkpoint_sha256": "0f71f24010b3fe8874a2319801008f49c00badb9af050c43b8aa8ed80122e466",
+  "evaluator_sha256": "e2c87f66f1001853f58b9e78a646fc4e089fc894d05fa6c215a8baf493a84055",
+  "checkout": "/home/ubuntu/mstok-runs/iclr-debug-joint-20260916",
+  "reference_dtype": "float64",
+  "reference_attention": "eager",
+  "tf32_allowed": false,
+  "memory_fraction": 0.15,
+  "gpu": 0,
+  "batch_size": 4,
+  "seeds": [
+    0,
+    1,
+    2,
+    3,
+    4
+  ],
+  "samples_per_seed": 128,
+  "sampling": "temperature-1-untruncated-supplied-true-level0",
+  "concurrent_with_training": true,
+  "retained_checkpoint": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/milestone-iter-17167.pt",
+  "created_utc": "2026-09-19T04:42:31.183086+00:00"
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/manual-eval-status.json

@@ -0,0 +1,6 @@
+{
+  "state": "complete",
+  "updated_utc": "2026-09-19T04:47:12.120382+00:00",
+  "controller_pid": 590459,
+  "summary": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/summary.json"
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/quality-summary.json

@@ -0,0 +1,87 @@
+{
+  "checkpoint_step": 17167,
+  "reference_dtype": "float64",
+  "tf32_allowed": false,
+  "unfiltered": {
+    "samples": 640,
+    "scored_tokens": 160949,
+    "pooled_token_weighted_ppl": 109.15588482181249,
+    "mean_entropy_nats": 4.363820267455618,
+    "fraction_samples_with_replacement_character": 0.665625,
+    "adjacent_repeated_word_fraction": 0.048007480092300596
+  },
+  "entropy_ge_4p5": {
+    "samples": 109,
+    "scored_tokens": 27286,
+    "pooled_token_weighted_ppl": 201.7171620400652,
+    "mean_entropy_nats": 4.582393683871207,
+    "fraction_samples_with_replacement_character": 0.5504587155963303,
+    "adjacent_repeated_word_fraction": 0.043738225687130025
+  },
+  "acceptance_rate": 0.1703125,
+  "comparison_step17167": {
+    "checkpoint_step": 17167,
+    "reference_dtype": "float64",
+    "threshold": 4.5,
+    "entropy_definition": "Empirical GPT-2 unigram entropy in nats within each generated sample",
+    "selection": "Filter existing 640 generations; no new generation; all accepted samples pooled",
+    "accepted": {
+      "samples": 115,
+      "scored_tokens": 28860,
+      "pooled_token_weighted_ppl": 189.63357736496337,
+      "mean_entropy_nats": 4.57744085314838,
+      "median_sample_ppl": 171.1510152167491,
+      "adjacent_repeated_word_fraction": 0.04295705405426942,
+      "samples_with_replacement_character": 63,
+      "fraction_samples_with_replacement_character": 0.5478260869565217
+    },
+    "unfiltered": {
+      "samples": 640,
+      "scored_tokens": 160898,
+      "pooled_token_weighted_ppl": 109.72590001293689,
+      "mean_entropy_nats": 4.3665410349745,
+      "median_sample_ppl": 99.25832710923365,
+      "adjacent_repeated_word_fraction": 0.04761292563004243,
+      "samples_with_replacement_character": 431,
+      "fraction_samples_with_replacement_character": 0.6734375
+    },
+    "acceptance_rate": 0.1796875,
+    "per_seed": [
+      {
+        "seed": 0,
+        "accepted": 26,
+        "proposed": 128,
+        "ppl": 198.79796628571316
+      },
+      {
+        "seed": 1,
+        "accepted": 21,
+        "proposed": 128,
+        "ppl": 171.26448994329286
+      },
+      {
+        "seed": 2,
+        "accepted": 25,
+        "proposed": 128,
+        "ppl": 201.94449304567118
+      },
+      {
+        "seed": 3,
+        "accepted": 21,
+        "proposed": 128,
+        "ppl": 201.29939871172186
+      },
+      {
+        "seed": 4,
+        "accepted": 22,
+        "proposed": 128,
+        "ppl": 174.01773580284262
+      }
+    ],
+    "mean_of_seed_ppls": 189.46481675784833,
+    "ppl_definition": "exp(sum saved FP64 reference NLL / sum scored tokens)",
+    "conditioning": "supplied true level-zero code",
+    "fp64_source": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64",
+    "entropy_source": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-diversity-audit/step_17167-samples.json"
+  }
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/report.md

@@ -0,0 +1,49 @@
+# Step 17,167 generation evaluation
+
+FP64 GPT-2 Large generation PPL: 109.261 +/- 2.332 standard error across five seeds.
+
+640 generations, temperature 1 without truncation, supplied true level-zero code. TF32 disabled.
+
+## Quality measures
+
+| Metric | All samples | Entropy >= 4.5 |
+|---|---:|---:|
+| Samples | 640.000 | 109.000 |
+| Pooled token-weighted PPL | 109.156 | 201.717 |
+| Mean entropy (nats) | 4.364 | 4.582 |
+| Samples with replacement character (%) | 66.562 | 55.046 |
+| Adjacent repeated words (%) | 4.801 | 4.374 |
+
+## First three seed-0 samples
+
+Samples shown in original order, without selecting for quality.
+
+### Seed 0, index 0 — PPL 522.05, entropy 4.688
+
+ismkel�mēs�n��ɰmmm.
+
+
+4 2�ěńm��ɛoz�asas)
+
+::54AAAd-ddlesy�a -
+
+
+C 8.0 �d Sndäanonu v l-azem���m�?)wam �ere
+exdó n���d p.Ì yllɹ (a) pr)The
+2 8 15th extreme 16,�g d fʍ� nadiom dnnt  hihd  in�r�,abhere�n� (a - t) sim
+n n lt. In such, you - - have the changellt't' in a site site least one ofon.a. The n' aren do questionst for e..�.m's's is v are early late in the last Online 9 That
+-575 18 5 5.guma a obj bphz �m�zím.
+
+Recent visiting history [ edit ]
+
+In September 8 July 2010, aond give Zie was performing at The New the in in Total
+
+### Seed 0, index 1 — PPL 68.08, entropy 4.295
+
+ want want about you. That’ss going to be it. But. I,�ll get then, I’ve got it it.”
+
+On the last few days he got well-known through. secondless — then and says.�He got other times this season. It’s only one thing the big guys where’s there reason being having a lot of those guy who serves on on needed on the issue as a guy, and and guy’s a guy he’s a guy. He’� not a lot of money. There� ofs not man and there’ss going to take a lot of running over,, while I said, ‘I can opt if somebody’s. That’s not anybody, for and he’s one of doing things. But I think that it’s not going to do you, and he’s really me or a very important thing. He’s been playing every single now now that we got to say we get a person whole whole right. They’ve already been done it little thing fight, if it’s not going to do what play on the game. That
+
+### Seed 0, index 2 — PPL 131.68, entropy 4.325
+
+ mean, it was different; physically on the interview with my own. When fans, I don't call scene tell you the same thing makes you And. And all what it's looks for me. It doesn't mean well, I don just just it, you do me. There's a of of time for moment to come again that I did like that that. I mean you, it's always as the me,No.?''s's like, "I'll'll it it out out?" then I'm like, 'Oh", do do 'Hey, I embarrassed you?'?".'ll like, I be like, 'Oh really, I'll do, that could if I was looking." So I thought, again in something that fell look out — I've pulled like,, you'll have to myself [-].]. I said, what I just like that because I " God, I'm to say that's back. You'll to look them what what. It It never been a tough, and that's's old old. Because old guy someone someone would you to come out just because you're just kid you say, I say.' That's not not, it's that they got to get that running. Every person would a any someone who

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/summarize-quality.py

@@ -0,0 +1,52 @@
+import json
+import math
+from pathlib import Path
+import re
+import statistics
+
+OUT = Path(__file__).resolve().parent
+rows=[]
+for seed in range(5):
+    result=json.loads((OUT/f'random-seed-{seed}.json').read_text())
+    assert result['reference_dtype']=='float64' and not result['tf32_allowed']
+    assert result['checkpoint_step']==17167 and result['scored_samples']==128
+    for i,text in enumerate(result['sample_generations']):
+        words=re.findall(r'\b\w+\b',text.lower())
+        row=dict(seed=seed,index=i,text=text,entropy_nats=result['per_sample_entropy_nats'][i],
+                 nll_sum=result['per_sample_nll_sum'][i],scored_tokens=result['per_sample_token_count'][i],
+                 ppl=result['per_sample_ppl'][i],has_replacement_character=int('\ufffd' in text),
+                 adjacent_repeated_word_fraction=sum(a==b for a,b in zip(words,words[1:]))/max(1,len(words)-1))
+        assert math.isfinite(row['ppl']) and row['scored_tokens']>0
+        rows.append(row)
+
+def metrics(group):
+    if not group:
+        return dict(samples=0)
+    return dict(samples=len(group),scored_tokens=sum(r['scored_tokens'] for r in group),
+                pooled_token_weighted_ppl=math.exp(sum(r['nll_sum'] for r in group)/sum(r['scored_tokens'] for r in group)),
+                mean_entropy_nats=statistics.mean(r['entropy_nats'] for r in group),
+                fraction_samples_with_replacement_character=statistics.mean(r['has_replacement_character'] for r in group),
+                adjacent_repeated_word_fraction=statistics.mean(r['adjacent_repeated_word_fraction'] for r in group))
+
+accepted=[r for r in rows if r['entropy_nats']>=4.5]
+quality=dict(checkpoint_step=17167,reference_dtype='float64',tf32_allowed=False,
+             unfiltered=metrics(rows),entropy_ge_4p5=metrics(accepted),acceptance_rate=len(accepted)/len(rows),
+             comparison_step17167=json.loads((OUT.parent/'step-17167-entropy-ge-4p5/summary.json').read_text()))
+(OUT/'quality-summary.json').write_text(json.dumps(quality,indent=2)+'\n')
+(OUT/'accepted-entropy-ge-4p5.json').write_text(json.dumps(accepted,indent=2)+'\n')
+summary=json.loads((OUT/'summary.json').read_text())
+summary.update(reference_dtype='float64',tf32_allowed=False,quality_summary=str(OUT/'quality-summary.json'),
+               generation_tf32_allowed=False,total_requested_samples=640,total_skipped_empty_samples=0)
+(OUT/'summary.json').write_text(json.dumps(summary,indent=2)+'\n')
+lines=['# Step 17,167 generation evaluation','',
+       f"FP64 GPT-2 Large generation PPL: {summary['mean_gen_ppl']:.3f} +/- {summary['gen_ppl_se']:.3f} standard error across five seeds.",
+       '', '640 generations, temperature 1 without truncation, supplied true level-zero code. TF32 disabled.',
+       '', '## Quality measures','', '| Metric | All samples | Entropy >= 4.5 |','|---|---:|---:|']
+for label,key,scale in [('Samples','samples',1),('Pooled token-weighted PPL','pooled_token_weighted_ppl',1),('Mean entropy (nats)','mean_entropy_nats',1),('Samples with replacement character (%)','fraction_samples_with_replacement_character',100),('Adjacent repeated words (%)','adjacent_repeated_word_fraction',100)]:
+    lines.append(f"| {label} | {quality['unfiltered'][key]*scale:.3f} | {quality['entropy_ge_4p5'].get(key,float('nan'))*scale:.3f} |")
+lines+=['','## First three seed-0 samples','', 'Samples shown in original order, without selecting for quality.']
+for row in rows[:3]:
+    lines+=['',f"### Seed {row['seed']}, index {row['index']} — PPL {row['ppl']:.2f}, entropy {row['entropy_nats']:.3f}",'',row['text']]
+(OUT/'report.md').write_text('\n'.join(lines)+'\n')
+print(json.dumps(summary,indent=2))
+print(json.dumps({k:v for k,v in quality.items() if k!='comparison_step17167'},indent=2))

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/summary.json

@@ -0,0 +1,41 @@
+{
+  "protocol": {
+    "sampling": "random",
+    "temperature": 1.0,
+    "top_k": 0,
+    "top_p": 1.0,
+    "n_provided_levels": 1,
+    "n_levels": 16,
+    "document_aware_validation_sampling": true
+  },
+  "reference_model": "gpt2-large",
+  "checkpoint": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/exports/step-17167/ncp.pt",
+  "checkpoint_step": 17167,
+  "seeds": [
+    0,
+    1,
+    2,
+    3,
+    4
+  ],
+  "samples_per_seed": 128,
+  "total_scored_samples": 640,
+  "total_scored_tokens": 160949,
+  "per_seed_mean_ppl": {
+    "0": 116.0033750052343,
+    "1": 112.64891583090473,
+    "2": 107.53348738655384,
+    "3": 107.66336444165039,
+    "4": 102.45701050430984
+  },
+  "mean_gen_ppl": 109.26123063373062,
+  "gen_ppl_se": 2.3320442361469507,
+  "mean_of_seed_median_ppl": 98.77863179069354,
+  "mean_entropy_nats": 4.363820267455619,
+  "reference_dtype": "float64",
+  "tf32_allowed": false,
+  "quality_summary": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/quality-summary.json",
+  "generation_tf32_allowed": false,
+  "total_requested_samples": 640,
+  "total_skipped_empty_samples": 0
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/milestone-iter-17167.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f71f24010b3fe8874a2319801008f49c00badb9af050c43b8aa8ed80122e466
+size 4879455585

iclr-debug-decoder-135b-20260916/provenance/hf-backup-step-17167-manifest.json

@@ -0,0 +1,131 @@
+{
+  "repository": "iskhare/iclr-debug",
+  "checkpoint_step": 17167,
+  "files": [
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/milestone-iter-17167.pt",
+      "bytes": 4879455585,
+      "sha256": "0f71f24010b3fe8874a2319801008f49c00badb9af050c43b8aa8ed80122e466"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/manifest.json",
+      "bytes": 28971,
+      "sha256": "53dc5b666685e03efc016e4311b89ab94bb6babe710405c28ae58356406582d0"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/git-provenance.json",
+      "bytes": 59,
+      "sha256": "96f601db3a051fd136086ca5f54c87ce2a72449e528d826a3b882541953e3fa2"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/config.yaml",
+      "bytes": 4827,
+      "sha256": "073bd5a69f1d0a39019efffd79f4b7bd645e0c1792edead7e31b524515ef134f"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/teacher-provenance.json",
+      "bytes": 270,
+      "sha256": "96e0063afe4211b1109509690a697fd8a7366941988c894f1027e1ead45938cf"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/summary.json",
+      "bytes": 1189,
+      "sha256": "244f84b5ba0fe32b7b3a36e669e9cdfaab0a874a649aaa1b20c87f8c2d3780a2"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/quality-summary.json",
+      "bytes": 2946,
+      "sha256": "c566202ac303fe467b47d5045e2eaf1bbbf0c50865c365a1846d650d89d5b6e1"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/report.md",
+      "bytes": 3199,
+      "sha256": "9fc9fc06602f9d3cb453a8ad72ace2544eac9d2f00ecdbdbe55332b60c8977e6"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/manifest.json",
+      "bytes": 944,
+      "sha256": "8756daa27a93c0754a6bda4186833ba163c9f95e3686e7b2262657dc5df5f3f6"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/evaluator.py",
+      "bytes": 7709,
+      "sha256": "e2c87f66f1001853f58b9e78a646fc4e089fc894d05fa6c215a8baf493a84055"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/controller.py",
+      "bytes": 3124,
+      "sha256": "57161030b9c01fef411ede348a7ec869a620f6687543aa2a7b0f298a8d55ef8a"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/summarize-quality.py",
+      "bytes": 3757,
+      "sha256": "ff838759520854d2ffbf2e5701172ba54b86eada2be52061e8b6141ff72fae06"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/accepted-entropy-ge-4p5.json",
+      "bytes": 143849,
+      "sha256": "6aecf269ead6b8053ed7aeb54114821c4cbb0defcac4fa809dff0aa1f348dd26"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/manual-eval-status.json",
+      "bytes": 252,
+      "sha256": "f90fe53b25058cda8f6867b868075db1cee4f4ffc0c7d18e325aedaf6ac81a76"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-0.json",
+      "bytes": 139237,
+      "sha256": "3b86548062c195b0a19f7d50ef1a2dcc92c0a211daccaf29b2593e134336fe7a"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-0.samples.json",
+      "bytes": 129318,
+      "sha256": "88b5f007bad75aef1a9b226aba45e82509560b3f6869f91717c30c4cbf345841"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-1.json",
+      "bytes": 137569,
+      "sha256": "9efd2b090ed86f2d483aea18d554fdf8b60efe0a81790ebbbf86622bb1252487"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-1.samples.json",
+      "bytes": 127647,
+      "sha256": "2c307023a4fda12822ed28c2dbbcd1fc5dbb87a76ac41f1c0c251ab194951a96"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-2.json",
+      "bytes": 137015,
+      "sha256": "66b3615172cd41666279134cd9c5bdb55285610e506333100acc5e86b999e640"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-2.samples.json",
+      "bytes": 127115,
+      "sha256": "834c87296ba5256677b8978ca659a4512d3eb4c24bf6e9da7a587ff47f971105"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-3.json",
+      "bytes": 139830,
+      "sha256": "439b93bb677adc43cad18f3663837787c1b680d5bd6fda82d49b7caf82ded0a0"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-3.samples.json",
+      "bytes": 129930,
+      "sha256": "6a99e26e1d8d8006aab694e543666152d0e58f5d71e81c6be6d82fd3b378838e"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-4.json",
+      "bytes": 138691,
+      "sha256": "3b0ec394f10deed0685f55501c083347fa4c27782d2c72e31e71a8f69cfbb73b"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-17167-fp64-rerun-20260919/random-seed-4.samples.json",
+      "bytes": 128775,
+      "sha256": "3680100a4c9389c6a98fe265db53fd519d82351b2ece304d6f3ed2a91bad4b01"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/checkpoint-17167-README.md",
+      "bytes": 1427,
+      "sha256": "fd964ecc83d8dc10607faba3386c468845beed869a216d3c41acbc5304093ee7"
+    }
+  ]
+}