Phase 0: mirror probe scripts + launcher pattern

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.pyc
+.DS_Store

README.md

@@ -0,0 +1,22 @@
+# ounce100m-code
+
+Canonical mirror of the scripts and configs for project `ounce100m`, so a Kaggle instance can fetch
+its own source instead of having it pasted into a kernel. The constitution lives in `MASTER_PROMPT.md`
+in the local workspace and is read-only.
+
+Layout:
+
+- `probes/` — Phase 0 platform probes and Phase 3 preflight scripts.
+- `config/` — frozen architecture / mix / eval configs (Phase 1 onward).
+
+**No credential ever appears in this repo.** Anything that needs the Hugging Face token reads it
+from the environment (`HF_TOKEN`) at runtime. A launcher kernel fetches a script here and execs it:
+
+```python
+import urllib.request
+URL = "https://huggingface.co/Cion-lab/ounce100m-code/resolve/main/probes/<name>.py"
+src = urllib.request.urlopen(URL, timeout=60).read().decode()
+exec(compile(src, "<name>.py", "exec"), {"__name__": "__main__"})
+```
+
+See `memory/ASSETS.md` in the local workspace for what each probe established.

probes/p0_cpu_probe.py

@@ -0,0 +1,230 @@
+# Phase 0 CPU probe. Runs on a Kaggle CPU instance (zero GPU quota).
+# Purpose: establish empirically what the main run will depend on --
+# in-job internet, pip behavior, image contents, disk/memory limits,
+# Hub file download, and the artifact-retrieval path. Writes one artifact
+# to /kaggle/working so retrieval can be verified by hash.
+# This file contains NO credentials and must never gain any.
+
+import hashlib
+import json
+import os
+import platform
+import shutil
+import socket
+import subprocess
+import sys
+import time
+
+R = {}
+T_START = time.time()
+MONO0 = time.monotonic()
+
+
+def step(name, fn):
+    a, b = time.monotonic(), time.time()
+    try:
+        R[name] = fn()
+    except Exception as e:
+        R[name] = {"error": f"{type(e).__name__}: {e}"}
+    R.setdefault("_timing", {})[name] = round(time.monotonic() - a, 2)
+    R["_timing"][name + "_epoch"] = round(b, 0)
+
+
+def run(cmd, timeout=90):
+    p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
+    return {"rc": p.returncode, "out": p.stdout[-3000:], "err": p.stderr[-1500:]}
+
+
+def get_http(url, timeout=25, nbytes=400):
+    import urllib.request
+
+    t = time.monotonic()
+    req = urllib.request.Request(url, headers={"User-Agent": "ounce100m-probe"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        body = r.read(nbytes)
+        return {
+            "status": r.status,
+            "ms": round((time.monotonic() - t) * 1000),
+            "bytes_head": len(body),
+            "final_url": r.url[:160],
+        }
+
+
+def pkg_versions():
+    from importlib.metadata import PackageNotFoundError, version
+
+    want = [
+        "torch", "torchvision", "transformers", "tokenizers", "datasets", "evaluate",
+        "accelerate", "trl", "numpy", "pandas", "pyarrow", "sentencepiece", "tiktoken",
+        "protobuf", "einops", "flash-attn", "xformers", "deepspeed", "bitsandbytes",
+        "huggingface-hub", "safetensors", "timm", "scipy", "sentence-transformers",
+        "wandb", "liger-kernel", "causal-conv1d", "mamba-ssm", "unsloth", "peft",
+    ]
+    out = {}
+    for n in want:
+        try:
+            out[n] = version(n)
+        except PackageNotFoundError:
+            pass
+    return out
+
+
+def cuda_info():
+    import torch
+
+    d = {
+        "torch": torch.__version__,
+        "cuda_built": torch.version.cuda,
+        "cudnn": torch.backends.cudnn.version(),
+        "mkl": torch.cuda.get_mkl_version() if hasattr(torch.cuda, "get_mkl_version") else None,
+        "allow_tf32_default": torch.backends.cuda.matmul.allow_tf32,
+        "arch_list": list(torch.cuda.get_arch_list()),
+        "device_count": torch.cuda.device_count(),
+        "cpu_threads": torch.get_num_threads(),
+    }
+    if torch.cuda.is_available():
+        props = []
+        for i in range(torch.cuda.device_count()):
+            p = torch.cuda.get_device_properties(i)
+            props.append({
+                "i": i, "name": p.name, "cc": f"{p.major}.{p.minor}",
+                "sm": p.multi_processor_count, "mem_GB": round(p.total_memory / 1024**3, 2),
+                "bf16_capable_cc": p.major >= 8,
+            })
+        d["devices"] = props
+    return d
+
+
+def precision_probe():
+    # Turing T4 = cc 7.5: fp16 tensor cores yes, bf16 no. Prove it rather than recall it.
+    import torch
+
+    out = {}
+    dev = "cuda" if torch.cuda.is_available() else "cpu"
+    for dt in (torch.float16, torch.bfloat16, torch.float32):
+        try:
+            x = torch.randn(1024, 1024, device=dev, dtype=dt)
+            y = (x @ x).float()
+            torch.cuda.synchronize() if dev == "cuda" else None
+            out[str(dt)] = {"ok": True, "checksum": round(float(y.abs().mean()), 4)}
+        except Exception as e:
+            out[str(dt)] = {"ok": False, "err": f"{type(e).__name__}: {e}"[:160]}
+    return out
+
+
+def net_checks():
+    out = {}
+    for host in ["huggingface.co", "cdn-lfs.huggingface.co", "pypi.org", "files.pythonhosted.org"]:
+        try:
+            infos = socket.getaddrinfo(host, 443)
+            out[host] = sorted({i[4][0] for i in infos})[:3]
+        except Exception as e:
+            out[host] = f"DNSFAIL {type(e).__name__}: {e}"
+    return out
+
+
+def hub_download():
+    url = "https://huggingface.co/datasets/wikimedia/wikipedia/resolve/main/README.md"
+    d = get_http(url)
+    try:
+        import datasets
+
+        t = time.monotonic()
+        ds = datasets.load_dataset("wikimedia/wikipedia", "20231101.en",
+                                  split="train", streaming=True)
+        row = next(iter(ds))
+        d["streaming"] = {
+            "ok": True, "ms_to_first_row": round((time.monotonic() - t) * 1000),
+            "keys": list(row.keys()), "title": str(row.get("title"))[:60],
+        }
+    except Exception as e:
+        d["streaming"] = {"ok": False, "err": f"{type(e).__name__}: {e}"[:300]}
+    return d
+
+
+def pip_probe():
+    # xxhash: small, not in the Kaggle image, useful later for shard hashing.
+    t = time.monotonic()
+    r = run("python -m pip install --quiet --disable-pip-version-check --no-input xxhash", timeout=240)
+    r["install_seconds"] = round(time.monotonic() - t, 1)
+    try:
+        import xxhash
+
+        r["import_ok"] = xxhash.xxh64(b"ounce100m").hexdigest()
+    except Exception as e:
+        r["import_ok"] = f"FAIL {e}"
+    return r
+
+
+def fs_probe():
+    out = {}
+    for p in ["/kaggle/working", "/kaggle/input", "/tmp", "/", "/dev/shm"]:
+        try:
+            u = shutil.disk_usage(p)
+            out[p] = {"total_GB": round(u.total / 1024**3, 1), "free_GB": round(u.free / 1024**3, 1)}
+        except Exception as e:
+            out[p] = f"ERR {e}"
+    with open("/proc/meminfo") as f:
+        mm = {l.split(":")[0]: l.split()[0] for l in f if ":" in l}
+    out["mem_MemTotal_MB"] = int(mm.get("MemTotal", 0)) // 1024
+    out["mem_MemAvailable_MB"] = int(mm.get("MemAvailable", 0)) // 1024
+    out["cgroup_v2_max"] = None
+    for lp in ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]:
+        if os.path.exists(lp):
+            out["cgroup_v2_max"] = open(lp).read().strip()
+            break
+    out["cpu_count_os"] = os.cpu_count()
+    try:
+        out["cpu_affinity"] = len(os.sched_getaffinity(0))
+    except Exception:
+        pass
+    out["nproc_smi"] = run("nvidia-smi -L", timeout=30)
+    out["df_h"] = run("df -h | head -20", timeout=30).get("out", "")
+    return out
+
+
+def artifact():
+    payload = json.dumps({"probe": "p0", "t": T_START, "tokens": 12345}).encode()
+    path = "ounce100m_p0_artifact.json"
+    with open(path, "wb") as f:
+        f.write(payload)
+    return {
+        "path": path, "abs": os.path.abspath(path), "size": len(payload),
+        "sha256": hashlib.sha256(payload).hexdigest(),
+        "cwd": os.getcwd(),
+        "working_listing": sorted(os.listdir("/kaggle/working")),
+    }
+
+
+step("env", lambda: {
+    "python": sys.version.split()[0], "platform": platform.platform(),
+    "hostname": socket.gethostname(), "user": os.environ.get("USER"),
+    "cwd": os.getcwd(), "start_epoch": T_START,
+})
+step("env_var_names", lambda: sorted(k for k in os.environ if any(
+    s in k.upper() for s in ["HF_", "KAGGLE_", "HUGGING", "TOKEN", "SECRET", "CUDA", "TORCH"])))
+step("packages", pkg_versions)
+step("net_dns", net_checks)
+step("http", lambda: {
+    "hub_api": get_http("https://huggingface.co/api/models?limit=1"),
+    "pypi_simple": get_http("https://pypi.org/simple/xxhash/"),
+    "google": get_http("https://www.google.com/generate_204"),
+})
+step("hub_download", hub_download)
+step("pip_install", pip_probe)
+step("filesystem", fs_probe)
+step("artifact", artifact)
+if os.environ.get("CUDA_VISIBLE_DEVICES") != "None":
+    step("cuda", cuda_info)
+    step("precision", precision_probe)
+else:
+    step("cuda", lambda: {"skipped": "CUDA_VISIBLE_DEVICES=None -> CPU shape"})
+
+R["_summary"] = {
+    "total_seconds": round(time.monotonic() - MONO0, 1),
+    "end_epoch": time.time(),
+    "errors_seen": sorted(k for k, v in R.items() if isinstance(v, dict) and "error" in v),
+}
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")

probes/p0_gpu_probe.py

@@ -0,0 +1,242 @@
+# Phase 0 GPU probe. Runs on the 2x Tesla T4 shape -- the only accelerator §2 permits.
+# Purpose: prove the GPU path end to end and settle facts the main run's schedule depends on:
+#   (1) the real accelerator(s) handed out, (2) Turing capability envelope (fp16 yes / bf16 no),
+#   (3) whether NCCL all-reduce actually works across both cards, (4) torch.compile and
+#       flash-attention viability on cc 7.5, (5) a first tokens/sec reading, and
+#       (6) wall-clock, so the quota delta around this job reveals the burn rate per hour.
+# No credentials here. Nothing is installed at runtime beyond what the image already has.
+
+import hashlib
+import json
+import os
+import platform
+import socket
+import subprocess
+import sys
+import time
+
+R = {}
+MONO0 = time.monotonic()
+EPOCH0 = time.time()
+
+
+def step(name, fn):
+    a = time.monotonic()
+    try:
+        R[name] = fn()
+    except Exception as e:
+        R[name] = {"error": f"{type(e).__name__}: {e}"}
+    R.setdefault("_timing", {})[name] = round(time.monotonic() - a, 2)
+
+
+def run(cmd, timeout=180):
+    p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
+    return {"rc": p.returncode, "out": p.stdout[-2500:], "err": p.stderr[-1500:]}
+
+
+def gpu_identity():
+    import shutil
+
+    d = {"CUDA_VISIBLE_DEVICES": os.environ.get("CUDA_VISIBLE_DEVICES"),
+         "KAGGLE_DOCKER_IMAGE": os.environ.get("KAGGLE_DOCKER_IMAGE"),
+         "python": sys.version.split()[0]}
+    for p in ["/kaggle/working", "/tmp"]:
+        u = shutil.disk_usage(p)
+        d[f"free_GB_{p.strip('/')}"] = round(u.free / 1024**3, 1)
+    d["smi_L"] = run("nvidia-smi -L", 60)
+    d["smi_query"] = run(
+        'nvidia-smi --query-gpu=index,name,driver_version,memory.total,compute_cap,'
+        'multiprocessor_count --format=csv', 60)
+    return d
+
+
+def torch_env():
+    import torch
+
+    out = {
+        "torch": torch.__version__, "cuda_built": torch.version.cuda,
+        "cudnn": torch.backends.cudnn.version(),
+        "nccl_built": torch.cuda.nccl.version() if hasattr(torch.cuda, "nccl") else None,
+        "arch_list": list(torch.cuda.get_arch_list()),
+        "device_count": torch.cuda.device_count(),
+        "tf32_default": torch.backends.cuda.matmul.allow_tf32,
+    }
+    devs = []
+    for i in range(torch.cuda.device_count()):
+        p = torch.cuda.get_device_properties(i)
+        devs.append({"i": i, "name": p.name, "cc": f"{p.major}.{p.minor}", "sm": p.multi_processor_count,
+                     "mem_GB": round(p.total_memory / 1024**3, 2)})
+    out["devices"] = devs
+    out["all_turing_T4"] = all(d["cc"] == "7.5" and "T4" in d["name"] for d in devs)
+    out["exactly_two"] = len(devs) == 2
+    for extra in ("triton", "flash_attn", "xformers", "transformers", "datasets", "accelerate", "trl"):
+        try:
+            from importlib.metadata import version
+
+            out[f"pkg_{extra}"] = version(extra if extra != "flash_attn" else "flash-attn")
+        except Exception:
+            out[f"pkg_{extra}"] = "absent"
+    return out
+
+
+def precision_and_gemm():
+    import torch
+
+    out = {}
+    for dt in (torch.float32, torch.float16, torch.bfloat16):
+        for tf32 in ([False] if dt != torch.float32 else [False, True]):
+            torch.backends.cuda.matmul.allow_tf32 = tf32
+            key = f"{str(dt).replace('torch.', '')}{'_tf32' if tf32 else ''}"
+            try:
+                t0 = time.monotonic()
+                a = torch.randn(2048, 2048, device="cuda", dtype=dt)
+                b = torch.randn(2048, 2048, device="cuda", dtype=dt)
+                for _ in range(10):
+                    c = a @ b
+                torch.cuda.synchronize()
+                dt_s = (time.monotonic() - t0) / 10
+                out[key] = {"ok": True, "ms_per_gemm": round(dt_s * 1000, 2),
+                            "tflops": round(2 * 2048**3 / dt_s / 1e12, 2),
+                            "checksum": round(float(c.float().abs().mean()), 4)}
+            except Exception as e:
+                out[key] = {"ok": False, "err": f"{type(e).__name__}: {e}"[:180]}
+    torch.backends.cuda.matmul.allow_tf32 = False
+    return out
+
+
+def sdpa_backends():
+    import torch
+
+    out = {}
+    for name in ("flash", "mem_efficient", "math"):
+        try:
+            fn = getattr(torch.backends.cuda, f"{name}_sdpa_enabled")
+            out[name + "_enabled"] = fn()
+        except Exception as e:
+            out[name + "_enabled"] = str(e)[:80]
+    from torch.nn.attention import SDPBackend, sdpa_kernel
+    q = torch.randn(2, 4, 128, 64, device="cuda", dtype=torch.float16)
+    for name in ("FLASH_ATTENTION", "EFFICIENT_ATTENTION", "MATH"):
+        try:
+            be = getattr(SDPBackend, name)
+            with sdpa_kernel(be):
+                o = torch.nn.functional.scaled_dot_product_attention(q, q, q)
+            torch.cuda.synchronize()
+            out[f"sdpa_{name}"] = {"ok": True, "shape": list(o.shape)}
+        except Exception as e:
+            out[f"sdpa_{name}"] = {"ok": False, "err": f"{type(e).__name__}: {e}"[:180]}
+    return out
+
+
+def ddp_smoke():
+    # The main run is 2 GPUs. If NCCL across these two T4s does not work, every plan that
+    # assumes data-parallel throughput is wrong -- prove it here on a toy, not at launch.
+    worker = r'''
+import os, json, time, torch, torch.distributed as dist
+dist.init_process_group("nccl")
+r = dist.get_rank()
+torch.cuda.set_device(r)
+x = torch.ones(1024, 1024, device=f"cuda:{r}") * (r + 1)
+t0 = time.monotonic()
+for _ in range(20):
+    y = x.clone()
+    dist.all_reduce(y)
+torch.cuda.synchronize()
+dt = (time.monotonic() - t0) / 20
+print("WORKER_JSON " + json.dumps({"rank": r, "world": dist.get_world_size(),
+      "allreduce_mean": float(y.mean()), "ms_per_allreduce": round(dt*1000, 3),
+      "backend": dist.get_backend(), "device": torch.cuda.get_device_name(r)}), flush=True)
+dist.destroy_process_group()
+'''
+    with open("ddp_worker.py", "w") as f:
+        f.write(worker)
+    t = time.monotonic()
+    p = subprocess.run(
+        [sys.executable, "-m", "torch.distributed.run", "--nproc_per_node=2", "--master_port=29511",
+         "ddp_worker.py"], capture_output=True, text=True, timeout=300,
+        env={**os.environ, "OMP_NUM_THREADS": "1"})
+    lines = [l for l in (p.stdout + p.stderr).splitlines() if l.startswith("WORKER_JSON ")]
+    return {"rc": p.returncode, "seconds": round(time.monotonic() - t, 1),
+            "workers": [json.loads(l[len("WORKER_JSON "):]) for l in lines],
+            "err_tail": p.stderr[-1200:]}
+
+
+def tiny_lm_steps():
+    # A few hundred steps of a real 10M-param causal LM, so tokens/sec is a measurement and not
+    # an extrapolation. Not a model-building exercise: config is deliberately small.
+    import torch
+    from transformers import LlamaConfig, LlamaForCausalLM
+
+    cfg = LlamaConfig(hidden_size=384, num_hidden_layers=6, num_attention_heads=6,
+                      num_key_value_heads=6, intermediate_size=1024, vocab_size=32000,
+                      max_position_embeddings=1024)
+    model = LlamaForCausalLM(cfg).cuda().to(torch.float16)
+    n = sum(p.numel() for p in model.parameters())
+    opt = torch.optim.AdamW(model.parameters(), lr=3e-4)
+    B, L = 8, 1024
+    ids = torch.randint(0, 32000, (B, L), device="cuda")
+    tg = torch.randint(0, 32000, (B, L), device="cuda")
+    model.train()
+    losses = []
+    warm = 5
+    t = time.monotonic()
+    for s in range(warm + 60):
+        opt.zero_grad(set_to_none=True)
+        with torch.autocast("cuda", dtype=torch.float16):
+            out = model(input_ids=ids, labels=tg)
+        loss = out.loss
+        loss.backward()
+        opt.step()
+        if s >= warm:
+            losses.append(round(float(loss), 4))
+    torch.cuda.synchronize()
+    el = time.monotonic() - t
+    tps = (60 * B * L) / el
+    return {"params": n, "loss_first": losses[0], "loss_last": losses[-1],
+            "losses_sampled": losses[::15], "seconds": round(el, 1),
+            "tokens_per_sec": round(tps), "peak_mem_GB": round(torch.cuda.max_memory_allocated() / 1024**3, 2),
+            "note": "single GPU, fp16 autocast, 8x1024 batch"}
+
+
+def compile_smoke():
+    import torch
+
+    try:
+        m = torch.nn.Sequential(torch.nn.Linear(512, 512), torch.nn.GELU(), torch.nn.Linear(512, 512)).cuda()
+        cm = torch.compile(m)
+        x = torch.randn(256, 512, device="cuda")
+        t0 = time.monotonic()
+        for _ in range(20):
+            y = cm(x)
+        torch.cuda.synchronize()
+        compile_s = time.monotonic() - t0
+        return {"ok": True, "warm_plus_20_steps_s": round(compile_s, 2),
+                "out_shape": list(y.shape)}
+    except Exception as e:
+        return {"ok": False, "err": f"{type(e).__name__}: {e}"[:300]}
+
+
+def artifact():
+    payload = json.dumps({"probe": "p0-gpu", "epoch0": EPOCH0, "host": socket.gethostname()}).encode()
+    open("ounce100m_p0_gpu_artifact.json", "wb").write(payload)
+    return {"size": len(payload), "sha256": hashlib.sha256(payload).hexdigest()}
+
+
+step("gpu_identity", gpu_identity)
+step("torch_env", torch_env)
+step("precision_gemm", precision_and_gemm)
+step("sdpa", sdpa_backends)
+step("ddp_nccl", ddp_smoke)
+step("compile", compile_smoke)
+step("tiny_lm", tiny_lm_steps)
+step("smi_after", lambda: run("nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used "
+                              "--format=csv", 60))
+step("artifact", artifact)
+
+R["_timing"]["total"] = round(time.monotonic() - MONO0, 1)
+R["_session"] = {"epoch_start": EPOCH0, "epoch_end": time.time(),
+                 "wall_seconds": round(time.time() - EPOCH0, 1),
+                 "hostname": socket.gethostname(), "platform": platform.platform()}
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")

probes/p0b_cpu_fs.py

@@ -0,0 +1,88 @@
+# Phase 0 CPU probe B. Follows up the one gap in p0_cpu_probe.py: its /proc/meminfo parser
+# crashed (E-003), so instance disk and RAM -- the numbers §3.13 depends on -- were never read.
+# Also answers a question the first probe raised: the CPU image ships torch 2.10.0+cpu, so
+# which image/torch a GPU session actually gets has to be observed from inside one.
+# Zero GPU quota. No credentials.
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+t0 = time.time()
+
+
+def meminfo():
+    out = {}
+    with open("/proc/meminfo") as f:
+        for line in f:
+            if ":" not in line:
+                continue
+            k, rest = line.split(":", 1)
+            parts = rest.split()
+            if parts:
+                out[k] = int(parts[0]) // 1024
+    return {k: out[k] for k in ("MemTotal", "MemAvailable", "MemFree", "SwapTotal", "Shmem") if k in out}
+
+
+def disk():
+    out = {}
+    for p in ["/kaggle/working", "/kaggle/input", "/tmp", "/.cache", "/root/.cache", "/dev/shm", "/usr"]:
+        try:
+            u = shutil.disk_usage(p)
+            out[p] = {"total_GB": round(u.total / 1024**3, 1), "used_GB": round(u.used / 1024**3, 1),
+                      "free_GB": round(u.free / 1024**3, 1)}
+        except Exception as e:
+            out[p] = f"missing ({type(e).__name__})"
+    return out
+
+
+def cgroup():
+    out = {}
+    for name, paths in {
+        "mem_max": ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"],
+        "cpu_max": ["/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"],
+    }.items():
+        for p in paths:
+            if os.path.exists(p):
+                out[name] = open(p).read().strip()
+                out[name + "_path"] = p
+                break
+        else:
+            out[name] = "not found"
+    return out
+
+
+R = {
+    "shape": {"CUDA_VISIBLE_DEVICES_present": "CUDA_VISIBLE_DEVICES" in os.environ,
+              "CUDA_VISIBLE_DEVICES": os.environ.get("CUDA_VISIBLE_DEVICES"),
+              "KAGGLE_DOCKER_IMAGE": os.environ.get("KAGGLE_DOCKER_IMAGE"),
+              "KAGGLE_KERNEL_RUN_TYPE": os.environ.get("KAGGLE_KERNEL_RUN_TYPE"),
+              "KAGGLE_CONTAINER_NAME": os.environ.get("KAGGLE_CONTAINER_NAME"),
+              "KAGGLE_GCP_ZONE": os.environ.get("KAGGLE_GCP_ZONE"),
+              "KAGGLE_KERNEL_INTEGRATIONS": os.environ.get("KAGGLE_KERNEL_INTEGRATIONS")},
+    "cpu": {"os_cpu_count": os.cpu_count(), "affinity": len(os.sched_getaffinity(0)),
+            "loadavg": os.getloadavg(),
+            "model": [l for l in open("/proc/cpuinfo") if "model name" in l][:1]},
+    "mem_MB": meminfo(),
+    "cgroup": cgroup(),
+    "disk": disk(),
+    "df_h": subprocess.run("df -h", shell=True, capture_output=True, text=True).stdout,
+    "du_cache": subprocess.run("du -sh /root/.cache/* 2>/dev/null | head -10", shell=True,
+                               capture_output=True, text=True).stdout,
+}
+import socket
+
+dns = {}
+for h in ["cdn-lfs.huggingface.co", "cdn-lfs-us-1.huggingface.co", "huggingface.co", "xethub.huggingface.co"]:
+    try:
+        dns[h] = sorted({i[4][0] for i in socket.getaddrinfo(h, 443)})[:2]
+    except Exception as e:
+        dns[h] = f"DNSFAIL {type(e).__name__}"
+R["dns"] = dns
+R["_seconds"] = round(time.time() - t0, 1)
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")

probes/p0c_fp16_ddp_probe.py

@@ -0,0 +1,275 @@
+# Phase 0 GPU probe C -- corrects two failures in probe B and settles the schedule's biggest risk.
+#
+# Why this exists:
+#   * probe B's tiny_lm ran the weights in pure fp16 with no GradScaler and the loss came back NaN,
+#     so its tokens/sec figure measured a broken recipe. The main run cannot discover that at 100M.
+#   * probe B's DDP step never ran: torchrun's two WORKER_JSON lines broke the parser, so whether
+#     NCCL works across these two specific T4s is STILL unproven, and the whole stack choice in
+#     Phase 1 assumes it does.
+#   * probe B's throughput was an 11.6M-weights model (35M params once a 50k-row embedding is
+#     counted), not the ~95M we will actually train, so it is the wrong order of magnitude to plan on.
+#
+# Nothing here is a model-building exercise: one config near the likely main-run shape, a few
+# hundred steps, a hard self-imposed deadline so this cannot run away.
+# Readable canonical copy: HF repo Cion-lab/ounce100m-code, path probes/p0c_fp16_ddp_probe.py
+# This file contains NO credentials and must never gain any.
+
+import json
+import os
+import socket
+import subprocess
+import sys
+import time
+
+R = {}
+MONO0 = time.monotonic()
+EPOCH0 = time.time()
+DEADLINE_S = 420  # ~7 min of work; aborts remaining measurements rather than burning quota
+
+import torch  # noqa: E402
+
+
+def remaining():
+    return DEADLINE_S - (time.monotonic() - MONO0)
+
+
+R["setup"] = {
+    "torch": torch.__version__, "cuda": torch.version.cuda,
+    "devices": [(torch.cuda.get_device_properties(i).name,
+                 f"{torch.cuda.get_device_properties(i).major}."
+                 f"{torch.cuda.get_device_properties(i).minor}")
+                for i in range(torch.cuda.device_count())],
+    "host": socket.gethostname(),
+    "KAGGLE_DOCKER_IMAGE": os.environ.get("KAGGLE_DOCKER_IMAGE"),
+}
+
+# --- the config: ~95M params with a 50k vocab and tied embeddings, i.e. main-run shaped ------
+H, NL, NH, NKV, FF, VOCAB, SEQ = 768, 12, 12, 3, 2048, 50257, 1024
+
+from transformers import LlamaConfig, LlamaForCausalLM  # noqa: E402
+
+CFG = LlamaConfig(
+    hidden_size=H, num_hidden_layers=NL, num_attention_heads=NH, num_key_value_heads=NKV,
+    intermediate_size=FF, vocab_size=VOCAB, max_position_embeddings=SEQ, tie_word_embeddings=True,
+)
+
+
+def make_model(attn="sdpa"):
+    m = LlamaForCausalLM(CFG, attn_implementation=attn)
+    return m.cuda()
+
+
+def n_params(m):
+    return sum(p.numel() for p in m.parameters())
+
+
+def step_run(model, mk_batch, steps, scaler=None, accum=1, deadline=None):
+    """Returns (tokens_per_sec, first_loss, last_loss, nan_count, peak_GB)."""
+    opt = torch.optim.AdamW(model.parameters(), lr=3e-4)
+    model.train()
+    losses, nans = [], 0
+    B = mk_batch().shape[0]
+    torch.cuda.synchronize()
+    t0 = time.monotonic()
+    for s in range(steps):
+        for micro in range(accum):
+            ids = mk_batch()
+            with torch.autocast("cuda", dtype=torch.float16):
+                loss = model(input_ids=ids, labels=ids).loss / accum
+            if scaler is not None:
+                scaler.scale(loss).backward()
+            else:
+                loss.backward()
+        if scaler is not None:
+            scaler.unscale_(opt)
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+            scaler.step(opt)
+            scaler.update()
+        else:
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+            opt.step()
+        opt.zero_grad(set_to_none=True)
+        lv = float(loss.detach())
+        losses.append(lv)
+        nans += int(lv != lv)
+        if deadline and time.monotonic() - MONO0 > deadline:
+            break
+    torch.cuda.synchronize()
+    el = time.monotonic() - t0
+    done = max(1, len(losses))
+    return {
+        "tokens_per_sec": round(done * B * SEQ / el),
+        "steps_done": len(losses),
+        "loss_first3": [round(v, 3) for v in losses[:3]],
+        "loss_last3": [round(v, 3) for v in losses[-3:]],
+        "nan_count": nans,
+        "peak_GB": round(torch.cuda.max_memory_allocated() / 1024**3, 2),
+        "seconds": round(el, 1),
+    }
+
+
+def batch_fn(micro_b):
+    def f():
+        return torch.randint(0, VOCAB, (micro_b, SEQ), device="cuda")
+    return f
+
+
+results = {}
+
+# 1. the recipe probe that failed before: fp32 master weights + fp16 autocast + GradScaler
+m = make_model()
+results["model_shape"] = {
+    "params_total": n_params(m),
+    "params_embedding": int(CFG.vocab_size * H * (2 if not CFG.tie_word_embeddings else 1)),
+    "config": {"H": H, "layers": NL, "heads": NH, "kv_heads": NKV, "ff": FF,
+               "vocab": VOCAB, "seq": SEQ},
+}
+del m
+torch.cuda.empty_cache()
+
+try:
+    m = make_model()
+    opt_probe = torch.optim.AdamW(m.parameters(), lr=3e-4)
+    sc = torch.amp.GradScaler("cuda")
+    b = torch.randint(0, VOCAB, (2, SEQ), device="cuda")
+    ok_fp16 = None
+    for _ in range(4):
+        opt_probe.zero_grad(set_to_none=True)
+        with torch.autocast("cuda", dtype=torch.float16):
+            loss = m(input_ids=b, labels=b).loss
+        sc.scale(loss).backward()
+        sc.step(opt_probe)
+        sc.update()
+        ok_fp16 = float(loss.detach())
+    results["fp16_with_scaler_sanity"] = {"loss": round(ok_fp16, 3), "finite": ok_fp16 == ok_fp16}
+    del m, opt_probe, sc
+    torch.cuda.empty_cache()
+except Exception as e:
+    results["fp16_with_scaler_sanity"] = {"error": f"{type(e).__name__}: {e}"[:200]}
+
+# 2. bf16 on Turing: expected to be unsupported or emulated. Record, do not assume.
+try:
+    m = make_model()
+    b = torch.randint(0, VOCAB, (2, SEQ), device="cuda")
+    t = time.monotonic()
+    with torch.autocast("cuda", dtype=torch.bfloat16):
+        loss = m(input_ids=b, labels=b).loss
+    torch.cuda.synchronize()
+    results["bf16_autocast"] = {"ok": True, "loss": round(float(loss.detach()), 3),
+                                "first_fwd_s": round(time.monotonic() - t, 2)}
+    del m
+    torch.cuda.empty_cache()
+except Exception as e:
+    results["bf16_autocast"] = {"ok": False, "err": f"{type(e).__name__}: {e}"[:200]}
+
+# 3. throughput sweep, single GPU, correct recipe, decreasing-deadline order
+m = make_model()
+scaler = torch.amp.GradScaler("cuda")
+for micro_b in (4, 8, 12):
+    if remaining() < 90:
+        results["sweep_stopped_early_at"] = micro_b
+        break
+    try:
+        results[f"1gpu_bs{micro_b}"] = step_run(
+            m, batch_fn(micro_b), steps=40, scaler=scaler, deadline=MONO0 + DEADLINE_S - 60)
+    except Exception as e:
+        results[f"1gpu_bs{micro_b}"] = {"error": f"{type(e).__name__}: {e}"[:200]}
+        torch.cuda.empty_cache()
+del m, scaler
+torch.cuda.empty_cache()
+R["single_gpu"] = results
+
+# 4. attention/compile variants at one batch size, to see what actually pays on cc 7.5
+variants = {}
+for name, kwargs in [("eager", {"attn": "eager"}), ("sdpa", {"attn": "sdpa"})]:
+    if remaining() < 90:
+        break
+    try:
+        m = make_model(kwargs["attn"])
+        sc = torch.amp.GradScaler("cuda")
+        variants[name] = step_run(m, batch_fn(4), steps=30, scaler=sc,
+                                  deadline=MONO0 + DEADLINE_S - 30)
+        del m, sc
+        torch.cuda.empty_cache()
+    except Exception as e:
+        variants[name] = {"error": f"{type(e).__name__}: {e}"[:200]}
+R["attn_variants"] = variants
+
+
+# 5. DDP across the two cards, via torchrun, parsed line-by-line so one stray line cannot
+#    swallow the whole measurement (the mistake that voided probe B's attempt).
+DDP_SRC = r'''
+import json, os, time, torch, torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+from transformers import LlamaConfig, LlamaForCausalLM
+
+dist.init_process_group("nccl")
+rank = dist.get_rank()
+world = dist.get_world_size()
+torch.cuda.set_device(rank)
+
+H, NL, NH, NKV, FF, VOCAB, SEQ = 768, 12, 12, 3, 2048, 50257, 1024
+cfg = LlamaConfig(hidden_size=H, num_hidden_layers=NL, num_attention_heads=NH,
+                  num_key_value_heads=NKV, intermediate_size=FF, vocab_size=VOCAB,
+                  max_position_embeddings=SEQ, tie_word_embeddings=True)
+torch.manual_seed(1234)
+model = LlamaForCausalLM(cfg, attn_implementation="sdpa").cuda().to(torch.float32)
+model = DDP(model, device_ids=[rank])
+opt = torch.optim.AdamW(model.parameters(), lr=3e-4)
+scaler = torch.amp.GradScaler("cuda")
+
+x = torch.randint(0, VOCAB, (4, SEQ), device="cuda")
+losses = []
+t0 = time.monotonic()
+for s in range(30):
+    opt.zero_grad(set_to_none=True)
+    with torch.autocast("cuda", dtype=torch.float16):
+        loss = model(input_ids=x, labels=x).loss
+    scaler.scale(loss).backward()
+    scaler.step(opt)
+    scaler.update()
+    losses.append(float(loss.detach()))
+torch.cuda.synchronize()
+el = time.monotonic() - t0
+out = {"rank": rank, "world": world, "backend": dist.get_backend(),
+       "device": torch.cuda.get_device_name(rank),
+       "params_per_rank": sum(p.numel() for p in model.parameters()),
+       "loss_first": round(losses[0], 3), "loss_last": round(losses[-1], 3),
+       "all_finite": all(v == v for v in losses),
+       "tokens_per_sec_per_rank": round(30 * 4 * SEQ / el),
+       "peak_GB": round(torch.cuda.max_memory_allocated() / 1024**3, 2)}
+print("RESULT_JSON:" + json.dumps(out), flush=True)
+dist.barrier()
+dist.destroy_process_group()
+'''
+
+ddp = {}
+if remaining() > 120:
+    with open("ddp_worker_c.py", "w") as f:
+        f.write(DDP_SRC)
+    t = time.monotonic()
+    p = subprocess.run(
+        [sys.executable, "-m", "torch.distributed.run", "--nproc_per_node=2",
+         "--master_port=29533", "ddp_worker_c.py"],
+        capture_output=True, text=True, timeout=300,
+        env={**os.environ, "OMP_NUM_THREADS": "2", "TORCH_CPP_LOG_LEVEL": "ERROR"})
+    workers = []
+    for line in (p.stdout + "\n" + p.stderr).splitlines():
+        marker = "RESULT_JSON:"
+        if marker in line:
+            try:
+                workers.append(json.loads(line.split(marker, 1)[1].strip()))
+            except Exception as e:
+                workers.append({"parse_error": str(e)[:120], "raw": line[:200]})
+    ddp = {"rc": p.returncode, "seconds": round(time.monotonic() - t, 1), "workers": workers,
+           "nccl_ok": (len(workers) == 2 and all(w.get("all_finite") for w in workers)
+                       and all(w.get("world") == 2 for w in workers)),
+           "stderr_tail": p.stderr[-900:]}
+else:
+    ddp = {"skipped": "deadline"}
+R["ddp"] = ddp
+R["_session"] = {"epoch_start": EPOCH0, "epoch_end": time.time(),
+                 "wall_seconds": round(time.time() - EPOCH0, 1), "deadline_s": DEADLINE_S}
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")