File size: 5,532 Bytes
419f250 b871251 419f250 b871251 419f250 b871251 419f250 b871251 419f250 b871251 419f250 b871251 419f250 b871251 419f250 67aa1dc 419f250 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | #!/usr/bin/env python3
"""Generate notebooks/NOEON_TRAIN_COLAB.ipynb (kept as a script so it is diffable)."""
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
nb = {
"cells": [],
"metadata": {
"accelerator": "GPU",
"kernelspec": {"display_name": "Python 3", "name": "python3"},
"language_info": {"name": "python"},
},
"nbformat": 4,
"nbformat_minor": 5,
}
def md(t):
nb["cells"].append({"cell_type": "markdown", "metadata": {},
"source": t.splitlines(keepends=True)})
def code(t):
nb["cells"].append({"cell_type": "code", "execution_count": None,
"metadata": {}, "outputs": [],
"source": t.splitlines(keepends=True)})
md("""# NOEON — training on Google Colab
NOEON unifies three capabilities — **vision**, **computer use** and
**generation** (image synthesis) behind trainable bridge adapters into a shared
2560-d space. The sources stay **frozen**; only NOEON's bridges + fusion core
are trained.
This notebook trains the bridges against the **real** frozen sources.
> Runtime -> Change runtime type -> **GPU (T4 or better)**.
Before you start, add a Colab secret named `HF_TOKEN` (key icon, left sidebar)
with a Hugging Face token that can read the sources. Do not paste tokens into
cells.""")
code("""# 1. Environment
!nvidia-smi | head -12
!pip -q install "torch>=2.2" transformers safetensors huggingface_hub pyyaml accelerate""")
code("""# 2. Get the code. Upload NOEON-FULL-MODEL/ (or clone your repo) into /content.
import os, sys, pathlib
ROOT = pathlib.Path("/content/NOEON-FULL-MODEL")
assert ROOT.exists(), "Upload the NOEON-FULL-MODEL folder to /content first."
sys.path.insert(0, str(ROOT))
os.chdir(ROOT)
print("cwd:", os.getcwd())
!ls""")
code("""# 3. Hugging Face token from Colab secrets (never paste it in a cell)
from google.colab import userdata
os.environ["HF_TOKEN"] = userdata.get("HF_TOKEN")
print("HF_TOKEN set:", bool(os.environ.get("HF_TOKEN")))""")
code("""# 4. Confirm the environment can train (GPU + resources)
!python scripts/attach_sources.py | tail -20""")
md("""## 5. Attach the frozen sources
This downloads ~10 GB for the vision slot alone. Run once; it caches in
`~/.cache/huggingface`.""")
code("""import torch
from noeon.encoders.hf_source import HFSourceEncoder
from noeon.architecture.model import Noeon
from noeon.core.config import NoeonConfig
device = "cuda" if torch.cuda.is_available() else "cpu"
model = Noeon(NoeonConfig()).to(device)
print("trainable params:", f"{model.trainable_parameters():,}")
# Load the frozen sources (eval mode, requires_grad=False -> never modified).
vl = HFSourceEncoder("vision", device=device).load()
cowork = HFSourceEncoder("computer_use", device=device).load()
print("sources attached. (generation is 8.7B; load it only with >=16 GB VRAM)")""")
md("""## 6. The training loop
`feature_fn` is the one task-specific piece. It returns
`(streams, targets, teacher)`:
* `streams` - `{source_key: (B, T, 2560) hidden states}` from the frozen encoders
* `targets` - `(B, 2560)` the generation head should produce
* `teacher` - `(B, 2560)` frozen-teacher signal for distillation (or `None`)
Below is a **reference** implementation for a text/caption task. Adapt the
prompts to your data; the mechanism is identical.""")
code("""from noeon.training.trainer import TrainConfig, train
import torch
PROMPTS = [
"Describe the interface shown in this screenshot.",
"What action should be taken next?",
"Generate an image of a red fox in snow.",
] * 8
def feature_fn():
batch = PROMPTS
streams = {}
with torch.no_grad():
for key, enc in (("vision", vl), ("computer_use", cowork)):
inputs = enc.processor(text=batch, return_tensors="pt", padding=True)
inputs = {k: v.to(device) for k, v in inputs.items()
if k in ("input_ids", "attention_mask")}
streams[key] = enc.encode(**inputs).float()
pooled = streams["vision"].mean(dim=1)
targets = torch.zeros_like(pooled)
teacher = torch.nn.functional.normalize(pooled, dim=-1)
return streams, targets, teacher
hist = train(model, feature_fn, str(ROOT / "weights/checkpoints"),
cfg=TrainConfig(steps=50, lr=1e-4, checkpoint_every=10, grad_accum=4),
resume=False, extra_meta={"stage": "phase3-real-sources"})
print("first loss", hist[0]["loss"], "last loss", hist[-1]["loss"])""")
md("""## 7. What this does and does not establish
* It **does** train NOEON's bridges + fusion core against the real frozen
sources, through the real shared 2560-d space.
* It **does not** by itself produce a finished assistant. The `targets`/`teacher`
above are a reference; a production run needs a real task objective and data
(instruction pairs for vision, GUI action traces for computer-use, image-conditioning
pairs for Flow).
* Nothing here modifies the source weights. Verify with
`python scripts/verify_integrity.py` at the end.
## 8. Publish (only when you decide)
```python
from huggingface_hub import HfApi
api = HfApi(token=os.environ["HF_TOKEN"])
api.upload_folder(folder_path=str(ROOT / "weights/checkpoints"),
repo_id="Frankenstein-Labs/NOEON", path_in_repo="checkpoints")
```""")
(ROOT / "notebooks").mkdir(exist_ok=True)
(ROOT / "notebooks" / "NOEON_TRAIN_COLAB.ipynb").write_text(
json.dumps(nb, indent=1))
print("wrote notebook with", len(nb["cells"]), "cells")
|