#!/usr/bin/env python3 """Generate notebooks/NOEON_TRAIN_COLAB.ipynb (kept as a script so it is diffable).""" import json from pathlib import Path ROOT = Path(__file__).resolve().parent.parent nb = { "cells": [], "metadata": { "accelerator": "GPU", "kernelspec": {"display_name": "Python 3", "name": "python3"}, "language_info": {"name": "python"}, }, "nbformat": 4, "nbformat_minor": 5, } def md(t): nb["cells"].append({"cell_type": "markdown", "metadata": {}, "source": t.splitlines(keepends=True)}) def code(t): nb["cells"].append({"cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": t.splitlines(keepends=True)}) md("""# NOEON — training on Google Colab NOEON unifies three capabilities — **vision**, **computer use** and **generation** (image synthesis) behind trainable bridge adapters into a shared 2560-d space. The sources stay **frozen**; only NOEON's bridges + fusion core are trained. This notebook trains the bridges against the **real** frozen sources. > Runtime -> Change runtime type -> **GPU (T4 or better)**. Before you start, add a Colab secret named `HF_TOKEN` (key icon, left sidebar) with a Hugging Face token that can read the sources. Do not paste tokens into cells.""") code("""# 1. Environment !nvidia-smi | head -12 !pip -q install "torch>=2.2" transformers safetensors huggingface_hub pyyaml accelerate""") code("""# 2. Get the code. Upload NOEON-FULL-MODEL/ (or clone your repo) into /content. import os, sys, pathlib ROOT = pathlib.Path("/content/NOEON-FULL-MODEL") assert ROOT.exists(), "Upload the NOEON-FULL-MODEL folder to /content first." sys.path.insert(0, str(ROOT)) os.chdir(ROOT) print("cwd:", os.getcwd()) !ls""") code("""# 3. Hugging Face token from Colab secrets (never paste it in a cell) from google.colab import userdata os.environ["HF_TOKEN"] = userdata.get("HF_TOKEN") print("HF_TOKEN set:", bool(os.environ.get("HF_TOKEN")))""") code("""# 4. Confirm the environment can train (GPU + resources) !python scripts/attach_sources.py | tail -20""") md("""## 5. Attach the frozen sources This downloads ~10 GB for the vision slot alone. Run once; it caches in `~/.cache/huggingface`.""") code("""import torch from noeon.encoders.hf_source import HFSourceEncoder from noeon.architecture.model import Noeon from noeon.core.config import NoeonConfig device = "cuda" if torch.cuda.is_available() else "cpu" model = Noeon(NoeonConfig()).to(device) print("trainable params:", f"{model.trainable_parameters():,}") # Load the frozen sources (eval mode, requires_grad=False -> never modified). vl = HFSourceEncoder("vision", device=device).load() cowork = HFSourceEncoder("computer_use", device=device).load() print("sources attached. (generation is 8.7B; load it only with >=16 GB VRAM)")""") md("""## 6. The training loop `feature_fn` is the one task-specific piece. It returns `(streams, targets, teacher)`: * `streams` - `{source_key: (B, T, 2560) hidden states}` from the frozen encoders * `targets` - `(B, 2560)` the generation head should produce * `teacher` - `(B, 2560)` frozen-teacher signal for distillation (or `None`) Below is a **reference** implementation for a text/caption task. Adapt the prompts to your data; the mechanism is identical.""") code("""from noeon.training.trainer import TrainConfig, train import torch PROMPTS = [ "Describe the interface shown in this screenshot.", "What action should be taken next?", "Generate an image of a red fox in snow.", ] * 8 def feature_fn(): batch = PROMPTS streams = {} with torch.no_grad(): for key, enc in (("vision", vl), ("computer_use", cowork)): inputs = enc.processor(text=batch, return_tensors="pt", padding=True) inputs = {k: v.to(device) for k, v in inputs.items() if k in ("input_ids", "attention_mask")} streams[key] = enc.encode(**inputs).float() pooled = streams["vision"].mean(dim=1) targets = torch.zeros_like(pooled) teacher = torch.nn.functional.normalize(pooled, dim=-1) return streams, targets, teacher hist = train(model, feature_fn, str(ROOT / "weights/checkpoints"), cfg=TrainConfig(steps=50, lr=1e-4, checkpoint_every=10, grad_accum=4), resume=False, extra_meta={"stage": "phase3-real-sources"}) print("first loss", hist[0]["loss"], "last loss", hist[-1]["loss"])""") md("""## 7. What this does and does not establish * It **does** train NOEON's bridges + fusion core against the real frozen sources, through the real shared 2560-d space. * It **does not** by itself produce a finished assistant. The `targets`/`teacher` above are a reference; a production run needs a real task objective and data (instruction pairs for vision, GUI action traces for computer-use, image-conditioning pairs for Flow). * Nothing here modifies the source weights. Verify with `python scripts/verify_integrity.py` at the end. ## 8. Publish (only when you decide) ```python from huggingface_hub import HfApi api = HfApi(token=os.environ["HF_TOKEN"]) api.upload_folder(folder_path=str(ROOT / "weights/checkpoints"), repo_id="Frankenstein-Labs/NOEON", path_in_repo="checkpoints") ```""") (ROOT / "notebooks").mkdir(exist_ok=True) (ROOT / "notebooks" / "NOEON_TRAIN_COLAB.ipynb").write_text( json.dumps(nb, indent=1)) print("wrote notebook with", len(nb["cells"]), "cells")