# coding=utf-8 # Copyright 2024 The Google Research Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This file contains function that coordinates the training and evaluation of the model.""" import functools import json import math import os import signal import socket import time import traceback from absl import logging from clu import metric_writers from flax import jax_utils from flax import linen as nn from flax.training import checkpoints import jax from jax import random import jax.numpy as jnp import numpy as np import tensorflow as tf import wandb from flax import serialization from train import data from train import evaluater from train import model from train import trainer def _merge_state_dict(target, saved): """Take `saved` wherever it has the key, keep `target` everywhere else.""" if not isinstance(target, dict): return saved out = {} for key, value in target.items(): if isinstance(saved, dict) and key in saved: out[key] = _merge_state_dict(value, saved[key]) else: out[key] = value return out def _new_leaf_names(target, saved, prefix=""): """Names in `target` that `saved` does not contain, for logging.""" if not isinstance(target, dict): return [] out = [] for key, value in target.items(): name = prefix + str(key) if not isinstance(saved, dict) or key not in saved: out.append(name) else: out.extend(_new_leaf_names(value, saved[key], prefix=name + "/")) return out def _restore_allowing_new_params(ckpt_loc, state): """Restore a checkpoint that predates some of the current parameters. Turning on the stochastic latents adds per-slot mean and log-sigma heads, so the parameter tree no longer matches a checkpoint saved without them and a plain restore fails on the structure mismatch. Every leaf the checkpoint does have -- including the Adam moments of the pre-existing weights and the step count -- is taken from disk; the new heads keep their fresh init, which is the identity on the mean and a constant sigma. """ raw = checkpoints.restore_checkpoint(ckpt_loc, target=None) if raw is None: return state target_sd = serialization.to_state_dict(state) added = _new_leaf_names(target_sd.get("params", {}), raw.get("params", {})) if added: print("[warm-start] not in checkpoint, using fresh init: " + ", ".join(sorted(added)), flush=True) return serialization.from_state_dict( state, _merge_state_dict(target_sd, raw)) def log_hyperparams_tb( config, model_config, initial_variables, tf_summary_writer ): """Log hyperparameters to TensorBoard. Args: config: experiment's ConfigDict model_config: model's ConfigDict initial_variables: initial hyperparameter values tf_summary_writer: SummaryWriter object. Returns: The SummaryWriter object and the config. """ # Calculate the total number of model parameters config.num_model_parameters = sum( x.size for x in jax.tree_util.tree_leaves(initial_variables) ) # Convert hyperparameters to tensors config_hyperparameters = [ tf.convert_to_tensor([k, str(v)]) for k, v in config.items() ] model_config_hyperparameters = [ tf.convert_to_tensor([k, str(v)]) for k, v in model_config.__dict__.items() ] # Log model hyperparameters to TensorBoard with tf_summary_writer.as_default(): tf.summary.text( "Model hyperparameters", tf.stack(model_config_hyperparameters), step=0 ) tf.summary.text( "Config hyperparameters", tf.stack(config_hyperparameters), step=0 ) return tf_summary_writer, config def _write_json(path, payload): """Atomic JSON write so a sidecar never reads a half-written file.""" tmp = path + ".tmp" with open(tmp, "w") as f: json.dump(payload, f, indent=2, sort_keys=True) f.write("\n") os.replace(tmp, path) def _write_heartbeat(workdir, payload): payload = dict(payload) payload.setdefault("host", socket.gethostname()) payload.setdefault("job_id", os.environ.get("SLURM_JOB_ID", "")) payload["unix_time"] = time.time() payload["iso_time"] = time.strftime("%Y-%m-%dT%H:%M:%S%z") slim = {k: payload[k] for k in ("event", "step", "stage", "loss", # val_acc = P(digit not in the stage-t candidate set) "val_acc", "puzzle_acc", # location order, tolerant to ties within a propagation wave "loc_wave", "loc_coverage", "loc_lcs", "loc_dup", "loc_acc", # superposition: excess 0 == uniform over the candidate set "val_excess", "val_mass", "val_spread", "val_excess_multi", "val_mass_multi", "val_out_multi", "val_kl_multi", "last_ckpt_step", "error") if k in payload} _write_json(os.path.join(workdir, "heartbeat.json"), slim) def _nest_bar(d, stage, base, top): """Nesting bar for step d-1 -> d, looser on the older steps. A single bar for every step is what the oscillator cannot satisfy: repairing one old latent pushes another under, so the all-steps AND never holds. Here the newest step (d == stage) carries `top` and the oldest (d == 2) carries `base`, ramping linearly between. base <= 0 or base >= top means no grading. """ if stage <= 2 or base <= 0.0 or base >= top: return top return base + (top - base) * float(d - 2) / float(stage - 2) def train_and_evaluate(config, workdir): """The training and evaluation loops for the model. Args: config: experiment's config dictionary. workdir: directory to use for logging. """ # Orbax checkpointing requires an absolute path. workdir = os.path.abspath(workdir) logging.info("Creating training and evaluator dataset iterator") curriculum = data.CurriculumState( stage=int(getattr(config, "curriculum_start_stage", 1)), max_stage=int(getattr(config, "curriculum_max_stage", 6))) train_data_iter = data.create_iter( config, config.minibatch_size, train=True, curriculum=curriculum) eval_data_iter = data.create_iter(config, config.minibatch_size, train=False) # ---- Nesting backtrack (stage replay keyed on the nest gate) ---------- # Frontier-only training optimizes the newest step and forgets the ones # that already graduated: 1->2 went 0.981 -> 0.79 within 40k steps of # entering stage 3, and 8x the data did not bring it back. So replay the # deficient depth instead of waiting. Two rules keep it from just trading # one step for another: it is a MIX (most steps stay on the frontier), and # exclusive repair only fires when a step falls hard. # # Replay needs that depth's own instances, not the frontier's: stage-3 # values are legal under the stage-2 mask but far sharper than q_2, so # training depth 2 on them would teach the over-commit we are trying to # remove. Hence a second iterator with its own curriculum object. nest_bt = bool(int(getattr(config, "nest_backtrack", 0))) nest_bt_ratio = float(getattr(config, "nest_backtrack_ratio", 0.25)) nest_bt_hard = float(getattr(config, "nest_backtrack_hard", 0.85)) nest_bt_max_excl = int(getattr(config, "nest_backtrack_max_excl", 0)) # Servo controller on the replay share, in place of the 0/1 flip. ratio is # the floor it never drops below (maintenance replay) and excl_max the # ceiling it never exceeds, so neither objective is ever fully abandoned. nest_bt_servo = bool(int(getattr(config, "nest_backtrack_servo", 0))) nest_bt_gain = float(getattr(config, "nest_backtrack_gain", 1.0)) nest_bt_excl_max = float(getattr(config, "nest_backtrack_excl_max", 1.0)) depth_mix = bool(int(getattr(config, "depth_mix", 0))) replay_curriculum, replay_data_iter, mix_ds = None, None, None if nest_bt: replay_curriculum = data.CurriculumState( stage=1, max_stage=int(config.curriculum_max_stage)) replay_data_iter = data.create_iter( config, config.minibatch_size, train=True, curriculum=replay_curriculum) if depth_mix: # Direct numpy sampler. Do not share a tf.data iterator across depths: # shuffle+prefetch mixed planes, and flipping stage rebuilt the # instance pair list (~10M rows) on every depth of every step. mix_ds = data.SudokuDataset(config, train=True, curriculum=None) print("depth-mix sampler: one SudokuDataset, per-depth instance " "override (no tf shuffle/prefetch)", flush=True) logging.info("Finished creating training dataset iterator") model_config = model.TransformerConfig( dtype=config.dtype, vocab_size=config.vocab_size, seq_len=config.seq_len, num_heads=config.num_heads, num_layers=config.num_layers, emb_dim=config.emb_dim, qkv_dim=config.qkv_dim, mlp_dim=config.mlp_dim, dropout_rate=config.dropout_rate, attention_dropout_rate=config.attention_dropout_rate, deterministic=False, num_latent_slots=int(config.num_latent_slots), inject_latents=bool(int(getattr(config, "recurrent_latent", 1))), latent_noise=bool(int(getattr(config, "latent_noise", 0))), latent_noise_logsigma0=float( getattr(config, "latent_noise_logsigma0", -4.0)), ) logging.info("train_config: %s", str(model_config.__dict__)) print(str(model_config.__dict__), flush=True) rng = jax.random.PRNGKey(config.seed) rng, init_rng, inference_rng = random.split(rng, num=3) # Initialize the model and get initial variables. Dummy latent arguments # are provided so the latent projector parameters are created at init. rng, dropout_rng = jax.random.split(rng) input_shape = (config.minibatch_size, config.seq_len) net = model.TransformerLMHeadModel(model_config) rng_keys = {"params": init_rng, "dropout": dropout_rng} K = int(config.num_latent_slots) dummy_latents = jnp.zeros( (config.minibatch_size, K, config.emb_dim), model_config.dtype) dummy_positions = jnp.zeros((config.minibatch_size, K), jnp.int32) dummy_active = jnp.zeros((config.minibatch_size, K), bool) sample_out, initial_variables = jax.jit( net.init_with_output )(rng_keys, jnp.ones(input_shape, jnp.int32), dummy_latents, dummy_positions, dummy_active) state, lr_scheduler_fn = trainer.get_state(config, net, initial_variables) # Resume-and-extend support: when resuming, start the training loop at the # restored optimizer step (not 0) so a larger config.max_steps continues the # cosine LR schedule cleanly instead of re-running steps or overshooting. start_step = 0 if config.resume_training: state = _restore_allowing_new_params(config.ckpt_loc, state) start_step = int(state.step) print("----------Restored model from", config.ckpt_loc, f"at step {start_step}-----------") writer = metric_writers.create_default_writer( workdir, asynchronous=False, just_logging=(jax.process_index() > 0)) tf_summary_writer = tf.summary.create_file_writer(workdir) logging.info("config: %s", str(config.__dict__)) state = jax_utils.replicate(state) dropout_rngs = jax.random.split(rng, jax.local_device_count()) def make_p_train_step(num_passes, inst_slot): return jax.pmap( functools.partial( trainer.train_step, config=model_config, hyperparams=config, learning_rate_fn=lr_scheduler_fn, num_passes=num_passes, inst_slot=inst_slot), axis_name="batch", donate_argnums=(0,)) def make_p_grad_step(num_passes, inst_slot): return jax.pmap( functools.partial( trainer.train_step, config=model_config, hyperparams=config, learning_rate_fn=lr_scheduler_fn, num_passes=num_passes, inst_slot=inst_slot, apply_update=False), axis_name="batch") # num_passes = latent recurrence depth for this stage, capped by the number # of latent slots (K=0 -> 0 passes -> no-latent control baseline). # When recurrence is disabled (control), force 0 passes so no hidden state # is ever fed back: the slots become static, independent per-stage readouts. recurrent = bool(int(getattr(config, "recurrent_latent", 1))) passes_per_stage = int(getattr(config, "passes_per_stage", 1)) def passes_for(stage): return min(passes_per_stage * stage, K) if recurrent else 0 p_train_step = make_p_train_step(passes_for(curriculum.stage), curriculum.stage - 1) # Replay steps run at an earlier depth, so they need their own compiled # step. Cached by depth; at most one extra variant is live at a time. replay_steps = {} def replay_step_for(t): # Same noisy train_step as the frontier: one epsilon per latent # hidden from 1..t, then readout at depth t and backprop into those # thoughts. Not a noiseless prefix. if t not in replay_steps: replay_steps[t] = make_p_train_step(passes_for(t), t - 1) return replay_steps[t] grad_steps = {} def grad_step_for(t): if t not in grad_steps: grad_steps[t] = make_p_grad_step(passes_for(t), t - 1) return grad_steps[t] p_apply = jax.pmap(trainer.apply_grads, axis_name="batch") latent_noise = bool(int(getattr(config, "latent_noise", 0))) if latent_noise: print(f"stochastic latents: {int(config.latent_mc_samples)} MC " f"samples (noise h1, pass, noise h2, ...), " f"objective={config.latent_mc_objective}, " f"sigma0={math.exp(float(config.latent_noise_logsigma0)):.4g}, " f"per-slot mean and diagonal covariance", flush=True) if depth_mix: print("depth-mix CE: each step averages grads over depths 1..stage, " "each on that depth's own instance plane", flush=True) p_eval_step = jax.pmap(functools.partial(evaluater.eval_step, config=model_config.replace(deterministic=True)), axis_name="batch") hooks, report_progress, train_metrics = trainer.get_metrics_report_progress( config, workdir, writer) tf_summary_writer, config = log_hyperparams_tb( config, model_config, initial_variables, tf_summary_writer ) # Instance-arm promotion is THREE gates, all required, no patience/plateau: # val_mass_multi >= promote_mass_threshold (support inside S) # val_spread >= promote_spread_threshold (uniform over S) # val_hbound >= promote_hbound_threshold (H(p) <= log|S| per cell) # S is the raw per-cell candidate set from the wave mask, not the filtered # instance support after constraint solving. promote_mass_threshold = float(getattr(config, "promote_acc_threshold", 0.85)) promote_spread_threshold = float( getattr(config, "promote_spread_threshold", 0.85)) promote_hbound_threshold = float( getattr(config, "promote_hbound_threshold", 0.0)) # When set, this is the ONLY value gate: CE(q||p) <= log|S| already implies # both "mass is inside S" and "no candidate was dropped", so mass*/spread*/ # hbound become diagnostics. See _inst_ce_stats in evaluater.py. # mode "mean" -> stage average must clear the bar (what training minimizes) # mode "frac" -> that fraction of individual cells must clear it (harder) promote_ceq_threshold = float(getattr(config, "promote_ceq_threshold", 0.0)) promote_ceq_mode = str(getattr(config, "promote_ceq_mode", "mean")) promote_ceq_margin = float(getattr(config, "promote_ceq_margin", 0.0)) promote_hinge_threshold = float( getattr(config, "promote_hinge_threshold", 0.0)) promote_hinge_slack = float(getattr(config, "promote_hinge_slack", 0.05)) # mode "kl" -> mean over puzzles of max(0, CE(q||p) - H(q)) <= delta. # The hinge above is satisfied by ANY p spread over S, since # CE(q||U_S) = log|S| identically, so it only certifies the support. # This one is zero only at p == q, which is what the final readout # being the unique solution actually requires. promote_kl_delta = float(getattr(config, "promote_kl_delta", 0.05)) # mode "eps" -> mean over puzzles of max(0, CE - log|S| + eps) <= slack. # The same family as "kl": CE <= log|S| - eps is KL <= headroom - eps, # where headroom = log|S| - H(q). Stated this way the tightening is # measured from the uniform-on-S ceiling rather than from q, so eps is # directly the number of nats of superposition the stage must give up. # eps must stay below the headroom or even p == q fails. promote_eps = float(getattr(config, "promote_eps", 0.0)) # Puzzle-level nesting, conjunctive with the CE/KL gate from stage 2. # 0 = off. Stage 1 has no previous latent, so the check is skipped. promote_nest_frac = float(getattr(config, "promote_nest_frac", 0.0)) promote_nest_base = float(getattr(config, "promote_nest_base", 0.0)) promote_keep_frac = float(getattr(config, "promote_keep_frac", 0.0)) # Hard ceiling on one stage, so a bar the model cannot reach cannot burn the # whole run. 0 = no ceiling (a stage then waits indefinitely). max_stage_steps = int(getattr(config, "max_stage_steps", 0)) # Applies to loc_wave (order up to wave ties), not the positional loc_acc. # 0 disables the gate. promote_loc_threshold = float(getattr(config, "promote_loc_threshold", 0.0)) promote_patience = int(getattr(config, "promote_patience_steps", 8000)) min_stage_steps = int(getattr(config, "min_stage_steps", 2000)) plateau_steps = int(getattr(config, "plateau_steps", 0)) plateau_delta = float(getattr(config, "plateau_delta", 0.005)) instance_mode = bool(getattr(config, "instance_dir", None)) per_stage_inset = {} per_stage_vexcess, per_stage_vmass, per_stage_vspread = {}, {}, {} per_stage_vexc_m, per_stage_vmass_m, per_stage_vkl_m = {}, {}, {} per_stage_vhb, per_stage_vhgap, per_stage_vhb_puz = {}, {}, {} per_stage_ceq, per_stage_hq, per_stage_ceq_ok = {}, {}, {} per_stage_ceq_gap, per_stage_ceq_margin = {}, {} per_stage_ceq_pop, per_stage_logm_pop = {}, {} per_stage_hinge, per_stage_hinge_floor = {}, {} per_stage_klq, per_stage_heps = {}, {} # Best frontier-depth accuracy seen in the current stage, and the step it # was last improved: the plateau detector's state. stage_best_acc = -1.0 stage_best_step = start_step ckpt_keep = int(getattr(config, "ckpt_keep", 100)) # Protected directory for per-stage checkpoints (never rolled off). stage_ckpt_dir = os.path.join(workdir, "stage_ckpts") stage_started_at = start_step last_ckpt_step = start_step stop_requested = {"flag": False} # Replay controller, updated at every eval: which earlier depth is below # the nest bar, and what fraction of steps to spend on it. bt_depth, bt_ratio_now, bt_steps = None, 0.0, 0 # bt_alt flips the replay between the two endpoints of the failing step; # bt_excl_steps is how long we have been exclusive, for the cap. bt_alt, bt_excl_steps = 0, 0 bt_rng = np.random.RandomState(int(config.seed) + 17) def _on_stop(signum, _frame): print(f"[signal] {signum} received; will save and exit after this step", flush=True) stop_requested["flag"] = True signal.signal(signal.SIGTERM, _on_stop) signal.signal(signal.SIGINT, _on_stop) _write_heartbeat(workdir, { "event": "start", "step": start_step, "stage": curriculum.stage, "max_steps": int(config.max_steps), }) with metric_writers.ensure_flushes(writer): for step in range(start_step, config.max_steps): if step%10000 == 0: print("Step:", step, flush=True) if depth_mix: if step == start_step: print(f"depth-mix first step {step}: sampling instance " f"planes 0..{int(curriculum.stage) - 1} " f"(python sampler, {int(config.minibatch_size)} / depth)", flush=True) state, metrics = trainer.train_depth_mix_step( grad_step_for, p_apply, config, state, step, dropout_rngs, mix_ds, curriculum.stage) if (step - start_step) < 3: print(f"depth-mix step {step} loss=" f"{float(np.asarray(metrics['loss']).reshape(-1)[0]):.4f} " f"ce={float(np.asarray(metrics['ce_loss']).reshape(-1)[0]):.4f} " f"depths=1..{curriculum.stage}", flush=True) elif bt_depth is not None and bt_rng.rand() < bt_ratio_now: state, metrics = trainer.train_one_step( replay_step_for(bt_depth), config, state, step, dropout_rngs, replay_data_iter) bt_steps += 1 else: state, metrics = trainer.train_one_step(p_train_step, config, state, step, dropout_rngs, train_data_iter) if latent_noise and step % 2000 == 0: def _first(key): return float(np.asarray(metrics[key]).reshape(-1)[0]) print(f"[noise] step {step} sigma={_first('latent_sigma'):.6f} " f"ce={_first('ce_loss'):.4f}", flush=True) for h in hooks: h(step) if math.isnan(metrics["loss"][0]): print("The loss function became nan: This might be due to the choice of hyperparameters.") _write_heartbeat(workdir, { "event": "nan_loss", "step": step, "stage": curriculum.stage, }) break if step % config.eval_every_steps == 0: try: eval_metrics = evaluater.get_eval_metrics( state, eval_data_iter, p_eval_step, config, curriculum=curriculum) except Exception: traceback.print_exc() _write_heartbeat(workdir, { "event": "eval_error", "step": step, "stage": curriculum.stage, "loss": float(metrics["loss"].mean()), "ce": float(metrics["ce_loss"].mean()), "error": traceback.format_exc()[-2000:], }) print("[eval] failed; skipping this eval and continuing", flush=True) eval_metrics = None if eval_metrics is not None: per_level = eval_metrics.pop("per_level_acc") per_slot = eval_metrics.pop("per_slot_acc", {}) per_slot_ch = eval_metrics.pop("per_slot_acc_changed", {}) per_stage_inset = eval_metrics.pop("per_stage_inset_acc", {}) per_stage_vexcess = eval_metrics.pop( "per_stage_val_excess", {}) per_stage_vmass = eval_metrics.pop("per_stage_val_mass", {}) per_stage_vspread = eval_metrics.pop( "per_stage_val_spread", {}) per_stage_vexc_m = eval_metrics.pop( "per_stage_val_excess_multi", {}) per_stage_vmass_m = eval_metrics.pop( "per_stage_val_mass_multi", {}) per_stage_vkl_m = eval_metrics.pop( "per_stage_val_kl_multi", {}) per_stage_vhb = eval_metrics.pop( "per_stage_val_hbound", {}) per_stage_vhgap = eval_metrics.pop( "per_stage_val_hgap", {}) per_stage_vhb_puz = eval_metrics.pop( "per_stage_val_hbound_puz", {}) per_stage_ceq = eval_metrics.pop("per_stage_val_ceq", {}) per_stage_hq = eval_metrics.pop("per_stage_val_hq", {}) per_stage_ceq_ok = eval_metrics.pop( "per_stage_val_ceq_ok", {}) per_stage_ceq_gap = eval_metrics.pop( "per_stage_val_ceq_gap", {}) per_stage_ceq_margin = eval_metrics.pop( "per_stage_val_ceq_margin", {}) per_stage_ceq_pop = eval_metrics.pop( "per_stage_val_ceq_pop", {}) per_stage_logm_pop = eval_metrics.pop( "per_stage_val_logm_pop", {}) per_stage_hinge_sol = eval_metrics.pop( "per_stage_val_hinge", {}) per_stage_hinge_floor_sol = eval_metrics.pop( "per_stage_val_hinge_floor", {}) # Train-matched readout (instance prefix, stage depth) is # the one the gate acts on; it is only filled for the # current stage, so fall back per stage. per_stage_hinge_inst = eval_metrics.pop( "per_stage_val_hinge_inst", {}) per_stage_hinge_floor_inst = eval_metrics.pop( "per_stage_val_hinge_floor_inst", {}) per_stage_klq_sol = eval_metrics.pop( "per_stage_val_klq", {}) per_stage_klq_inst = eval_metrics.pop( "per_stage_val_klq_inst", {}) per_stage_heps_sol = eval_metrics.pop( "per_stage_val_hinge_eps", {}) per_stage_heps_inst = eval_metrics.pop( "per_stage_val_hinge_eps_inst", {}) eval_metrics.pop("per_stage_val_ceq_pop_inst", {}) eval_metrics.pop("per_stage_val_hq_pop_inst", {}) eval_metrics.pop("per_stage_val_ceq_margin_inst", {}) def _pick(inst, sol): out = dict(sol) for s, v in inst.items(): if v is not None and v >= 0: out[s] = v return out per_stage_hinge = _pick(per_stage_hinge_inst, per_stage_hinge_sol) per_stage_hinge_floor = _pick(per_stage_hinge_floor_inst, per_stage_hinge_floor_sol) per_stage_klq = _pick(per_stage_klq_inst, per_stage_klq_sol) per_stage_heps = _pick(per_stage_heps_inst, per_stage_heps_sol) eval_metrics.pop("per_stage_val_hq_pop", {}) eval_metrics.pop("per_stage_val_ceq_puz", {}) eval_metrics.pop("per_stage_val_ce", {}) eval_metrics.pop("per_stage_val_floor", {}) nest_step = eval_metrics.pop("nest_step", {}) nest_A = eval_metrics.pop("nest_A", {}) nest_S = eval_metrics.pop("nest_S", {}) keep_depth = eval_metrics.pop("keep_depth", {}) per_bin = eval_metrics.pop("per_bin_acc", {}) def _m(key): v = eval_metrics.get(key, []) return round(float(np.mean(v)), 4) if len(v) else -1.0 def _s(d): return round(float(d.get(curriculum.stage, -1.0)), 4) # val_acc = P(digit not in stage-t candidate set), # teacher-forced so location cannot contaminate it. # 0 = all mass inside S; chance at stage 0 is ~0.59. mass = _s(per_stage_vmass) val_out = round(1.0 - mass, 4) if mass >= 0 else -1.0 mass_m = _s(per_stage_vmass_m) val_out_m = round(1.0 - mass_m, 4) if mass_m >= 0 else -1.0 # ---- Replay controller ------------------------------- # Pick the worst EARLIER step that is under the nest bar # (nest_step is keyed by the later depth, so key d is the # step d-1 -> d, and d == stage is the frontier's own # step, which frontier training already serves). if nest_bt: # Keep-first: if any active depth dropped the true # digit, exclusive-replay the FIRST such depth (later # slots cannot put it back). When every depth is 1.0, # fall through to nest repair (if that gate is on) or # the frontier. # Under the servo, trip on the gate's own keep bar # rather than on any value below 1.0: keep_d dips to # 0.99 routinely, and treating that as a hard failure # hands every such eval back to bang-bang. keep_trig = (promote_keep_frac if (nest_bt_servo and promote_keep_frac > 0.0) else 1.0) keep_behind = { int(d): float(v) for d, v in keep_depth.items() if 1 <= int(d) <= curriculum.stage and float(v) >= 0.0 and float(v) < keep_trig} if keep_behind: target = min(keep_behind) bt_ratio_now = 1.0 bt_excl_steps += int(config.eval_every_steps) bt_depth = target if replay_curriculum is not None: replay_curriculum.stage = target else: top = nest_bt_hard if nest_bt_hard > 0.0 else 0.85 bars = { d: _nest_bar(d, curriculum.stage, promote_nest_base, top) for d, v in nest_step.items() if 2 <= d < curriculum.stage and v >= 0} behind = {d: nest_step[d] for d, b in bars.items() if nest_step[d] < b} if nest_bt_servo and promote_nest_frac > 0.0 and bars: # Servo, not a flip. Replay raises the earlier # steps' nest and the hinge together; frontier # lowers both. A 0/1 controller therefore # overshoots each bar in turn -- exclusive # repair took the hinge to 0.21 while a single # frontier interval dropped 2->3 from 0.93 to # 0.48. Moving the share by the signed error # lets it settle on a share where both hold, # if one exists. nest_err = 0.0 for d, b in bars.items(): if b > 0.0: nest_err = max( nest_err, (b - nest_step[d]) / b) nest_err = min(nest_err, 1.0) hinge_ctl = _s(per_stage_hinge) floor_ctl = _s(per_stage_hinge_floor) hinge_bar_ctl = ( promote_hinge_threshold if promote_hinge_threshold > 0.0 else max(floor_ctl, 0.0) + promote_hinge_slack) hinge_err = 0.0 if hinge_ctl >= 0.0 and hinge_bar_ctl > 0.0: hinge_err = min(1.0, max( 0.0, (hinge_ctl - hinge_bar_ctl) / hinge_bar_ctl)) bt_ratio_now = float(np.clip( bt_ratio_now + nest_bt_gain * (nest_err - hinge_err), nest_bt_ratio, nest_bt_excl_max)) # Alternate endpoints of the failing step, or # hold the weakest step when nothing is behind # -- frontier-only is what let the graduated # steps decay in the first place. worst = min(bars, key=lambda d: nest_step[d] - bars[d]) if behind: bt_alt += 1 target = (worst if (bt_alt % 2 == 0) else max(1, worst - 1)) else: target = worst bt_depth = target bt_excl_steps = ( bt_excl_steps + int(config.eval_every_steps) if bt_ratio_now > 0.9 else 0) if replay_curriculum is not None: replay_curriculum.stage = target elif behind and promote_nest_frac > 0.0: worst = min(behind, key=behind.get) bt_alt += 1 target = (worst if (bt_alt % 2 == 0) else max(1, worst - 1)) bt_ratio_now = 1.0 bt_excl_steps += int(config.eval_every_steps) bt_depth = target if replay_curriculum is not None: replay_curriculum.stage = target else: bt_depth, bt_ratio_now = None, 0.0 bt_excl_steps = 0 # Short eval line. Everything else stays in the heartbeat # / tensorboard. nest_steps is the one extra: the AND # hides which latent grew. print(step, "eval", "stage", curriculum.stage, "CE", _s(per_stage_ceq_pop), "log|S|", _s(per_stage_logm_pop), "hinge", _s(per_stage_hinge), "nest", _m("nest_clean"), "nest_d", {f"{s-1}->{s}": round(float(v), 3) for s, v in nest_step.items() if v >= 0}, "keep", _m("keep_all"), "keep_d", {int(d): round(float(v), 3) for d, v in keep_depth.items() if float(v) >= 0 and 1 <= int(d) <= curriculum.stage}, # Only when replay is on: which depth is being # repaired, at what share of steps, and how many # replay steps so far. Without it a nest move is # uninterpretable. *(("bt", f"{bt_depth}@{bt_ratio_now:.2f}", "bt_steps", bt_steps) if nest_bt else ()), flush=True) _write_heartbeat(workdir, { "event": "eval", "step": int(step), "stage": int(curriculum.stage), "loss": round(float(metrics["loss"].mean()), 6), "ce": round(float(metrics["ce_loss"].mean()), 6), "val_acc": val_out_m, "puzzle_acc": _m("acc_complete_puzzle"), # Permutation-tolerant location signals; loc_acc is the # old positional match, kept only for continuity. "loc_wave": _m("loc_wave"), "loc_coverage": _m("loc_coverage"), "loc_lcs": _m("loc_lcs"), "loc_dup": _m("loc_dup"), "loc_acc": _m("loc_acc"), # Superposition: 0 excess == uniform over the stage's # candidate set; spread 1.0 == no mode collapse. "val_excess": _s(per_stage_vexcess), "val_mass": _s(per_stage_vmass), "val_spread": _s(per_stage_vspread), # |S|>=2 cells only: excess_multi == log(1/mass_multi) # + kl_multi, so it is the single number that falls only # when leakage and non-uniformity both fall. "val_excess_multi": _s(per_stage_vexc_m), "val_mass_multi": mass_m, "val_out_multi": val_out_m, "val_kl_multi": _s(per_stage_vkl_m), # Entropy ceiling H(p) <= log|S|: fraction of |S|>=2 # cells that satisfy it, the mean margin in nats, and # the stricter all-cells-in-the-puzzle view. "val_hbound": _s(per_stage_vhb), "val_hgap": _s(per_stage_vhgap), "val_hbound_puz": _s(per_stage_vhb_puz), # The promotion criterion and the two numbers that # bracket it: H(q) <= ceq should hold, ceq <= log|S| is # the gate, and ceq_gap is the signed slack. "val_ceq": _s(per_stage_ceq_pop), "val_logm": _s(per_stage_logm_pop), "val_ceq_margin": _s(per_stage_ceq_margin), "val_hq": _s(per_stage_hq), "val_ceq_ok": _s(per_stage_ceq_ok), "val_ceq_gap": _s(per_stage_ceq_gap), # Per-puzzle hinge, and the floor at p == q that bounds # how low it can go on this data. "val_hinge": _s(per_stage_hinge), "val_hinge_floor": _s(per_stage_hinge_floor), "val_klq": _s(per_stage_klq), "last_ckpt_step": int(last_ckpt_step), }) with tf_summary_writer.as_default(): tf.summary.scalar("loss", metrics["loss"].mean(), step=step) tf.summary.scalar("ce_loss", metrics["ce_loss"].mean(), step=step) tf.summary.scalar("aux_bce_loss", metrics["aux_loss"].mean(), step=step) tf.summary.scalar("leak_loss", metrics["leak_loss"].mean(), step=step) tf.summary.scalar( "learning rate", metrics["learning_rate"].mean(), step=step ) tf.summary.scalar("curriculum_stage", curriculum.stage, step=step) log_dict = {'loss': metrics["loss"].mean(), 'learning rate': metrics["learning_rate"].mean()} for key in eval_metrics.keys(): vals = eval_metrics[key] if not vals: continue tf.summary.scalar( "eval_" + key, np.array(vals).mean(), step=step ) log_dict[ "eval_" + key ] = np.array(vals).mean() for lvl, v in per_level.items(): if v >= 0: tf.summary.scalar(f"eval_acc_level_{lvl}", v, step=step) log_dict[f"eval_acc_level_{lvl}"] = v for s, v in per_slot.items(): if v >= 0: tf.summary.scalar(f"eval_cand_depth_{s}", v, step=step) log_dict[f"eval_cand_depth_{s}"] = v for s, v in per_slot_ch.items(): if v >= 0: tf.summary.scalar(f"eval_cand_depth_chg_{s}", v, step=step) if config.use_wandb: wandb.log(log_dict, step=step) # ---- Curriculum promotion ---- # Instance arm: THREE gates, all required, no plateau, no # patience. The model must actually learn the current stage's # representation before the next wave is introduced. # 1. val_mass_multi >= mass_bar: support is inside S # 2. val_spread >= spread_bar: support is uniform on S # 3. val_hbound >= hbound_bar: H(p) <= log|S| on that # fraction of cells. Checked per CELL and then counted, # because the inequality is per cell: comparing mean # H(p) to mean log|S| would let comfortable cells mask # violating ones. Catches diffuse leak that mass* is # blind to, since spreading a fixed leak over many # illegal digits costs mass the same but entropy more. # S is the raw wave-solver candidate set at that cell, NOT # the support of the filtered instances after constraints. rounds_curric = str( getattr(config, "data_curriculum", "none")) == "rounds" has_frontier = int(config.num_latent_slots) > 0 or rounds_curric if has_frontier and curriculum.stage < curriculum.max_stage: t = curriculum.stage loc_now = _m("loc_wave") mass_m = float(per_stage_vmass_m.get(t, -1.0)) spread_now = float(per_stage_vspread.get(t, -1.0)) hbound_now = float(per_stage_vhb.get(t, -1.0)) if instance_mode: frontier_acc = mass_m elif int(config.num_latent_slots) > 0: frontier_acc = per_slot_ch.get(t, -1.0) if frontier_acc < 0: frontier_acc = per_slot.get(t, -1.0) else: frontier_acc = per_bin.get(t, -1.0) if frontier_acc < 0: seen = [v for b, v in per_bin.items() if b <= t and v >= 0] frontier_acc = (float(np.mean(seen)) if seen else -1.0) steps_in_stage = step - stage_started_at measured = frontier_acc >= 0 if measured and frontier_acc > stage_best_acc + plateau_delta: stage_best_acc = frontier_acc stage_best_step = step loc_ready = (promote_loc_threshold <= 0.0 or loc_now >= promote_loc_threshold) ceq_ok_now = float(per_stage_ceq_ok.get(t, -1.0)) ceq_margin_now = float( per_stage_ceq_margin.get(t, -99.0)) hinge_now = float(per_stage_hinge.get(t, -1.0)) klq_now = float(per_stage_klq.get(t, -1.0)) heps_now = float(per_stage_heps.get(t, -1.0)) if instance_mode and promote_ceq_threshold > 0.0: # Single criterion. CE(q||p) <= log|S| is violated by # leak and by collapse alike, so the mass and spread # bars would only add mutually infeasible constraints # on top of it (see probe_entropy_ceiling.py). if promote_ceq_mode == "frac": hit_threshold = ( ceq_ok_now >= 0 and loc_ready and ceq_ok_now >= promote_ceq_threshold) elif promote_ceq_mode == "hinge": # Same comparison as "mean", but a puzzle still # above its own bar can no longer be cancelled # by puzzles that have slack to spare. The bar is # relative to the p==q floor unless an absolute # one was given, since no model can drive the # hinge below that floor on this data. floor_now = float( per_stage_hinge_floor.get(t, 0.0)) hinge_bar = ( promote_hinge_threshold if promote_hinge_threshold > 0.0 else max(floor_now, 0.0) + promote_hinge_slack) hit_threshold = ( hinge_now >= 0.0 and loc_ready and hinge_now <= hinge_bar) elif promote_ceq_mode == "kl": hit_threshold = ( klq_now >= 0.0 and loc_ready and klq_now <= promote_kl_delta) elif promote_ceq_mode == "eps": hit_threshold = ( heps_now >= 0.0 and loc_ready and heps_now <= promote_hinge_slack) else: hit_threshold = ( ceq_margin_now > -99.0 and loc_ready and ceq_margin_now >= promote_ceq_margin) # Nesting is a second required gate from stage 2: # the whole prefix 1->2->...->s must not grow the # nucleus on more than (1 - frac) of puzzles. # nest_clean is that all-steps figure; nest_min # is the worst single step. Stage 1 is skipped. if promote_nest_frac > 0.0 and t >= 2: if promote_nest_base > 0.0: # Per-step bars, loosest on the oldest # latent. nest_clean is a puzzle-level AND # over every step, so it cannot express # "the newest step must be tight and the # first one may have drifted". nest_ok = True for d in range(2, t + 1): v = float(nest_step.get(d, -1.0)) if v < 0.0: continue if v < _nest_bar(d, t, promote_nest_base, promote_nest_frac): nest_ok = False break hit_threshold = hit_threshold and nest_ok else: nest_now = _m("nest_clean") hit_threshold = ( hit_threshold and nest_now >= promote_nest_frac) # Nesting alone permits a tidy collapse onto the # WRONG grid: the sets shrink, they just stop # containing the answer. keep_all is the fraction of # puzzles whose every cell still holds the unique # solution at every depth, so requiring 1.0 means no # eval puzzle has been made unsolvable. if promote_keep_frac > 0.0: # keep_all = puzzles that hold the truth at # EVERY cell and EVERY active depth. Also # require each earlier latent on its own, so a # later slot cannot hide a depth-1 leak. keep_now = _m("keep_all") keep_d_min = keep_now for d, v in keep_depth.items(): if 1 <= int(d) <= t and float(v) >= 0.0: keep_d_min = min(keep_d_min, float(v)) hit_threshold = ( hit_threshold and keep_now >= promote_keep_frac and keep_d_min >= promote_keep_frac) stalled = False patience_over = False elif instance_mode: mass_ready = mass_m >= promote_mass_threshold spread_ready = spread_now >= promote_spread_threshold hbound_ready = (promote_hbound_threshold <= 0.0 or hbound_now >= promote_hbound_threshold) hit_threshold = (measured and loc_ready and mass_ready and spread_ready and hbound_ready) # No plateau, no patience: stay on this stage until # both superposition gates clear. stalled = False patience_over = False else: hit_threshold = (measured and loc_ready and frontier_acc >= promote_mass_threshold) stalled = (measured and loc_ready and plateau_steps > 0 and (step - stage_best_step) >= plateau_steps) patience_over = (promote_patience > 0 and steps_in_stage >= promote_patience) capped = (max_stage_steps > 0 and steps_in_stage >= max_stage_steps) if steps_in_stage >= min_stage_steps and ( hit_threshold or stalled or patience_over or capped): reason = ("threshold" if hit_threshold else "plateau" if stalled else "patience" if patience_over else "cap") curriculum.stage += 1 stage_started_at = step stage_best_acc = -1.0 stage_best_step = step p_train_step = make_p_train_step( passes_for(curriculum.stage), curriculum.stage - 1) what = ("round-bin" if int(config.num_latent_slots) == 0 else "depth") if instance_mode and promote_ceq_threshold > 0.0: extra = (f" ceq={float(per_stage_ceq_pop.get(t, -1)):.4f}" f" log|S|={float(per_stage_logm_pop.get(t, -1)):.4f}" f" margin={ceq_margin_now:+.4f}" f" ceq_ok={ceq_ok_now:.3f}" f" hinge={hinge_now:.4f}" f" hfloor=" f"{float(per_stage_hinge_floor.get(t, -1)):.4f}" f" klq={klq_now:.4f}" f" heps={heps_now:.4f}" f" eps={promote_eps:.3f}" f" nest={_m('nest_clean'):.3f}" f" nest_min={_m('nest_min'):.3f}") elif instance_mode: extra = (f" mass*={mass_m:.3f}" f" spread*={spread_now:.3f}" f" hbound={hbound_now:.3f}") else: extra = f" acc={frontier_acc:.3f}" print(f"[curriculum] step {step}: promote to stage " f"{curriculum.stage} ({reason}; graduated {what} " f"{t}{extra} after {steps_in_stage} steps); " f"latent passes={curriculum.stage}, " f"pool/snapshots now 1..{curriculum.stage}", flush=True) if config.save_checkpoint: unrep_state = jax_utils.unreplicate(state) # Rolling checkpoint in the main workdir. checkpoints.save_checkpoint_multiprocess( workdir, unrep_state, step, keep=ckpt_keep, overwrite=True) # Protected copy: the model *entering* this stage, # kept permanently under stage_ckpts/ (never rolled # off), so every stage's checkpoint survives. checkpoints.save_checkpoint_multiprocess( stage_ckpt_dir, unrep_state, step, keep=100, overwrite=True, prefix=f"stage{curriculum.stage}_") last_ckpt_step = step _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "stage", "step": int(step), "stage": int(curriculum.stage), "reason": reason, "workdir": workdir, "stage_ckpt_dir": stage_ckpt_dir, "keep_snapshot": True, }) _write_heartbeat(workdir, { "event": "promote", "step": int(step), "stage": int(curriculum.stage), "reason": reason, "graduated_acc": round(float(frontier_acc), 6), "last_ckpt_step": int(last_ckpt_step), }) if config.save_checkpoint and step > 0 and step % config.save_every_steps == 0: checkpoints.save_checkpoint_multiprocess( workdir, jax_utils.unreplicate(state), step, keep=ckpt_keep, overwrite=True ) last_ckpt_step = step _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "periodic", "step": int(step), "stage": int(curriculum.stage), "workdir": workdir, "keep_snapshot": (step % 50000 == 0), }) _write_heartbeat(workdir, { "event": "ckpt", "step": int(step), "stage": int(curriculum.stage), "last_ckpt_step": int(last_ckpt_step), }) if stop_requested["flag"]: print(f"[signal] stopping at step {step}", flush=True) if config.save_checkpoint: checkpoints.save_checkpoint_multiprocess( workdir, jax_utils.unreplicate(state), step, keep=ckpt_keep, overwrite=True) last_ckpt_step = step _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "signal", "step": int(step), "stage": int(curriculum.stage), "workdir": workdir, "keep_snapshot": True, }) _write_heartbeat(workdir, { "event": "stopped", "step": int(step), "stage": int(curriculum.stage), "last_ckpt_step": int(last_ckpt_step), }) break # Final checkpoint at the end of training if config.save_checkpoint and not stop_requested["flag"]: checkpoints.save_checkpoint_multiprocess( workdir, jax_utils.unreplicate(state), config.max_steps, keep=ckpt_keep, overwrite=True) _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "final", "step": int(config.max_steps), "stage": int(curriculum.stage), "workdir": workdir, "keep_snapshot": True, }) _write_heartbeat(workdir, { "event": "done", "step": int(last_ckpt_step), "stage": int(curriculum.stage), "last_ckpt_step": int(last_ckpt_step), })