autoregressive

backend.py

@@ -79,25 +79,35 @@ N_CHUNKS = 24            # fixed sequence length (measures) -- matches training
 CONTINUE_MEASURES = 8
 
 # Noise level the already-written measures are held at while the new ones are
-# sampled. 0.0 hands the past over clean, which is both the Diffusion Forcing
-# reference implementation's shipped default (its scheduling matrix pads history
-# with zeros; the `stabilization` variable exists but is set to 0) and the right
-# default here.
+# sampled.
 #
-# The DF paper only says a small nonzero level 0<k<<K can stabilize rollout, with
-# no number. The concrete win reported elsewhere -- GameNGen's noise-augmented
-# context, max level 0.7 over 10 buckets -- is a TRAINING-time intervention, and
-# that paper notes results are already much improved at inference with no added
-# noise. This model gets that training for free: FM.loss draws an independent
-# t ~ sigmoid(randn) per measure, so it has already seen every measure at every
-# noise level and is calibrated for a past it should not fully trust.
+# NOT 0.0, even though clean history is what the Diffusion Forcing reference
+# implementation ships. That implementation trains on DISCRETE noise indices
+# drawn uniformly over [0, K), so index 0 -- perfectly clean -- is an ordinary
+# training input for it. This model does not: FM.loss draws
 #
-# So raise this only if repeated Continues visibly drift or harden into a loop.
-# Useful range is 0.1-0.2, NOT lower: under sigmoid(randn) only 0.005% of training
-# tokens landed below t=0.02 (vs 1.4% below 0.1, 8.3% below 0.2), so very small
-# levels are the undertrained tail rather than a gentler version of the same
-# thing. Above ~0.3 the context starts genuinely washing out.
-CONTINUE_CTX_NOISE = 0.0
+#     t = torch.sigmoid(torch.randn(B, T))
+#
+# a continuous logit-normal, which has ZERO density at 0 and never gets near it.
+# Over a 300k-step run (grad_accum 4 x batch 32 x 24 measures ~= 9.2e8 draws) the
+# smallest t the model ever saw is about 0.0025, and t < 0.001 never occurred at
+# all. t=0.0 is therefore off the end of the training support, and t is not a
+# passive tag: it feeds the per-measure AdaLN modulation (`t = t + style + bpm`),
+# so an out-of-distribution value on the 16 context measures corrupts their
+# features and, through attention, the free measures with them.
+#
+# 0.05 is the smallest value that is comfortably supported -- roughly 1.5e6
+# training draws landed below it -- while leaving the context essentially intact
+# (alpha=0.95, sigma=0.05). Draws below each level over that run:
+#
+#     t < 0.001              0        t < 0.05      1,490,876
+#     t < 0.01           1,993        t < 0.10     12,904,433
+#
+# Raising it toward 0.1-0.2 is also well supported and is the knob to reach for
+# if repeated Continues drift or harden into a loop (Diffusion Forcing's
+# stabilization result: a model that half-distrusts its own past compounds error
+# more slowly). Above ~0.3 the context genuinely washes out.
+CONTINUE_CTX_NOISE = 0.05
 
 # Vanilla history guidance (HG-v) weight for Continue. CFG steers toward the
 # conditioning signals; this steers toward THE PAST:

real_backend.py

@@ -295,10 +295,27 @@ def fm_df_sample(
                 v_none = _predict(x_none, t_none)
                 v = v_none + omega * (v - v_none)
 
-            x_t = x_t + v * (t_next - t_cur)
+            # (t_cur - t_next), NOT (t_next - t_cur). The net predicts -dx_t/dt
+            # (FM.loss's target is A(t)x + B(t)noise == x - (1-sigma_min)noise,
+            # commented "-dxt/dt"), and t runs 1 -> 0, so the descending step
+            # dt = t_next - t_cur is negative and the two negatives cancel:
+            #     dx = (dx_t/dt) * dt = (-v) * (t_next - t_cur) = v * (t_cur - t_next)
+            # This is exactly what FMEulerSampler.sample does. Getting it
+            # backwards integrates the flow the wrong way and returns noise.
+            x_t = x_t + v * (t_cur - t_next)
             # the Euler step moved the context too; put it back untouched.
             x_t = torch.where(keep, ctx_x, x_t)
-    return x_t
+
+    # Hand back the CLEAN context, not ctx_x. ctx_x is an input the model reads;
+    # it is not an answer. The caller stores this whole tensor as tr.latents and
+    # the next continuation slices its context out of it, so returning the noised
+    # copy would re-noise the same measures on every press: after k continues the
+    # oldest surviving measure has been scaled by alpha(ctx_noise)^k with k
+    # independent noise draws stacked on it (0.95^k at the 0.05 default -- 0.77
+    # by the fifth press) and has drifted well off the latent manifold. That
+    # decays into mush no matter how good the sampler is. It also means the
+    # re-decoded context measures are the real ones rather than degraded ones.
+    return torch.where(keep, known_x0, x_t)
 
 
 def _check_stage(m, stage, path, model_args):