Fastfill-test / code /fastfill /v2 /tests /test_accumulation_tail.py
XXXpilar's picture
FastFill v4 base (step 6552): LoRA adapter, geometry heads, tokenizer, training code, predict.py
e6f80f3 verified
Raw History Blame Contribute Delete
10.5 kB
"""Actual Accelerator training: every flushed window averages its microbatches."""
from pathlib import Path
from datetime import timedelta
import json
import os
import sys
import pytest
import torch
from fastfill.v2 import train
from fastfill.v2.tests.test_active_objective_training import _data, _row, _yaw_config
class _ScalarModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.yaw_logits_head = torch.nn.Parameter(torch.zeros(()))
def forward(self, **kwargs):
return self.yaw_logits_head
def save_pretrained(self, directory):
directory = Path(directory)
directory.mkdir(parents=True)
torch.save(self.state_dict(), directory / "scalar.pt")
class _ScalarCriterion:
def __init__(self, coefficients):
self.coefficients = coefficients
def __call__(self, prediction, batch):
values = [self.coefficients[int(row["scene_id"])] for row in batch["provenance"]]
coefficient = sum(values) / len(values)
loss = prediction * coefficient
zero = prediction * 0.
active = sum(value != 0. for value in values)
return {"loss": loss, "yaw_cls": loss, "counts": {"yaw": active},
"active_objective_count_local": active,
"term_sums": {**dict.fromkeys(train.TERMS, zero.detach()), "yaw_cls": loss.detach()},
"term_counts": {**dict.fromkeys(train.TERMS, 0), "yaw_cls": active},
**{key: zero for key in ("position", "size", "yaw_reg", "box", "collision", "boundary")}}
def _patch_scalar_training(patches, coefficients, observed, *, scaled=False):
# Keep actual preflight, Accelerator.prepare/backward, accumulation, clipping,
# logging and optimizer dispatch. Replace only model/objective with a known
# constant derivative and use lr=0 to observe gradients without changing it.
original_loader = train.DataLoader
patches.setattr(train, "DataLoader", lambda rows, **kw:
original_loader(rows, **{**kw, "shuffle": False}))
patches.setattr(train, "build_model", lambda config: _ScalarModel())
patches.setattr(train, "GeometryCriterion", lambda config: _ScalarCriterion(coefficients))
class RecordingSGD(torch.optim.SGD):
def step(self, closure=None):
observed.append(float(self.param_groups[0]["params"][0].grad.detach()))
return super().step(closure)
patches.setattr(train.torch.optim, "AdamW", lambda params, **kw: RecordingSGD(list(params), lr=0.))
if scaled:
import accelerate
original_accelerator = accelerate.Accelerator
class CpuScaledAccelerator(original_accelerator):
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Real torch AMP scaler on CPU exercises scaled backward,
# Accelerate unscale/clip and GradScaler optimizer dispatch.
self.scaler = torch.amp.GradScaler("cpu", init_scale=16.)
def unscale_gradients(self, optimizer=None):
# Accelerate enables fp16 unscale on accelerator devices only;
# adapt that device gate to the real CPU GradScaler test.
optimizers = self._optimizers if optimizer is None else [optimizer]
for prepared in optimizers:
self.scaler.unscale_(prepared.optimizer)
patches.setattr(accelerate, "Accelerator", CpuScaledAccelerator)
def _run_scalar(tmp_path, monkeypatch, coefficients, *, accumulation=2, clip=100., scaled=False, steps=None):
observed = []
_patch_scalar_training(monkeypatch, coefficients, observed, scaled=scaled)
config = _yaw_config()
updates = sum(any(coefficients[i:i + accumulation])
for i in range(0, len(coefficients), accumulation))
config["training"] = {**config["training"], "gradient_accumulation_steps": accumulation,
"steps": updates if steps is None else steps, "clip_grad_norm": clip}
rows = [_row(str(index)) for index in range(len(coefficients))]
logs = train.run_training(config, _data(tmp_path, rows), tmp_path / "run")
manifest = json.loads((tmp_path / "run/run_manifest.json").read_text())
return observed, logs, manifest
@pytest.mark.parametrize("count,accumulation,expected", [
(3, 2, [1., 1.]), (4, 2, [1., 1.]), (3, 1, [1., 1., 1.]),
(1, 3, [1.]), (5, 3, [1., 1.]),
])
def test_actual_run_training_averages_complete_and_tail_windows(tmp_path, monkeypatch, count, accumulation, expected):
observed, logs, manifest = _run_scalar(tmp_path, monkeypatch, [1.] * count, accumulation=accumulation)
assert observed == pytest.approx(expected)
assert len(logs) == manifest["steps_completed"] == len(expected)
def test_actual_tail_counts_empty_label_microbatches_in_declared_mean(tmp_path, monkeypatch):
observed, logs, _ = _run_scalar(tmp_path, monkeypatch, [2., 0., 2., 2., 0.], accumulation=3)
# Both windows count every microbatch, including those with no labels.
assert observed == pytest.approx([4. / 3., 1.])
assert [record["active_objective_count"] for record in logs] == [2, 1]
def test_actual_tail_retains_prior_supervision_and_counts_empty_last_batch(tmp_path, monkeypatch):
observed, logs, _ = _run_scalar(tmp_path, monkeypatch, [1., 1., 1., 2., 0.], accumulation=3)
assert observed == pytest.approx([1., 1.])
assert [record["active_objective_count"] for record in logs] == [3, 1]
def test_actual_empty_complete_window_does_not_reweight_later_tail(tmp_path, monkeypatch):
observed, logs, manifest = _run_scalar(tmp_path, monkeypatch, [0., 0., 1.])
assert observed == pytest.approx([1.])
assert logs[0]["active_objective_count"] == 1
assert manifest["skipped_no_objective_windows"] == 1
@pytest.mark.parametrize("scaled", [False, True])
def test_actual_tail_rescale_precedes_clipping_and_amp_unscale(tmp_path, monkeypatch, scaled):
observed, _, _ = _run_scalar(tmp_path, monkeypatch, [.8, .8, .8], clip=.75, scaled=scaled)
assert observed == pytest.approx([.75, .75], rel=2e-6)
def test_amp_overflow_does_not_count_as_completed_optimizer_step(tmp_path, monkeypatch):
# The first finite scalar loss has an overflowing *scaled* derivative, so
# GradScaler skips its optimizer update. The following window is finite and
# must become completed step 1; an attempted overflow is not a training step.
observed, logs, manifest = _run_scalar(
tmp_path, monkeypatch, [1e38, 1.], accumulation=1, scaled=True, steps=1)
assert observed == pytest.approx([1.])
assert [record["step"] for record in logs] == [1]
assert manifest["steps_completed"] == 1
assert manifest["skipped_gradient_overflow_windows"] == 1
def test_actual_partial_window_is_flushed_each_epoch(tmp_path, monkeypatch):
observed, logs, _ = _run_scalar(tmp_path, monkeypatch, [1., 1., 3.], steps=4)
# Each epoch ends with its own one-microbatch mean. No tail is combined
# with the next epoch or used to reweight the previous complete window.
assert observed == pytest.approx([1., 3., 1., 3.])
assert [record["step"] for record in logs] == [1, 2, 3, 4]
def _distributed_tail_worker(rank, rendezvous, directory, mode):
from torch import distributed as dist
os.environ.update(RANK=str(rank), WORLD_SIZE="2", LOCAL_RANK=str(rank), LOCAL_WORLD_SIZE="2",
OMP_NUM_THREADS="1", MASTER_ADDR="127.0.0.1", MASTER_PORT="29514")
dist.init_process_group("gloo", init_method=rendezvous, rank=rank, world_size=2,
timeout=timedelta(seconds=20))
patches = pytest.MonkeyPatch()
try:
if sys.platform == "darwin":
patches.setattr(torch._C, "_get_accelerator", lambda: torch.device("cpu"))
root = Path(directory)
coefficients = {"all_valid": [1.] * 6, "peer_empty": [0., 2.] * 3,
"tail_one_rank_empty": [1., 1., 1., 1., 2., 0.],
"empty_tail": [1., 1., 1., 1., 0., 0.]}[mode]
observed = []
_patch_scalar_training(patches, coefficients, observed)
if rank == 0:
_data(root, [_row(str(index)) for index in range(6)])
dist.barrier()
config = _yaw_config()
config["training"] = {**config["training"], "gradient_accumulation_steps": 2,
"steps": 2, "clip_grad_norm": 100.}
logs = train.run_training(config, root / "train.jsonl", root / "run")
# Real prepared loader has three microbatches per rank: one complete
# K=2 window and one single-batch tail. DDP averages rank derivatives.
assert observed == pytest.approx([1., 1.])
assert [record["accumulation_microbatches"] for record in logs] == (
[2, 2] if mode == "empty_tail" else [2, 1])
expected_counts = {"all_valid": [4, 2], "peer_empty": [2, 1],
"tail_one_rank_empty": [4, 1], "empty_tail": [4, 4]}[mode]
assert [record["active_objective_count"] for record in logs] == expected_counts
if rank == 0:
manifest = json.loads((root / "run/run_manifest.json").read_text())
assert manifest["world_size"] == 2
assert manifest["skipped_no_objective_windows"] == int(mode == "empty_tail")
(root / f"rank-{rank}.json").write_text(json.dumps({
"mode": mode, "optimizer_gradients": observed,
"microbatches": [record["accumulation_microbatches"] for record in logs],
"global_objective_counts": expected_counts}, indent=2) + "\n")
finally:
patches.undo()
if dist.is_initialized():
dist.destroy_process_group()
@pytest.mark.skipif(not torch.distributed.is_available() or not torch.distributed.is_gloo_available(),
reason="requires Gloo")
@pytest.mark.parametrize("mode", ["all_valid", "peer_empty", "tail_one_rank_empty", "empty_tail"])
def test_actual_two_rank_complete_and_tail_means(tmp_path, monkeypatch, mode):
monkeypatch.setenv("GLOO_SOCKET_IFNAME", "lo0" if sys.platform == "darwin" else "lo")
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "") # CPU Gloo ranks: with one visible GPU, rank 1 would ask for cuda:1
torch.multiprocessing.spawn(_distributed_tail_worker,
args=("file://" + str(tmp_path / "rendezvous"), str(tmp_path), mode), nprocs=2, join=True)
assert all((tmp_path / f"rank-{rank}.json").exists() for rank in range(2))