Instructions to use XXXpilar/Fastfill-test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use XXXpilar/Fastfill-test with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
Download code/fastfill/v2/tests/test_accumulation_tail.py from XXXpilar/Fastfill-test: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/XXXpilar/Fastfill-test/resolve/main/code/fastfill/v2/tests/test_accumulation_tail.py
- Command line
-
hf download hf://XXXpilar/Fastfill-test/code/fastfill/v2/tests/test_accumulation_tail.py
-
curl -L -o test_accumulation_tail.py https://huggingface.co/XXXpilar/Fastfill-test/resolve/main/code/fastfill/v2/tests/test_accumulation_tail.py
10.5 kB
| """Actual Accelerator training: every flushed window averages its microbatches.""" | |
| from pathlib import Path | |
| from datetime import timedelta | |
| import json | |
| import os | |
| import sys | |
| import pytest | |
| import torch | |
| from fastfill.v2 import train | |
| from fastfill.v2.tests.test_active_objective_training import _data, _row, _yaw_config | |
| class _ScalarModel(torch.nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.yaw_logits_head = torch.nn.Parameter(torch.zeros(())) | |
| def forward(self, **kwargs): | |
| return self.yaw_logits_head | |
| def save_pretrained(self, directory): | |
| directory = Path(directory) | |
| directory.mkdir(parents=True) | |
| torch.save(self.state_dict(), directory / "scalar.pt") | |
| class _ScalarCriterion: | |
| def __init__(self, coefficients): | |
| self.coefficients = coefficients | |
| def __call__(self, prediction, batch): | |
| values = [self.coefficients[int(row["scene_id"])] for row in batch["provenance"]] | |
| coefficient = sum(values) / len(values) | |
| loss = prediction * coefficient | |
| zero = prediction * 0. | |
| active = sum(value != 0. for value in values) | |
| return {"loss": loss, "yaw_cls": loss, "counts": {"yaw": active}, | |
| "active_objective_count_local": active, | |
| "term_sums": {**dict.fromkeys(train.TERMS, zero.detach()), "yaw_cls": loss.detach()}, | |
| "term_counts": {**dict.fromkeys(train.TERMS, 0), "yaw_cls": active}, | |
| **{key: zero for key in ("position", "size", "yaw_reg", "box", "collision", "boundary")}} | |
| def _patch_scalar_training(patches, coefficients, observed, *, scaled=False): | |
| # Keep actual preflight, Accelerator.prepare/backward, accumulation, clipping, | |
| # logging and optimizer dispatch. Replace only model/objective with a known | |
| # constant derivative and use lr=0 to observe gradients without changing it. | |
| original_loader = train.DataLoader | |
| patches.setattr(train, "DataLoader", lambda rows, **kw: | |
| original_loader(rows, **{**kw, "shuffle": False})) | |
| patches.setattr(train, "build_model", lambda config: _ScalarModel()) | |
| patches.setattr(train, "GeometryCriterion", lambda config: _ScalarCriterion(coefficients)) | |
| class RecordingSGD(torch.optim.SGD): | |
| def step(self, closure=None): | |
| observed.append(float(self.param_groups[0]["params"][0].grad.detach())) | |
| return super().step(closure) | |
| patches.setattr(train.torch.optim, "AdamW", lambda params, **kw: RecordingSGD(list(params), lr=0.)) | |
| if scaled: | |
| import accelerate | |
| original_accelerator = accelerate.Accelerator | |
| class CpuScaledAccelerator(original_accelerator): | |
| def __init__(self, **kwargs): | |
| super().__init__(**kwargs) | |
| # Real torch AMP scaler on CPU exercises scaled backward, | |
| # Accelerate unscale/clip and GradScaler optimizer dispatch. | |
| self.scaler = torch.amp.GradScaler("cpu", init_scale=16.) | |
| def unscale_gradients(self, optimizer=None): | |
| # Accelerate enables fp16 unscale on accelerator devices only; | |
| # adapt that device gate to the real CPU GradScaler test. | |
| optimizers = self._optimizers if optimizer is None else [optimizer] | |
| for prepared in optimizers: | |
| self.scaler.unscale_(prepared.optimizer) | |
| patches.setattr(accelerate, "Accelerator", CpuScaledAccelerator) | |
| def _run_scalar(tmp_path, monkeypatch, coefficients, *, accumulation=2, clip=100., scaled=False, steps=None): | |
| observed = [] | |
| _patch_scalar_training(monkeypatch, coefficients, observed, scaled=scaled) | |
| config = _yaw_config() | |
| updates = sum(any(coefficients[i:i + accumulation]) | |
| for i in range(0, len(coefficients), accumulation)) | |
| config["training"] = {**config["training"], "gradient_accumulation_steps": accumulation, | |
| "steps": updates if steps is None else steps, "clip_grad_norm": clip} | |
| rows = [_row(str(index)) for index in range(len(coefficients))] | |
| logs = train.run_training(config, _data(tmp_path, rows), tmp_path / "run") | |
| manifest = json.loads((tmp_path / "run/run_manifest.json").read_text()) | |
| return observed, logs, manifest | |
| def test_actual_run_training_averages_complete_and_tail_windows(tmp_path, monkeypatch, count, accumulation, expected): | |
| observed, logs, manifest = _run_scalar(tmp_path, monkeypatch, [1.] * count, accumulation=accumulation) | |
| assert observed == pytest.approx(expected) | |
| assert len(logs) == manifest["steps_completed"] == len(expected) | |
| def test_actual_tail_counts_empty_label_microbatches_in_declared_mean(tmp_path, monkeypatch): | |
| observed, logs, _ = _run_scalar(tmp_path, monkeypatch, [2., 0., 2., 2., 0.], accumulation=3) | |
| # Both windows count every microbatch, including those with no labels. | |
| assert observed == pytest.approx([4. / 3., 1.]) | |
| assert [record["active_objective_count"] for record in logs] == [2, 1] | |
| def test_actual_tail_retains_prior_supervision_and_counts_empty_last_batch(tmp_path, monkeypatch): | |
| observed, logs, _ = _run_scalar(tmp_path, monkeypatch, [1., 1., 1., 2., 0.], accumulation=3) | |
| assert observed == pytest.approx([1., 1.]) | |
| assert [record["active_objective_count"] for record in logs] == [3, 1] | |
| def test_actual_empty_complete_window_does_not_reweight_later_tail(tmp_path, monkeypatch): | |
| observed, logs, manifest = _run_scalar(tmp_path, monkeypatch, [0., 0., 1.]) | |
| assert observed == pytest.approx([1.]) | |
| assert logs[0]["active_objective_count"] == 1 | |
| assert manifest["skipped_no_objective_windows"] == 1 | |
| def test_actual_tail_rescale_precedes_clipping_and_amp_unscale(tmp_path, monkeypatch, scaled): | |
| observed, _, _ = _run_scalar(tmp_path, monkeypatch, [.8, .8, .8], clip=.75, scaled=scaled) | |
| assert observed == pytest.approx([.75, .75], rel=2e-6) | |
| def test_amp_overflow_does_not_count_as_completed_optimizer_step(tmp_path, monkeypatch): | |
| # The first finite scalar loss has an overflowing *scaled* derivative, so | |
| # GradScaler skips its optimizer update. The following window is finite and | |
| # must become completed step 1; an attempted overflow is not a training step. | |
| observed, logs, manifest = _run_scalar( | |
| tmp_path, monkeypatch, [1e38, 1.], accumulation=1, scaled=True, steps=1) | |
| assert observed == pytest.approx([1.]) | |
| assert [record["step"] for record in logs] == [1] | |
| assert manifest["steps_completed"] == 1 | |
| assert manifest["skipped_gradient_overflow_windows"] == 1 | |
| def test_actual_partial_window_is_flushed_each_epoch(tmp_path, monkeypatch): | |
| observed, logs, _ = _run_scalar(tmp_path, monkeypatch, [1., 1., 3.], steps=4) | |
| # Each epoch ends with its own one-microbatch mean. No tail is combined | |
| # with the next epoch or used to reweight the previous complete window. | |
| assert observed == pytest.approx([1., 3., 1., 3.]) | |
| assert [record["step"] for record in logs] == [1, 2, 3, 4] | |
| def _distributed_tail_worker(rank, rendezvous, directory, mode): | |
| from torch import distributed as dist | |
| os.environ.update(RANK=str(rank), WORLD_SIZE="2", LOCAL_RANK=str(rank), LOCAL_WORLD_SIZE="2", | |
| OMP_NUM_THREADS="1", MASTER_ADDR="127.0.0.1", MASTER_PORT="29514") | |
| dist.init_process_group("gloo", init_method=rendezvous, rank=rank, world_size=2, | |
| timeout=timedelta(seconds=20)) | |
| patches = pytest.MonkeyPatch() | |
| try: | |
| if sys.platform == "darwin": | |
| patches.setattr(torch._C, "_get_accelerator", lambda: torch.device("cpu")) | |
| root = Path(directory) | |
| coefficients = {"all_valid": [1.] * 6, "peer_empty": [0., 2.] * 3, | |
| "tail_one_rank_empty": [1., 1., 1., 1., 2., 0.], | |
| "empty_tail": [1., 1., 1., 1., 0., 0.]}[mode] | |
| observed = [] | |
| _patch_scalar_training(patches, coefficients, observed) | |
| if rank == 0: | |
| _data(root, [_row(str(index)) for index in range(6)]) | |
| dist.barrier() | |
| config = _yaw_config() | |
| config["training"] = {**config["training"], "gradient_accumulation_steps": 2, | |
| "steps": 2, "clip_grad_norm": 100.} | |
| logs = train.run_training(config, root / "train.jsonl", root / "run") | |
| # Real prepared loader has three microbatches per rank: one complete | |
| # K=2 window and one single-batch tail. DDP averages rank derivatives. | |
| assert observed == pytest.approx([1., 1.]) | |
| assert [record["accumulation_microbatches"] for record in logs] == ( | |
| [2, 2] if mode == "empty_tail" else [2, 1]) | |
| expected_counts = {"all_valid": [4, 2], "peer_empty": [2, 1], | |
| "tail_one_rank_empty": [4, 1], "empty_tail": [4, 4]}[mode] | |
| assert [record["active_objective_count"] for record in logs] == expected_counts | |
| if rank == 0: | |
| manifest = json.loads((root / "run/run_manifest.json").read_text()) | |
| assert manifest["world_size"] == 2 | |
| assert manifest["skipped_no_objective_windows"] == int(mode == "empty_tail") | |
| (root / f"rank-{rank}.json").write_text(json.dumps({ | |
| "mode": mode, "optimizer_gradients": observed, | |
| "microbatches": [record["accumulation_microbatches"] for record in logs], | |
| "global_objective_counts": expected_counts}, indent=2) + "\n") | |
| finally: | |
| patches.undo() | |
| if dist.is_initialized(): | |
| dist.destroy_process_group() | |
| def test_actual_two_rank_complete_and_tail_means(tmp_path, monkeypatch, mode): | |
| monkeypatch.setenv("GLOO_SOCKET_IFNAME", "lo0" if sys.platform == "darwin" else "lo") | |
| monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "") # CPU Gloo ranks: with one visible GPU, rank 1 would ask for cuda:1 | |
| torch.multiprocessing.spawn(_distributed_tail_worker, | |
| args=("file://" + str(tmp_path / "rendezvous"), str(tmp_path), mode), nprocs=2, join=True) | |
| assert all((tmp_path / f"rank-{rank}.json").exists() for rank in range(2)) | |