"""Optional Inductor training callable; the original model owns all checkpoint weights.""" from __future__ import annotations import os from pathlib import Path import torch from ..tokenizers.multimodal import Sample def real_warmup_samples(sources, lengths): """Select actual examples across every cache, preserving each row's backing file.""" selected={length:None for length in lengths} for name,source in sources.items(): for task in source.available(): for row in source.rows[task]: for length in lengths: old=selected[length] if row['length']-1<=length and (old is None or row['length']>=old[2]['length']): selected[length]=(name,source,row) out={} for length,pick in selected.items(): if pick is None:raise RuntimeError(f'No real training example fits compile bucket {length}') name,source,row=pick out[length]=(name,Sample(source.tokens[row['offset']:row['offset']+row['length']],row['target_start'])) return out def compile_loss(model, cfg: dict): if not cfg.get("compile", False): return model.loss backend = cfg.get("compile_backend", "inductor") options = dict(cfg.get("compile_options", {})) if backend == "inductor" and os.name == "nt": # Process-local, short cache/kernel names avoid Windows MAX_PATH errors. os.environ.setdefault("TORCHINDUCTOR_CACHE_DIR", str(Path("outputs/ti").resolve())) options.setdefault("triton.descriptive_names", False) kwargs = {"backend": backend, "dynamic": cfg.get("compile_dynamic", False), "fullgraph": cfg.get("compile_fullgraph", False)} mode = cfg.get("compile_mode", "default") if mode != "default": if options: raise ValueError("choose either a compile mode or compile options, not both") kwargs["mode"] = mode if options: kwargs["options"] = options return torch.compile(model.loss, **kwargs)