"""Pure planning for a traceable, coverage-preserving batch migration.""" from __future__ import annotations import hashlib import json import math import random from pathlib import Path SEED = 26100601 def rank(sid, salt): return hashlib.sha256(f"{SEED}:{salt}:{sid}".encode()).hexdigest() def evaluation_ids(selection, count=60): """Keep the original 12 (and their seeds), add balanced fixed train cases.""" if count not in (60, 300): raise ValueError("Evaluation requires 60 or 300 selected training sessions") rows = selection["sessions"] existing = list(selection["probe_session_ids"]) if len(rows) != 300 or len(set(selection["session_ids"])) != 300: raise ValueError("Expected the frozen 300-session training set") by_id = {r["session_id"]: r for r in rows} if len(existing) != 12 or len(set(existing)) != 12 or not set(existing) <= by_id.keys(): raise ValueError("Invalid legacy probes") chosen = set(existing) keys = sorted({(r["session_type"], r["background_language"]) for r in rows}) if len(keys) != 12: raise ValueError("Expected six types and two languages") if count == 300: fixed_sixty = evaluation_ids(selection, 60) # Seeds depend on list position. Keep the full comparison panel, not # just the legacy twelve, stable in the final all-training evaluation. return fixed_sixty + sorted(by_id.keys() - set(fixed_sixty), key=lambda s: rank(s, "all")) for cell, (profile, language) in enumerate(keys): bucket = [r for r in rows if (r["session_type"], r["background_language"]) == (profile, language)] # 14/60 high-volume cases, close to 72/300; each cell has 5 cases. high_target = 2 if cell in (0, 7) else 1 for high, target in ((True, high_target), (False, 5 - high_target)): group = [r for r in bucket if bool(r["background_high"]) == high] already = sum(r["session_id"] in chosen for r in group) candidates = sorted((r["session_id"] for r in group if r["session_id"] not in chosen), key=lambda s: rank(s, "probe")) if already > target or len(candidates) < target - already: raise ValueError("Insufficient balanced training examples") chosen.update(candidates[:target - already]) result = existing + sorted(chosen - set(existing), key=lambda s: rank(s, "probe-order")) assert len(result) == count return result def remaining_plan(selection, cursor, ordered, batch_size): if batch_size not in (2, 4, 8, 16): raise ValueError("Audited candidate batches are 2, 4, 8, 16") if cursor["epoch"] != 2: raise ValueError("This migration is scoped to the remaining third epoch") pending = list(ordered[cursor["consumed_sources_in_epoch"]:]) if not pending or len(set(pending)) != len(pending): raise ValueError("Invalid remaining-source queue") rows = selection["sessions"] pending.sort(key=lambda i: (rows[i]["dense_steps"], rows[i]["session_id"])) groups = [pending[i:i + batch_size] for i in range(0, len(pending), batch_size)] random.Random(SEED).shuffle(groups) lengths = [max(rows[i]["dense_steps"] for i in group) for group in groups] return {"batch_size": batch_size, "source_indices": groups, "group_steps": lengths, "source_count": len(pending), "remaining_updates": sum(lengths), "source_windows": sum(rows[i]["dense_steps"] for i in pending), "parent_step": cursor["global_step"], "final_optimizer_step": cursor["global_step"] + sum(lengths), "original_final_step": cursor["steps_per_epoch"] * 3, "policy": "same remaining sources exactly once; length-grouped; partial last batch retained"} def mapped_scheduler_step(step, parent_step, new_updates, original_final_step): if new_updates <= 0 or original_final_step <= parent_step: raise ValueError("Invalid continuation schedule") fraction = min(1.0, max(0.0, (step - parent_step) / new_updates)) return parent_step + fraction * (original_final_step - parent_step) def choose_batch(results, memory_limit_gib=100.0): valid = [r for r in results if r.get("status") == "passed" and 0 < r["peak_reserved_gib"] < memory_limit_gib and math.isfinite(r["active_windows_per_second"])] if not valid: raise ValueError("No safe measured training batch") fastest = max(r["active_windows_per_second"] for r in valid) # Prefer the smaller effective batch if speed is effectively tied. return min((r for r in valid if r["active_windows_per_second"] >= .95 * fastest), key=lambda r: r["batch_size"]) def write_json(path, value): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n") temporary.replace(path)