code-HFD_lzk / scripts /third_party_strategy_plan.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
5.04 kB
"""Pure planning for a traceable, coverage-preserving batch migration."""
from __future__ import annotations
import hashlib
import json
import math
import random
from pathlib import Path
SEED = 26100601
def rank(sid, salt):
return hashlib.sha256(f"{SEED}:{salt}:{sid}".encode()).hexdigest()
def evaluation_ids(selection, count=60):
"""Keep the original 12 (and their seeds), add balanced fixed train cases."""
if count not in (60, 300):
raise ValueError("Evaluation requires 60 or 300 selected training sessions")
rows = selection["sessions"]
existing = list(selection["probe_session_ids"])
if len(rows) != 300 or len(set(selection["session_ids"])) != 300:
raise ValueError("Expected the frozen 300-session training set")
by_id = {r["session_id"]: r for r in rows}
if len(existing) != 12 or len(set(existing)) != 12 or not set(existing) <= by_id.keys():
raise ValueError("Invalid legacy probes")
chosen = set(existing)
keys = sorted({(r["session_type"], r["background_language"]) for r in rows})
if len(keys) != 12:
raise ValueError("Expected six types and two languages")
if count == 300:
fixed_sixty = evaluation_ids(selection, 60)
# Seeds depend on list position. Keep the full comparison panel, not
# just the legacy twelve, stable in the final all-training evaluation.
return fixed_sixty + sorted(by_id.keys() - set(fixed_sixty), key=lambda s: rank(s, "all"))
for cell, (profile, language) in enumerate(keys):
bucket = [r for r in rows if (r["session_type"], r["background_language"]) == (profile, language)]
# 14/60 high-volume cases, close to 72/300; each cell has 5 cases.
high_target = 2 if cell in (0, 7) else 1
for high, target in ((True, high_target), (False, 5 - high_target)):
group = [r for r in bucket if bool(r["background_high"]) == high]
already = sum(r["session_id"] in chosen for r in group)
candidates = sorted((r["session_id"] for r in group if r["session_id"] not in chosen),
key=lambda s: rank(s, "probe"))
if already > target or len(candidates) < target - already:
raise ValueError("Insufficient balanced training examples")
chosen.update(candidates[:target - already])
result = existing + sorted(chosen - set(existing), key=lambda s: rank(s, "probe-order"))
assert len(result) == count
return result
def remaining_plan(selection, cursor, ordered, batch_size):
if batch_size not in (2, 4, 8, 16):
raise ValueError("Audited candidate batches are 2, 4, 8, 16")
if cursor["epoch"] != 2:
raise ValueError("This migration is scoped to the remaining third epoch")
pending = list(ordered[cursor["consumed_sources_in_epoch"]:])
if not pending or len(set(pending)) != len(pending):
raise ValueError("Invalid remaining-source queue")
rows = selection["sessions"]
pending.sort(key=lambda i: (rows[i]["dense_steps"], rows[i]["session_id"]))
groups = [pending[i:i + batch_size] for i in range(0, len(pending), batch_size)]
random.Random(SEED).shuffle(groups)
lengths = [max(rows[i]["dense_steps"] for i in group) for group in groups]
return {"batch_size": batch_size, "source_indices": groups, "group_steps": lengths,
"source_count": len(pending), "remaining_updates": sum(lengths),
"source_windows": sum(rows[i]["dense_steps"] for i in pending),
"parent_step": cursor["global_step"],
"final_optimizer_step": cursor["global_step"] + sum(lengths),
"original_final_step": cursor["steps_per_epoch"] * 3,
"policy": "same remaining sources exactly once; length-grouped; partial last batch retained"}
def mapped_scheduler_step(step, parent_step, new_updates, original_final_step):
if new_updates <= 0 or original_final_step <= parent_step:
raise ValueError("Invalid continuation schedule")
fraction = min(1.0, max(0.0, (step - parent_step) / new_updates))
return parent_step + fraction * (original_final_step - parent_step)
def choose_batch(results, memory_limit_gib=100.0):
valid = [r for r in results if r.get("status") == "passed"
and 0 < r["peak_reserved_gib"] < memory_limit_gib
and math.isfinite(r["active_windows_per_second"])]
if not valid:
raise ValueError("No safe measured training batch")
fastest = max(r["active_windows_per_second"] for r in valid)
# Prefer the smaller effective batch if speed is effectively tied.
return min((r for r in valid if r["active_windows_per_second"] >= .95 * fastest),
key=lambda r: r["batch_size"])
def write_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
temporary.replace(path)