Download scripts/third_party_strategy_plan.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 5.04 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_strategy_plan.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/third_party_strategy_plan.py
-
curl -L -o third_party_strategy_plan.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_strategy_plan.py
5.04 kB
| """Pure planning for a traceable, coverage-preserving batch migration.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import math | |
| import random | |
| from pathlib import Path | |
| SEED = 26100601 | |
| def rank(sid, salt): | |
| return hashlib.sha256(f"{SEED}:{salt}:{sid}".encode()).hexdigest() | |
| def evaluation_ids(selection, count=60): | |
| """Keep the original 12 (and their seeds), add balanced fixed train cases.""" | |
| if count not in (60, 300): | |
| raise ValueError("Evaluation requires 60 or 300 selected training sessions") | |
| rows = selection["sessions"] | |
| existing = list(selection["probe_session_ids"]) | |
| if len(rows) != 300 or len(set(selection["session_ids"])) != 300: | |
| raise ValueError("Expected the frozen 300-session training set") | |
| by_id = {r["session_id"]: r for r in rows} | |
| if len(existing) != 12 or len(set(existing)) != 12 or not set(existing) <= by_id.keys(): | |
| raise ValueError("Invalid legacy probes") | |
| chosen = set(existing) | |
| keys = sorted({(r["session_type"], r["background_language"]) for r in rows}) | |
| if len(keys) != 12: | |
| raise ValueError("Expected six types and two languages") | |
| if count == 300: | |
| fixed_sixty = evaluation_ids(selection, 60) | |
| # Seeds depend on list position. Keep the full comparison panel, not | |
| # just the legacy twelve, stable in the final all-training evaluation. | |
| return fixed_sixty + sorted(by_id.keys() - set(fixed_sixty), key=lambda s: rank(s, "all")) | |
| for cell, (profile, language) in enumerate(keys): | |
| bucket = [r for r in rows if (r["session_type"], r["background_language"]) == (profile, language)] | |
| # 14/60 high-volume cases, close to 72/300; each cell has 5 cases. | |
| high_target = 2 if cell in (0, 7) else 1 | |
| for high, target in ((True, high_target), (False, 5 - high_target)): | |
| group = [r for r in bucket if bool(r["background_high"]) == high] | |
| already = sum(r["session_id"] in chosen for r in group) | |
| candidates = sorted((r["session_id"] for r in group if r["session_id"] not in chosen), | |
| key=lambda s: rank(s, "probe")) | |
| if already > target or len(candidates) < target - already: | |
| raise ValueError("Insufficient balanced training examples") | |
| chosen.update(candidates[:target - already]) | |
| result = existing + sorted(chosen - set(existing), key=lambda s: rank(s, "probe-order")) | |
| assert len(result) == count | |
| return result | |
| def remaining_plan(selection, cursor, ordered, batch_size): | |
| if batch_size not in (2, 4, 8, 16): | |
| raise ValueError("Audited candidate batches are 2, 4, 8, 16") | |
| if cursor["epoch"] != 2: | |
| raise ValueError("This migration is scoped to the remaining third epoch") | |
| pending = list(ordered[cursor["consumed_sources_in_epoch"]:]) | |
| if not pending or len(set(pending)) != len(pending): | |
| raise ValueError("Invalid remaining-source queue") | |
| rows = selection["sessions"] | |
| pending.sort(key=lambda i: (rows[i]["dense_steps"], rows[i]["session_id"])) | |
| groups = [pending[i:i + batch_size] for i in range(0, len(pending), batch_size)] | |
| random.Random(SEED).shuffle(groups) | |
| lengths = [max(rows[i]["dense_steps"] for i in group) for group in groups] | |
| return {"batch_size": batch_size, "source_indices": groups, "group_steps": lengths, | |
| "source_count": len(pending), "remaining_updates": sum(lengths), | |
| "source_windows": sum(rows[i]["dense_steps"] for i in pending), | |
| "parent_step": cursor["global_step"], | |
| "final_optimizer_step": cursor["global_step"] + sum(lengths), | |
| "original_final_step": cursor["steps_per_epoch"] * 3, | |
| "policy": "same remaining sources exactly once; length-grouped; partial last batch retained"} | |
| def mapped_scheduler_step(step, parent_step, new_updates, original_final_step): | |
| if new_updates <= 0 or original_final_step <= parent_step: | |
| raise ValueError("Invalid continuation schedule") | |
| fraction = min(1.0, max(0.0, (step - parent_step) / new_updates)) | |
| return parent_step + fraction * (original_final_step - parent_step) | |
| def choose_batch(results, memory_limit_gib=100.0): | |
| valid = [r for r in results if r.get("status") == "passed" | |
| and 0 < r["peak_reserved_gib"] < memory_limit_gib | |
| and math.isfinite(r["active_windows_per_second"])] | |
| if not valid: | |
| raise ValueError("No safe measured training batch") | |
| fastest = max(r["active_windows_per_second"] for r in valid) | |
| # Prefer the smaller effective batch if speed is effectively tied. | |
| return min((r for r in valid if r["active_windows_per_second"] >= .95 * fastest), | |
| key=lambda r: r["batch_size"]) | |
| def write_json(path, value): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(path.suffix + ".tmp") | |
| temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n") | |
| temporary.replace(path) | |