Download scripts/third_party_scale_plan.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 4.39 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_scale_plan.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/third_party_scale_plan.py
-
curl -L -o third_party_scale_plan.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_scale_plan.py
4.39 kB
| """Full eligible third-party coverage and isolated evaluation planning.""" | |
| from __future__ import annotations | |
| from collections import Counter | |
| import hashlib | |
| import math | |
| import random | |
| SEED = 26100802 | |
| PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", "continuous", "same_voice_aside") | |
| def rank(sid, salt): | |
| return hashlib.sha256(f"{SEED}:{salt}:{sid}".encode()).hexdigest() | |
| def fixed_panel(rows, per_cell=10): | |
| """Fixed class/language panel; not selected by model score or duration.""" | |
| selected = [] | |
| for profile in PROFILES: | |
| for language in ("zh", "en"): | |
| bucket = [r for r in rows if (r["session_type"], r["background_language"]) == (profile, language)] | |
| if not bucket: | |
| raise ValueError(f"Missing evaluation cell {profile}/{language}") | |
| ordered = sorted(bucket, key=lambda r: rank(r["session_id"], "evaluation")) | |
| # Keep louder backgrounds represented without imposing impossible | |
| # balanced quotas on the small held-out same-voice class. | |
| loud = [r for r in ordered if r["background_high"]] | |
| normal = [r for r in ordered if not r["background_high"]] | |
| n = min(per_cell, len(ordered)) | |
| chosen = loud[:min(len(loud), max(1, round(n * .25)))] | |
| chosen += normal[:n - len(chosen)] | |
| used = {r["session_id"] for r in chosen} | |
| chosen += [r for r in ordered if r["session_id"] not in used][:n - len(chosen)] | |
| selected.extend(r["session_id"] for r in chosen) | |
| if len(set(selected)) != len(selected): | |
| raise ValueError("Duplicate panel session") | |
| return selected | |
| def coverage_plan(selection, *, epochs=1, batch_size=16): | |
| if type(epochs) is not int or not 1 <= epochs <= 3 or batch_size != 16: | |
| raise ValueError("Scale stage supports 1–3 complete passes at batch16") | |
| rows = selection["sessions"] | |
| ids = [r["session_id"] for r in rows] | |
| if (selection.get("split") != "train" or selection.get("scaleup") is not True | |
| or ids != selection["session_ids"] or len(ids) < 300 or len(set(ids)) != len(ids)): | |
| raise ValueError("Expected an explicit unique full training selection") | |
| for r in rows: | |
| if (r.get("split", "train") != "train" or r["session_type"] not in PROFILES | |
| or type(r["dense_steps"]) is not int or r["dense_steps"] <= 0 | |
| or type(r["semantic_steps"]) is not int or not 0 < r["semantic_steps"] <= r["dense_steps"]): | |
| raise ValueError("Invalid train-only coverage row") | |
| ordered = sorted(range(len(rows)), key=lambda i: (rows[i]["dense_steps"], ids[i])) | |
| groups = [ordered[i:i + batch_size] for i in range(0, len(rows), batch_size)] | |
| result = [] | |
| for epoch in range(1, epochs + 1): | |
| current = [list(g) for g in groups] | |
| random.Random(SEED + epoch * 1_000_003).shuffle(current) | |
| steps = [max(rows[i]["dense_steps"] for i in g) for g in current] | |
| result.append({"epoch": epoch, "source_indices": current, "group_steps": steps, | |
| "optimizer_updates": sum(steps), "source_count": len(rows), | |
| "action_samples": sum(r["dense_steps"] for r in rows), | |
| "control_samples": sum(r["semantic_steps"] for r in rows)}) | |
| return {"schema": 1, "batch_size": batch_size, "epochs": result, "seed": SEED, | |
| "max_optimizer_updates": sum(e["optimizer_updates"] for e in result), | |
| "max_epochs": epochs, "train_count": len(rows), | |
| "policy": "all eligible original train sessions once per epoch; no replacement/cuts/loss change"} | |
| def review_ids(rows, seed=SEED): | |
| rng = random.Random(seed) | |
| chosen = [] | |
| for profile in PROFILES: | |
| for language in ("zh", "en"): | |
| for high in (False, True): | |
| pool = sorted((r for r in rows if r["session_type"] == profile | |
| and r["background_language"] == language and r["background_high"] == high), key=lambda r: r["session_id"]) | |
| if pool: | |
| chosen.append(rng.choice(pool)["session_id"]) | |
| return chosen | |
| def safe_relative_path(root, value): | |
| path = (root / value).resolve() | |
| if path == root.resolve() or not path.is_relative_to(root.resolve()): | |
| raise ValueError("Payload path escaped production root") | |
| return path | |