File size: 4,387 Bytes
55e0599
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
"""Full eligible third-party coverage and isolated evaluation planning."""
from __future__ import annotations

from collections import Counter
import hashlib
import math
import random

SEED = 26100802
PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", "continuous", "same_voice_aside")


def rank(sid, salt):
    return hashlib.sha256(f"{SEED}:{salt}:{sid}".encode()).hexdigest()


def fixed_panel(rows, per_cell=10):
    """Fixed class/language panel; not selected by model score or duration."""
    selected = []
    for profile in PROFILES:
        for language in ("zh", "en"):
            bucket = [r for r in rows if (r["session_type"], r["background_language"]) == (profile, language)]
            if not bucket:
                raise ValueError(f"Missing evaluation cell {profile}/{language}")
            ordered = sorted(bucket, key=lambda r: rank(r["session_id"], "evaluation"))
            # Keep louder backgrounds represented without imposing impossible
            # balanced quotas on the small held-out same-voice class.
            loud = [r for r in ordered if r["background_high"]]
            normal = [r for r in ordered if not r["background_high"]]
            n = min(per_cell, len(ordered))
            chosen = loud[:min(len(loud), max(1, round(n * .25)))]
            chosen += normal[:n - len(chosen)]
            used = {r["session_id"] for r in chosen}
            chosen += [r for r in ordered if r["session_id"] not in used][:n - len(chosen)]
            selected.extend(r["session_id"] for r in chosen)
    if len(set(selected)) != len(selected):
        raise ValueError("Duplicate panel session")
    return selected


def coverage_plan(selection, *, epochs=1, batch_size=16):
    if type(epochs) is not int or not 1 <= epochs <= 3 or batch_size != 16:
        raise ValueError("Scale stage supports 1–3 complete passes at batch16")
    rows = selection["sessions"]
    ids = [r["session_id"] for r in rows]
    if (selection.get("split") != "train" or selection.get("scaleup") is not True
            or ids != selection["session_ids"] or len(ids) < 300 or len(set(ids)) != len(ids)):
        raise ValueError("Expected an explicit unique full training selection")
    for r in rows:
        if (r.get("split", "train") != "train" or r["session_type"] not in PROFILES
                or type(r["dense_steps"]) is not int or r["dense_steps"] <= 0
                or type(r["semantic_steps"]) is not int or not 0 < r["semantic_steps"] <= r["dense_steps"]):
            raise ValueError("Invalid train-only coverage row")
    ordered = sorted(range(len(rows)), key=lambda i: (rows[i]["dense_steps"], ids[i]))
    groups = [ordered[i:i + batch_size] for i in range(0, len(rows), batch_size)]
    result = []
    for epoch in range(1, epochs + 1):
        current = [list(g) for g in groups]
        random.Random(SEED + epoch * 1_000_003).shuffle(current)
        steps = [max(rows[i]["dense_steps"] for i in g) for g in current]
        result.append({"epoch": epoch, "source_indices": current, "group_steps": steps,
                       "optimizer_updates": sum(steps), "source_count": len(rows),
                       "action_samples": sum(r["dense_steps"] for r in rows),
                       "control_samples": sum(r["semantic_steps"] for r in rows)})
    return {"schema": 1, "batch_size": batch_size, "epochs": result, "seed": SEED,
            "max_optimizer_updates": sum(e["optimizer_updates"] for e in result),
            "max_epochs": epochs, "train_count": len(rows),
            "policy": "all eligible original train sessions once per epoch; no replacement/cuts/loss change"}


def review_ids(rows, seed=SEED):
    rng = random.Random(seed)
    chosen = []
    for profile in PROFILES:
        for language in ("zh", "en"):
            for high in (False, True):
                pool = sorted((r for r in rows if r["session_type"] == profile
                    and r["background_language"] == language and r["background_high"] == high), key=lambda r: r["session_id"])
                if pool:
                    chosen.append(rng.choice(pool)["session_id"])
    return chosen


def safe_relative_path(root, value):
    path = (root / value).resolve()
    if path == root.resolve() or not path.is_relative_to(root.resolve()):
        raise ValueError("Payload path escaped production root")
    return path