File size: 4,598 Bytes
bcc8d35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | #!/usr/bin/env python
"""Build every corpus this project trains on, in the only legal order.
python scripts/build_data.py --out <build root> [--tokenizer <dir>] [--curation-target N]
Stages: eval sets -> decontamination hash set -> SFT -> Arm A -> Arm B.
Each refuses to run without its upstream receipt; the final manifest
(`<out>/manifest.json`) pins every content hash, every measured count against
its pinned value, reject and contamination counts, the length-bias baseline,
the delimiter ids, the tokenizer hash, and the dataset revisions. Sections
11/12 quote that manifest in their launch banner and refuse to start without
it.
Host safety (this box, 186GB, NO swap, shared): `apply_resource_guards(
heavy=True)` runs the preflight and takes the exclusive job flock BEFORE
anything heavy, so this build cannot run concurrently with a training job --
that exact overlap killed the predecessor's first SFT run. Every corpus is
streamed; nothing is held resident.
"""
from __future__ import annotations
import argparse
import json
import logging
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from align.utils.guard import apply_resource_guards # noqa: E402
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger("build_data")
DEFAULT_OUT = "/apps/development/rlhf/nano-mistral-o1/data"
TOKENIZER_SNAPSHOT = (
"/apps/voxtral/.cache/huggingface/hub/models--mistralai--Mistral-7B-v0.3/"
"snapshots/caa1feb0e54d415e2df31207e5f4e273e33509b1"
)
#: Byte-identical tokenizer files vendored in-repo (assets/tokenizer/.../PROVENANCE.md).
#: The HF-cache snapshot is NOT on every box; the build must not die on a missing
#: cache when the same four files are in the repo.
VENDORED_TOKENIZER = str(
Path(__file__).resolve().parent.parent / "assets" / "tokenizer" / "mistral-7b-v0.3"
)
def default_tokenizer_dir() -> str:
"""The HF snapshot if it holds a tokenizer.json, else the vendored copy."""
for c in (TOKENIZER_SNAPSHOT, VENDORED_TOKENIZER):
if (Path(c) / "tokenizer.json").is_file():
return c
return TOKENIZER_SNAPSHOT # fails loudly in load_tokenizer, naming the path
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument(
"--out", default=DEFAULT_OUT, help="build root; every path under it is asserted onto /apps"
)
ap.add_argument("--tokenizer", default=default_tokenizer_dir())
ap.add_argument(
"--curation-target",
type=int,
default=None,
help="Arm B curated pair count (default: the pinned 12,000)",
)
ap.add_argument(
"--max-len",
type=int,
default=2048,
help="build cutoff, deliberately above the 1024 training "
"seq_len: see PinnedYields.max_len for the measured "
"length distribution behind this choice",
)
ap.add_argument("--held-out-frac", type=float, default=0.05)
ap.add_argument("--seed", type=int, default=0)
ap.add_argument(
"--strict-pins",
default=True,
action=argparse.BooleanOptionalAction,
help="assert every measured count against its pinned "
"value. --no-strict-pins is RE-MEASUREMENT MODE: it "
"stamps pins_verified=false into the manifest and "
"sections 11/12 refuse to launch against it.",
)
ap.add_argument(
"--strict-host-check",
default=True,
action=argparse.BooleanOptionalAction,
help="disable ONLY with a deliberate reason; the check is "
"the guard that did not exist on 2026-08-12",
)
args = ap.parse_args()
apply_resource_guards(
cpu_threads=8, heavy=True, label="build_data", strict_host_check=args.strict_host_check
)
from align.data.build import PinnedYields, build_all
from align.data.sources import DataSources
from align.tokenizer import load_tokenizer
tokenizer = load_tokenizer(args.tokenizer)
sources = DataSources.from_local_hub()
kwargs = {
"max_len": args.max_len,
"held_out_frac": args.held_out_frac,
"seed": args.seed,
"strict": args.strict_pins,
}
if args.curation_target is not None:
kwargs["arm_b_curated"] = args.curation_target
pins = PinnedYields(**kwargs)
manifest = build_all(sources, args.out, tokenizer, pins=pins)
print(json.dumps(manifest, indent=1))
logger.info("build complete: %s", Path(args.out) / "manifest.json")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|