h3-t53 / scripts /prep_cache_only.sh

Upload scripts/prep_cache_only.sh with huggingface_hub

898375c verified 28 days ago

6.97 kB

#!/usr/bin/env bash
# box/prep_cache_only.sh — CPU-tier prep per inbox 20:52 (coordinator-owned; does NOT touch train_from_cache.sh).
# Downloads + sha-verifies + unpacks task-53's caches and the t53 dataset root into Home (persists across the
# CPU->GPU switch), regenerates the toml with the tar's mkds.py, dry-parses every path the toml references,
# checks pylib/trainer/box scripts, and writes $T/CPU_PREP_DONE with the checklist + the exact train command.
# Never starts the trainer, never touches a runtime, never prints tokens. Fail closed => CACHE_VERIFY_FAIL.
set -uo pipefail
T=${T:-/home/aistudio/t48y}
CHAR=${CHAR:?usage: CHAR=yier|bubu box/prep_cache_only.sh}
case $CHAR in yier|bubu) ;; *) echo "PREP_ABORT bad char"; exit 2;; esac
B=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
ROOT53=$T/root/t53; DL=$ROOT53/dl; DS=$ROOT53/ds; CROOT=$ROOT53/cache/$CHAR
HF=${HF_BASE:-https://hf-mirror.com/vftfnn/h3-t53/resolve/main}
case $CHAR in yier) TAR_PREFIX=${TAR_PREFIX:-1215441406};; *) TAR_PREFIX=${TAR_PREFIX:-172db2474c};; esac
LOGS=$T/logs; mkdir -p "$LOGS" "$DL" "$DS" "$ROOT53/cache"
exec >>"$LOGS/prep_cache_only.log" 2>&1 </dev/null
say(){ printf '%s %s\n' "$(date -Is)" "$*"; }
sha(){ sha256sum "$1" 2>/dev/null | cut -d' ' -f1; }
fail(){ say "PREP_FAIL: $1"; printf 'CACHE_VERIFY_FAIL %s %s\n' "$(date -Is)" "$1" > "$T/CACHE_VERIFY_FAIL"; exit 3; }
dl(){ local dst=$1 url=$2 t rc
  [[ -s $dst ]] && { say "  dl SKIP $(basename "$dst") present $(stat -c %s "$dst") B"; return 0; }
  for t in 1 2 3; do curl -fsSL -C - --retry 2 --max-time 3000 -o "$dst" "$url" >/dev/null 2>&1; rc=$?
    say "  dl try=$t rc=$rc bytes=$(stat -c %s "$dst" 2>/dev/null || echo 0) $(basename "$url")"; (( rc == 0 )) && return 0; sleep 5; done
  return 1; }
say "PREP_START char=$CHAR T=$T"
dl "$DL/cache_manifest_${CHAR}.json" "$HF/cache/cache_manifest_${CHAR}.json" || fail "manifest download"
python3 -c "import json,sys;json.load(open(sys.argv[1]))" "$DL/cache_manifest_${CHAR}.json" || fail "manifest json"
dl "$DL/cache_${CHAR}.tar" "$HF/cache/cache_${CHAR}.tar" || fail "cache tar download"
dl "$DL/t53_${CHAR}.tar.gz" "$HF/t53_${CHAR}.tar.gz" || fail "dataset tar download"
TAR_SHA=$(sha "$DL/cache_${CHAR}.tar"); say "  tar sha=$TAR_SHA"
[[ $TAR_SHA == ${TAR_PREFIX}* ]] || fail "tar sha prefix != $TAR_PREFIX"
tar -xzf "$DL/t53_${CHAR}.tar.gz" -C "$DS" || fail "dataset unpack"
[[ -f $DS/items.jsonl ]] || { inner=$(find "$DS" -maxdepth 2 -name items.jsonl | head -1); [[ -n $inner ]] && mv "$(dirname "$inner")"/* "$DS/" || fail "no items.jsonl"; }
tar -xf "$DL/cache_${CHAR}.tar" -C "$ROOT53/cache" || fail "cache unpack"
if [[ ! -d $CROOT/b512 ]]; then found=$(find "$ROOT53/cache" -maxdepth 3 -type d -name b512 | head -1)
  [[ -n $found ]] || fail "no b512"; par=$(dirname "$found"); [[ $par != "$CROOT" ]] && { mkdir -p "$CROOT"; mv "$par"/b512 "$CROOT"/; [[ -d $par/b768 ]] && mv "$par"/b768 "$CROOT"/; }; fi
[[ -d $CROOT/b512 && -d $CROOT/b768 ]] || fail "cache layout"
python3 - "$DL/cache_manifest_${CHAR}.json" <<'PY' > "$DL/want_shas.txt" || fail "manifest parse"
import json, sys
m = json.load(open(sys.argv[1]))
files = m.get("files") or m.get("cache_files") or m.get("cache") or {}
pairs = []
if isinstance(files, list):
    for e in files:
        if isinstance(e, dict):
            s = e.get("sha256") or e.get("sha"); p = e.get("path")
            if isinstance(s, str) and len(s) == 64 and isinstance(p, str): pairs.append((p, s))
elif isinstance(files, dict):
    for k, v in files.items():
        s = v.get("sha256") if isinstance(v, dict) else v
        if isinstance(s, str) and len(s) == 64: pairs.append((k, s))
if not pairs: pairs = [(k, v) for k, v in m.items() if isinstance(v, str) and len(v) == 64]
print("\n".join("%s %s" % p for p in pairs))
PY
NF=$(grep -c . "$DL/want_shas.txt" || true); (( NF > 0 )) || fail "manifest has no per-file shas"
OK=0
while read -r rel want; do p="$CROOT/$rel"
  [[ -f $p ]] || p="$ROOT53/cache/$rel"          # manifest rel paths may or may not carry the char dir
  [[ -f $p ]] || p="$CROOT/${rel#"$CHAR"/}"
  [[ -f $p ]] || fail "missing $rel"
  got=$(sha "$p"); [[ $got == "$want" ]] || fail "sha mismatch $rel"; OK=$((OK+1)); done < "$DL/want_shas.txt"
say "  sha-verified $OK/$NF cache files"
COUNTS=$(python3 - "$DS/items.jsonl" "$CROOT" <<'PY' || fail "count assertion"
import json, os, sys
per = {}
for line in open(sys.argv[1], encoding="utf-8"):
    line = line.strip()
    if line:
        per[str(json.loads(line).get("bucket") or "")] = per.get(str(json.loads(line).get("bucket") or ""), 0) + 1
out = []; total = 0
for b in ("512", "768"):
    n = per.get(b, 0); d = os.path.join(sys.argv[2], "b" + b)
    fs = os.listdir(d) if os.path.isdir(d) else []
    lat = len([f for f in fs if f.endswith(".safetensors") and not f.endswith("_mmh3_te.safetensors")])
    tex = len([f for f in fs if f.endswith("_mmh3_te.safetensors")])
    if n == 0 and lat == 0 and tex == 0: continue
    if not (lat == n and tex == n): print("FAIL b%s items=%d lat=%d tex=%d" % (b, n, lat, tex)); sys.exit(1)
    out.append("%s:%d" % (b, n)); total += n
if total == 0: print("FAIL none"); sys.exit(1)
print("OK items=%d buckets=%s" % (total, ",".join(out)))
PY
)
case $COUNTS in OK*) say "  counts: $COUNTS";; *) fail "counts: $COUNTS";; esac
NITEMS=$(printf '%s' "$COUNTS" | sed -nE 's/^OK items=([0-9]+).*/\1/p')
# stack presence (CPU-checkable): pylib + trainer + box scripts
[[ -d $T/root/pylib && -d $T/root/trainer/src ]] || fail "pylib/trainer missing on box"
[[ -f $B/train_char.sh && -f $B/t48f_env.sh && -f $B/mk_toml_char.py ]] || fail "box scripts missing"
python3 -c "import sys; sys.path.insert(0,'$T/root/pylib'); import torch, safetensors; print('  cpu import ok', torch.__version__)" || fail "pylib import"
TRIG=yier3d; [[ $CHAR == bubu ]] && TRIG=bubu3d
python3 "$DS/mkds.py" --pack "$DS" --char "$CHAR" --trigger "$TRIG" --buckets 512,768 --refs-list 2 \
  --shard 0/1 --out-dir "$DS" --cache-root "$CROOT" || fail "mkds.py"
TOML=$DS/dataset.toml; [[ -f $TOML ]] || fail "no toml"
python3 - "$TOML" <<'PY' || fail "toml paths missing"
import os, re, sys
toml = open(sys.argv[1], encoding="utf-8").read()
paths = [p for _, p in re.findall(r'(?:^|\s)(["\']?)(/(?:home|kaggle)[^"\'\n]+)\1', toml)]
missing = [p for p in paths if not os.path.exists(p)]
print("  toml paths checked=%d missing=%d" % (len(paths), len(missing)))
for m in missing: print("  MISSING", m)
sys.exit(1 if missing else 0)
PY
TRAIN_CMD="env CHAR=$CHAR T=$T MODE=full FROM_CACHE=1 SKIP_CACHE=1 PACKDIR=$DS T48_CACHE_ROOT=$CROOT MAX_STEPS=1000 SAVE_EVERY=250 BASE_STEPS=0 RESUME=0 bash $T/box/train_char.sh"
{ printf 'CPU_PREP_DONE %s %s\n' "$CHAR" "$(date -Is)"
  printf 'items=%s cache_files=%s tar_sha=%s\n' "$NITEMS" "$OK" "$TAR_SHA"
  printf 'toml=%s\n' "$TOML"
  printf 'cmd=%s\n' "$TRAIN_CMD"; } > "$T/CPU_PREP_DONE"
say "CPU_PREP_DONE char=$CHAR items=$NITEMS files=$OK tar_sha=$TAR_SHA"
say "TRAIN_CMD=$TRAIN_CMD"

	#!/usr/bin/env bash
	# box/prep_cache_only.sh — CPU-tier prep per inbox 20:52 (coordinator-owned; does NOT touch train_from_cache.sh).
	# Downloads + sha-verifies + unpacks task-53's caches and the t53 dataset root into Home (persists across the
	# CPU->GPU switch), regenerates the toml with the tar's mkds.py, dry-parses every path the toml references,
	# checks pylib/trainer/box scripts, and writes $T/CPU_PREP_DONE with the checklist + the exact train command.
	# Never starts the trainer, never touches a runtime, never prints tokens. Fail closed => CACHE_VERIFY_FAIL.
	set -uo pipefail
	T=${T:-/home/aistudio/t48y}
	CHAR=${CHAR:?usage: CHAR=yier\|bubu box/prep_cache_only.sh}
	case $CHAR in yier\|bubu) ;; *) echo "PREP_ABORT bad char"; exit 2;; esac
	B=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
	ROOT53=$T/root/t53; DL=$ROOT53/dl; DS=$ROOT53/ds; CROOT=$ROOT53/cache/$CHAR
	HF=${HF_BASE:-https://hf-mirror.com/vftfnn/h3-t53/resolve/main}
	case $CHAR in yier) TAR_PREFIX=${TAR_PREFIX:-1215441406};; *) TAR_PREFIX=${TAR_PREFIX:-172db2474c};; esac
	LOGS=$T/logs; mkdir -p "$LOGS" "$DL" "$DS" "$ROOT53/cache"
	exec >>"$LOGS/prep_cache_only.log" 2>&1 </dev/null
	say(){ printf '%s %s\n' "$(date -Is)" "$*"; }
	sha(){ sha256sum "$1" 2>/dev/null \| cut -d' ' -f1; }
	fail(){ say "PREP_FAIL: $1"; printf 'CACHE_VERIFY_FAIL %s %s\n' "$(date -Is)" "$1" > "$T/CACHE_VERIFY_FAIL"; exit 3; }
	dl(){ local dst=$1 url=$2 t rc
	[[ -s $dst ]] && { say " dl SKIP $(basename "$dst") present $(stat -c %s "$dst") B"; return 0; }
	for t in 1 2 3; do curl -fsSL -C - --retry 2 --max-time 3000 -o "$dst" "$url" >/dev/null 2>&1; rc=$?
	say " dl try=$t rc=$rc bytes=$(stat -c %s "$dst" 2>/dev/null \|\| echo 0) $(basename "$url")"; (( rc == 0 )) && return 0; sleep 5; done
	return 1; }
	say "PREP_START char=$CHAR T=$T"
	dl "$DL/cache_manifest_${CHAR}.json" "$HF/cache/cache_manifest_${CHAR}.json" \|\| fail "manifest download"
	python3 -c "import json,sys;json.load(open(sys.argv[1]))" "$DL/cache_manifest_${CHAR}.json" \|\| fail "manifest json"
	dl "$DL/cache_${CHAR}.tar" "$HF/cache/cache_${CHAR}.tar" \|\| fail "cache tar download"
	dl "$DL/t53_${CHAR}.tar.gz" "$HF/t53_${CHAR}.tar.gz" \|\| fail "dataset tar download"
	TAR_SHA=$(sha "$DL/cache_${CHAR}.tar"); say " tar sha=$TAR_SHA"
	[[ $TAR_SHA == ${TAR_PREFIX}* ]] \|\| fail "tar sha prefix != $TAR_PREFIX"
	tar -xzf "$DL/t53_${CHAR}.tar.gz" -C "$DS" \|\| fail "dataset unpack"
	[[ -f $DS/items.jsonl ]] \|\| { inner=$(find "$DS" -maxdepth 2 -name items.jsonl \| head -1); [[ -n $inner ]] && mv "$(dirname "$inner")"/* "$DS/" \|\| fail "no items.jsonl"; }
	tar -xf "$DL/cache_${CHAR}.tar" -C "$ROOT53/cache" \|\| fail "cache unpack"
	if [[ ! -d $CROOT/b512 ]]; then found=$(find "$ROOT53/cache" -maxdepth 3 -type d -name b512 \| head -1)
	[[ -n $found ]] \|\| fail "no b512"; par=$(dirname "$found"); [[ $par != "$CROOT" ]] && { mkdir -p "$CROOT"; mv "$par"/b512 "$CROOT"/; [[ -d $par/b768 ]] && mv "$par"/b768 "$CROOT"/; }; fi
	[[ -d $CROOT/b512 && -d $CROOT/b768 ]] \|\| fail "cache layout"
	python3 - "$DL/cache_manifest_${CHAR}.json" <<'PY' > "$DL/want_shas.txt" \|\| fail "manifest parse"
	import json, sys
	m = json.load(open(sys.argv[1]))
	files = m.get("files") or m.get("cache_files") or m.get("cache") or {}
	pairs = []
	if isinstance(files, list):
	for e in files:
	if isinstance(e, dict):
	s = e.get("sha256") or e.get("sha"); p = e.get("path")
	if isinstance(s, str) and len(s) == 64 and isinstance(p, str): pairs.append((p, s))
	elif isinstance(files, dict):
	for k, v in files.items():
	s = v.get("sha256") if isinstance(v, dict) else v
	if isinstance(s, str) and len(s) == 64: pairs.append((k, s))
	if not pairs: pairs = [(k, v) for k, v in m.items() if isinstance(v, str) and len(v) == 64]
	print("\n".join("%s %s" % p for p in pairs))
	PY
	NF=$(grep -c . "$DL/want_shas.txt" \|\| true); (( NF > 0 )) \|\| fail "manifest has no per-file shas"
	OK=0
	while read -r rel want; do p="$CROOT/$rel"
	[[ -f $p ]] \|\| p="$ROOT53/cache/$rel" # manifest rel paths may or may not carry the char dir
	[[ -f $p ]] \|\| p="$CROOT/${rel#"$CHAR"/}"
	[[ -f $p ]] \|\| fail "missing $rel"
	got=$(sha "$p"); [[ $got == "$want" ]] \|\| fail "sha mismatch $rel"; OK=$((OK+1)); done < "$DL/want_shas.txt"
	say " sha-verified $OK/$NF cache files"
	COUNTS=$(python3 - "$DS/items.jsonl" "$CROOT" <<'PY' \|\| fail "count assertion"
	import json, os, sys
	per = {}
	for line in open(sys.argv[1], encoding="utf-8"):
	line = line.strip()
	if line:
	per[str(json.loads(line).get("bucket") or "")] = per.get(str(json.loads(line).get("bucket") or ""), 0) + 1
	out = []; total = 0
	for b in ("512", "768"):
	n = per.get(b, 0); d = os.path.join(sys.argv[2], "b" + b)
	fs = os.listdir(d) if os.path.isdir(d) else []
	lat = len([f for f in fs if f.endswith(".safetensors") and not f.endswith("_mmh3_te.safetensors")])
	tex = len([f for f in fs if f.endswith("_mmh3_te.safetensors")])
	if n == 0 and lat == 0 and tex == 0: continue
	if not (lat == n and tex == n): print("FAIL b%s items=%d lat=%d tex=%d" % (b, n, lat, tex)); sys.exit(1)
	out.append("%s:%d" % (b, n)); total += n
	if total == 0: print("FAIL none"); sys.exit(1)
	print("OK items=%d buckets=%s" % (total, ",".join(out)))
	PY
	)
	case $COUNTS in OK) say " counts: $COUNTS";; ) fail "counts: $COUNTS";; esac
	NITEMS=$(printf '%s' "$COUNTS" \| sed -nE 's/^OK items=([0-9]+).*/\1/p')
	# stack presence (CPU-checkable): pylib + trainer + box scripts
	[[ -d $T/root/pylib && -d $T/root/trainer/src ]] \|\| fail "pylib/trainer missing on box"
	[[ -f $B/train_char.sh && -f $B/t48f_env.sh && -f $B/mk_toml_char.py ]] \|\| fail "box scripts missing"
	python3 -c "import sys; sys.path.insert(0,'$T/root/pylib'); import torch, safetensors; print(' cpu import ok', torch.__version__)" \|\| fail "pylib import"
	TRIG=yier3d; [[ $CHAR == bubu ]] && TRIG=bubu3d
	python3 "$DS/mkds.py" --pack "$DS" --char "$CHAR" --trigger "$TRIG" --buckets 512,768 --refs-list 2 \
	--shard 0/1 --out-dir "$DS" --cache-root "$CROOT" \|\| fail "mkds.py"
	TOML=$DS/dataset.toml; [[ -f $TOML ]] \|\| fail "no toml"
	python3 - "$TOML" <<'PY' \|\| fail "toml paths missing"
	import os, re, sys
	toml = open(sys.argv[1], encoding="utf-8").read()
	paths = [p for _, p in re.findall(r'(?:^\|\s)(["\']?)(/(?:home\|kaggle)[^"\'\n]+)\1', toml)]
	missing = [p for p in paths if not os.path.exists(p)]
	print(" toml paths checked=%d missing=%d" % (len(paths), len(missing)))
	for m in missing: print(" MISSING", m)
	sys.exit(1 if missing else 0)
	PY
	TRAIN_CMD="env CHAR=$CHAR T=$T MODE=full FROM_CACHE=1 SKIP_CACHE=1 PACKDIR=$DS T48_CACHE_ROOT=$CROOT MAX_STEPS=1000 SAVE_EVERY=250 BASE_STEPS=0 RESUME=0 bash $T/box/train_char.sh"
	{ printf 'CPU_PREP_DONE %s %s\n' "$CHAR" "$(date -Is)"
	printf 'items=%s cache_files=%s tar_sha=%s\n' "$NITEMS" "$OK" "$TAR_SHA"
	printf 'toml=%s\n' "$TOML"
	printf 'cmd=%s\n' "$TRAIN_CMD"; } > "$T/CPU_PREP_DONE"
	say "CPU_PREP_DONE char=$CHAR items=$NITEMS files=$OK tar_sha=$TAR_SHA"
	say "TRAIN_CMD=$TRAIN_CMD"