| # Build the vimeml-tiny-zh-v1 training data once, where the frozen export lives (docs/zh-CN/training-zh-v1-plan.md): | |
| # verify export-r1 against SHA256SUMS -> measure pools -> encode 8K and 12K -> eight arm mixtures, | |
| # then write configs/data-zh13.sha256, the list the cluster checks after downloading. | |
| # Needs outputs/zh-corpus/export-r1 and the formal tokenizers in artifacts/tokenizers/zh-v13. | |
| # Finished steps are skipped on a rerun; a half-written output directory must be removed by hand. | |
| set -euo pipefail | |
| cd "$(dirname "$0")/../.." | |
| unset PYTHONPATH | |
| python_bin="${VIMEML_PYTHON:-python3}" | |
| export_dir="outputs/zh-corpus/export-r1" | |
| ( cd "$export_dir" && sha256sum --quiet -c SHA256SUMS ) | |
| echo "export-r1 matches SHA256SUMS" | |
| if [[ ! -f artifacts/training-data/zh13-pools/pools.json ]]; then | |
| "$python_bin" -I -X utf8 -u scripts/tokenizer/encode_zh13.py --no-verify-inputs measure | |
| fi | |
| for vocab in 8192 12288; do | |
| tag=$([[ $vocab == 8192 ]] && echo 8k || echo 12k) | |
| if [[ -f "artifacts/training-data/zh13-chat-${tag}-c128/manifest.json" ]]; then | |
| echo "zh13 ${tag} stores already built"; continue | |
| fi | |
| "$python_bin" -I -X utf8 -u scripts/tokenizer/encode_zh13.py --no-verify-inputs encode --vocab "$vocab" | |
| done | |
| arms=(12k-fresh-s1 12k-fresh-s2 12k-fresh-s3 12k-ep5 12k-ep20 8k-fresh 8k-ep5 8k-ep20) | |
| for arm in "${arms[@]}"; do | |
| "$python_bin" -X utf8 -u scripts/corpus/prepare_zh13.py --arm "$arm" | tail -1 | |
| done | |
| # Everything the cluster downloads, relative to the repository root. run.json only holds wall time. | |
| find artifacts/tokenizers/zh-v13 artifacts/training-data/zh13-* artifacts/token-data/zh13-* -type f \ | |
| ! -name run.json ! -name '*.tmp' | LC_ALL=C sort | xargs sha256sum | sed 's/ \*/ /' > configs/data-zh13.sha256 | |
| echo "wrote configs/data-zh13.sha256 ($(wc -l < configs/data-zh13.sha256) files)" | |