File size: 1,718 Bytes
81cffe7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | """Train the vimeml-tiny-zh-v1 8K and 12K tokenizers on the frozen export.
Formal runs read configs/tokenizer-zh13.toml, require the export to be frozen
(manifest.json + SHA256SUMS) and hash every input against SHA256SUMS. --unfrozen is
for local smoke tests only and must write somewhere other than the formal directory.
usage:
python -I -X utf8 scripts/tokenizer/train_zh13.py
python -I -X utf8 scripts/tokenizer/train_zh13.py --unfrozen --output OUT [--export DIR]
"""
import argparse
from pathlib import Path
import sys
import tomllib
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
from vimeml.tokenizer.zh13 import run # noqa: E402
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--config", type=Path, default=ROOT / "configs/tokenizer-zh13.toml")
parser.add_argument("--export", type=Path)
parser.add_argument("--output", type=Path)
parser.add_argument("--unfrozen", action="store_true", help="smoke test on an unfrozen export")
parser.add_argument("--no-verify-inputs", action="store_true", help="check names in SHA256SUMS but skip hashing")
args = parser.parse_args()
config = tomllib.loads(args.config.read_text("utf-8"))
export = args.export or ROOT / config["export_dir"]
output = args.output or ROOT / config["output_dir"]
if args.unfrozen and output.resolve() == (ROOT / config["output_dir"]).resolve():
parser.error("--unfrozen must not write the formal tokenizer directory")
run(config, export, output, frozen=not args.unfrozen, verify_inputs=not args.no_verify_inputs)
if __name__ == "__main__":
main()
|