File size: 3,432 Bytes
4e09985
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
"""Fine-tune Laya on the Banking77 training set with Laya's own `finetune`.

The settings are Laya's defaults with two changes, both fixed before any fine-tuned model was scored:

- `head_max_len` goes from 192 to 512 tokens. The 77 options take 356 tokens; at 192 Laya cuts each option short
  and four pairs of them read the same.
- `target_error` goes from 0.10 to 0.01. Laya fits a confidence threshold on training messages it held back, so
  that the answers above it are about 99% right. The rest are meant for a person.

Laya holds back 400 training messages (its `calib_max`) to fit its temperature and that threshold, and trains on the
other 9,603.

Run it with `python -m banking77.train --seed 0`.
"""

from __future__ import annotations

import argparse
import dataclasses
import json
import os
import time
from pathlib import Path

from banking77.data import data_dir, home, sha256_file

BASE_REPO = "convaiinnovations/laya"
BASE_REVISION = "85cc74ddae6109077ebe2853a9b3077fb6606a78"
BASE_WEIGHTS_SHA256 = "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c"
HEAD_MAX_LEN = 512
TARGET_ERROR = 0.01


def base_dir(path: str | None = None) -> str:
    """The pinned base checkpoint: downloaded from the Hub at `BASE_REVISION`, or a local copy of it.

    Either way its weights must match `BASE_WEIGHTS_SHA256`, so a local copy cannot quietly be another revision.
    A local copy can also be named by `LAYA_B77_BASE`.
    """
    path = path or os.environ.get("LAYA_B77_BASE") or None
    if path is None:
        from huggingface_hub import snapshot_download

        path = snapshot_download(BASE_REPO, revision=BASE_REVISION, allow_patterns=[
            "rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*"])
    got = sha256_file(Path(path) / "model.safetensors")
    if got != BASE_WEIGHTS_SHA256:
        raise RuntimeError("base weights in %s are %s, expected %s" % (path, got, BASE_WEIGHTS_SHA256))
    return str(path)


def config(seed: int):
    from laya.train import TrainConfig

    return TrainConfig(seed=seed, head_max_len=HEAD_MAX_LEN, target_error=TARGET_ERROR)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--seed", type=int, default=0)
    parser.add_argument("--name", default=None, help="run name (default: seed<N>)")
    parser.add_argument("--base", default=None, help="a local copy of the base checkpoint (default: download it)")
    args = parser.parse_args()

    from laya.train import finetune

    name = args.name or "seed%d" % args.seed
    out = home() / "runs" / name
    out.mkdir(parents=True, exist_ok=True)
    cfg = config(args.seed)
    base = base_dir(args.base)
    t = time.perf_counter()
    summary = finetune(str(data_dir() / "train.jsonl"), base, str(out / "model"), config=cfg)
    seconds = time.perf_counter() - t
    record = {"name": name, "seed": args.seed, "seconds": seconds, "base": {"repo": BASE_REPO,
              "revision": BASE_REVISION, "weights_sha256": BASE_WEIGHTS_SHA256},
              "config": dataclasses.asdict(cfg), "summary": summary}
    with open(out / "train.json", "w", encoding="utf-8") as f:
        json.dump(record, f, indent=2, default=str)
    print("trained %s in %.0f s; model in %s" % (name, seconds, out / "model"))


if __name__ == "__main__":
    os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
    main()