File size: 2,017 Bytes
431ffa3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# -*- coding: utf-8 -*-
"""MorphBPE JSON v1 serialization."""

from __future__ import annotations

import json
from typing import Any

SCHEMA_VERSION = 1


def dump(tokenizer: Any, path: str, extra_config: dict[str, Any] | None = None) -> None:
    """Write a MorphBPETokenizer to disk using v1 schema."""
    merges_sorted = sorted(tokenizer.merges.items(), key=lambda item: item[1][1])
    merges_arr = [
        {"pair": [left, right], "merged": merged, "rank": rank}
        for (left, right), (merged, rank) in merges_sorted
    ]
    payload = {
        "version": SCHEMA_VERSION,
        "type": "morphbpe",
        "vocab": tokenizer.vocab,
        "merges": merges_arr,
        "special_tokens": {"unk": "<unk>"},
        "config": {
            "boundary_constrained": True,
            "min_boundary_confidence": getattr(
                tokenizer, "min_boundary_confidence", 0.60
            ),
            "seed_alphabet": getattr(tokenizer, "seed_alphabet", False),
            **(extra_config or {}),
        },
    }
    with open(path, "w", encoding="utf-8") as f:
        json.dump(payload, f, ensure_ascii=False, indent=2)


def load(path: str) -> dict[str, Any]:
    with open(path, "r", encoding="utf-8") as f:
        payload = json.load(f)
    version = payload.get("version")
    if version is not None and int(version) != SCHEMA_VERSION:
        raise ValueError(
            f"unsupported morphbpe schema version: {version}; expected {SCHEMA_VERSION}"
        )
    return payload


def merges_from_payload(
    payload: dict[str, Any],
) -> dict[tuple[str, str], tuple[str, int]]:
    merges: dict[tuple[str, str], tuple[str, int]] = {}
    for i, item in enumerate(payload.get("merges", [])):
        if isinstance(item, dict):
            left, right = item["pair"]
            merged = item.get("merged", left + right)
            rank = int(item.get("rank", i))
        else:
            left, right, merged, rank = item
        merges[(left, right)] = (merged, rank)
    return merges