Commit ยท
cfb4cf5
0
Parent(s):
genmai 1.0.0
Browse files- .gitattributes +35 -0
- README.md +123 -0
- config.json +36 -0
- model-coreml.zip +3 -0
- model.safetensors +3 -0
- tokenizer.json +0 -0
- tokenizer_config.json +14 -0
.gitattributes
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-sa-4.0
|
| 3 |
+
language: ja
|
| 4 |
+
base_model: Miwa-Keita/zenz-v3.2-small-gguf
|
| 5 |
+
pipeline_tag: text-generation
|
| 6 |
+
tags:
|
| 7 |
+
- japanese
|
| 8 |
+
- input-method
|
| 9 |
+
- kana-kanji-conversion
|
| 10 |
+
---
|
| 11 |
+
# genmai-90M
|
| 12 |
+
|
| 13 |
+
A small Japanese **kana-kanji conversion** model: it converts a reading (katakana) into Japanese text, optionally
|
| 14 |
+
conditioned on the text before the cursor. It is the light-weight variant of
|
| 15 |
+
[subdiox/genmai-310M](https://huggingface.co/subdiox/genmai-310M), distilled from it, meant for devices with less
|
| 16 |
+
memory (e.g. iPhone). It uses the same prompt format, tokenizer and file layout as genmai-310M.
|
| 17 |
+
|
| 18 |
+
- Architecture: GPT-2, character-level vocabulary (6,000), 12 layers, 768 hidden, 90M parameters
|
| 19 |
+
- Base model: zenz-v3.2-small ([Miwa-Keita/zenz-v3.2-small-gguf](https://huggingface.co/Miwa-Keita/zenz-v3.2-small-gguf),
|
| 20 |
+
Apache-2.0), dequantized from the Q5_K_M GGUF
|
| 21 |
+
- Teacher: genmai-310M
|
| 22 |
+
|
| 23 |
+
## Files
|
| 24 |
+
|
| 25 |
+
Releases are git tags (`v1.0.0`); use a tag to pin a version. `main` is the latest release.
|
| 26 |
+
|
| 27 |
+
| file | contents |
|
| 28 |
+
|---|---|
|
| 29 |
+
| `model.safetensors` | fp32 weights in the Hugging Face GPT-2 layout (362 MB), for transformers |
|
| 30 |
+
| `config.json` | GPT-2 config; the prompt format is recorded as `"genmai_prompt_format": "zenzlc"` |
|
| 31 |
+
| `tokenizer.json`, `tokenizer_config.json` | the tokenizer |
|
| 32 |
+
| `model-coreml.zip` | what the genmai input method runs: the model as a stateful Core ML package for the Apple Neural Engine (82 MB; see below): `genmai.mlpackage` (int8 weights) and `genmai_coreml.json` (its static shapes: `format` `genmai-coreml-1`, `prompt_slots` 256, `output_steps` 64, `rows` 5, `prefill_chunk` 32) |
|
| 33 |
+
|
| 34 |
+
```python
|
| 35 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 36 |
+
m = AutoModelForCausalLM.from_pretrained("subdiox/genmai-90M", revision="v1.0.0")
|
| 37 |
+
tok = AutoTokenizer.from_pretrained("subdiox/genmai-90M", revision="v1.0.0")
|
| 38 |
+
x = tok("<s>\uee00" + "ใใใใใฌใคใฟใคใฉใทใค" + "\uee01", return_tensors="pt", add_special_tokens=False)
|
| 39 |
+
y = m.generate(**x, num_beams=5, num_return_sequences=5, length_penalty=0.0, max_new_tokens=64)
|
| 40 |
+
print([tok.decode(s[x.input_ids.shape[1]:], skip_special_tokens=True) for s in y]) # ๆฏใฏๆญฏใ็ใใใใ, ...
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
Rank beams by the plain sum of log-probabilities (`length_penalty=0.0`): with transformers' default length
|
| 44 |
+
normalisation, long degenerate outputs such as ใใใใใฌใคใฟใคใใใ can win.
|
| 45 |
+
|
| 46 |
+
## Prompt format (`zenzlc`)
|
| 47 |
+
|
| 48 |
+
Left context first (up to the last 500 characters), then the reading, then the output:
|
| 49 |
+
|
| 50 |
+
```
|
| 51 |
+
<s>[U+EE02]{left context}[U+EE00]{reading in katakana}[U+EE01]{output}</s>
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
`[U+EE0x]` are the private-use marker characters U+EE02 (context), U+EE00 (input) and U+EE01 (output).
|
| 55 |
+
Without context the `[U+EE02]{left context}` part is omitted. Spaces are written as U+3000 and newlines are
|
| 56 |
+
removed. Decode with beam search (5 beams) and stop at EOS; long readings are split at ใใ into chunks of at most
|
| 57 |
+
40 characters, each decoded after the previous output.
|
| 58 |
+
|
| 59 |
+
## Core ML (Apple Neural Engine)
|
| 60 |
+
|
| 61 |
+
`genmai.mlpackage` in `model-coreml.zip` is the same model converted with coremltools 9 (macOS 15 / iOS 18 or later) for the
|
| 62 |
+
genmai input method's Core ML decoder. It is one multifunction package with two functions that share one set of
|
| 63 |
+
states (per layer, the keys and values in fp16):
|
| 64 |
+
|
| 65 |
+
- `prefill`: `ids`, `pos` (1, 32) int32, `start` (1,) int32, `mask` (32, 256) fp16 additive โ `logp` (32, 6000):
|
| 66 |
+
writes 32 prompt tokens to prompt slots `start ..< start + 32` and returns the log-probabilities after each.
|
| 67 |
+
- `decode`: `tok` (5, 1) int32, `pos` (1,) int32, `wsel` (320, 5) / `keep` (320, 1) fp16, `pmask` (1, 256) /
|
| 68 |
+
`smask` (5, 320) fp16 additive โ `logp` (5, 6000): one token for each of 5 beams. Beam r's token of step t is
|
| 69 |
+
stored at output slot t ยท 5 + r (`wsel` is the one-hot placement, `keep` = 1 โ its row sums); each beam attends to
|
| 70 |
+
the prompt slots in `pmask` and to its ancestors' output slots in `smask`, so reordering beams copies nothing.
|
| 71 |
+
|
| 72 |
+
Linear weights are int8 (symmetric, per channel), the rest fp16. Prompts are limited to 256 tokens and outputs to
|
| 73 |
+
64 tokens per chunk. The conversion script is `Tools/coreml/genmai_coreml.py` of
|
| 74 |
+
[genmai-ime](https://github.com/subdiox/genmai-ime).
|
| 75 |
+
|
| 76 |
+
On an M4 Pro (5 beams): the same top-1 as the weights in 8 bits on the GPU for 101 / 101 everyday sentences,
|
| 77 |
+
199 / 200 AJIMEE items and 27 / 28 hard sentences; about 4 ms per decode step on the Neural Engine; process footprint
|
| 78 |
+
about 50โ90 MB. The first load compiles the model for
|
| 79 |
+
the Neural Engine (about 20 s; macOS caches the result).
|
| 80 |
+
|
| 81 |
+
## Training
|
| 82 |
+
|
| 83 |
+
Knowledge distillation from genmai-310M: at every output position the loss is
|
| 84 |
+
0.5 ยท cross-entropy to the reference + 0.5 ยท KL divergence to the teacher's next-character distribution (both models
|
| 85 |
+
share the vocabulary), plus an auxiliary next-character language-model loss (weight 0.3) on the text before the
|
| 86 |
+
cursor. Stages:
|
| 87 |
+
|
| 88 |
+
1. ~3M automatically read sentences (Wikipedia, FineWeb-2 Japanese, Aozora Bunko, Diet minutes) and sentences
|
| 89 |
+
containing rare Mozc dictionary words;
|
| 90 |
+
2. ~3.2M further Wikipedia / FineWeb-2 sentences;
|
| 91 |
+
3. final stage on curated data (hand-checked items, synthetic and homophone sentences generated with Claude,
|
| 92 |
+
LLM-judged corrections) with replay of the earlier data.
|
| 93 |
+
|
| 94 |
+
## Evaluation
|
| 95 |
+
|
| 96 |
+
Top-1 accuracy, beam size 5; candidates whose reading does not match the input are dropped. AJIMEE, everyday
|
| 97 |
+
sentences and ime_v1 dev are judged leniently (orthographic variants accepted); homophones = word-level accuracy
|
| 98 |
+
on held-out homophone sentences; hard = 28 sentences written to stress an IME (exact match up to orthography).
|
| 99 |
+
|
| 100 |
+
| set | genmai-90M (bf16) | genmai-310M (bf16) | Zenzai (zenz-v3.2-small, azooKey settings) |
|
| 101 |
+
|---|---|---|---|
|
| 102 |
+
| AJIMEE-Bench (200) | 91.0 % | 95.0 % | 85.5 % |
|
| 103 |
+
| everyday sentences (101) | 98.0 % | 98.0 % | 97.0 % |
|
| 104 |
+
| ime_v1 dev (544) | 94.3 % | 93.6 % | โ |
|
| 105 |
+
| homophone sentences (1,607) | 89.6 % | 93.1 % | โ |
|
| 106 |
+
| average of the four | 93.2 % | 94.9 % | โ |
|
| 107 |
+
| hard IME sentences (28) | 22 / 28 | 22 / 28 | โ |
|
| 108 |
+
|
| 109 |
+
Speed (RTX 3090, bf16, 5 beams, everyday sentences): median 18 ms per conversion vs. 36 ms for genmai-310M. In the
|
| 110 |
+
input method (Core ML, M4 Pro), the conversion key answers in a median 16 / 22 ms (everyday sentences / AJIMEE).
|
| 111 |
+
|
| 112 |
+
## Limitations
|
| 113 |
+
|
| 114 |
+
- Compared with genmai-310M it is weaker on knowledge-heavy text (rare proper nouns, technical terms) and on fine
|
| 115 |
+
homophone distinctions (about 3โ4 points lower on AJIMEE and the homophone set).
|
| 116 |
+
- Readings that repeat one sound (ใใใใฎใใใใใใใใงใใใใใ) tend to map every occurrence to the same word.
|
| 117 |
+
- The model copies the left context strongly: a wrong conversion earlier in the context can be repeated.
|
| 118 |
+
|
| 119 |
+
## License and attribution
|
| 120 |
+
|
| 121 |
+
CC BY-SA 4.0. Base model zenz-v3.2-small by Miwa-Keita (Apache-2.0). The training data include Wikipedia
|
| 122 |
+
(CC BY-SA), FineWeb-2 (ODC-By), Aozora Bunko, the National Diet minutes, and the Mozc dictionary (BSD-3-Clause,
|
| 123 |
+
Copyright Google Inc.; contains data derived from IPAdic, Copyright Nara Institute of Science and Technology).
|
config.json
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu_pytorch_tanh",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.0,
|
| 8 |
+
"bos_token_id": 2,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.0,
|
| 11 |
+
"eos_token_id": 3,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 9.999999747378752e-06,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_ctx": 1024,
|
| 16 |
+
"n_embd": 768,
|
| 17 |
+
"n_head": 12,
|
| 18 |
+
"n_inner": null,
|
| 19 |
+
"n_layer": 12,
|
| 20 |
+
"n_positions": 1024,
|
| 21 |
+
"pad_token_id": 1,
|
| 22 |
+
"reorder_and_upcast_attn": false,
|
| 23 |
+
"resid_pdrop": 0.0,
|
| 24 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 25 |
+
"scale_attn_weights": true,
|
| 26 |
+
"summary_activation": null,
|
| 27 |
+
"summary_first_dropout": 0.1,
|
| 28 |
+
"summary_proj_to_labels": true,
|
| 29 |
+
"summary_type": "cls_index",
|
| 30 |
+
"summary_use_proj": true,
|
| 31 |
+
"tie_word_embeddings": true,
|
| 32 |
+
"transformers_version": "5.20.0",
|
| 33 |
+
"use_cache": false,
|
| 34 |
+
"vocab_size": 6000,
|
| 35 |
+
"genmai_prompt_format": "zenzlc"
|
| 36 |
+
}
|
model-coreml.zip
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b1fa077ee4e3fce5a5c80c5fa851aa736e6d4934dc3eec378bf04d86149b2324
|
| 3 |
+
size 82220338
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:537a4f57d815e7cee8aa179b7196683597469d216a0686ac67a78d197d4a05b1
|
| 3 |
+
size 361816704
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"bos_token": "<s>",
|
| 5 |
+
"clean_up_tokenization_spaces": true,
|
| 6 |
+
"eos_token": "</s>",
|
| 7 |
+
"errors": "replace",
|
| 8 |
+
"is_local": true,
|
| 9 |
+
"local_files_only": false,
|
| 10 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 11 |
+
"pad_token": "[PAD]",
|
| 12 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 13 |
+
"unk_token": "[UNK]"
|
| 14 |
+
}
|