anudit commited on
Commit
c394b65
·
verified ·
1 Parent(s): 877e203

Release tinyemo model and inference code

Browse files
Files changed (17) hide show
  1. .gitattributes +1 -33
  2. README.md +86 -0
  3. THIRD_PARTY_NOTICES.md +66 -0
  4. __init__.py +0 -0
  5. config.json +832 -0
  6. en_features.py +69 -0
  7. export_release.py +38 -0
  8. model.onnx +3 -0
  9. model.py +141 -0
  10. model.safetensors +3 -0
  11. normalize.py +36 -0
  12. quant.py +41 -0
  13. results_v2.json +323 -0
  14. runtime.py +167 -0
  15. tiny_emo.bin +3 -0
  16. tiny_tokenizer.bin +3 -0
  17. tiny_tokenizer.py +78 -0
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
  *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
  *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  *.onnx filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
2
  *.safetensors filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
README.md ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ library_name: pytorch
5
+ tags:
6
+ - emoji
7
+ - text-classification
8
+ - onnx
9
+ - safetensors
10
+ base_model: desert-ant-labs/emo
11
+ license: other
12
+ license_name: desert-ant-labs-source-available
13
+ license_link: https://license.desertant.com/
14
+ ---
15
+
16
+ # tinyemo
17
+
18
+ Tinyemo is an English emoji suggestion model distilled from [Emo](https://huggingface.co/desert-ant-labs/emo). It predicts one of 812 emoji labels from a short phrase. It normalizes text to ASCII letters and digits and uses the last four words; it is intended for short input, not general sentiment analysis.
19
+
20
+ The evaluated distribution is `tiny_emo.bin` and `tiny_tokenizer.bin`, together **495,502 bytes**. `model.onnx` and `model.safetensors` contain the same quantized-then-dequantized weights in float32, so they are larger; they are not int4/int8 files. They output logits, not text or emoji. The model uses custom preprocessing and is **not** a Transformers `AutoModel` or `AutoTokenizer`.
21
+
22
+ ## Usage
23
+
24
+ Download the files into a directory named `tinyemo`, then run Python from its parent directory. The reference runtime requires NumPy:
25
+
26
+ ```python
27
+ from tinyemo.runtime import TinyEmoRuntime
28
+
29
+ model = TinyEmoRuntime("tinyemo/tiny_emo.bin")
30
+ print(model.suggest("walk the dog", 3))
31
+ ```
32
+
33
+ `suggest` returns `(emoji, probability)` pairs, defaults to one result, clamps the requested count to 1–3, and returns an empty list for input that normalizes to empty. Probabilities are softmax scores, not calibrated confidence estimates.
34
+
35
+ ### ONNX Runtime
36
+
37
+ Install `numpy` and `onnxruntime`. The supplied runtime provides the exact tokenizer, normalization and hashed n-gram preprocessing. `config.json` contains the label order and architecture settings.
38
+
39
+ ```python
40
+ import numpy as np
41
+ import onnxruntime as ort
42
+ from tinyemo.runtime import TinyEmoRuntime
43
+
44
+ pre = TinyEmoRuntime("tinyemo/tiny_emo.bin")
45
+ session = ort.InferenceSession("tinyemo/model.onnx", providers=["CPUExecutionProvider"])
46
+ text = "walk the dog"
47
+ ids, mask, buckets, signs = pre.encode_batch([text])
48
+ count = (signs[:, :, 0] != 0).sum(axis=1).astype(np.float32)
49
+ logits = session.run(None, {
50
+ "tok_ids": ids, "tok_mask": mask, "ng_bucket": buckets,
51
+ "ng_sign": signs.astype(np.float32), "ng_count": count,
52
+ })[0]
53
+ print(pre.labels[int(logits[0].argmax())])
54
+ ```
55
+
56
+ The ONNX graph takes a dynamic batch dimension, with token arrays `[batch, 16]`, n-gram arrays `[batch, 96, 2]`, and `ng_count` `[batch]`. `tok_ids` and `ng_bucket` are int64, `tok_mask` is bool, and `ng_sign` and `ng_count` are float32. Preprocessing is outside the graph.
57
+
58
+ ### PyTorch and safetensors
59
+
60
+ Install `torch`, `safetensors`, and `numpy`:
61
+
62
+ ```python
63
+ import json
64
+ from safetensors.torch import load_file
65
+ from tinyemo.model import TinyEmo, TinyEmoConfig
66
+
67
+ with open("tinyemo/config.json", encoding="utf-8") as f:
68
+ config = json.load(f)
69
+ model = TinyEmo(TinyEmoConfig(**config["config"]))
70
+ model.load_state_dict(load_file("tinyemo/model.safetensors"))
71
+ model.eval()
72
+ ```
73
+
74
+ Use the same five preprocessed inputs shown above, converted to PyTorch tensors, to obtain logits. The Python implementation is included in this repository. `export_release.py` rebuilds both formats from the TEM2 file; it requires PyTorch and ONNX.
75
+
76
+ ## Evaluation and limitations
77
+
78
+ The original quantized runtime scored 87.7% top-1 (95% bootstrap interval 84.5–90.9%) and 97.1% top-3 (95.2–98.7%) on 375 hand-labelled phrases with accepted alternatives. Emo scored 88.8% and 97.1%, respectively. The paired top-1 difference was −1.1 percentage points (95% interval −4.3 to +2.1); this interval does not establish equivalence. Model selection used held-out training-pool agreement, while gold was logged during training.
79
+
80
+ On 6,000 held-out snippets from GoEmotions, DailyDialog and tweet_eval, top-1 agreement with the teacher was 62.2%; that measures agreement, not accuracy. Rare nouns such as “koala” remain a weakness. The gold set is small and author-labelled; these results may not generalize to other inputs. Earlier v1 results used a different test set.
81
+
82
+ Training data included short snippets from GoEmotions, DailyDialog, tweet_eval, SST-2, emotion, Tatoeba and Unicode CLDR, plus contexts built from the teacher's vocabulary. The gold phrases were excluded from the training pool, although constituent words could appear elsewhere. The model was trained on teacher predictions; no LLM was used for teacher labelling.
83
+
84
+ ## License and attribution
85
+
86
+ Tinyemo was distilled from Emo, which is distributed under the [Desert Ant Labs Source-Available License](https://license.desertant.com/). Review those terms and the licenses of the source datasets before distribution or commercial use. The `other` metadata records the teacher's license and does not assert additional rights for this derivative. See `THIRD_PARTY_NOTICES.md` for upstream notices. Training data is not included in this release folder.
THIRD_PARTY_NOTICES.md ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Third-party notices: Emo
2
+
3
+ Emo's weights build on the work below. Each license applies to its own work, and the
4
+ Desert Ant Labs Source-Available License doesn't override it. Training data isn't redistributed here.
5
+
6
+ ## Models
7
+
8
+ - **potion-multilingual-128M**, MinishLab. MIT License. Emo's embedding and tokenizer derive from it, reduced and pruned.
9
+ https://huggingface.co/minishlab/potion-multilingual-128M
10
+ - **bge-m3**, BAAI. MIT License. potion-multilingual-128M was distilled from it.
11
+ https://huggingface.co/BAAI/bge-m3
12
+
13
+ ## Training data
14
+
15
+ - **Unicode CLDR**, Unicode, Inc.. Unicode License Agreement for Data Files and Software (2016). Emoji annotations in many languages.
16
+ https://cldr.unicode.org
17
+
18
+ ## License texts
19
+
20
+ ### MIT License
21
+
22
+ ```text
23
+ MIT License
24
+
25
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
26
+ associated documentation files (the "Software"), to deal in the Software without restriction, including
27
+ without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
28
+ copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
29
+ following conditions:
30
+
31
+ The above copyright notice and this permission notice shall be included in all copies or substantial
32
+ portions of the Software.
33
+
34
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
35
+ LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
36
+ EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
37
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
38
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
39
+ ```
40
+
41
+ ### Unicode License Agreement for Data Files and Software (2016)
42
+
43
+ ```text
44
+ UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
45
+
46
+ Unicode Data Files include all data files under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and http://www.unicode.org/utility/trac/browser/.
47
+
48
+ Unicode Data Files do not include PDF online code charts under the directory http://www.unicode.org/Public/.
49
+
50
+ Software includes any source code published in the Unicode Standard or under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and http://www.unicode.org/utility/trac/browser/.
51
+
52
+ NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
53
+
54
+ COPYRIGHT AND PERMISSION NOTICE
55
+
56
+ Copyright © 1991-2016 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in http://www.unicode.org/copyright.html.
57
+
58
+ Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that either
59
+
60
+ (a) this copyright and permission notice appear with all copies of the Data Files or Software, or
61
+ (b) this copyright and permission notice appear in associated Documentation.
62
+
63
+ THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE.
64
+
65
+ Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder.
66
+ ```
__init__.py ADDED
File without changes
config.json ADDED
@@ -0,0 +1,832 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "tinyemo",
3
+ "architecture": "TinyEmo",
4
+ "config": {
5
+ "vocab": 2048,
6
+ "n_buckets": 16384,
7
+ "ng_dim": 16,
8
+ "tok_dim": 48,
9
+ "d_model": 128,
10
+ "n_layers": 1,
11
+ "n_heads": 4,
12
+ "ffn": 256,
13
+ "head_hidden": 128,
14
+ "n_labels": 812,
15
+ "max_tokens": 16,
16
+ "max_features": 96
17
+ },
18
+ "labels": [
19
+ "↩️",
20
+ "⌚",
21
+ "⌨️",
22
+ "⏰",
23
+ "⏱️",
24
+ "⏲️",
25
+ "⏳",
26
+ "⏸️",
27
+ "☀️",
28
+ "☂️",
29
+ "☎️",
30
+ "☔",
31
+ "☕",
32
+ "♌",
33
+ "♟️",
34
+ "♻️",
35
+ "⚓",
36
+ "⚖️",
37
+ "⚙️",
38
+ "⚠️",
39
+ "⚡",
40
+ "⚰️",
41
+ "⚽",
42
+ "⚾",
43
+ "⛈️",
44
+ "⛪",
45
+ "⛰️",
46
+ "⛳",
47
+ "⛵",
48
+ "⛸️",
49
+ "⛺",
50
+ "⛽",
51
+ "✂️",
52
+ "✅",
53
+ "✈️",
54
+ "✉️",
55
+ "✍️",
56
+ "✏️",
57
+ "✝️",
58
+ "✨",
59
+ "❄️",
60
+ "❌",
61
+ "❓",
62
+ "❤️",
63
+ "➕",
64
+ "⭐",
65
+ "🅿️",
66
+ "🆔",
67
+ "🆕",
68
+ "🆘",
69
+ "🇺🇸",
70
+ "🌀",
71
+ "🌃",
72
+ "🌅",
73
+ "🌈",
74
+ "🌊",
75
+ "🌌",
76
+ "🌍",
77
+ "🌐",
78
+ "🌕",
79
+ "🌙",
80
+ "🌠",
81
+ "🌡️",
82
+ "🌧️",
83
+ "🌪️",
84
+ "🌬️",
85
+ "🌮",
86
+ "🌯",
87
+ "🌰",
88
+ "🌱",
89
+ "🌳",
90
+ "🌵",
91
+ "🌶️",
92
+ "🌷",
93
+ "🌸",
94
+ "🌹",
95
+ "🌻",
96
+ "🌼",
97
+ "🌽",
98
+ "🌾",
99
+ "🌿",
100
+ "🍀",
101
+ "🍂",
102
+ "🍃",
103
+ "🍄",
104
+ "🍅",
105
+ "🍇",
106
+ "🍉",
107
+ "🍊",
108
+ "🍌",
109
+ "🍎",
110
+ "🍓",
111
+ "🍔",
112
+ "🍕",
113
+ "🍖",
114
+ "🍗",
115
+ "🍚",
116
+ "🍛",
117
+ "🍜",
118
+ "🍝",
119
+ "🍞",
120
+ "🍣",
121
+ "🍤",
122
+ "🍦",
123
+ "🍩",
124
+ "🍪",
125
+ "🍫",
126
+ "🍬",
127
+ "🍯",
128
+ "🍰",
129
+ "🍱",
130
+ "🍲",
131
+ "🍳",
132
+ "🍵",
133
+ "🍶",
134
+ "🍷",
135
+ "🍼",
136
+ "🍽️",
137
+ "🍾",
138
+ "🍿",
139
+ "🎀",
140
+ "🎁",
141
+ "🎂",
142
+ "🎃",
143
+ "🎄",
144
+ "🎆",
145
+ "🎈",
146
+ "🎉",
147
+ "🎊",
148
+ "🎒",
149
+ "🎓",
150
+ "🎙️",
151
+ "🎞️",
152
+ "🎟️",
153
+ "🎣",
154
+ "🎤",
155
+ "🎥",
156
+ "🎧",
157
+ "🎨",
158
+ "🎩",
159
+ "🎪",
160
+ "🎬",
161
+ "🎭",
162
+ "🎮",
163
+ "🎯",
164
+ "🎲",
165
+ "🎳",
166
+ "🎵",
167
+ "🎷",
168
+ "🎸",
169
+ "🎺",
170
+ "🎾",
171
+ "🎿",
172
+ "🏀",
173
+ "🏁",
174
+ "🏃",
175
+ "🏆",
176
+ "🏈",
177
+ "🏊",
178
+ "🏋️",
179
+ "🏌️",
180
+ "🏐",
181
+ "🏒",
182
+ "🏓",
183
+ "🏕️",
184
+ "🏖️",
185
+ "🏘️",
186
+ "🏙️",
187
+ "🏛️",
188
+ "🏜️",
189
+ "🏞️",
190
+ "🏠",
191
+ "🏢",
192
+ "🏥",
193
+ "🏦",
194
+ "🏧",
195
+ "🏨",
196
+ "🏪",
197
+ "🏫",
198
+ "🏷️",
199
+ "🏺",
200
+ "🐇",
201
+ "🐈",
202
+ "🐌",
203
+ "🐍",
204
+ "🐔",
205
+ "🐕",
206
+ "🐘",
207
+ "🐙",
208
+ "🐛",
209
+ "🐜",
210
+ "🐝",
211
+ "🐞",
212
+ "🐠",
213
+ "🐢",
214
+ "🐦",
215
+ "🐧",
216
+ "🐩",
217
+ "🐬",
218
+ "🐭",
219
+ "🐳",
220
+ "🐷",
221
+ "🐸",
222
+ "🐻",
223
+ "🐼",
224
+ "🐾",
225
+ "👀",
226
+ "👁️",
227
+ "👂",
228
+ "👋",
229
+ "👓",
230
+ "👔",
231
+ "👕",
232
+ "👗",
233
+ "👜",
234
+ "👟",
235
+ "👢",
236
+ "👣",
237
+ "👥",
238
+ "👫",
239
+ "👰",
240
+ "👶",
241
+ "👻",
242
+ "👽",
243
+ "💀",
244
+ "💃",
245
+ "💄",
246
+ "💅",
247
+ "💆",
248
+ "💇",
249
+ "💈",
250
+ "💉",
251
+ "💊",
252
+ "💌",
253
+ "💍",
254
+ "💎",
255
+ "💐",
256
+ "💑",
257
+ "💒",
258
+ "💡",
259
+ "💦",
260
+ "💧",
261
+ "💨",
262
+ "💪",
263
+ "💬",
264
+ "💭",
265
+ "💰",
266
+ "💳",
267
+ "💻",
268
+ "💼",
269
+ "💾",
270
+ "💿",
271
+ "📀",
272
+ "📁",
273
+ "📄",
274
+ "📅",
275
+ "📇",
276
+ "📈",
277
+ "📉",
278
+ "📊",
279
+ "📋",
280
+ "📌",
281
+ "📍",
282
+ "📎",
283
+ "📏",
284
+ "📐",
285
+ "📑",
286
+ "📒",
287
+ "📓",
288
+ "📔",
289
+ "📚",
290
+ "📜",
291
+ "📝",
292
+ "📞",
293
+ "📡",
294
+ "📢",
295
+ "📦",
296
+ "📬",
297
+ "📮",
298
+ "📰",
299
+ "📱",
300
+ "📶",
301
+ "📷",
302
+ "📸",
303
+ "📹",
304
+ "📺",
305
+ "📻",
306
+ "📿",
307
+ "🔁",
308
+ "🔄",
309
+ "🔊",
310
+ "🔋",
311
+ "🔌",
312
+ "🔍",
313
+ "🔑",
314
+ "🔒",
315
+ "🔔",
316
+ "🔗",
317
+ "🔢",
318
+ "🔥",
319
+ "🔦",
320
+ "🔧",
321
+ "🔨",
322
+ "🔩",
323
+ "🔪",
324
+ "🔬",
325
+ "🔭",
326
+ "🔮",
327
+ "🕊️",
328
+ "🕌",
329
+ "🕐",
330
+ "🕒",
331
+ "🕯️",
332
+ "🕷️",
333
+ "🕸️",
334
+ "🖋️",
335
+ "🖌️",
336
+ "🖥️",
337
+ "🖨️",
338
+ "🖱️",
339
+ "🗂️",
340
+ "🗃️",
341
+ "🗄️",
342
+ "🗑️",
343
+ "🗝️",
344
+ "🗣️",
345
+ "🗺️",
346
+ "🗽",
347
+ "😊",
348
+ "😋",
349
+ "😌",
350
+ "😔",
351
+ "😠",
352
+ "😢",
353
+ "😤",
354
+ "😫",
355
+ "😱",
356
+ "😴",
357
+ "😷",
358
+ "🙏",
359
+ "🚀",
360
+ "🚂",
361
+ "🚄",
362
+ "🚆",
363
+ "🚇",
364
+ "🚉",
365
+ "🚌",
366
+ "🚐",
367
+ "🚕",
368
+ "🚗",
369
+ "🚚",
370
+ "🚜",
371
+ "🚢",
372
+ "🚣",
373
+ "🚤",
374
+ "🚦",
375
+ "🚨",
376
+ "🚪",
377
+ "🚫",
378
+ "🚰",
379
+ "🚲",
380
+ "🚶",
381
+ "🚽",
382
+ "🚿",
383
+ "🛀",
384
+ "🛁",
385
+ "🛂",
386
+ "🛋️",
387
+ "🛍️",
388
+ "🛏️",
389
+ "🛒",
390
+ "🛗",
391
+ "🛠️",
392
+ "🛡️",
393
+ "🛣️",
394
+ "🛫",
395
+ "🛶",
396
+ "🛸",
397
+ "🤒",
398
+ "🤔",
399
+ "🤕",
400
+ "🤖",
401
+ "🤝",
402
+ "🤧",
403
+ "🤫",
404
+ "🥁",
405
+ "🥂",
406
+ "🥊",
407
+ "🥋",
408
+ "🥐",
409
+ "🥑",
410
+ "🥒",
411
+ "🥓",
412
+ "🥔",
413
+ "🥕",
414
+ "🥖",
415
+ "🥗",
416
+ "🥘",
417
+ "🥚",
418
+ "🥛",
419
+ "🥞",
420
+ "🥡",
421
+ "🥢",
422
+ "🥣",
423
+ "🥤",
424
+ "🥦",
425
+ "🥧",
426
+ "🥩",
427
+ "🥪",
428
+ "🥫",
429
+ "🥬",
430
+ "🥭",
431
+ "🥶",
432
+ "🥾",
433
+ "🦁",
434
+ "🦃",
435
+ "🦄",
436
+ "🦅",
437
+ "🦈",
438
+ "🦉",
439
+ "🦊",
440
+ "🦋",
441
+ "🦌",
442
+ "🦎",
443
+ "🦐",
444
+ "🦒",
445
+ "🦜",
446
+ "🦞",
447
+ "🦠",
448
+ "🦵",
449
+ "🦶",
450
+ "🦷",
451
+ "🧀",
452
+ "🧁",
453
+ "🧂",
454
+ "🧃",
455
+ "🧅",
456
+ "🧈",
457
+ "🧊",
458
+ "🧑",
459
+ "🧑‍⚕️",
460
+ "🧑‍🍳",
461
+ "🧑‍🏫",
462
+ "🧑‍💻",
463
+ "🧑‍💼",
464
+ "🧑‍🔧",
465
+ "🧑‍🧑‍🧒‍🧒",
466
+ "🧑‍🧒",
467
+ "🧒",
468
+ "🧓",
469
+ "🧖",
470
+ "🧗",
471
+ "🧘",
472
+ "🧙",
473
+ "🧠",
474
+ "🧢",
475
+ "🧣",
476
+ "🧤",
477
+ "🧥",
478
+ "🧦",
479
+ "🧩",
480
+ "🧪",
481
+ "🧬",
482
+ "🧭",
483
+ "🧮",
484
+ "🧯",
485
+ "🧰",
486
+ "🧱",
487
+ "🧳",
488
+ "🧴",
489
+ "🧵",
490
+ "🧶",
491
+ "🧸",
492
+ "🧹",
493
+ "🧺",
494
+ "🧻",
495
+ "🧼",
496
+ "🧽",
497
+ "🧾",
498
+ "🩸",
499
+ "🩹",
500
+ "🩺",
501
+ "🩻",
502
+ "🪁",
503
+ "🪄",
504
+ "🪐",
505
+ "🪑",
506
+ "🪒",
507
+ "🪓",
508
+ "🪟",
509
+ "🪡",
510
+ "🪥",
511
+ "🪪",
512
+ "🪴",
513
+ "🪵",
514
+ "🫄",
515
+ "🫒",
516
+ "🫖",
517
+ "🫘",
518
+ "🫙",
519
+ "☃️",
520
+ "☄️",
521
+ "☘️",
522
+ "♨️",
523
+ "⚔️",
524
+ "⚗️",
525
+ "⚱️",
526
+ "⛏️",
527
+ "⛑️",
528
+ "⛓️",
529
+ "⛓️‍💥",
530
+ "⛩️",
531
+ "⛲",
532
+ "⛴️",
533
+ "✌️",
534
+ "✒️",
535
+ "🌁",
536
+ "🌇",
537
+ "🌉",
538
+ "🌋",
539
+ "🌥️",
540
+ "🌨️",
541
+ "🌫️",
542
+ "🌭",
543
+ "🌴",
544
+ "🌺",
545
+ "🍁",
546
+ "🍄‍🟫",
547
+ "🍆",
548
+ "🍈",
549
+ "🍋",
550
+ "🍋‍🟩",
551
+ "🍍",
552
+ "🍐",
553
+ "🍑",
554
+ "🍒",
555
+ "🍘",
556
+ "🍙",
557
+ "🍟",
558
+ "🍠",
559
+ "🍡",
560
+ "🍢",
561
+ "🍥",
562
+ "🍧",
563
+ "🍭",
564
+ "🍮",
565
+ "🎇",
566
+ "🎐",
567
+ "🎖️",
568
+ "🎠",
569
+ "🎡",
570
+ "🎢",
571
+ "🎰",
572
+ "🎱",
573
+ "🎻",
574
+ "🎽",
575
+ "🏅",
576
+ "🏉",
577
+ "🏍️",
578
+ "🏎️",
579
+ "🏏",
580
+ "🏑",
581
+ "🏗️",
582
+ "🏝️",
583
+ "🏟️",
584
+ "🏤",
585
+ "🏩",
586
+ "🏬",
587
+ "🏭",
588
+ "🏮",
589
+ "🏰",
590
+ "🏸",
591
+ "🏹",
592
+ "🐀",
593
+ "🐂",
594
+ "🐃",
595
+ "🐄",
596
+ "🐅",
597
+ "🐆",
598
+ "🐊",
599
+ "🐎",
600
+ "🐏",
601
+ "🐐",
602
+ "🐑",
603
+ "🐒",
604
+ "🐓",
605
+ "🐗",
606
+ "🐚",
607
+ "🐡",
608
+ "🐣",
609
+ "🐤",
610
+ "🐨",
611
+ "🐪",
612
+ "🐲",
613
+ "🐺",
614
+ "🐿️",
615
+ "👃",
616
+ "👄",
617
+ "👅",
618
+ "👑",
619
+ "👒",
620
+ "👖",
621
+ "👘",
622
+ "👙",
623
+ "👝",
624
+ "👠",
625
+ "👨‍✈️",
626
+ "💂",
627
+ "💋",
628
+ "💣",
629
+ "💤",
630
+ "💥",
631
+ "💮",
632
+ "💴",
633
+ "💶",
634
+ "💷",
635
+ "💽",
636
+ "📕",
637
+ "📗",
638
+ "📙",
639
+ "📟",
640
+ "📠",
641
+ "📣",
642
+ "📼",
643
+ "📽️",
644
+ "🔖",
645
+ "🔫",
646
+ "🕋",
647
+ "🕍",
648
+ "🕎",
649
+ "🕰️",
650
+ "🕳️",
651
+ "🕵️",
652
+ "🕹️",
653
+ "🖇️",
654
+ "🖊️",
655
+ "🖍️",
656
+ "🖲️",
657
+ "🗒️",
658
+ "🗜️",
659
+ "🗡️",
660
+ "🗳️",
661
+ "🗼",
662
+ "🗿",
663
+ "🚁",
664
+ "🚃",
665
+ "🚈",
666
+ "🚊",
667
+ "🚎",
668
+ "🚏",
669
+ "🚑",
670
+ "🚒",
671
+ "🚓",
672
+ "🚙",
673
+ "🚛",
674
+ "🚝",
675
+ "🚞",
676
+ "🚟",
677
+ "🚠",
678
+ "🚡",
679
+ "🚧",
680
+ "🚬",
681
+ "🛎️",
682
+ "🛕",
683
+ "🛖",
684
+ "🛝",
685
+ "🛞",
686
+ "🛟",
687
+ "🛢️",
688
+ "🛤️",
689
+ "🛥️",
690
+ "🛰️",
691
+ "🛳️",
692
+ "🛴",
693
+ "🛵",
694
+ "🛷",
695
+ "🛹",
696
+ "🛺",
697
+ "🛼",
698
+ "🤵",
699
+ "🤿",
700
+ "🥀",
701
+ "🥄",
702
+ "🥅",
703
+ "🥌",
704
+ "🥍",
705
+ "🥎",
706
+ "🥏",
707
+ "🥙",
708
+ "🥜",
709
+ "🥝",
710
+ "🥟",
711
+ "🥠",
712
+ "🥥",
713
+ "🥨",
714
+ "🥮",
715
+ "🥯",
716
+ "🥷",
717
+ "🥻",
718
+ "🥽",
719
+ "🦀",
720
+ "🦂",
721
+ "🦆",
722
+ "🦇",
723
+ "🦍",
724
+ "🦏",
725
+ "🦑",
726
+ "🦓",
727
+ "🦔",
728
+ "🦗",
729
+ "🦘",
730
+ "🦙",
731
+ "🦚",
732
+ "🦛",
733
+ "🦝",
734
+ "🦟",
735
+ "🦡",
736
+ "🦢",
737
+ "🦥",
738
+ "🦦",
739
+ "🦧",
740
+ "🦨",
741
+ "🦩",
742
+ "🦪",
743
+ "🦫",
744
+ "🦬",
745
+ "🦭",
746
+ "🦯",
747
+ "🦴",
748
+ "🦺",
749
+ "🦼",
750
+ "🦾",
751
+ "🦿",
752
+ "🧄",
753
+ "🧆",
754
+ "🧇",
755
+ "🧉",
756
+ "🧋",
757
+ "🧑‍🎓",
758
+ "🧑‍🎤",
759
+ "🧑‍🎨",
760
+ "🧑‍🏭",
761
+ "🧑‍🔬",
762
+ "🧑‍🚀",
763
+ "🧑‍🚒",
764
+ "🧕",
765
+ "🧧",
766
+ "🧨",
767
+ "🧫",
768
+ "🧲",
769
+ "🧷",
770
+ "🩰",
771
+ "🩱",
772
+ "🩲",
773
+ "🩳",
774
+ "🩴",
775
+ "🩼",
776
+ "🪀",
777
+ "🪂",
778
+ "🪃",
779
+ "🪅",
780
+ "🪆",
781
+ "🪇",
782
+ "🪈",
783
+ "🪉",
784
+ "🪊",
785
+ "🪎",
786
+ "🪏",
787
+ "🪔",
788
+ "🪕",
789
+ "🪗",
790
+ "🪙",
791
+ "🪚",
792
+ "🪛",
793
+ "🪜",
794
+ "🪝",
795
+ "🪞",
796
+ "🪠",
797
+ "🪢",
798
+ "🪣",
799
+ "🪤",
800
+ "🪦",
801
+ "🪧",
802
+ "🪨",
803
+ "🪩",
804
+ "🪭",
805
+ "🪮",
806
+ "🪰",
807
+ "🪱",
808
+ "🪳",
809
+ "🪶",
810
+ "🪷",
811
+ "🪸",
812
+ "🪹",
813
+ "🪻",
814
+ "🪼",
815
+ "🪽",
816
+ "🪿",
817
+ "🫀",
818
+ "🫁",
819
+ "🫍",
820
+ "🫎",
821
+ "🫏",
822
+ "🫐",
823
+ "🫓",
824
+ "🫔",
825
+ "🫕",
826
+ "🫚",
827
+ "🫛",
828
+ "🫜",
829
+ "🫦",
830
+ "🫧"
831
+ ]
832
+ }
en_features.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """English-only hashed n-gram features for the student (input is `normalize`d).
2
+
3
+ Per word: the whole word, plus 3..5-char grams with boundary markers. Between
4
+ neighbouring words: a word bigram. Each feature is hashed twice (FNV-1a 64);
5
+ the bucket comes from bits 20..34 so it stays well mixed for any power-of-two
6
+ bucket count <= 32768, and the sign from bit 63.
7
+
8
+ `hash16` returns the 15-bit raw bucket (`bucket = raw & (n_buckets - 1)`) and the
9
+ sign so the prepared dataset does not depend on the final bucket count.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ FNV_OFFSET = 0xCBF29CE484222325
15
+ FNV_PRIME = 0x100000001B3
16
+ MASK64 = (1 << 64) - 1
17
+ SEEDS = (0x9E3779B97F4A7C15, 0xC2B2AE3D27D4EB4F)
18
+ N_HASHES = len(SEEDS)
19
+ RAW_BITS = 15
20
+
21
+
22
+ def fnv64(s: str, seed: int) -> int:
23
+ h = (FNV_OFFSET ^ seed) & MASK64
24
+ for b in s.encode("utf-8"):
25
+ h = ((h ^ b) * FNV_PRIME) & MASK64
26
+ return h
27
+
28
+
29
+ def features(norm_text: str) -> list[str]:
30
+ words = norm_text.split()
31
+ out: list[str] = []
32
+ for w in words:
33
+ out.append("w:" + w)
34
+ s = "<" + w + ">"
35
+ for n in (3, 4, 5):
36
+ for i in range(len(s) - n + 1):
37
+ out.append("g:" + s[i : i + n])
38
+ for a, b in zip(words, words[1:]):
39
+ out.append("b:" + a + " " + b)
40
+ return out or ["w:\x00"]
41
+
42
+
43
+ _CACHE: dict[str, tuple[tuple[int, int], tuple[int, int]]] = {}
44
+
45
+
46
+ def hash16(feat: str) -> tuple[tuple[int, int], tuple[int, int]]:
47
+ """((raw0, sign0), (raw1, sign1)) with raw in [0, 2**15)."""
48
+ r = _CACHE.get(feat)
49
+ if r is None:
50
+ hs = [fnv64(feat, sd) for sd in SEEDS]
51
+ r = tuple(
52
+ (((h >> 20) & ((1 << RAW_BITS) - 1)), 1 if (h >> 63) & 1 else -1) for h in hs
53
+ ) # type: ignore[assignment]
54
+ _CACHE[feat] = r
55
+ return r
56
+
57
+
58
+ def featurize(norm_text: str, max_features: int):
59
+ """-> (raw[n,2] int16, sign[n,2] int8) for the first `max_features` features."""
60
+ import numpy as np
61
+
62
+ fs = features(norm_text)[:max_features]
63
+ raw = np.zeros((len(fs), N_HASHES), np.int16)
64
+ sign = np.zeros((len(fs), N_HASHES), np.int8)
65
+ for i, f in enumerate(fs):
66
+ for k, (r, s) in enumerate(hash16(f)):
67
+ raw[i, k] = r
68
+ sign[i, k] = s
69
+ return raw, sign
export_release.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Rebuild standard exports from the evaluated TEM2 artifact."""
2
+ import json
3
+ import struct
4
+ from pathlib import Path
5
+
6
+ import numpy as np
7
+ import torch
8
+
9
+ from tinyemo.model import TinyEmo, TinyEmoConfig
10
+ from tinyemo.runtime import TinyEmoRuntime
11
+
12
+ HERE = Path(__file__).resolve().parent
13
+ rt = TinyEmoRuntime(str(HERE / 'tiny_emo.bin'))
14
+ model = TinyEmo(TinyEmoConfig(**rt.meta))
15
+ model.load_state_dict({k: torch.from_numpy(v.copy()) for k, v in rt.w.items()})
16
+ model.eval()
17
+ (HERE / 'config.json').write_text(json.dumps({'model_type': 'tinyemo', 'architecture': 'TinyEmo', 'config': rt.meta, 'labels': rt.labels}, ensure_ascii=False, indent=2) + '\n')
18
+
19
+ # Safetensors: 8-byte header length, JSON header, then contiguous little-endian data.
20
+ tensors = {k: v.detach().cpu().numpy().astype('<f4') for k, v in model.state_dict().items()}
21
+ header, chunks, offset = {'__metadata__': {'format': 'pt', 'source': 'dequantized TEM2 artifact'}}, [], 0
22
+ for name, arr in tensors.items():
23
+ blob = arr.tobytes(order='C')
24
+ header[name] = {'dtype': 'F32', 'shape': list(arr.shape), 'data_offsets': [offset, offset + len(blob)]}
25
+ chunks.append(blob)
26
+ offset += len(blob)
27
+ h = json.dumps(header, separators=(',', ':')).encode('utf-8')
28
+ h += b' ' * (-len(h) % 8)
29
+ (HERE / 'model.safetensors').write_bytes(struct.pack('<Q', len(h)) + h + b''.join(chunks))
30
+
31
+ ids, mask, buckets, signs = rt.encode_batch(['walk the dog', 'happy birthday'])
32
+ inputs = (torch.from_numpy(ids), torch.from_numpy(mask), torch.from_numpy(buckets),
33
+ torch.from_numpy(signs).float(), torch.from_numpy((signs[:, :, 0] != 0).sum(1)).float())
34
+ torch.onnx.export(model, inputs, str(HERE / 'model.onnx'), opset_version=17,
35
+ input_names=['tok_ids', 'tok_mask', 'ng_bucket', 'ng_sign', 'ng_count'], output_names=['logits'],
36
+ dynamic_axes={name: {0: 'batch'} for name in ['tok_ids', 'tok_mask', 'ng_bucket', 'ng_sign', 'ng_count', 'logits']},
37
+ do_constant_folding=True, dynamo=False)
38
+ print('Exported', HERE)
model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:505c3830435468d192e8bef4e655facc0a5a35c6a4a6fdd68424a3920e3ef241
3
+ size 2580586
model.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input.
2
+
3
+ 1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm
4
+ 2. token stream: int4 token embeddings -> projection -> learned positions ->
5
+ small transformer -> attention pool
6
+
7
+ The streams are concatenated and classified into Emo's 812 emoji labels.
8
+ `set_qat(True)` fake-quantises every weight matrix exactly the way `export.py`
9
+ stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated
10
+ is what ships.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import asdict, dataclass
16
+
17
+ import torch
18
+ import torch.nn as nn
19
+ import torch.nn.functional as F
20
+
21
+ from .quant import fake_quant
22
+
23
+
24
+ @dataclass
25
+ class TinyEmoConfig:
26
+ vocab: int = 2048
27
+ n_buckets: int = 16384 # power of two, <= 32768
28
+ ng_dim: int = 16
29
+ tok_dim: int = 48
30
+ d_model: int = 96
31
+ n_layers: int = 2
32
+ n_heads: int = 2
33
+ ffn: int = 192
34
+ head_hidden: int = 112
35
+ n_labels: int = 812
36
+ max_tokens: int = 16
37
+ max_features: int = 96
38
+ dropout: float = 0.0
39
+
40
+ def to_dict(self) -> dict:
41
+ return asdict(self)
42
+
43
+
44
+ class QLinear(nn.Linear):
45
+ qat = False
46
+
47
+ def forward(self, x):
48
+ w = fake_quant(self.weight, "int8_row") if self.qat else self.weight
49
+ return F.linear(x, w, self.bias)
50
+
51
+
52
+ class QEmbedding(nn.Embedding):
53
+ qat = False
54
+ mode = "int4_row"
55
+
56
+ def forward(self, ids):
57
+ w = fake_quant(self.weight, self.mode) if self.qat else self.weight
58
+ return F.embedding(ids, w)
59
+
60
+
61
+ class TinyEmo(nn.Module):
62
+ def __init__(self, cfg: TinyEmoConfig):
63
+ super().__init__()
64
+ assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two"
65
+ self.cfg = cfg
66
+ self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim)
67
+ self.tok_emb.mode = "int4_row"
68
+ self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False)
69
+ self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02)
70
+ self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim)
71
+ self.ng_emb.mode = "int4_tensor"
72
+ nn.init.normal_(self.ng_emb.weight, std=0.1)
73
+ self.ng_norm = nn.LayerNorm(cfg.ng_dim)
74
+ self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)])
75
+ self.final_norm = nn.LayerNorm(cfg.d_model)
76
+ self.pool_proj = QLinear(cfg.d_model, cfg.d_model)
77
+ self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02)
78
+ self.pool_norm = nn.LayerNorm(cfg.d_model)
79
+ self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden)
80
+ self.head2 = QLinear(cfg.head_hidden, cfg.n_labels)
81
+ self.drop = nn.Dropout(cfg.dropout)
82
+
83
+ def set_qat(self, on: bool) -> None:
84
+ for m in self.modules():
85
+ if isinstance(m, (QLinear, QEmbedding)):
86
+ m.qat = on
87
+
88
+ def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count):
89
+ cfg = self.cfg
90
+ # ---- n-gram stream ----------------------------------------------
91
+ e = self.ng_emb(ng_bucket % cfg.n_buckets) # B,F,H,ng
92
+ e = e * ng_sign.unsqueeze(-1).to(e.dtype)
93
+ ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype)
94
+ ng = self.ng_norm(ng)
95
+
96
+ # ---- token stream -----------------------------------------------
97
+ t = tok_ids.shape[1]
98
+ x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t]
99
+ key_mask = tok_mask.unsqueeze(1).unsqueeze(1) # B,1,1,T
100
+ for blk in self.blocks:
101
+ x = blk(x, key_mask)
102
+ x = self.final_norm(x)
103
+ score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5)
104
+ score = score.masked_fill(~tok_mask, -1e9)
105
+ a = torch.softmax(score, dim=1)
106
+ pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1))
107
+
108
+ z = torch.cat([ng, pooled], dim=-1)
109
+ z = self.drop(F.gelu(self.head1(z)))
110
+ return self.head2(z)
111
+
112
+
113
+ class _Block(nn.Module):
114
+ def __init__(self, cfg: TinyEmoConfig):
115
+ super().__init__()
116
+ d = cfg.d_model
117
+ self.n_heads = cfg.n_heads
118
+ self.norm1 = nn.LayerNorm(d)
119
+ self.qkv = QLinear(d, 3 * d)
120
+ self.proj = QLinear(d, d)
121
+ self.norm2 = nn.LayerNorm(d)
122
+ self.ff1 = QLinear(d, cfg.ffn)
123
+ self.ff2 = QLinear(cfg.ffn, d)
124
+ self.drop = nn.Dropout(cfg.dropout)
125
+
126
+ def forward(self, x, key_mask):
127
+ b, t, d = x.shape
128
+ h = self.norm1(x)
129
+ qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4)
130
+ q, k, v = qkv.unbind(0)
131
+ score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5)
132
+ score = score.masked_fill(~key_mask, -1e9)
133
+ a = self.drop(torch.softmax(score, dim=-1))
134
+ o = (a @ v).transpose(1, 2).reshape(b, t, d)
135
+ x = x + self.drop(self.proj(o))
136
+ h = self.norm2(x)
137
+ return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h)))))
138
+
139
+
140
+ def count_params(model: nn.Module) -> int:
141
+ return sum(p.numel() for p in model.parameters())
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:760e0816343f5eda2e910442d460ffe2113124089b6a2e533d5b6206dc5a47fe
3
+ size 2568904
normalize.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Input normalisation shared by training, evaluation and the runtime.
2
+
3
+ Every text the student ever sees (and every text the teacher is asked to label)
4
+ goes through `normalize`:
5
+
6
+ * NFKD, accents stripped, lower-cased
7
+ * apostrophes inside words removed ("don't" -> "dont")
8
+ * everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...)
9
+ * at most MAX_WORDS words kept - the *last* ones, because the words nearest
10
+ the cursor carry the intent when someone is typing
11
+ * each word capped at MAX_WORD_LEN characters
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import re
17
+ import unicodedata
18
+
19
+ MAX_WORDS = 4
20
+ MAX_WORD_LEN = 20
21
+
22
+ _APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"})
23
+ _INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])")
24
+ _NON_ALNUM = re.compile(r"[^a-z0-9]+")
25
+
26
+
27
+ def words_of(text: str) -> list[str]:
28
+ """Normalised words of `text`, *without* the max-words truncation."""
29
+ t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower()
30
+ t = "".join(c for c in t if not unicodedata.combining(c))
31
+ t = _INNER_APOS.sub("", t)
32
+ return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()]
33
+
34
+
35
+ def normalize(text: str, max_words: int = MAX_WORDS) -> str:
36
+ return " ".join(words_of(text)[-max_words:])
quant.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Quantisation shared by QAT (torch) and the exporter, so both agree exactly.
2
+
3
+ Modes
4
+ int8_row : symmetric int8, one f16 scale per row (linear layers)
5
+ int4_row : symmetric int4 (-7..7), one f16 scale per row (token embeddings)
6
+ int4_tensor: symmetric int4, one f16 scale per tensor, scale = CLIP_STD * std / 7
7
+ (large hashed n-gram table - per-row scales would cost 2 B/row)
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import torch
13
+
14
+ CLIP_STD = 4.0
15
+ MODES = {"int8_row": (8, True), "int4_row": (4, True), "int4_tensor": (4, False)}
16
+
17
+
18
+ def _scale(w: torch.Tensor, mode: str) -> torch.Tensor:
19
+ bits, per_row = MODES[mode]
20
+ qmax = 2 ** (bits - 1) - 1
21
+ if per_row:
22
+ s = w.abs().amax(dim=1, keepdim=True) / qmax
23
+ else:
24
+ s = (w.std() * CLIP_STD / qmax).reshape(1, 1)
25
+ # scales are stored as f16
26
+ return s.clamp(min=1e-5).half().float()
27
+
28
+
29
+ def quantize(w: torch.Tensor, mode: str) -> tuple[torch.Tensor, torch.Tensor]:
30
+ """-> (integer tensor, scale tensor) with `w ~= q * scale`."""
31
+ bits, _ = MODES[mode]
32
+ qmax = 2 ** (bits - 1) - 1
33
+ s = _scale(w, mode)
34
+ q = torch.clamp(torch.round(w / s), -qmax, qmax)
35
+ return q, s
36
+
37
+
38
+ def fake_quant(w: torch.Tensor, mode: str) -> torch.Tensor:
39
+ """Straight-through fake quantisation."""
40
+ q, s = quantize(w.detach(), mode)
41
+ return w + (q * s - w).detach()
results_v2.json ADDED
@@ -0,0 +1,323 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "n_gold": 375,
3
+ "n_heldout": 6000,
4
+ "emo": {
5
+ "size_bytes": 10955933,
6
+ "gold1": [
7
+ 0.888,
8
+ 0.8533333333333334,
9
+ 0.92
10
+ ],
11
+ "gold3": [
12
+ 0.9706666666666667,
13
+ 0.952,
14
+ 0.9866666666666667
15
+ ]
16
+ },
17
+ "tiny": {
18
+ "size_bytes": 495502,
19
+ "gold1": [
20
+ 0.8773333333333333,
21
+ 0.8452666666666667,
22
+ 0.9093333333333333
23
+ ],
24
+ "gold3": [
25
+ 0.9706666666666667,
26
+ 0.952,
27
+ 0.9866666666666667
28
+ ],
29
+ "agree1": [
30
+ 0.6218333333333333,
31
+ 0.6098333333333333,
32
+ 0.6341666666666667
33
+ ],
34
+ "agree3": [
35
+ 0.812,
36
+ 0.8019958333333334,
37
+ 0.822
38
+ ],
39
+ "s1_in_t3": [
40
+ 0.8221666666666667,
41
+ 0.8125,
42
+ 0.8321666666666667
43
+ ]
44
+ },
45
+ "gold_misses": [
46
+ {
47
+ "text": "good night",
48
+ "gold": "😴",
49
+ "student": "✨",
50
+ "emo": "🌙"
51
+ },
52
+ {
53
+ "text": "starry night",
54
+ "gold": "🌌",
55
+ "student": "✨",
56
+ "emo": "✨"
57
+ },
58
+ {
59
+ "text": "got a cold",
60
+ "gold": "🤧",
61
+ "student": "❄️",
62
+ "emo": "🤧"
63
+ },
64
+ {
65
+ "text": "doctor visit",
66
+ "gold": "🩺",
67
+ "student": "🧑‍⚕️",
68
+ "emo": "🧑‍⚕️"
69
+ },
70
+ {
71
+ "text": "headache",
72
+ "gold": "🤕",
73
+ "student": "🧠",
74
+ "emo": "🧠"
75
+ },
76
+ {
77
+ "text": "hot dog",
78
+ "gold": "🌭",
79
+ "student": "🐕",
80
+ "emo": "🐕"
81
+ },
82
+ {
83
+ "text": "lemon juice",
84
+ "gold": "🍋",
85
+ "student": "🧃",
86
+ "emo": "🍋"
87
+ },
88
+ {
89
+ "text": "lollipop",
90
+ "gold": "🍭",
91
+ "student": "🎨",
92
+ "emo": "🍭"
93
+ },
94
+ {
95
+ "text": "hourglass",
96
+ "gold": "⏳",
97
+ "student": "⏰",
98
+ "emo": "🍦"
99
+ },
100
+ {
101
+ "text": "sos",
102
+ "gold": "🆘",
103
+ "student": "✨",
104
+ "emo": "🆘"
105
+ },
106
+ {
107
+ "text": "new feature",
108
+ "gold": "🆕",
109
+ "student": "✨",
110
+ "emo": "✨"
111
+ },
112
+ {
113
+ "text": "stock market",
114
+ "gold": "📈",
115
+ "student": "🛒",
116
+ "emo": "🛒"
117
+ },
118
+ {
119
+ "text": "bar chart",
120
+ "gold": "📊",
121
+ "student": "🍷",
122
+ "emo": "🗺️"
123
+ },
124
+ {
125
+ "text": "coding all day",
126
+ "gold": "🧑‍💻",
127
+ "student": "📅",
128
+ "emo": "💻"
129
+ },
130
+ {
131
+ "text": "darts",
132
+ "gold": "🎯",
133
+ "student": "📅",
134
+ "emo": "🎯"
135
+ },
136
+ {
137
+ "text": "koala",
138
+ "gold": "🐨",
139
+ "student": "🛋️",
140
+ "emo": "🐨"
141
+ },
142
+ {
143
+ "text": "octopus",
144
+ "gold": "🐙",
145
+ "student": "😴",
146
+ "emo": "🐙"
147
+ },
148
+ {
149
+ "text": "ladybug",
150
+ "gold": "🐞",
151
+ "student": "💰",
152
+ "emo": "🐞"
153
+ },
154
+ {
155
+ "text": "ant",
156
+ "gold": "🐜",
157
+ "student": "⚠️",
158
+ "emo": "🐜"
159
+ },
160
+ {
161
+ "text": "owl",
162
+ "gold": "🦉",
163
+ "student": "📚",
164
+ "emo": "🦉"
165
+ },
166
+ {
167
+ "text": "peacock",
168
+ "gold": "🦚",
169
+ "student": "🎨",
170
+ "emo": "🦚"
171
+ },
172
+ {
173
+ "text": "swan",
174
+ "gold": "🦢",
175
+ "student": "✨",
176
+ "emo": "🧒"
177
+ },
178
+ {
179
+ "text": "sunflower",
180
+ "gold": "🌻",
181
+ "student": "☀️",
182
+ "emo": "🌻"
183
+ },
184
+ {
185
+ "text": "plant parent",
186
+ "gold": "🪴",
187
+ "student": "🐦",
188
+ "emo": "🪴"
189
+ },
190
+ {
191
+ "text": "wedding dress",
192
+ "gold": "👰",
193
+ "student": "💒",
194
+ "emo": "👗"
195
+ },
196
+ {
197
+ "text": "post office",
198
+ "gold": "🏤",
199
+ "student": "🏢",
200
+ "emo": "📮"
201
+ },
202
+ {
203
+ "text": "fire truck",
204
+ "gold": "🚒",
205
+ "student": "🚚",
206
+ "emo": "🚚"
207
+ },
208
+ {
209
+ "text": "police car",
210
+ "gold": "🚓",
211
+ "student": "🚗",
212
+ "emo": "🚗"
213
+ },
214
+ {
215
+ "text": "screwdriver",
216
+ "gold": "🪛",
217
+ "student": "🔧",
218
+ "emo": "🔧"
219
+ },
220
+ {
221
+ "text": "magnifying glass",
222
+ "gold": "🔍",
223
+ "student": "✨",
224
+ "emo": "🍦"
225
+ },
226
+ {
227
+ "text": "snowman",
228
+ "gold": "☃️",
229
+ "student": "❄️",
230
+ "emo": "☃️"
231
+ },
232
+ {
233
+ "text": "listen to jazz music",
234
+ "gold": "🎵",
235
+ "student": "🎷",
236
+ "emo": "🎷"
237
+ },
238
+ {
239
+ "text": "cook pasta for dinner",
240
+ "gold": "🍳",
241
+ "student": "🧑‍🍳",
242
+ "emo": "🍝"
243
+ },
244
+ {
245
+ "text": "drink an espresso",
246
+ "gold": "☕",
247
+ "student": "🥤",
248
+ "emo": "☕"
249
+ },
250
+ {
251
+ "text": "water the houseplants",
252
+ "gold": "🌱",
253
+ "student": "💧",
254
+ "emo": "💧"
255
+ },
256
+ {
257
+ "text": "clean the kitchen",
258
+ "gold": "🧹",
259
+ "student": "🧽",
260
+ "emo": "✨"
261
+ },
262
+ {
263
+ "text": "celebrate the promotion",
264
+ "gold": "🎉",
265
+ "student": "🥂",
266
+ "emo": "🥂"
267
+ },
268
+ {
269
+ "text": "meditate for ten minutes",
270
+ "gold": "🧘",
271
+ "student": "⏱️",
272
+ "emo": "🧘"
273
+ },
274
+ {
275
+ "text": "plant some flowers",
276
+ "gold": "🌻",
277
+ "student": "🌷",
278
+ "emo": "🌷"
279
+ },
280
+ {
281
+ "text": "look at the stars",
282
+ "gold": "🌌",
283
+ "student": "✨",
284
+ "emo": "✨"
285
+ },
286
+ {
287
+ "text": "file my taxes",
288
+ "gold": "🧾",
289
+ "student": "📁",
290
+ "emo": "💰"
291
+ },
292
+ {
293
+ "text": "call an ambulance",
294
+ "gold": "🚑",
295
+ "student": "📞",
296
+ "emo": "📞"
297
+ },
298
+ {
299
+ "text": "call the police",
300
+ "gold": "🚓",
301
+ "student": "🚨",
302
+ "emo": "📞"
303
+ },
304
+ {
305
+ "text": "write in my journal",
306
+ "gold": "📓",
307
+ "student": "📔",
308
+ "emo": "📔"
309
+ },
310
+ {
311
+ "text": "go to class",
312
+ "gold": "🏫",
313
+ "student": "📚",
314
+ "emo": "📚"
315
+ },
316
+ {
317
+ "text": "clean the bathroom",
318
+ "gold": "🧽",
319
+ "student": "🚽",
320
+ "emo": "🚽"
321
+ }
322
+ ]
323
+ }
runtime.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Dependency-free (NumPy) inference for the exported TinyEmo artifact.
2
+
3
+ rt = TinyEmoRuntime("tiny_emo/artifacts_v2/tiny_emo.bin")
4
+ rt.suggest("Walk the dog!!") # -> [("🐕", 0.31)] (default: 1)
5
+ rt.suggest("Walk the dog!!", 3) # -> up to 3 (clamped to 1..3)
6
+
7
+ Input is normalised first (lower-case, punctuation/emoji stripped, last 4 words).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import math
14
+ import struct
15
+ from pathlib import Path
16
+
17
+ import numpy as np
18
+
19
+ from .en_features import N_HASHES, featurize
20
+ from .normalize import MAX_WORDS, normalize
21
+ from .tiny_tokenizer import TinyTokenizer
22
+
23
+ MAX_OUTPUT = 3
24
+
25
+
26
+ def _gelu(x: np.ndarray) -> np.ndarray:
27
+ # exact GELU with an A&S 7.1.26 erf (|err| < 1.5e-7)
28
+ z = x / math.sqrt(2.0)
29
+ a = np.abs(z)
30
+ t = 1.0 / (1.0 + 0.3275911 * a)
31
+ poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))))
32
+ erf = np.sign(z) * (1.0 - poly * np.exp(-a * a))
33
+ return 0.5 * x * (1.0 + erf)
34
+
35
+
36
+ def _softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
37
+ x = x - x.max(axis=axis, keepdims=True)
38
+ e = np.exp(x)
39
+ return e / e.sum(axis=axis, keepdims=True)
40
+
41
+
42
+ def _ln(x, w, b, eps=1e-5):
43
+ m = x.mean(-1, keepdims=True)
44
+ v = ((x - m) ** 2).mean(-1, keepdims=True)
45
+ return (x - m) / np.sqrt(v + eps) * w + b
46
+
47
+
48
+ def _unpack_nibbles(buf: bytes, n: int) -> np.ndarray:
49
+ b = np.frombuffer(buf, np.uint8)
50
+ q = np.empty(b.size * 2, np.int8)
51
+ q[0::2] = (b & 0x0F).astype(np.int8) - 8
52
+ q[1::2] = (b >> 4).astype(np.int8) - 8
53
+ return q[:n]
54
+
55
+
56
+ class TinyEmoRuntime:
57
+ def __init__(self, weights_path: str, tokenizer_path: str | None = None):
58
+ data = Path(weights_path).read_bytes()
59
+ if data[:4] != b"TEM2":
60
+ raise ValueError("bad model file")
61
+ (ml,) = struct.unpack_from("<H", data, 4)
62
+ self.meta = json.loads(data[6 : 6 + ml])
63
+ off = 6 + ml
64
+ (ll,) = struct.unpack_from("<I", data, off)
65
+ off += 4
66
+ self.labels = data[off : off + ll].decode("utf-8").split("\n")
67
+ off += ll
68
+ self.w: dict[str, np.ndarray] = {}
69
+ while off < len(data):
70
+ nl = data[off]
71
+ name = data[off + 1 : off + 1 + nl].decode()
72
+ off += 1 + nl
73
+ nd = data[off]
74
+ dims = struct.unpack_from("<%dH" % nd, data, off + 1)
75
+ off += 1 + 2 * nd
76
+ mode = data[off]
77
+ off += 1
78
+ n = int(np.prod(dims))
79
+ if mode == 0:
80
+ arr = np.frombuffer(data, "<f2", n, off).astype(np.float32)
81
+ off += 2 * n
82
+ else:
83
+ ns = dims[0] if mode in (1, 2) else 1
84
+ scales = np.frombuffer(data, "<f2", ns, off).astype(np.float32)
85
+ off += 2 * ns
86
+ if mode == 1:
87
+ q = np.frombuffer(data, np.int8, n, off).astype(np.float32)
88
+ off += n
89
+ else:
90
+ nb = (n + 1) // 2
91
+ q = _unpack_nibbles(data[off : off + nb], n).astype(np.float32)
92
+ off += nb
93
+ arr = q.reshape(dims) * (scales[:, None] if mode in (1, 2) else scales[0])
94
+ self.w[name] = arr.reshape(dims)
95
+ tok = tokenizer_path or str(Path(weights_path).with_name("tiny_tokenizer.bin"))
96
+ self.tok = TinyTokenizer.load(tok)
97
+ m = self.meta
98
+ self.d, self.heads, self.T, self.F = m["d_model"], m["n_heads"], m["max_tokens"], m["max_features"]
99
+
100
+ # ------------------------------------------------------------------
101
+ def encode_batch(self, texts: list[str]):
102
+ """Normalised texts -> (tok_ids[B,T], tok_mask[B,T], ng_raw[B,F,H], ng_sign[B,F,H])."""
103
+ b = len(texts)
104
+ ids = np.zeros((b, self.T), np.int64)
105
+ mask = np.zeros((b, self.T), bool)
106
+ raw = np.zeros((b, self.F, N_HASHES), np.int64)
107
+ sign = np.zeros((b, self.F, N_HASHES), np.int8)
108
+ for i, text in enumerate(texts):
109
+ t = self.tok.encode(normalize(text), self.T)
110
+ ids[i, : len(t)] = t
111
+ mask[i, : len(t)] = True
112
+ r, s = featurize(normalize(text), self.F)
113
+ raw[i, : len(r)] = r
114
+ sign[i, : len(s)] = s
115
+ return ids, mask, raw, sign
116
+
117
+ def _block(self, x, mask, i):
118
+ w, p = self.w, f"blocks.{i}."
119
+ b, t, d = x.shape
120
+ hd = d // self.heads
121
+ h = _ln(x, w[p + "norm1.weight"], w[p + "norm1.bias"])
122
+ qkv = h @ w[p + "qkv.weight"].T + w[p + "qkv.bias"]
123
+ q, k, v = qkv.reshape(b, t, 3, self.heads, hd).transpose(2, 0, 3, 1, 4)
124
+ s = q @ k.transpose(0, 1, 3, 2) / math.sqrt(hd)
125
+ s = np.where(mask[:, None, None, :], s, -1e9)
126
+ o = (_softmax(s) @ v).transpose(0, 2, 1, 3).reshape(b, t, d)
127
+ x = x + o @ w[p + "proj.weight"].T + w[p + "proj.bias"]
128
+ h = _ln(x, w[p + "norm2.weight"], w[p + "norm2.bias"])
129
+ h = _gelu(h @ w[p + "ff1.weight"].T + w[p + "ff1.bias"])
130
+ return x + h @ w[p + "ff2.weight"].T + w[p + "ff2.bias"]
131
+
132
+ def forward(self, ids, mask, raw, sign) -> np.ndarray:
133
+ w, m = self.w, self.meta
134
+ # token stream
135
+ x = w["tok_emb.weight"][ids] @ w["tok_proj.weight"].T + w["pos_emb"][: ids.shape[1]]
136
+ for i in range(m["n_layers"]):
137
+ x = self._block(x, mask, i)
138
+ x = _ln(x, w["final_norm.weight"], w["final_norm.bias"])
139
+ proj = np.tanh(x) @ w["pool_proj.weight"].T + w["pool_proj.bias"]
140
+ score = np.where(mask, proj @ w["pool_q"] / math.sqrt(self.d), -1e9)
141
+ pooled = (_softmax(score, 1)[:, :, None] * x).sum(1)
142
+ pooled = _ln(pooled, w["pool_norm.weight"], w["pool_norm.bias"])
143
+ # n-gram stream
144
+ e = w["ng_emb.weight"][raw % m["n_buckets"]] * sign[..., None] # B,F,H,ng
145
+ count = np.maximum((sign[:, :, 0] != 0).sum(1), 1).astype(np.float32)
146
+ ng = _ln(e.sum((1, 2)) / count[:, None], w["ng_norm.weight"], w["ng_norm.bias"])
147
+ z = _gelu(np.concatenate([ng, pooled], -1) @ w["head1.weight"].T + w["head1.bias"])
148
+ return z @ w["head2.weight"].T + w["head2.bias"]
149
+
150
+ # ------------------------------------------------------------------
151
+ def logits_batch(self, texts: list[str], bs: int = 512) -> np.ndarray:
152
+ out = []
153
+ for i in range(0, len(texts), bs):
154
+ out.append(self.forward(*self.encode_batch(texts[i : i + bs])))
155
+ return np.concatenate(out) if out else np.zeros((0, self.meta["n_labels"]), np.float32)
156
+
157
+ def logits(self, text: str) -> np.ndarray:
158
+ return self.logits_batch([text])[0]
159
+
160
+ def suggest(self, text: str, limit: int = 1) -> list[tuple[str, float]]:
161
+ """Up to `limit` emoji suggestions (1-3, default 1); [] for empty input."""
162
+ limit = max(1, min(MAX_OUTPUT, int(limit)))
163
+ if not normalize(text):
164
+ return []
165
+ p = _softmax(self.logits(text))
166
+ order = np.argsort(-p)[:limit]
167
+ return [(self.labels[i], float(p[i])) for i in order]
tiny_emo.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db9a48131f0ce0fef9a14e9cbe23ef704773657911bc297994703048da101db5
3
+ size 480147
tiny_tokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78efe82d45399faff2e849e4fb68b110b848c0eb07769239ceecfc08e2dc04d6
3
+ size 15355
tiny_tokenizer.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Compact English-only unigram tokenizer ("ETK2").
2
+
3
+ Input is already `normalize`d ([a-z0-9] words separated by single spaces), so no
4
+ byte fallback or unicode handling is needed. Word boundaries are marked with
5
+ `_` (SentencePiece's `▁`).
6
+
7
+ File layout: b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 )
8
+ Piece 0 is <unk>.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import struct
14
+
15
+ MAGIC = b"ETK2"
16
+ SCORE_SCALE = 8.0
17
+ UNK_SCORE = -30.0
18
+
19
+
20
+ class TinyTokenizer:
21
+ def __init__(self, data: bytes):
22
+ if data[:4] != MAGIC:
23
+ raise ValueError("bad tokenizer file")
24
+ (n,) = struct.unpack_from("<H", data, 4)
25
+ off = 6
26
+ self.pieces: list[str] = []
27
+ self.scores: list[float] = []
28
+ for _ in range(n):
29
+ ln = data[off]
30
+ off += 1
31
+ self.pieces.append(data[off : off + ln].decode("ascii"))
32
+ off += ln
33
+ self.scores.append(struct.unpack_from("<b", data, off)[0] / SCORE_SCALE)
34
+ off += 1
35
+ self.index = {p: i for i, p in enumerate(self.pieces) if i > 0}
36
+ self.unk_id = 0
37
+ self.max_len = max(len(p) for p in self.pieces)
38
+
39
+ @classmethod
40
+ def load(cls, path: str) -> "TinyTokenizer":
41
+ with open(path, "rb") as f:
42
+ return cls(f.read())
43
+
44
+ @staticmethod
45
+ def dump(pieces: list[str], scores: list[float]) -> bytes:
46
+ out = bytearray(MAGIC + struct.pack("<H", len(pieces)))
47
+ for p, s in zip(pieces, scores):
48
+ b = p.encode("ascii")
49
+ q = max(-128, min(127, round(s * SCORE_SCALE)))
50
+ out += struct.pack("<B", len(b)) + b + struct.pack("<b", q)
51
+ return bytes(out)
52
+
53
+ def encode(self, norm_text: str, max_tokens: int | None = None) -> list[int]:
54
+ s = "_" + norm_text.replace(" ", "_")
55
+ n = len(s)
56
+ if n <= 1:
57
+ return []
58
+ best = [-1e18] * (n + 1)
59
+ best[0] = 0.0
60
+ bp = [0] * (n + 1)
61
+ bid = [self.unk_id] * (n + 1)
62
+ for i in range(1, n + 1):
63
+ for j in range(max(0, i - self.max_len), i):
64
+ tid = self.index.get(s[j:i])
65
+ if tid is not None:
66
+ sc = best[j] + self.scores[tid]
67
+ if sc > best[i]:
68
+ best[i], bp[i], bid[i] = sc, j, tid
69
+ sc = best[i - 1] + UNK_SCORE
70
+ if sc > best[i]:
71
+ best[i], bp[i], bid[i] = sc, i - 1, self.unk_id
72
+ ids: list[int] = []
73
+ i = n
74
+ while i > 0:
75
+ ids.append(bid[i])
76
+ i = bp[i]
77
+ ids.reverse()
78
+ return ids if max_tokens is None else ids[-max_tokens:]