Release tinyemo model and inference code
Browse files- .gitattributes +1 -33
- README.md +86 -0
- THIRD_PARTY_NOTICES.md +66 -0
- __init__.py +0 -0
- config.json +832 -0
- en_features.py +69 -0
- export_release.py +38 -0
- model.onnx +3 -0
- model.py +141 -0
- model.safetensors +3 -0
- normalize.py +36 -0
- quant.py +41 -0
- results_v2.json +323 -0
- runtime.py +167 -0
- tiny_emo.bin +3 -0
- tiny_tokenizer.bin +3 -0
- tiny_tokenizer.py +78 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- en
|
| 4 |
+
library_name: pytorch
|
| 5 |
+
tags:
|
| 6 |
+
- emoji
|
| 7 |
+
- text-classification
|
| 8 |
+
- onnx
|
| 9 |
+
- safetensors
|
| 10 |
+
base_model: desert-ant-labs/emo
|
| 11 |
+
license: other
|
| 12 |
+
license_name: desert-ant-labs-source-available
|
| 13 |
+
license_link: https://license.desertant.com/
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
# tinyemo
|
| 17 |
+
|
| 18 |
+
Tinyemo is an English emoji suggestion model distilled from [Emo](https://huggingface.co/desert-ant-labs/emo). It predicts one of 812 emoji labels from a short phrase. It normalizes text to ASCII letters and digits and uses the last four words; it is intended for short input, not general sentiment analysis.
|
| 19 |
+
|
| 20 |
+
The evaluated distribution is `tiny_emo.bin` and `tiny_tokenizer.bin`, together **495,502 bytes**. `model.onnx` and `model.safetensors` contain the same quantized-then-dequantized weights in float32, so they are larger; they are not int4/int8 files. They output logits, not text or emoji. The model uses custom preprocessing and is **not** a Transformers `AutoModel` or `AutoTokenizer`.
|
| 21 |
+
|
| 22 |
+
## Usage
|
| 23 |
+
|
| 24 |
+
Download the files into a directory named `tinyemo`, then run Python from its parent directory. The reference runtime requires NumPy:
|
| 25 |
+
|
| 26 |
+
```python
|
| 27 |
+
from tinyemo.runtime import TinyEmoRuntime
|
| 28 |
+
|
| 29 |
+
model = TinyEmoRuntime("tinyemo/tiny_emo.bin")
|
| 30 |
+
print(model.suggest("walk the dog", 3))
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
`suggest` returns `(emoji, probability)` pairs, defaults to one result, clamps the requested count to 1–3, and returns an empty list for input that normalizes to empty. Probabilities are softmax scores, not calibrated confidence estimates.
|
| 34 |
+
|
| 35 |
+
### ONNX Runtime
|
| 36 |
+
|
| 37 |
+
Install `numpy` and `onnxruntime`. The supplied runtime provides the exact tokenizer, normalization and hashed n-gram preprocessing. `config.json` contains the label order and architecture settings.
|
| 38 |
+
|
| 39 |
+
```python
|
| 40 |
+
import numpy as np
|
| 41 |
+
import onnxruntime as ort
|
| 42 |
+
from tinyemo.runtime import TinyEmoRuntime
|
| 43 |
+
|
| 44 |
+
pre = TinyEmoRuntime("tinyemo/tiny_emo.bin")
|
| 45 |
+
session = ort.InferenceSession("tinyemo/model.onnx", providers=["CPUExecutionProvider"])
|
| 46 |
+
text = "walk the dog"
|
| 47 |
+
ids, mask, buckets, signs = pre.encode_batch([text])
|
| 48 |
+
count = (signs[:, :, 0] != 0).sum(axis=1).astype(np.float32)
|
| 49 |
+
logits = session.run(None, {
|
| 50 |
+
"tok_ids": ids, "tok_mask": mask, "ng_bucket": buckets,
|
| 51 |
+
"ng_sign": signs.astype(np.float32), "ng_count": count,
|
| 52 |
+
})[0]
|
| 53 |
+
print(pre.labels[int(logits[0].argmax())])
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
The ONNX graph takes a dynamic batch dimension, with token arrays `[batch, 16]`, n-gram arrays `[batch, 96, 2]`, and `ng_count` `[batch]`. `tok_ids` and `ng_bucket` are int64, `tok_mask` is bool, and `ng_sign` and `ng_count` are float32. Preprocessing is outside the graph.
|
| 57 |
+
|
| 58 |
+
### PyTorch and safetensors
|
| 59 |
+
|
| 60 |
+
Install `torch`, `safetensors`, and `numpy`:
|
| 61 |
+
|
| 62 |
+
```python
|
| 63 |
+
import json
|
| 64 |
+
from safetensors.torch import load_file
|
| 65 |
+
from tinyemo.model import TinyEmo, TinyEmoConfig
|
| 66 |
+
|
| 67 |
+
with open("tinyemo/config.json", encoding="utf-8") as f:
|
| 68 |
+
config = json.load(f)
|
| 69 |
+
model = TinyEmo(TinyEmoConfig(**config["config"]))
|
| 70 |
+
model.load_state_dict(load_file("tinyemo/model.safetensors"))
|
| 71 |
+
model.eval()
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
Use the same five preprocessed inputs shown above, converted to PyTorch tensors, to obtain logits. The Python implementation is included in this repository. `export_release.py` rebuilds both formats from the TEM2 file; it requires PyTorch and ONNX.
|
| 75 |
+
|
| 76 |
+
## Evaluation and limitations
|
| 77 |
+
|
| 78 |
+
The original quantized runtime scored 87.7% top-1 (95% bootstrap interval 84.5–90.9%) and 97.1% top-3 (95.2–98.7%) on 375 hand-labelled phrases with accepted alternatives. Emo scored 88.8% and 97.1%, respectively. The paired top-1 difference was −1.1 percentage points (95% interval −4.3 to +2.1); this interval does not establish equivalence. Model selection used held-out training-pool agreement, while gold was logged during training.
|
| 79 |
+
|
| 80 |
+
On 6,000 held-out snippets from GoEmotions, DailyDialog and tweet_eval, top-1 agreement with the teacher was 62.2%; that measures agreement, not accuracy. Rare nouns such as “koala” remain a weakness. The gold set is small and author-labelled; these results may not generalize to other inputs. Earlier v1 results used a different test set.
|
| 81 |
+
|
| 82 |
+
Training data included short snippets from GoEmotions, DailyDialog, tweet_eval, SST-2, emotion, Tatoeba and Unicode CLDR, plus contexts built from the teacher's vocabulary. The gold phrases were excluded from the training pool, although constituent words could appear elsewhere. The model was trained on teacher predictions; no LLM was used for teacher labelling.
|
| 83 |
+
|
| 84 |
+
## License and attribution
|
| 85 |
+
|
| 86 |
+
Tinyemo was distilled from Emo, which is distributed under the [Desert Ant Labs Source-Available License](https://license.desertant.com/). Review those terms and the licenses of the source datasets before distribution or commercial use. The `other` metadata records the teacher's license and does not assert additional rights for this derivative. See `THIRD_PARTY_NOTICES.md` for upstream notices. Training data is not included in this release folder.
|
THIRD_PARTY_NOTICES.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Third-party notices: Emo
|
| 2 |
+
|
| 3 |
+
Emo's weights build on the work below. Each license applies to its own work, and the
|
| 4 |
+
Desert Ant Labs Source-Available License doesn't override it. Training data isn't redistributed here.
|
| 5 |
+
|
| 6 |
+
## Models
|
| 7 |
+
|
| 8 |
+
- **potion-multilingual-128M**, MinishLab. MIT License. Emo's embedding and tokenizer derive from it, reduced and pruned.
|
| 9 |
+
https://huggingface.co/minishlab/potion-multilingual-128M
|
| 10 |
+
- **bge-m3**, BAAI. MIT License. potion-multilingual-128M was distilled from it.
|
| 11 |
+
https://huggingface.co/BAAI/bge-m3
|
| 12 |
+
|
| 13 |
+
## Training data
|
| 14 |
+
|
| 15 |
+
- **Unicode CLDR**, Unicode, Inc.. Unicode License Agreement for Data Files and Software (2016). Emoji annotations in many languages.
|
| 16 |
+
https://cldr.unicode.org
|
| 17 |
+
|
| 18 |
+
## License texts
|
| 19 |
+
|
| 20 |
+
### MIT License
|
| 21 |
+
|
| 22 |
+
```text
|
| 23 |
+
MIT License
|
| 24 |
+
|
| 25 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
| 26 |
+
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
| 27 |
+
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 28 |
+
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
| 29 |
+
following conditions:
|
| 30 |
+
|
| 31 |
+
The above copyright notice and this permission notice shall be included in all copies or substantial
|
| 32 |
+
portions of the Software.
|
| 33 |
+
|
| 34 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
| 35 |
+
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
| 36 |
+
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
| 37 |
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
| 38 |
+
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
### Unicode License Agreement for Data Files and Software (2016)
|
| 42 |
+
|
| 43 |
+
```text
|
| 44 |
+
UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
|
| 45 |
+
|
| 46 |
+
Unicode Data Files include all data files under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and http://www.unicode.org/utility/trac/browser/.
|
| 47 |
+
|
| 48 |
+
Unicode Data Files do not include PDF online code charts under the directory http://www.unicode.org/Public/.
|
| 49 |
+
|
| 50 |
+
Software includes any source code published in the Unicode Standard or under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and http://www.unicode.org/utility/trac/browser/.
|
| 51 |
+
|
| 52 |
+
NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
|
| 53 |
+
|
| 54 |
+
COPYRIGHT AND PERMISSION NOTICE
|
| 55 |
+
|
| 56 |
+
Copyright © 1991-2016 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in http://www.unicode.org/copyright.html.
|
| 57 |
+
|
| 58 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that either
|
| 59 |
+
|
| 60 |
+
(a) this copyright and permission notice appear with all copies of the Data Files or Software, or
|
| 61 |
+
(b) this copyright and permission notice appear in associated Documentation.
|
| 62 |
+
|
| 63 |
+
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE.
|
| 64 |
+
|
| 65 |
+
Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder.
|
| 66 |
+
```
|
__init__.py
ADDED
|
File without changes
|
config.json
ADDED
|
@@ -0,0 +1,832 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "tinyemo",
|
| 3 |
+
"architecture": "TinyEmo",
|
| 4 |
+
"config": {
|
| 5 |
+
"vocab": 2048,
|
| 6 |
+
"n_buckets": 16384,
|
| 7 |
+
"ng_dim": 16,
|
| 8 |
+
"tok_dim": 48,
|
| 9 |
+
"d_model": 128,
|
| 10 |
+
"n_layers": 1,
|
| 11 |
+
"n_heads": 4,
|
| 12 |
+
"ffn": 256,
|
| 13 |
+
"head_hidden": 128,
|
| 14 |
+
"n_labels": 812,
|
| 15 |
+
"max_tokens": 16,
|
| 16 |
+
"max_features": 96
|
| 17 |
+
},
|
| 18 |
+
"labels": [
|
| 19 |
+
"↩️",
|
| 20 |
+
"⌚",
|
| 21 |
+
"⌨️",
|
| 22 |
+
"⏰",
|
| 23 |
+
"⏱️",
|
| 24 |
+
"⏲️",
|
| 25 |
+
"⏳",
|
| 26 |
+
"⏸️",
|
| 27 |
+
"☀️",
|
| 28 |
+
"☂️",
|
| 29 |
+
"☎️",
|
| 30 |
+
"☔",
|
| 31 |
+
"☕",
|
| 32 |
+
"♌",
|
| 33 |
+
"♟️",
|
| 34 |
+
"♻️",
|
| 35 |
+
"⚓",
|
| 36 |
+
"⚖️",
|
| 37 |
+
"⚙️",
|
| 38 |
+
"⚠️",
|
| 39 |
+
"⚡",
|
| 40 |
+
"⚰️",
|
| 41 |
+
"⚽",
|
| 42 |
+
"⚾",
|
| 43 |
+
"⛈️",
|
| 44 |
+
"⛪",
|
| 45 |
+
"⛰️",
|
| 46 |
+
"⛳",
|
| 47 |
+
"⛵",
|
| 48 |
+
"⛸️",
|
| 49 |
+
"⛺",
|
| 50 |
+
"⛽",
|
| 51 |
+
"✂️",
|
| 52 |
+
"✅",
|
| 53 |
+
"✈️",
|
| 54 |
+
"✉️",
|
| 55 |
+
"✍️",
|
| 56 |
+
"✏️",
|
| 57 |
+
"✝️",
|
| 58 |
+
"✨",
|
| 59 |
+
"❄️",
|
| 60 |
+
"❌",
|
| 61 |
+
"❓",
|
| 62 |
+
"❤️",
|
| 63 |
+
"➕",
|
| 64 |
+
"⭐",
|
| 65 |
+
"🅿️",
|
| 66 |
+
"🆔",
|
| 67 |
+
"🆕",
|
| 68 |
+
"🆘",
|
| 69 |
+
"🇺🇸",
|
| 70 |
+
"🌀",
|
| 71 |
+
"🌃",
|
| 72 |
+
"🌅",
|
| 73 |
+
"🌈",
|
| 74 |
+
"🌊",
|
| 75 |
+
"🌌",
|
| 76 |
+
"🌍",
|
| 77 |
+
"🌐",
|
| 78 |
+
"🌕",
|
| 79 |
+
"🌙",
|
| 80 |
+
"🌠",
|
| 81 |
+
"🌡️",
|
| 82 |
+
"🌧️",
|
| 83 |
+
"🌪️",
|
| 84 |
+
"🌬️",
|
| 85 |
+
"🌮",
|
| 86 |
+
"🌯",
|
| 87 |
+
"🌰",
|
| 88 |
+
"🌱",
|
| 89 |
+
"🌳",
|
| 90 |
+
"🌵",
|
| 91 |
+
"🌶️",
|
| 92 |
+
"🌷",
|
| 93 |
+
"🌸",
|
| 94 |
+
"🌹",
|
| 95 |
+
"🌻",
|
| 96 |
+
"🌼",
|
| 97 |
+
"🌽",
|
| 98 |
+
"🌾",
|
| 99 |
+
"🌿",
|
| 100 |
+
"🍀",
|
| 101 |
+
"🍂",
|
| 102 |
+
"🍃",
|
| 103 |
+
"🍄",
|
| 104 |
+
"🍅",
|
| 105 |
+
"🍇",
|
| 106 |
+
"🍉",
|
| 107 |
+
"🍊",
|
| 108 |
+
"🍌",
|
| 109 |
+
"🍎",
|
| 110 |
+
"🍓",
|
| 111 |
+
"🍔",
|
| 112 |
+
"🍕",
|
| 113 |
+
"🍖",
|
| 114 |
+
"🍗",
|
| 115 |
+
"🍚",
|
| 116 |
+
"🍛",
|
| 117 |
+
"🍜",
|
| 118 |
+
"🍝",
|
| 119 |
+
"🍞",
|
| 120 |
+
"🍣",
|
| 121 |
+
"🍤",
|
| 122 |
+
"🍦",
|
| 123 |
+
"🍩",
|
| 124 |
+
"🍪",
|
| 125 |
+
"🍫",
|
| 126 |
+
"🍬",
|
| 127 |
+
"🍯",
|
| 128 |
+
"🍰",
|
| 129 |
+
"🍱",
|
| 130 |
+
"🍲",
|
| 131 |
+
"🍳",
|
| 132 |
+
"🍵",
|
| 133 |
+
"🍶",
|
| 134 |
+
"🍷",
|
| 135 |
+
"🍼",
|
| 136 |
+
"🍽️",
|
| 137 |
+
"🍾",
|
| 138 |
+
"🍿",
|
| 139 |
+
"🎀",
|
| 140 |
+
"🎁",
|
| 141 |
+
"🎂",
|
| 142 |
+
"🎃",
|
| 143 |
+
"🎄",
|
| 144 |
+
"🎆",
|
| 145 |
+
"🎈",
|
| 146 |
+
"🎉",
|
| 147 |
+
"🎊",
|
| 148 |
+
"🎒",
|
| 149 |
+
"🎓",
|
| 150 |
+
"🎙️",
|
| 151 |
+
"🎞️",
|
| 152 |
+
"🎟️",
|
| 153 |
+
"🎣",
|
| 154 |
+
"🎤",
|
| 155 |
+
"🎥",
|
| 156 |
+
"🎧",
|
| 157 |
+
"🎨",
|
| 158 |
+
"🎩",
|
| 159 |
+
"🎪",
|
| 160 |
+
"🎬",
|
| 161 |
+
"🎭",
|
| 162 |
+
"🎮",
|
| 163 |
+
"🎯",
|
| 164 |
+
"🎲",
|
| 165 |
+
"🎳",
|
| 166 |
+
"🎵",
|
| 167 |
+
"🎷",
|
| 168 |
+
"🎸",
|
| 169 |
+
"🎺",
|
| 170 |
+
"🎾",
|
| 171 |
+
"🎿",
|
| 172 |
+
"🏀",
|
| 173 |
+
"🏁",
|
| 174 |
+
"🏃",
|
| 175 |
+
"🏆",
|
| 176 |
+
"🏈",
|
| 177 |
+
"🏊",
|
| 178 |
+
"🏋️",
|
| 179 |
+
"🏌️",
|
| 180 |
+
"🏐",
|
| 181 |
+
"🏒",
|
| 182 |
+
"🏓",
|
| 183 |
+
"🏕️",
|
| 184 |
+
"🏖️",
|
| 185 |
+
"🏘️",
|
| 186 |
+
"🏙️",
|
| 187 |
+
"🏛️",
|
| 188 |
+
"🏜️",
|
| 189 |
+
"🏞️",
|
| 190 |
+
"🏠",
|
| 191 |
+
"🏢",
|
| 192 |
+
"🏥",
|
| 193 |
+
"🏦",
|
| 194 |
+
"🏧",
|
| 195 |
+
"🏨",
|
| 196 |
+
"🏪",
|
| 197 |
+
"🏫",
|
| 198 |
+
"🏷️",
|
| 199 |
+
"🏺",
|
| 200 |
+
"🐇",
|
| 201 |
+
"🐈",
|
| 202 |
+
"🐌",
|
| 203 |
+
"🐍",
|
| 204 |
+
"🐔",
|
| 205 |
+
"🐕",
|
| 206 |
+
"🐘",
|
| 207 |
+
"🐙",
|
| 208 |
+
"🐛",
|
| 209 |
+
"🐜",
|
| 210 |
+
"🐝",
|
| 211 |
+
"🐞",
|
| 212 |
+
"🐠",
|
| 213 |
+
"🐢",
|
| 214 |
+
"🐦",
|
| 215 |
+
"🐧",
|
| 216 |
+
"🐩",
|
| 217 |
+
"🐬",
|
| 218 |
+
"🐭",
|
| 219 |
+
"🐳",
|
| 220 |
+
"🐷",
|
| 221 |
+
"🐸",
|
| 222 |
+
"🐻",
|
| 223 |
+
"🐼",
|
| 224 |
+
"🐾",
|
| 225 |
+
"👀",
|
| 226 |
+
"👁️",
|
| 227 |
+
"👂",
|
| 228 |
+
"👋",
|
| 229 |
+
"👓",
|
| 230 |
+
"👔",
|
| 231 |
+
"👕",
|
| 232 |
+
"👗",
|
| 233 |
+
"👜",
|
| 234 |
+
"👟",
|
| 235 |
+
"👢",
|
| 236 |
+
"👣",
|
| 237 |
+
"👥",
|
| 238 |
+
"👫",
|
| 239 |
+
"👰",
|
| 240 |
+
"👶",
|
| 241 |
+
"👻",
|
| 242 |
+
"👽",
|
| 243 |
+
"💀",
|
| 244 |
+
"💃",
|
| 245 |
+
"💄",
|
| 246 |
+
"💅",
|
| 247 |
+
"💆",
|
| 248 |
+
"💇",
|
| 249 |
+
"💈",
|
| 250 |
+
"💉",
|
| 251 |
+
"💊",
|
| 252 |
+
"💌",
|
| 253 |
+
"💍",
|
| 254 |
+
"💎",
|
| 255 |
+
"💐",
|
| 256 |
+
"💑",
|
| 257 |
+
"💒",
|
| 258 |
+
"💡",
|
| 259 |
+
"💦",
|
| 260 |
+
"💧",
|
| 261 |
+
"💨",
|
| 262 |
+
"💪",
|
| 263 |
+
"💬",
|
| 264 |
+
"💭",
|
| 265 |
+
"💰",
|
| 266 |
+
"💳",
|
| 267 |
+
"💻",
|
| 268 |
+
"💼",
|
| 269 |
+
"💾",
|
| 270 |
+
"💿",
|
| 271 |
+
"📀",
|
| 272 |
+
"📁",
|
| 273 |
+
"📄",
|
| 274 |
+
"📅",
|
| 275 |
+
"📇",
|
| 276 |
+
"📈",
|
| 277 |
+
"📉",
|
| 278 |
+
"📊",
|
| 279 |
+
"📋",
|
| 280 |
+
"📌",
|
| 281 |
+
"📍",
|
| 282 |
+
"📎",
|
| 283 |
+
"📏",
|
| 284 |
+
"📐",
|
| 285 |
+
"📑",
|
| 286 |
+
"📒",
|
| 287 |
+
"📓",
|
| 288 |
+
"📔",
|
| 289 |
+
"📚",
|
| 290 |
+
"📜",
|
| 291 |
+
"📝",
|
| 292 |
+
"📞",
|
| 293 |
+
"📡",
|
| 294 |
+
"📢",
|
| 295 |
+
"📦",
|
| 296 |
+
"📬",
|
| 297 |
+
"📮",
|
| 298 |
+
"📰",
|
| 299 |
+
"📱",
|
| 300 |
+
"📶",
|
| 301 |
+
"📷",
|
| 302 |
+
"📸",
|
| 303 |
+
"📹",
|
| 304 |
+
"📺",
|
| 305 |
+
"📻",
|
| 306 |
+
"📿",
|
| 307 |
+
"🔁",
|
| 308 |
+
"🔄",
|
| 309 |
+
"🔊",
|
| 310 |
+
"🔋",
|
| 311 |
+
"🔌",
|
| 312 |
+
"🔍",
|
| 313 |
+
"🔑",
|
| 314 |
+
"🔒",
|
| 315 |
+
"🔔",
|
| 316 |
+
"🔗",
|
| 317 |
+
"🔢",
|
| 318 |
+
"🔥",
|
| 319 |
+
"🔦",
|
| 320 |
+
"🔧",
|
| 321 |
+
"🔨",
|
| 322 |
+
"🔩",
|
| 323 |
+
"🔪",
|
| 324 |
+
"🔬",
|
| 325 |
+
"🔭",
|
| 326 |
+
"🔮",
|
| 327 |
+
"🕊️",
|
| 328 |
+
"🕌",
|
| 329 |
+
"🕐",
|
| 330 |
+
"🕒",
|
| 331 |
+
"🕯️",
|
| 332 |
+
"🕷️",
|
| 333 |
+
"🕸️",
|
| 334 |
+
"🖋️",
|
| 335 |
+
"🖌️",
|
| 336 |
+
"🖥️",
|
| 337 |
+
"🖨️",
|
| 338 |
+
"🖱️",
|
| 339 |
+
"🗂️",
|
| 340 |
+
"🗃️",
|
| 341 |
+
"🗄️",
|
| 342 |
+
"🗑️",
|
| 343 |
+
"🗝️",
|
| 344 |
+
"🗣️",
|
| 345 |
+
"🗺️",
|
| 346 |
+
"🗽",
|
| 347 |
+
"😊",
|
| 348 |
+
"😋",
|
| 349 |
+
"😌",
|
| 350 |
+
"😔",
|
| 351 |
+
"😠",
|
| 352 |
+
"😢",
|
| 353 |
+
"😤",
|
| 354 |
+
"😫",
|
| 355 |
+
"😱",
|
| 356 |
+
"😴",
|
| 357 |
+
"😷",
|
| 358 |
+
"🙏",
|
| 359 |
+
"🚀",
|
| 360 |
+
"🚂",
|
| 361 |
+
"🚄",
|
| 362 |
+
"🚆",
|
| 363 |
+
"🚇",
|
| 364 |
+
"🚉",
|
| 365 |
+
"🚌",
|
| 366 |
+
"🚐",
|
| 367 |
+
"🚕",
|
| 368 |
+
"🚗",
|
| 369 |
+
"🚚",
|
| 370 |
+
"🚜",
|
| 371 |
+
"🚢",
|
| 372 |
+
"🚣",
|
| 373 |
+
"🚤",
|
| 374 |
+
"🚦",
|
| 375 |
+
"🚨",
|
| 376 |
+
"🚪",
|
| 377 |
+
"🚫",
|
| 378 |
+
"🚰",
|
| 379 |
+
"🚲",
|
| 380 |
+
"🚶",
|
| 381 |
+
"🚽",
|
| 382 |
+
"🚿",
|
| 383 |
+
"🛀",
|
| 384 |
+
"🛁",
|
| 385 |
+
"🛂",
|
| 386 |
+
"🛋️",
|
| 387 |
+
"🛍️",
|
| 388 |
+
"🛏️",
|
| 389 |
+
"🛒",
|
| 390 |
+
"🛗",
|
| 391 |
+
"🛠️",
|
| 392 |
+
"🛡️",
|
| 393 |
+
"🛣️",
|
| 394 |
+
"🛫",
|
| 395 |
+
"🛶",
|
| 396 |
+
"🛸",
|
| 397 |
+
"🤒",
|
| 398 |
+
"🤔",
|
| 399 |
+
"🤕",
|
| 400 |
+
"🤖",
|
| 401 |
+
"🤝",
|
| 402 |
+
"🤧",
|
| 403 |
+
"🤫",
|
| 404 |
+
"🥁",
|
| 405 |
+
"🥂",
|
| 406 |
+
"🥊",
|
| 407 |
+
"🥋",
|
| 408 |
+
"🥐",
|
| 409 |
+
"🥑",
|
| 410 |
+
"🥒",
|
| 411 |
+
"🥓",
|
| 412 |
+
"🥔",
|
| 413 |
+
"🥕",
|
| 414 |
+
"🥖",
|
| 415 |
+
"🥗",
|
| 416 |
+
"🥘",
|
| 417 |
+
"🥚",
|
| 418 |
+
"🥛",
|
| 419 |
+
"🥞",
|
| 420 |
+
"🥡",
|
| 421 |
+
"🥢",
|
| 422 |
+
"🥣",
|
| 423 |
+
"🥤",
|
| 424 |
+
"🥦",
|
| 425 |
+
"🥧",
|
| 426 |
+
"🥩",
|
| 427 |
+
"🥪",
|
| 428 |
+
"🥫",
|
| 429 |
+
"🥬",
|
| 430 |
+
"🥭",
|
| 431 |
+
"🥶",
|
| 432 |
+
"🥾",
|
| 433 |
+
"🦁",
|
| 434 |
+
"🦃",
|
| 435 |
+
"🦄",
|
| 436 |
+
"🦅",
|
| 437 |
+
"🦈",
|
| 438 |
+
"🦉",
|
| 439 |
+
"🦊",
|
| 440 |
+
"🦋",
|
| 441 |
+
"🦌",
|
| 442 |
+
"🦎",
|
| 443 |
+
"🦐",
|
| 444 |
+
"🦒",
|
| 445 |
+
"🦜",
|
| 446 |
+
"🦞",
|
| 447 |
+
"🦠",
|
| 448 |
+
"🦵",
|
| 449 |
+
"🦶",
|
| 450 |
+
"🦷",
|
| 451 |
+
"🧀",
|
| 452 |
+
"🧁",
|
| 453 |
+
"🧂",
|
| 454 |
+
"🧃",
|
| 455 |
+
"🧅",
|
| 456 |
+
"🧈",
|
| 457 |
+
"🧊",
|
| 458 |
+
"🧑",
|
| 459 |
+
"🧑⚕️",
|
| 460 |
+
"🧑🍳",
|
| 461 |
+
"🧑🏫",
|
| 462 |
+
"🧑💻",
|
| 463 |
+
"🧑💼",
|
| 464 |
+
"🧑🔧",
|
| 465 |
+
"🧑🧑🧒🧒",
|
| 466 |
+
"🧑🧒",
|
| 467 |
+
"🧒",
|
| 468 |
+
"🧓",
|
| 469 |
+
"🧖",
|
| 470 |
+
"🧗",
|
| 471 |
+
"🧘",
|
| 472 |
+
"🧙",
|
| 473 |
+
"🧠",
|
| 474 |
+
"🧢",
|
| 475 |
+
"🧣",
|
| 476 |
+
"🧤",
|
| 477 |
+
"🧥",
|
| 478 |
+
"🧦",
|
| 479 |
+
"🧩",
|
| 480 |
+
"🧪",
|
| 481 |
+
"🧬",
|
| 482 |
+
"🧭",
|
| 483 |
+
"🧮",
|
| 484 |
+
"🧯",
|
| 485 |
+
"🧰",
|
| 486 |
+
"🧱",
|
| 487 |
+
"🧳",
|
| 488 |
+
"🧴",
|
| 489 |
+
"🧵",
|
| 490 |
+
"🧶",
|
| 491 |
+
"🧸",
|
| 492 |
+
"🧹",
|
| 493 |
+
"🧺",
|
| 494 |
+
"🧻",
|
| 495 |
+
"🧼",
|
| 496 |
+
"🧽",
|
| 497 |
+
"🧾",
|
| 498 |
+
"🩸",
|
| 499 |
+
"🩹",
|
| 500 |
+
"🩺",
|
| 501 |
+
"🩻",
|
| 502 |
+
"🪁",
|
| 503 |
+
"🪄",
|
| 504 |
+
"🪐",
|
| 505 |
+
"🪑",
|
| 506 |
+
"🪒",
|
| 507 |
+
"🪓",
|
| 508 |
+
"🪟",
|
| 509 |
+
"🪡",
|
| 510 |
+
"🪥",
|
| 511 |
+
"🪪",
|
| 512 |
+
"🪴",
|
| 513 |
+
"🪵",
|
| 514 |
+
"🫄",
|
| 515 |
+
"🫒",
|
| 516 |
+
"🫖",
|
| 517 |
+
"🫘",
|
| 518 |
+
"🫙",
|
| 519 |
+
"☃️",
|
| 520 |
+
"☄️",
|
| 521 |
+
"☘️",
|
| 522 |
+
"♨️",
|
| 523 |
+
"⚔️",
|
| 524 |
+
"⚗️",
|
| 525 |
+
"⚱️",
|
| 526 |
+
"⛏️",
|
| 527 |
+
"⛑️",
|
| 528 |
+
"⛓️",
|
| 529 |
+
"⛓️💥",
|
| 530 |
+
"⛩️",
|
| 531 |
+
"⛲",
|
| 532 |
+
"⛴️",
|
| 533 |
+
"✌️",
|
| 534 |
+
"✒️",
|
| 535 |
+
"🌁",
|
| 536 |
+
"🌇",
|
| 537 |
+
"🌉",
|
| 538 |
+
"🌋",
|
| 539 |
+
"🌥️",
|
| 540 |
+
"🌨️",
|
| 541 |
+
"🌫️",
|
| 542 |
+
"🌭",
|
| 543 |
+
"🌴",
|
| 544 |
+
"🌺",
|
| 545 |
+
"🍁",
|
| 546 |
+
"🍄🟫",
|
| 547 |
+
"🍆",
|
| 548 |
+
"🍈",
|
| 549 |
+
"🍋",
|
| 550 |
+
"🍋🟩",
|
| 551 |
+
"🍍",
|
| 552 |
+
"🍐",
|
| 553 |
+
"🍑",
|
| 554 |
+
"🍒",
|
| 555 |
+
"🍘",
|
| 556 |
+
"🍙",
|
| 557 |
+
"🍟",
|
| 558 |
+
"🍠",
|
| 559 |
+
"🍡",
|
| 560 |
+
"🍢",
|
| 561 |
+
"🍥",
|
| 562 |
+
"🍧",
|
| 563 |
+
"🍭",
|
| 564 |
+
"🍮",
|
| 565 |
+
"🎇",
|
| 566 |
+
"🎐",
|
| 567 |
+
"🎖️",
|
| 568 |
+
"🎠",
|
| 569 |
+
"🎡",
|
| 570 |
+
"🎢",
|
| 571 |
+
"🎰",
|
| 572 |
+
"🎱",
|
| 573 |
+
"🎻",
|
| 574 |
+
"🎽",
|
| 575 |
+
"🏅",
|
| 576 |
+
"🏉",
|
| 577 |
+
"🏍️",
|
| 578 |
+
"🏎️",
|
| 579 |
+
"🏏",
|
| 580 |
+
"🏑",
|
| 581 |
+
"🏗️",
|
| 582 |
+
"🏝️",
|
| 583 |
+
"🏟️",
|
| 584 |
+
"🏤",
|
| 585 |
+
"🏩",
|
| 586 |
+
"🏬",
|
| 587 |
+
"🏭",
|
| 588 |
+
"🏮",
|
| 589 |
+
"🏰",
|
| 590 |
+
"🏸",
|
| 591 |
+
"🏹",
|
| 592 |
+
"🐀",
|
| 593 |
+
"🐂",
|
| 594 |
+
"🐃",
|
| 595 |
+
"🐄",
|
| 596 |
+
"🐅",
|
| 597 |
+
"🐆",
|
| 598 |
+
"🐊",
|
| 599 |
+
"🐎",
|
| 600 |
+
"🐏",
|
| 601 |
+
"🐐",
|
| 602 |
+
"🐑",
|
| 603 |
+
"🐒",
|
| 604 |
+
"🐓",
|
| 605 |
+
"🐗",
|
| 606 |
+
"🐚",
|
| 607 |
+
"🐡",
|
| 608 |
+
"🐣",
|
| 609 |
+
"🐤",
|
| 610 |
+
"🐨",
|
| 611 |
+
"🐪",
|
| 612 |
+
"🐲",
|
| 613 |
+
"🐺",
|
| 614 |
+
"🐿️",
|
| 615 |
+
"👃",
|
| 616 |
+
"👄",
|
| 617 |
+
"👅",
|
| 618 |
+
"👑",
|
| 619 |
+
"👒",
|
| 620 |
+
"👖",
|
| 621 |
+
"👘",
|
| 622 |
+
"👙",
|
| 623 |
+
"👝",
|
| 624 |
+
"👠",
|
| 625 |
+
"👨✈️",
|
| 626 |
+
"💂",
|
| 627 |
+
"💋",
|
| 628 |
+
"💣",
|
| 629 |
+
"💤",
|
| 630 |
+
"💥",
|
| 631 |
+
"💮",
|
| 632 |
+
"💴",
|
| 633 |
+
"💶",
|
| 634 |
+
"💷",
|
| 635 |
+
"💽",
|
| 636 |
+
"📕",
|
| 637 |
+
"📗",
|
| 638 |
+
"📙",
|
| 639 |
+
"📟",
|
| 640 |
+
"📠",
|
| 641 |
+
"📣",
|
| 642 |
+
"📼",
|
| 643 |
+
"📽️",
|
| 644 |
+
"🔖",
|
| 645 |
+
"🔫",
|
| 646 |
+
"🕋",
|
| 647 |
+
"🕍",
|
| 648 |
+
"🕎",
|
| 649 |
+
"🕰️",
|
| 650 |
+
"🕳️",
|
| 651 |
+
"🕵️",
|
| 652 |
+
"🕹️",
|
| 653 |
+
"🖇️",
|
| 654 |
+
"🖊️",
|
| 655 |
+
"🖍️",
|
| 656 |
+
"🖲️",
|
| 657 |
+
"🗒️",
|
| 658 |
+
"🗜️",
|
| 659 |
+
"🗡️",
|
| 660 |
+
"🗳️",
|
| 661 |
+
"🗼",
|
| 662 |
+
"🗿",
|
| 663 |
+
"🚁",
|
| 664 |
+
"🚃",
|
| 665 |
+
"🚈",
|
| 666 |
+
"🚊",
|
| 667 |
+
"🚎",
|
| 668 |
+
"🚏",
|
| 669 |
+
"🚑",
|
| 670 |
+
"🚒",
|
| 671 |
+
"🚓",
|
| 672 |
+
"🚙",
|
| 673 |
+
"🚛",
|
| 674 |
+
"🚝",
|
| 675 |
+
"🚞",
|
| 676 |
+
"🚟",
|
| 677 |
+
"🚠",
|
| 678 |
+
"🚡",
|
| 679 |
+
"🚧",
|
| 680 |
+
"🚬",
|
| 681 |
+
"🛎️",
|
| 682 |
+
"🛕",
|
| 683 |
+
"🛖",
|
| 684 |
+
"🛝",
|
| 685 |
+
"🛞",
|
| 686 |
+
"🛟",
|
| 687 |
+
"🛢️",
|
| 688 |
+
"🛤️",
|
| 689 |
+
"🛥️",
|
| 690 |
+
"🛰️",
|
| 691 |
+
"🛳️",
|
| 692 |
+
"🛴",
|
| 693 |
+
"🛵",
|
| 694 |
+
"🛷",
|
| 695 |
+
"🛹",
|
| 696 |
+
"🛺",
|
| 697 |
+
"🛼",
|
| 698 |
+
"🤵",
|
| 699 |
+
"🤿",
|
| 700 |
+
"🥀",
|
| 701 |
+
"🥄",
|
| 702 |
+
"🥅",
|
| 703 |
+
"🥌",
|
| 704 |
+
"🥍",
|
| 705 |
+
"🥎",
|
| 706 |
+
"🥏",
|
| 707 |
+
"🥙",
|
| 708 |
+
"🥜",
|
| 709 |
+
"🥝",
|
| 710 |
+
"🥟",
|
| 711 |
+
"🥠",
|
| 712 |
+
"🥥",
|
| 713 |
+
"🥨",
|
| 714 |
+
"🥮",
|
| 715 |
+
"🥯",
|
| 716 |
+
"🥷",
|
| 717 |
+
"🥻",
|
| 718 |
+
"🥽",
|
| 719 |
+
"🦀",
|
| 720 |
+
"🦂",
|
| 721 |
+
"🦆",
|
| 722 |
+
"🦇",
|
| 723 |
+
"🦍",
|
| 724 |
+
"🦏",
|
| 725 |
+
"🦑",
|
| 726 |
+
"🦓",
|
| 727 |
+
"🦔",
|
| 728 |
+
"🦗",
|
| 729 |
+
"🦘",
|
| 730 |
+
"🦙",
|
| 731 |
+
"🦚",
|
| 732 |
+
"🦛",
|
| 733 |
+
"🦝",
|
| 734 |
+
"🦟",
|
| 735 |
+
"🦡",
|
| 736 |
+
"🦢",
|
| 737 |
+
"🦥",
|
| 738 |
+
"🦦",
|
| 739 |
+
"🦧",
|
| 740 |
+
"🦨",
|
| 741 |
+
"🦩",
|
| 742 |
+
"🦪",
|
| 743 |
+
"🦫",
|
| 744 |
+
"🦬",
|
| 745 |
+
"🦭",
|
| 746 |
+
"🦯",
|
| 747 |
+
"🦴",
|
| 748 |
+
"🦺",
|
| 749 |
+
"🦼",
|
| 750 |
+
"🦾",
|
| 751 |
+
"🦿",
|
| 752 |
+
"🧄",
|
| 753 |
+
"🧆",
|
| 754 |
+
"🧇",
|
| 755 |
+
"🧉",
|
| 756 |
+
"🧋",
|
| 757 |
+
"🧑🎓",
|
| 758 |
+
"🧑🎤",
|
| 759 |
+
"🧑🎨",
|
| 760 |
+
"🧑🏭",
|
| 761 |
+
"🧑🔬",
|
| 762 |
+
"🧑🚀",
|
| 763 |
+
"🧑🚒",
|
| 764 |
+
"🧕",
|
| 765 |
+
"🧧",
|
| 766 |
+
"🧨",
|
| 767 |
+
"🧫",
|
| 768 |
+
"🧲",
|
| 769 |
+
"🧷",
|
| 770 |
+
"🩰",
|
| 771 |
+
"🩱",
|
| 772 |
+
"🩲",
|
| 773 |
+
"🩳",
|
| 774 |
+
"🩴",
|
| 775 |
+
"🩼",
|
| 776 |
+
"🪀",
|
| 777 |
+
"🪂",
|
| 778 |
+
"🪃",
|
| 779 |
+
"🪅",
|
| 780 |
+
"🪆",
|
| 781 |
+
"🪇",
|
| 782 |
+
"🪈",
|
| 783 |
+
"",
|
| 784 |
+
"",
|
| 785 |
+
"",
|
| 786 |
+
"",
|
| 787 |
+
"🪔",
|
| 788 |
+
"🪕",
|
| 789 |
+
"🪗",
|
| 790 |
+
"🪙",
|
| 791 |
+
"🪚",
|
| 792 |
+
"🪛",
|
| 793 |
+
"🪜",
|
| 794 |
+
"🪝",
|
| 795 |
+
"🪞",
|
| 796 |
+
"🪠",
|
| 797 |
+
"🪢",
|
| 798 |
+
"🪣",
|
| 799 |
+
"🪤",
|
| 800 |
+
"🪦",
|
| 801 |
+
"🪧",
|
| 802 |
+
"🪨",
|
| 803 |
+
"🪩",
|
| 804 |
+
"🪭",
|
| 805 |
+
"🪮",
|
| 806 |
+
"🪰",
|
| 807 |
+
"🪱",
|
| 808 |
+
"🪳",
|
| 809 |
+
"🪶",
|
| 810 |
+
"🪷",
|
| 811 |
+
"🪸",
|
| 812 |
+
"🪹",
|
| 813 |
+
"🪻",
|
| 814 |
+
"🪼",
|
| 815 |
+
"🪽",
|
| 816 |
+
"🪿",
|
| 817 |
+
"🫀",
|
| 818 |
+
"🫁",
|
| 819 |
+
"",
|
| 820 |
+
"🫎",
|
| 821 |
+
"🫏",
|
| 822 |
+
"🫐",
|
| 823 |
+
"🫓",
|
| 824 |
+
"🫔",
|
| 825 |
+
"🫕",
|
| 826 |
+
"🫚",
|
| 827 |
+
"🫛",
|
| 828 |
+
"",
|
| 829 |
+
"🫦",
|
| 830 |
+
"🫧"
|
| 831 |
+
]
|
| 832 |
+
}
|
en_features.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""English-only hashed n-gram features for the student (input is `normalize`d).
|
| 2 |
+
|
| 3 |
+
Per word: the whole word, plus 3..5-char grams with boundary markers. Between
|
| 4 |
+
neighbouring words: a word bigram. Each feature is hashed twice (FNV-1a 64);
|
| 5 |
+
the bucket comes from bits 20..34 so it stays well mixed for any power-of-two
|
| 6 |
+
bucket count <= 32768, and the sign from bit 63.
|
| 7 |
+
|
| 8 |
+
`hash16` returns the 15-bit raw bucket (`bucket = raw & (n_buckets - 1)`) and the
|
| 9 |
+
sign so the prepared dataset does not depend on the final bucket count.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
FNV_OFFSET = 0xCBF29CE484222325
|
| 15 |
+
FNV_PRIME = 0x100000001B3
|
| 16 |
+
MASK64 = (1 << 64) - 1
|
| 17 |
+
SEEDS = (0x9E3779B97F4A7C15, 0xC2B2AE3D27D4EB4F)
|
| 18 |
+
N_HASHES = len(SEEDS)
|
| 19 |
+
RAW_BITS = 15
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def fnv64(s: str, seed: int) -> int:
|
| 23 |
+
h = (FNV_OFFSET ^ seed) & MASK64
|
| 24 |
+
for b in s.encode("utf-8"):
|
| 25 |
+
h = ((h ^ b) * FNV_PRIME) & MASK64
|
| 26 |
+
return h
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def features(norm_text: str) -> list[str]:
|
| 30 |
+
words = norm_text.split()
|
| 31 |
+
out: list[str] = []
|
| 32 |
+
for w in words:
|
| 33 |
+
out.append("w:" + w)
|
| 34 |
+
s = "<" + w + ">"
|
| 35 |
+
for n in (3, 4, 5):
|
| 36 |
+
for i in range(len(s) - n + 1):
|
| 37 |
+
out.append("g:" + s[i : i + n])
|
| 38 |
+
for a, b in zip(words, words[1:]):
|
| 39 |
+
out.append("b:" + a + " " + b)
|
| 40 |
+
return out or ["w:\x00"]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
_CACHE: dict[str, tuple[tuple[int, int], tuple[int, int]]] = {}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def hash16(feat: str) -> tuple[tuple[int, int], tuple[int, int]]:
|
| 47 |
+
"""((raw0, sign0), (raw1, sign1)) with raw in [0, 2**15)."""
|
| 48 |
+
r = _CACHE.get(feat)
|
| 49 |
+
if r is None:
|
| 50 |
+
hs = [fnv64(feat, sd) for sd in SEEDS]
|
| 51 |
+
r = tuple(
|
| 52 |
+
(((h >> 20) & ((1 << RAW_BITS) - 1)), 1 if (h >> 63) & 1 else -1) for h in hs
|
| 53 |
+
) # type: ignore[assignment]
|
| 54 |
+
_CACHE[feat] = r
|
| 55 |
+
return r
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def featurize(norm_text: str, max_features: int):
|
| 59 |
+
"""-> (raw[n,2] int16, sign[n,2] int8) for the first `max_features` features."""
|
| 60 |
+
import numpy as np
|
| 61 |
+
|
| 62 |
+
fs = features(norm_text)[:max_features]
|
| 63 |
+
raw = np.zeros((len(fs), N_HASHES), np.int16)
|
| 64 |
+
sign = np.zeros((len(fs), N_HASHES), np.int8)
|
| 65 |
+
for i, f in enumerate(fs):
|
| 66 |
+
for k, (r, s) in enumerate(hash16(f)):
|
| 67 |
+
raw[i, k] = r
|
| 68 |
+
sign[i, k] = s
|
| 69 |
+
return raw, sign
|
export_release.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rebuild standard exports from the evaluated TEM2 artifact."""
|
| 2 |
+
import json
|
| 3 |
+
import struct
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from tinyemo.model import TinyEmo, TinyEmoConfig
|
| 10 |
+
from tinyemo.runtime import TinyEmoRuntime
|
| 11 |
+
|
| 12 |
+
HERE = Path(__file__).resolve().parent
|
| 13 |
+
rt = TinyEmoRuntime(str(HERE / 'tiny_emo.bin'))
|
| 14 |
+
model = TinyEmo(TinyEmoConfig(**rt.meta))
|
| 15 |
+
model.load_state_dict({k: torch.from_numpy(v.copy()) for k, v in rt.w.items()})
|
| 16 |
+
model.eval()
|
| 17 |
+
(HERE / 'config.json').write_text(json.dumps({'model_type': 'tinyemo', 'architecture': 'TinyEmo', 'config': rt.meta, 'labels': rt.labels}, ensure_ascii=False, indent=2) + '\n')
|
| 18 |
+
|
| 19 |
+
# Safetensors: 8-byte header length, JSON header, then contiguous little-endian data.
|
| 20 |
+
tensors = {k: v.detach().cpu().numpy().astype('<f4') for k, v in model.state_dict().items()}
|
| 21 |
+
header, chunks, offset = {'__metadata__': {'format': 'pt', 'source': 'dequantized TEM2 artifact'}}, [], 0
|
| 22 |
+
for name, arr in tensors.items():
|
| 23 |
+
blob = arr.tobytes(order='C')
|
| 24 |
+
header[name] = {'dtype': 'F32', 'shape': list(arr.shape), 'data_offsets': [offset, offset + len(blob)]}
|
| 25 |
+
chunks.append(blob)
|
| 26 |
+
offset += len(blob)
|
| 27 |
+
h = json.dumps(header, separators=(',', ':')).encode('utf-8')
|
| 28 |
+
h += b' ' * (-len(h) % 8)
|
| 29 |
+
(HERE / 'model.safetensors').write_bytes(struct.pack('<Q', len(h)) + h + b''.join(chunks))
|
| 30 |
+
|
| 31 |
+
ids, mask, buckets, signs = rt.encode_batch(['walk the dog', 'happy birthday'])
|
| 32 |
+
inputs = (torch.from_numpy(ids), torch.from_numpy(mask), torch.from_numpy(buckets),
|
| 33 |
+
torch.from_numpy(signs).float(), torch.from_numpy((signs[:, :, 0] != 0).sum(1)).float())
|
| 34 |
+
torch.onnx.export(model, inputs, str(HERE / 'model.onnx'), opset_version=17,
|
| 35 |
+
input_names=['tok_ids', 'tok_mask', 'ng_bucket', 'ng_sign', 'ng_count'], output_names=['logits'],
|
| 36 |
+
dynamic_axes={name: {0: 'batch'} for name in ['tok_ids', 'tok_mask', 'ng_bucket', 'ng_sign', 'ng_count', 'logits']},
|
| 37 |
+
do_constant_folding=True, dynamo=False)
|
| 38 |
+
print('Exported', HERE)
|
model.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:505c3830435468d192e8bef4e655facc0a5a35c6a4a6fdd68424a3920e3ef241
|
| 3 |
+
size 2580586
|
model.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input.
|
| 2 |
+
|
| 3 |
+
1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm
|
| 4 |
+
2. token stream: int4 token embeddings -> projection -> learned positions ->
|
| 5 |
+
small transformer -> attention pool
|
| 6 |
+
|
| 7 |
+
The streams are concatenated and classified into Emo's 812 emoji labels.
|
| 8 |
+
`set_qat(True)` fake-quantises every weight matrix exactly the way `export.py`
|
| 9 |
+
stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated
|
| 10 |
+
is what ships.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from dataclasses import asdict, dataclass
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
import torch.nn.functional as F
|
| 20 |
+
|
| 21 |
+
from .quant import fake_quant
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@dataclass
|
| 25 |
+
class TinyEmoConfig:
|
| 26 |
+
vocab: int = 2048
|
| 27 |
+
n_buckets: int = 16384 # power of two, <= 32768
|
| 28 |
+
ng_dim: int = 16
|
| 29 |
+
tok_dim: int = 48
|
| 30 |
+
d_model: int = 96
|
| 31 |
+
n_layers: int = 2
|
| 32 |
+
n_heads: int = 2
|
| 33 |
+
ffn: int = 192
|
| 34 |
+
head_hidden: int = 112
|
| 35 |
+
n_labels: int = 812
|
| 36 |
+
max_tokens: int = 16
|
| 37 |
+
max_features: int = 96
|
| 38 |
+
dropout: float = 0.0
|
| 39 |
+
|
| 40 |
+
def to_dict(self) -> dict:
|
| 41 |
+
return asdict(self)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class QLinear(nn.Linear):
|
| 45 |
+
qat = False
|
| 46 |
+
|
| 47 |
+
def forward(self, x):
|
| 48 |
+
w = fake_quant(self.weight, "int8_row") if self.qat else self.weight
|
| 49 |
+
return F.linear(x, w, self.bias)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class QEmbedding(nn.Embedding):
|
| 53 |
+
qat = False
|
| 54 |
+
mode = "int4_row"
|
| 55 |
+
|
| 56 |
+
def forward(self, ids):
|
| 57 |
+
w = fake_quant(self.weight, self.mode) if self.qat else self.weight
|
| 58 |
+
return F.embedding(ids, w)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class TinyEmo(nn.Module):
|
| 62 |
+
def __init__(self, cfg: TinyEmoConfig):
|
| 63 |
+
super().__init__()
|
| 64 |
+
assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two"
|
| 65 |
+
self.cfg = cfg
|
| 66 |
+
self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim)
|
| 67 |
+
self.tok_emb.mode = "int4_row"
|
| 68 |
+
self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False)
|
| 69 |
+
self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02)
|
| 70 |
+
self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim)
|
| 71 |
+
self.ng_emb.mode = "int4_tensor"
|
| 72 |
+
nn.init.normal_(self.ng_emb.weight, std=0.1)
|
| 73 |
+
self.ng_norm = nn.LayerNorm(cfg.ng_dim)
|
| 74 |
+
self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)])
|
| 75 |
+
self.final_norm = nn.LayerNorm(cfg.d_model)
|
| 76 |
+
self.pool_proj = QLinear(cfg.d_model, cfg.d_model)
|
| 77 |
+
self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02)
|
| 78 |
+
self.pool_norm = nn.LayerNorm(cfg.d_model)
|
| 79 |
+
self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden)
|
| 80 |
+
self.head2 = QLinear(cfg.head_hidden, cfg.n_labels)
|
| 81 |
+
self.drop = nn.Dropout(cfg.dropout)
|
| 82 |
+
|
| 83 |
+
def set_qat(self, on: bool) -> None:
|
| 84 |
+
for m in self.modules():
|
| 85 |
+
if isinstance(m, (QLinear, QEmbedding)):
|
| 86 |
+
m.qat = on
|
| 87 |
+
|
| 88 |
+
def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count):
|
| 89 |
+
cfg = self.cfg
|
| 90 |
+
# ---- n-gram stream ----------------------------------------------
|
| 91 |
+
e = self.ng_emb(ng_bucket % cfg.n_buckets) # B,F,H,ng
|
| 92 |
+
e = e * ng_sign.unsqueeze(-1).to(e.dtype)
|
| 93 |
+
ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype)
|
| 94 |
+
ng = self.ng_norm(ng)
|
| 95 |
+
|
| 96 |
+
# ---- token stream -----------------------------------------------
|
| 97 |
+
t = tok_ids.shape[1]
|
| 98 |
+
x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t]
|
| 99 |
+
key_mask = tok_mask.unsqueeze(1).unsqueeze(1) # B,1,1,T
|
| 100 |
+
for blk in self.blocks:
|
| 101 |
+
x = blk(x, key_mask)
|
| 102 |
+
x = self.final_norm(x)
|
| 103 |
+
score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5)
|
| 104 |
+
score = score.masked_fill(~tok_mask, -1e9)
|
| 105 |
+
a = torch.softmax(score, dim=1)
|
| 106 |
+
pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1))
|
| 107 |
+
|
| 108 |
+
z = torch.cat([ng, pooled], dim=-1)
|
| 109 |
+
z = self.drop(F.gelu(self.head1(z)))
|
| 110 |
+
return self.head2(z)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class _Block(nn.Module):
|
| 114 |
+
def __init__(self, cfg: TinyEmoConfig):
|
| 115 |
+
super().__init__()
|
| 116 |
+
d = cfg.d_model
|
| 117 |
+
self.n_heads = cfg.n_heads
|
| 118 |
+
self.norm1 = nn.LayerNorm(d)
|
| 119 |
+
self.qkv = QLinear(d, 3 * d)
|
| 120 |
+
self.proj = QLinear(d, d)
|
| 121 |
+
self.norm2 = nn.LayerNorm(d)
|
| 122 |
+
self.ff1 = QLinear(d, cfg.ffn)
|
| 123 |
+
self.ff2 = QLinear(cfg.ffn, d)
|
| 124 |
+
self.drop = nn.Dropout(cfg.dropout)
|
| 125 |
+
|
| 126 |
+
def forward(self, x, key_mask):
|
| 127 |
+
b, t, d = x.shape
|
| 128 |
+
h = self.norm1(x)
|
| 129 |
+
qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4)
|
| 130 |
+
q, k, v = qkv.unbind(0)
|
| 131 |
+
score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5)
|
| 132 |
+
score = score.masked_fill(~key_mask, -1e9)
|
| 133 |
+
a = self.drop(torch.softmax(score, dim=-1))
|
| 134 |
+
o = (a @ v).transpose(1, 2).reshape(b, t, d)
|
| 135 |
+
x = x + self.drop(self.proj(o))
|
| 136 |
+
h = self.norm2(x)
|
| 137 |
+
return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h)))))
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def count_params(model: nn.Module) -> int:
|
| 141 |
+
return sum(p.numel() for p in model.parameters())
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:760e0816343f5eda2e910442d460ffe2113124089b6a2e533d5b6206dc5a47fe
|
| 3 |
+
size 2568904
|
normalize.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Input normalisation shared by training, evaluation and the runtime.
|
| 2 |
+
|
| 3 |
+
Every text the student ever sees (and every text the teacher is asked to label)
|
| 4 |
+
goes through `normalize`:
|
| 5 |
+
|
| 6 |
+
* NFKD, accents stripped, lower-cased
|
| 7 |
+
* apostrophes inside words removed ("don't" -> "dont")
|
| 8 |
+
* everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...)
|
| 9 |
+
* at most MAX_WORDS words kept - the *last* ones, because the words nearest
|
| 10 |
+
the cursor carry the intent when someone is typing
|
| 11 |
+
* each word capped at MAX_WORD_LEN characters
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import unicodedata
|
| 18 |
+
|
| 19 |
+
MAX_WORDS = 4
|
| 20 |
+
MAX_WORD_LEN = 20
|
| 21 |
+
|
| 22 |
+
_APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"})
|
| 23 |
+
_INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])")
|
| 24 |
+
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def words_of(text: str) -> list[str]:
|
| 28 |
+
"""Normalised words of `text`, *without* the max-words truncation."""
|
| 29 |
+
t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower()
|
| 30 |
+
t = "".join(c for c in t if not unicodedata.combining(c))
|
| 31 |
+
t = _INNER_APOS.sub("", t)
|
| 32 |
+
return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def normalize(text: str, max_words: int = MAX_WORDS) -> str:
|
| 36 |
+
return " ".join(words_of(text)[-max_words:])
|
quant.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Quantisation shared by QAT (torch) and the exporter, so both agree exactly.
|
| 2 |
+
|
| 3 |
+
Modes
|
| 4 |
+
int8_row : symmetric int8, one f16 scale per row (linear layers)
|
| 5 |
+
int4_row : symmetric int4 (-7..7), one f16 scale per row (token embeddings)
|
| 6 |
+
int4_tensor: symmetric int4, one f16 scale per tensor, scale = CLIP_STD * std / 7
|
| 7 |
+
(large hashed n-gram table - per-row scales would cost 2 B/row)
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
CLIP_STD = 4.0
|
| 15 |
+
MODES = {"int8_row": (8, True), "int4_row": (4, True), "int4_tensor": (4, False)}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _scale(w: torch.Tensor, mode: str) -> torch.Tensor:
|
| 19 |
+
bits, per_row = MODES[mode]
|
| 20 |
+
qmax = 2 ** (bits - 1) - 1
|
| 21 |
+
if per_row:
|
| 22 |
+
s = w.abs().amax(dim=1, keepdim=True) / qmax
|
| 23 |
+
else:
|
| 24 |
+
s = (w.std() * CLIP_STD / qmax).reshape(1, 1)
|
| 25 |
+
# scales are stored as f16
|
| 26 |
+
return s.clamp(min=1e-5).half().float()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def quantize(w: torch.Tensor, mode: str) -> tuple[torch.Tensor, torch.Tensor]:
|
| 30 |
+
"""-> (integer tensor, scale tensor) with `w ~= q * scale`."""
|
| 31 |
+
bits, _ = MODES[mode]
|
| 32 |
+
qmax = 2 ** (bits - 1) - 1
|
| 33 |
+
s = _scale(w, mode)
|
| 34 |
+
q = torch.clamp(torch.round(w / s), -qmax, qmax)
|
| 35 |
+
return q, s
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def fake_quant(w: torch.Tensor, mode: str) -> torch.Tensor:
|
| 39 |
+
"""Straight-through fake quantisation."""
|
| 40 |
+
q, s = quantize(w.detach(), mode)
|
| 41 |
+
return w + (q * s - w).detach()
|
results_v2.json
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_gold": 375,
|
| 3 |
+
"n_heldout": 6000,
|
| 4 |
+
"emo": {
|
| 5 |
+
"size_bytes": 10955933,
|
| 6 |
+
"gold1": [
|
| 7 |
+
0.888,
|
| 8 |
+
0.8533333333333334,
|
| 9 |
+
0.92
|
| 10 |
+
],
|
| 11 |
+
"gold3": [
|
| 12 |
+
0.9706666666666667,
|
| 13 |
+
0.952,
|
| 14 |
+
0.9866666666666667
|
| 15 |
+
]
|
| 16 |
+
},
|
| 17 |
+
"tiny": {
|
| 18 |
+
"size_bytes": 495502,
|
| 19 |
+
"gold1": [
|
| 20 |
+
0.8773333333333333,
|
| 21 |
+
0.8452666666666667,
|
| 22 |
+
0.9093333333333333
|
| 23 |
+
],
|
| 24 |
+
"gold3": [
|
| 25 |
+
0.9706666666666667,
|
| 26 |
+
0.952,
|
| 27 |
+
0.9866666666666667
|
| 28 |
+
],
|
| 29 |
+
"agree1": [
|
| 30 |
+
0.6218333333333333,
|
| 31 |
+
0.6098333333333333,
|
| 32 |
+
0.6341666666666667
|
| 33 |
+
],
|
| 34 |
+
"agree3": [
|
| 35 |
+
0.812,
|
| 36 |
+
0.8019958333333334,
|
| 37 |
+
0.822
|
| 38 |
+
],
|
| 39 |
+
"s1_in_t3": [
|
| 40 |
+
0.8221666666666667,
|
| 41 |
+
0.8125,
|
| 42 |
+
0.8321666666666667
|
| 43 |
+
]
|
| 44 |
+
},
|
| 45 |
+
"gold_misses": [
|
| 46 |
+
{
|
| 47 |
+
"text": "good night",
|
| 48 |
+
"gold": "😴",
|
| 49 |
+
"student": "✨",
|
| 50 |
+
"emo": "🌙"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"text": "starry night",
|
| 54 |
+
"gold": "🌌",
|
| 55 |
+
"student": "✨",
|
| 56 |
+
"emo": "✨"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"text": "got a cold",
|
| 60 |
+
"gold": "🤧",
|
| 61 |
+
"student": "❄️",
|
| 62 |
+
"emo": "🤧"
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"text": "doctor visit",
|
| 66 |
+
"gold": "🩺",
|
| 67 |
+
"student": "🧑⚕️",
|
| 68 |
+
"emo": "🧑⚕️"
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"text": "headache",
|
| 72 |
+
"gold": "🤕",
|
| 73 |
+
"student": "🧠",
|
| 74 |
+
"emo": "🧠"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"text": "hot dog",
|
| 78 |
+
"gold": "🌭",
|
| 79 |
+
"student": "🐕",
|
| 80 |
+
"emo": "🐕"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"text": "lemon juice",
|
| 84 |
+
"gold": "🍋",
|
| 85 |
+
"student": "🧃",
|
| 86 |
+
"emo": "🍋"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"text": "lollipop",
|
| 90 |
+
"gold": "🍭",
|
| 91 |
+
"student": "🎨",
|
| 92 |
+
"emo": "🍭"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"text": "hourglass",
|
| 96 |
+
"gold": "⏳",
|
| 97 |
+
"student": "⏰",
|
| 98 |
+
"emo": "🍦"
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
"text": "sos",
|
| 102 |
+
"gold": "🆘",
|
| 103 |
+
"student": "✨",
|
| 104 |
+
"emo": "🆘"
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"text": "new feature",
|
| 108 |
+
"gold": "🆕",
|
| 109 |
+
"student": "✨",
|
| 110 |
+
"emo": "✨"
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"text": "stock market",
|
| 114 |
+
"gold": "📈",
|
| 115 |
+
"student": "🛒",
|
| 116 |
+
"emo": "🛒"
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"text": "bar chart",
|
| 120 |
+
"gold": "📊",
|
| 121 |
+
"student": "🍷",
|
| 122 |
+
"emo": "🗺️"
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"text": "coding all day",
|
| 126 |
+
"gold": "🧑💻",
|
| 127 |
+
"student": "📅",
|
| 128 |
+
"emo": "💻"
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"text": "darts",
|
| 132 |
+
"gold": "🎯",
|
| 133 |
+
"student": "📅",
|
| 134 |
+
"emo": "🎯"
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"text": "koala",
|
| 138 |
+
"gold": "🐨",
|
| 139 |
+
"student": "🛋️",
|
| 140 |
+
"emo": "🐨"
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"text": "octopus",
|
| 144 |
+
"gold": "🐙",
|
| 145 |
+
"student": "😴",
|
| 146 |
+
"emo": "🐙"
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"text": "ladybug",
|
| 150 |
+
"gold": "🐞",
|
| 151 |
+
"student": "💰",
|
| 152 |
+
"emo": "🐞"
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"text": "ant",
|
| 156 |
+
"gold": "🐜",
|
| 157 |
+
"student": "⚠️",
|
| 158 |
+
"emo": "🐜"
|
| 159 |
+
},
|
| 160 |
+
{
|
| 161 |
+
"text": "owl",
|
| 162 |
+
"gold": "🦉",
|
| 163 |
+
"student": "📚",
|
| 164 |
+
"emo": "🦉"
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"text": "peacock",
|
| 168 |
+
"gold": "🦚",
|
| 169 |
+
"student": "🎨",
|
| 170 |
+
"emo": "🦚"
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"text": "swan",
|
| 174 |
+
"gold": "🦢",
|
| 175 |
+
"student": "✨",
|
| 176 |
+
"emo": "🧒"
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"text": "sunflower",
|
| 180 |
+
"gold": "🌻",
|
| 181 |
+
"student": "☀️",
|
| 182 |
+
"emo": "🌻"
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"text": "plant parent",
|
| 186 |
+
"gold": "🪴",
|
| 187 |
+
"student": "🐦",
|
| 188 |
+
"emo": "🪴"
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"text": "wedding dress",
|
| 192 |
+
"gold": "👰",
|
| 193 |
+
"student": "💒",
|
| 194 |
+
"emo": "👗"
|
| 195 |
+
},
|
| 196 |
+
{
|
| 197 |
+
"text": "post office",
|
| 198 |
+
"gold": "🏤",
|
| 199 |
+
"student": "🏢",
|
| 200 |
+
"emo": "📮"
|
| 201 |
+
},
|
| 202 |
+
{
|
| 203 |
+
"text": "fire truck",
|
| 204 |
+
"gold": "🚒",
|
| 205 |
+
"student": "🚚",
|
| 206 |
+
"emo": "🚚"
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"text": "police car",
|
| 210 |
+
"gold": "🚓",
|
| 211 |
+
"student": "🚗",
|
| 212 |
+
"emo": "🚗"
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"text": "screwdriver",
|
| 216 |
+
"gold": "🪛",
|
| 217 |
+
"student": "🔧",
|
| 218 |
+
"emo": "🔧"
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"text": "magnifying glass",
|
| 222 |
+
"gold": "🔍",
|
| 223 |
+
"student": "✨",
|
| 224 |
+
"emo": "🍦"
|
| 225 |
+
},
|
| 226 |
+
{
|
| 227 |
+
"text": "snowman",
|
| 228 |
+
"gold": "☃️",
|
| 229 |
+
"student": "❄️",
|
| 230 |
+
"emo": "☃️"
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"text": "listen to jazz music",
|
| 234 |
+
"gold": "🎵",
|
| 235 |
+
"student": "🎷",
|
| 236 |
+
"emo": "🎷"
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"text": "cook pasta for dinner",
|
| 240 |
+
"gold": "🍳",
|
| 241 |
+
"student": "🧑🍳",
|
| 242 |
+
"emo": "🍝"
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"text": "drink an espresso",
|
| 246 |
+
"gold": "☕",
|
| 247 |
+
"student": "🥤",
|
| 248 |
+
"emo": "☕"
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"text": "water the houseplants",
|
| 252 |
+
"gold": "🌱",
|
| 253 |
+
"student": "💧",
|
| 254 |
+
"emo": "💧"
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"text": "clean the kitchen",
|
| 258 |
+
"gold": "🧹",
|
| 259 |
+
"student": "🧽",
|
| 260 |
+
"emo": "✨"
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"text": "celebrate the promotion",
|
| 264 |
+
"gold": "🎉",
|
| 265 |
+
"student": "🥂",
|
| 266 |
+
"emo": "🥂"
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"text": "meditate for ten minutes",
|
| 270 |
+
"gold": "🧘",
|
| 271 |
+
"student": "⏱️",
|
| 272 |
+
"emo": "🧘"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"text": "plant some flowers",
|
| 276 |
+
"gold": "🌻",
|
| 277 |
+
"student": "🌷",
|
| 278 |
+
"emo": "🌷"
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"text": "look at the stars",
|
| 282 |
+
"gold": "🌌",
|
| 283 |
+
"student": "✨",
|
| 284 |
+
"emo": "✨"
|
| 285 |
+
},
|
| 286 |
+
{
|
| 287 |
+
"text": "file my taxes",
|
| 288 |
+
"gold": "🧾",
|
| 289 |
+
"student": "📁",
|
| 290 |
+
"emo": "💰"
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"text": "call an ambulance",
|
| 294 |
+
"gold": "🚑",
|
| 295 |
+
"student": "📞",
|
| 296 |
+
"emo": "📞"
|
| 297 |
+
},
|
| 298 |
+
{
|
| 299 |
+
"text": "call the police",
|
| 300 |
+
"gold": "🚓",
|
| 301 |
+
"student": "🚨",
|
| 302 |
+
"emo": "📞"
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"text": "write in my journal",
|
| 306 |
+
"gold": "📓",
|
| 307 |
+
"student": "📔",
|
| 308 |
+
"emo": "📔"
|
| 309 |
+
},
|
| 310 |
+
{
|
| 311 |
+
"text": "go to class",
|
| 312 |
+
"gold": "🏫",
|
| 313 |
+
"student": "📚",
|
| 314 |
+
"emo": "📚"
|
| 315 |
+
},
|
| 316 |
+
{
|
| 317 |
+
"text": "clean the bathroom",
|
| 318 |
+
"gold": "🧽",
|
| 319 |
+
"student": "🚽",
|
| 320 |
+
"emo": "🚽"
|
| 321 |
+
}
|
| 322 |
+
]
|
| 323 |
+
}
|
runtime.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Dependency-free (NumPy) inference for the exported TinyEmo artifact.
|
| 2 |
+
|
| 3 |
+
rt = TinyEmoRuntime("tiny_emo/artifacts_v2/tiny_emo.bin")
|
| 4 |
+
rt.suggest("Walk the dog!!") # -> [("🐕", 0.31)] (default: 1)
|
| 5 |
+
rt.suggest("Walk the dog!!", 3) # -> up to 3 (clamped to 1..3)
|
| 6 |
+
|
| 7 |
+
Input is normalised first (lower-case, punctuation/emoji stripped, last 4 words).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import struct
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
from .en_features import N_HASHES, featurize
|
| 20 |
+
from .normalize import MAX_WORDS, normalize
|
| 21 |
+
from .tiny_tokenizer import TinyTokenizer
|
| 22 |
+
|
| 23 |
+
MAX_OUTPUT = 3
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _gelu(x: np.ndarray) -> np.ndarray:
|
| 27 |
+
# exact GELU with an A&S 7.1.26 erf (|err| < 1.5e-7)
|
| 28 |
+
z = x / math.sqrt(2.0)
|
| 29 |
+
a = np.abs(z)
|
| 30 |
+
t = 1.0 / (1.0 + 0.3275911 * a)
|
| 31 |
+
poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))))
|
| 32 |
+
erf = np.sign(z) * (1.0 - poly * np.exp(-a * a))
|
| 33 |
+
return 0.5 * x * (1.0 + erf)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
|
| 37 |
+
x = x - x.max(axis=axis, keepdims=True)
|
| 38 |
+
e = np.exp(x)
|
| 39 |
+
return e / e.sum(axis=axis, keepdims=True)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _ln(x, w, b, eps=1e-5):
|
| 43 |
+
m = x.mean(-1, keepdims=True)
|
| 44 |
+
v = ((x - m) ** 2).mean(-1, keepdims=True)
|
| 45 |
+
return (x - m) / np.sqrt(v + eps) * w + b
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _unpack_nibbles(buf: bytes, n: int) -> np.ndarray:
|
| 49 |
+
b = np.frombuffer(buf, np.uint8)
|
| 50 |
+
q = np.empty(b.size * 2, np.int8)
|
| 51 |
+
q[0::2] = (b & 0x0F).astype(np.int8) - 8
|
| 52 |
+
q[1::2] = (b >> 4).astype(np.int8) - 8
|
| 53 |
+
return q[:n]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class TinyEmoRuntime:
|
| 57 |
+
def __init__(self, weights_path: str, tokenizer_path: str | None = None):
|
| 58 |
+
data = Path(weights_path).read_bytes()
|
| 59 |
+
if data[:4] != b"TEM2":
|
| 60 |
+
raise ValueError("bad model file")
|
| 61 |
+
(ml,) = struct.unpack_from("<H", data, 4)
|
| 62 |
+
self.meta = json.loads(data[6 : 6 + ml])
|
| 63 |
+
off = 6 + ml
|
| 64 |
+
(ll,) = struct.unpack_from("<I", data, off)
|
| 65 |
+
off += 4
|
| 66 |
+
self.labels = data[off : off + ll].decode("utf-8").split("\n")
|
| 67 |
+
off += ll
|
| 68 |
+
self.w: dict[str, np.ndarray] = {}
|
| 69 |
+
while off < len(data):
|
| 70 |
+
nl = data[off]
|
| 71 |
+
name = data[off + 1 : off + 1 + nl].decode()
|
| 72 |
+
off += 1 + nl
|
| 73 |
+
nd = data[off]
|
| 74 |
+
dims = struct.unpack_from("<%dH" % nd, data, off + 1)
|
| 75 |
+
off += 1 + 2 * nd
|
| 76 |
+
mode = data[off]
|
| 77 |
+
off += 1
|
| 78 |
+
n = int(np.prod(dims))
|
| 79 |
+
if mode == 0:
|
| 80 |
+
arr = np.frombuffer(data, "<f2", n, off).astype(np.float32)
|
| 81 |
+
off += 2 * n
|
| 82 |
+
else:
|
| 83 |
+
ns = dims[0] if mode in (1, 2) else 1
|
| 84 |
+
scales = np.frombuffer(data, "<f2", ns, off).astype(np.float32)
|
| 85 |
+
off += 2 * ns
|
| 86 |
+
if mode == 1:
|
| 87 |
+
q = np.frombuffer(data, np.int8, n, off).astype(np.float32)
|
| 88 |
+
off += n
|
| 89 |
+
else:
|
| 90 |
+
nb = (n + 1) // 2
|
| 91 |
+
q = _unpack_nibbles(data[off : off + nb], n).astype(np.float32)
|
| 92 |
+
off += nb
|
| 93 |
+
arr = q.reshape(dims) * (scales[:, None] if mode in (1, 2) else scales[0])
|
| 94 |
+
self.w[name] = arr.reshape(dims)
|
| 95 |
+
tok = tokenizer_path or str(Path(weights_path).with_name("tiny_tokenizer.bin"))
|
| 96 |
+
self.tok = TinyTokenizer.load(tok)
|
| 97 |
+
m = self.meta
|
| 98 |
+
self.d, self.heads, self.T, self.F = m["d_model"], m["n_heads"], m["max_tokens"], m["max_features"]
|
| 99 |
+
|
| 100 |
+
# ------------------------------------------------------------------
|
| 101 |
+
def encode_batch(self, texts: list[str]):
|
| 102 |
+
"""Normalised texts -> (tok_ids[B,T], tok_mask[B,T], ng_raw[B,F,H], ng_sign[B,F,H])."""
|
| 103 |
+
b = len(texts)
|
| 104 |
+
ids = np.zeros((b, self.T), np.int64)
|
| 105 |
+
mask = np.zeros((b, self.T), bool)
|
| 106 |
+
raw = np.zeros((b, self.F, N_HASHES), np.int64)
|
| 107 |
+
sign = np.zeros((b, self.F, N_HASHES), np.int8)
|
| 108 |
+
for i, text in enumerate(texts):
|
| 109 |
+
t = self.tok.encode(normalize(text), self.T)
|
| 110 |
+
ids[i, : len(t)] = t
|
| 111 |
+
mask[i, : len(t)] = True
|
| 112 |
+
r, s = featurize(normalize(text), self.F)
|
| 113 |
+
raw[i, : len(r)] = r
|
| 114 |
+
sign[i, : len(s)] = s
|
| 115 |
+
return ids, mask, raw, sign
|
| 116 |
+
|
| 117 |
+
def _block(self, x, mask, i):
|
| 118 |
+
w, p = self.w, f"blocks.{i}."
|
| 119 |
+
b, t, d = x.shape
|
| 120 |
+
hd = d // self.heads
|
| 121 |
+
h = _ln(x, w[p + "norm1.weight"], w[p + "norm1.bias"])
|
| 122 |
+
qkv = h @ w[p + "qkv.weight"].T + w[p + "qkv.bias"]
|
| 123 |
+
q, k, v = qkv.reshape(b, t, 3, self.heads, hd).transpose(2, 0, 3, 1, 4)
|
| 124 |
+
s = q @ k.transpose(0, 1, 3, 2) / math.sqrt(hd)
|
| 125 |
+
s = np.where(mask[:, None, None, :], s, -1e9)
|
| 126 |
+
o = (_softmax(s) @ v).transpose(0, 2, 1, 3).reshape(b, t, d)
|
| 127 |
+
x = x + o @ w[p + "proj.weight"].T + w[p + "proj.bias"]
|
| 128 |
+
h = _ln(x, w[p + "norm2.weight"], w[p + "norm2.bias"])
|
| 129 |
+
h = _gelu(h @ w[p + "ff1.weight"].T + w[p + "ff1.bias"])
|
| 130 |
+
return x + h @ w[p + "ff2.weight"].T + w[p + "ff2.bias"]
|
| 131 |
+
|
| 132 |
+
def forward(self, ids, mask, raw, sign) -> np.ndarray:
|
| 133 |
+
w, m = self.w, self.meta
|
| 134 |
+
# token stream
|
| 135 |
+
x = w["tok_emb.weight"][ids] @ w["tok_proj.weight"].T + w["pos_emb"][: ids.shape[1]]
|
| 136 |
+
for i in range(m["n_layers"]):
|
| 137 |
+
x = self._block(x, mask, i)
|
| 138 |
+
x = _ln(x, w["final_norm.weight"], w["final_norm.bias"])
|
| 139 |
+
proj = np.tanh(x) @ w["pool_proj.weight"].T + w["pool_proj.bias"]
|
| 140 |
+
score = np.where(mask, proj @ w["pool_q"] / math.sqrt(self.d), -1e9)
|
| 141 |
+
pooled = (_softmax(score, 1)[:, :, None] * x).sum(1)
|
| 142 |
+
pooled = _ln(pooled, w["pool_norm.weight"], w["pool_norm.bias"])
|
| 143 |
+
# n-gram stream
|
| 144 |
+
e = w["ng_emb.weight"][raw % m["n_buckets"]] * sign[..., None] # B,F,H,ng
|
| 145 |
+
count = np.maximum((sign[:, :, 0] != 0).sum(1), 1).astype(np.float32)
|
| 146 |
+
ng = _ln(e.sum((1, 2)) / count[:, None], w["ng_norm.weight"], w["ng_norm.bias"])
|
| 147 |
+
z = _gelu(np.concatenate([ng, pooled], -1) @ w["head1.weight"].T + w["head1.bias"])
|
| 148 |
+
return z @ w["head2.weight"].T + w["head2.bias"]
|
| 149 |
+
|
| 150 |
+
# ------------------------------------------------------------------
|
| 151 |
+
def logits_batch(self, texts: list[str], bs: int = 512) -> np.ndarray:
|
| 152 |
+
out = []
|
| 153 |
+
for i in range(0, len(texts), bs):
|
| 154 |
+
out.append(self.forward(*self.encode_batch(texts[i : i + bs])))
|
| 155 |
+
return np.concatenate(out) if out else np.zeros((0, self.meta["n_labels"]), np.float32)
|
| 156 |
+
|
| 157 |
+
def logits(self, text: str) -> np.ndarray:
|
| 158 |
+
return self.logits_batch([text])[0]
|
| 159 |
+
|
| 160 |
+
def suggest(self, text: str, limit: int = 1) -> list[tuple[str, float]]:
|
| 161 |
+
"""Up to `limit` emoji suggestions (1-3, default 1); [] for empty input."""
|
| 162 |
+
limit = max(1, min(MAX_OUTPUT, int(limit)))
|
| 163 |
+
if not normalize(text):
|
| 164 |
+
return []
|
| 165 |
+
p = _softmax(self.logits(text))
|
| 166 |
+
order = np.argsort(-p)[:limit]
|
| 167 |
+
return [(self.labels[i], float(p[i])) for i in order]
|
tiny_emo.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:db9a48131f0ce0fef9a14e9cbe23ef704773657911bc297994703048da101db5
|
| 3 |
+
size 480147
|
tiny_tokenizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:78efe82d45399faff2e849e4fb68b110b848c0eb07769239ceecfc08e2dc04d6
|
| 3 |
+
size 15355
|
tiny_tokenizer.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compact English-only unigram tokenizer ("ETK2").
|
| 2 |
+
|
| 3 |
+
Input is already `normalize`d ([a-z0-9] words separated by single spaces), so no
|
| 4 |
+
byte fallback or unicode handling is needed. Word boundaries are marked with
|
| 5 |
+
`_` (SentencePiece's `▁`).
|
| 6 |
+
|
| 7 |
+
File layout: b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 )
|
| 8 |
+
Piece 0 is <unk>.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import struct
|
| 14 |
+
|
| 15 |
+
MAGIC = b"ETK2"
|
| 16 |
+
SCORE_SCALE = 8.0
|
| 17 |
+
UNK_SCORE = -30.0
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class TinyTokenizer:
|
| 21 |
+
def __init__(self, data: bytes):
|
| 22 |
+
if data[:4] != MAGIC:
|
| 23 |
+
raise ValueError("bad tokenizer file")
|
| 24 |
+
(n,) = struct.unpack_from("<H", data, 4)
|
| 25 |
+
off = 6
|
| 26 |
+
self.pieces: list[str] = []
|
| 27 |
+
self.scores: list[float] = []
|
| 28 |
+
for _ in range(n):
|
| 29 |
+
ln = data[off]
|
| 30 |
+
off += 1
|
| 31 |
+
self.pieces.append(data[off : off + ln].decode("ascii"))
|
| 32 |
+
off += ln
|
| 33 |
+
self.scores.append(struct.unpack_from("<b", data, off)[0] / SCORE_SCALE)
|
| 34 |
+
off += 1
|
| 35 |
+
self.index = {p: i for i, p in enumerate(self.pieces) if i > 0}
|
| 36 |
+
self.unk_id = 0
|
| 37 |
+
self.max_len = max(len(p) for p in self.pieces)
|
| 38 |
+
|
| 39 |
+
@classmethod
|
| 40 |
+
def load(cls, path: str) -> "TinyTokenizer":
|
| 41 |
+
with open(path, "rb") as f:
|
| 42 |
+
return cls(f.read())
|
| 43 |
+
|
| 44 |
+
@staticmethod
|
| 45 |
+
def dump(pieces: list[str], scores: list[float]) -> bytes:
|
| 46 |
+
out = bytearray(MAGIC + struct.pack("<H", len(pieces)))
|
| 47 |
+
for p, s in zip(pieces, scores):
|
| 48 |
+
b = p.encode("ascii")
|
| 49 |
+
q = max(-128, min(127, round(s * SCORE_SCALE)))
|
| 50 |
+
out += struct.pack("<B", len(b)) + b + struct.pack("<b", q)
|
| 51 |
+
return bytes(out)
|
| 52 |
+
|
| 53 |
+
def encode(self, norm_text: str, max_tokens: int | None = None) -> list[int]:
|
| 54 |
+
s = "_" + norm_text.replace(" ", "_")
|
| 55 |
+
n = len(s)
|
| 56 |
+
if n <= 1:
|
| 57 |
+
return []
|
| 58 |
+
best = [-1e18] * (n + 1)
|
| 59 |
+
best[0] = 0.0
|
| 60 |
+
bp = [0] * (n + 1)
|
| 61 |
+
bid = [self.unk_id] * (n + 1)
|
| 62 |
+
for i in range(1, n + 1):
|
| 63 |
+
for j in range(max(0, i - self.max_len), i):
|
| 64 |
+
tid = self.index.get(s[j:i])
|
| 65 |
+
if tid is not None:
|
| 66 |
+
sc = best[j] + self.scores[tid]
|
| 67 |
+
if sc > best[i]:
|
| 68 |
+
best[i], bp[i], bid[i] = sc, j, tid
|
| 69 |
+
sc = best[i - 1] + UNK_SCORE
|
| 70 |
+
if sc > best[i]:
|
| 71 |
+
best[i], bp[i], bid[i] = sc, i - 1, self.unk_id
|
| 72 |
+
ids: list[int] = []
|
| 73 |
+
i = n
|
| 74 |
+
while i > 0:
|
| 75 |
+
ids.append(bid[i])
|
| 76 |
+
i = bp[i]
|
| 77 |
+
ids.reverse()
|
| 78 |
+
return ids if max_tokens is None else ids[-max_tokens:]
|