Add Cite 68M (68m/ package, web-68m/ browser build), card
Browse files- 68m/cite.json +26 -0
- 68m/encoder/config.json +76 -0
- 68m/tokenizer/tokenizer.json +0 -0
- 68m/tokenizer/tokenizer_config.json +17 -0
- 68m/weights.safetensors +3 -0
- README.md +30 -1
- THIRD_PARTY_NOTICES.md +2 -0
- web-68m/cite-web.json +46 -0
- web-68m/model.onnx +3 -0
- web-68m/tokenizer/tokenizer.json +0 -0
- web-68m/tokenizer/tokenizer_config.json +17 -0
- web-68m/vocab_map.bin +3 -0
68m/cite.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "cite-package-1",
|
| 3 |
+
"kept": 41655,
|
| 4 |
+
"marker": "mask",
|
| 5 |
+
"max_tokens": 7999,
|
| 6 |
+
"platt": {
|
| 7 |
+
"bias": -0.12439946444662703,
|
| 8 |
+
"slope": 0.6250998652235936
|
| 9 |
+
},
|
| 10 |
+
"readout": "mlm_head",
|
| 11 |
+
"source": {
|
| 12 |
+
"base_model": {
|
| 13 |
+
"repo": "jhu-clsp/ettin-encoder-68m",
|
| 14 |
+
"revision": "ac19ae4bc51093b31c475665ac872a936d056cc2"
|
| 15 |
+
},
|
| 16 |
+
"experiment": "experiments/110-cite-68m",
|
| 17 |
+
"student": "cjdd3b/angle-core-private-evals/110-cite-68m-soup75"
|
| 18 |
+
},
|
| 19 |
+
"storage": "token embeddings int8 per row; other weights fp16; computed in fp32",
|
| 20 |
+
"thresholds": {
|
| 21 |
+
"contributing": 0.17,
|
| 22 |
+
"needed": 0.79,
|
| 23 |
+
"needed_rule": "lowest threshold with graf_calibration empty-set accuracy >= 0.93 (104 amendment), per variant"
|
| 24 |
+
},
|
| 25 |
+
"vocabulary_full": 50368
|
| 26 |
+
}
|
68m/encoder/config.json
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"ModernBertModel"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 41651,
|
| 8 |
+
"causal_mask": false,
|
| 9 |
+
"classifier_activation": "gelu",
|
| 10 |
+
"classifier_bias": false,
|
| 11 |
+
"classifier_dropout": 0.0,
|
| 12 |
+
"classifier_pooling": "mean",
|
| 13 |
+
"cls_token_id": 41651,
|
| 14 |
+
"decoder_bias": true,
|
| 15 |
+
"deterministic_flash_attn": false,
|
| 16 |
+
"dtype": "float32",
|
| 17 |
+
"embedding_dropout": 0.0,
|
| 18 |
+
"eos_token_id": 41652,
|
| 19 |
+
"global_attn_every_n_layers": 3,
|
| 20 |
+
"gradient_checkpointing": false,
|
| 21 |
+
"hidden_activation": "gelu",
|
| 22 |
+
"hidden_size": 512,
|
| 23 |
+
"initializer_cutoff_factor": 2.0,
|
| 24 |
+
"initializer_range": 0.02,
|
| 25 |
+
"intermediate_size": 768,
|
| 26 |
+
"is_causal": false,
|
| 27 |
+
"layer_norm_eps": 1e-05,
|
| 28 |
+
"layer_types": [
|
| 29 |
+
"full_attention",
|
| 30 |
+
"sliding_attention",
|
| 31 |
+
"sliding_attention",
|
| 32 |
+
"full_attention",
|
| 33 |
+
"sliding_attention",
|
| 34 |
+
"sliding_attention",
|
| 35 |
+
"full_attention",
|
| 36 |
+
"sliding_attention",
|
| 37 |
+
"sliding_attention",
|
| 38 |
+
"full_attention",
|
| 39 |
+
"sliding_attention",
|
| 40 |
+
"sliding_attention",
|
| 41 |
+
"full_attention",
|
| 42 |
+
"sliding_attention",
|
| 43 |
+
"sliding_attention",
|
| 44 |
+
"full_attention",
|
| 45 |
+
"sliding_attention",
|
| 46 |
+
"sliding_attention",
|
| 47 |
+
"full_attention"
|
| 48 |
+
],
|
| 49 |
+
"local_attention": 128,
|
| 50 |
+
"max_position_embeddings": 7999,
|
| 51 |
+
"mlp_bias": false,
|
| 52 |
+
"mlp_dropout": 0.0,
|
| 53 |
+
"model_type": "modernbert",
|
| 54 |
+
"norm_bias": false,
|
| 55 |
+
"norm_eps": 1e-05,
|
| 56 |
+
"num_attention_heads": 8,
|
| 57 |
+
"num_hidden_layers": 19,
|
| 58 |
+
"pad_token_id": 41653,
|
| 59 |
+
"position_embedding_type": "sans_pos",
|
| 60 |
+
"rope_parameters": {
|
| 61 |
+
"full_attention": {
|
| 62 |
+
"rope_theta": 160000.0,
|
| 63 |
+
"rope_type": "default"
|
| 64 |
+
},
|
| 65 |
+
"sliding_attention": {
|
| 66 |
+
"rope_theta": 160000.0,
|
| 67 |
+
"rope_type": "default"
|
| 68 |
+
}
|
| 69 |
+
},
|
| 70 |
+
"sep_token_id": 41652,
|
| 71 |
+
"sparse_pred_ignore_index": -100,
|
| 72 |
+
"sparse_prediction": false,
|
| 73 |
+
"tie_word_embeddings": true,
|
| 74 |
+
"transformers_version": "5.22.1",
|
| 75 |
+
"vocab_size": 41655
|
| 76 |
+
}
|
68m/tokenizer/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
68m/tokenizer/tokenizer_config.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"clean_up_tokenization_spaces": true,
|
| 4 |
+
"cls_token": "[CLS]",
|
| 5 |
+
"is_local": false,
|
| 6 |
+
"local_files_only": false,
|
| 7 |
+
"mask_token": "[MASK]",
|
| 8 |
+
"model_input_names": [
|
| 9 |
+
"input_ids",
|
| 10 |
+
"attention_mask"
|
| 11 |
+
],
|
| 12 |
+
"model_max_length": 8192,
|
| 13 |
+
"pad_token": "[PAD]",
|
| 14 |
+
"sep_token": "[SEP]",
|
| 15 |
+
"tokenizer_class": "TokenizersBackend",
|
| 16 |
+
"unk_token": "[UNK]"
|
| 17 |
+
}
|
68m/weights.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:441f386660492ddb915c9db88133809248b08307336a316a6f9ad0733fc6e0a5
|
| 3 |
+
size 106946686
|
README.md
CHANGED
|
@@ -27,7 +27,7 @@ calibrated probability per paragraph and two thresholds (`needed`,
|
|
| 27 |
| | |
|
| 28 |
| --- | --- |
|
| 29 |
| **Task** | Paragraph citation / needed-blocks |
|
| 30 |
-
| **Size** | ~42 MB weights (trimmed vocab, int8 embeddings, fp16 rest) |
|
| 31 |
| **Runtime** | CPU (PyTorch); browser (ONNX Runtime WASM) |
|
| 32 |
| **Weights** | [v0.3.0](https://huggingface.co/localangle/cite/tree/v0.3.0) · [`web/`](https://huggingface.co/localangle/cite/tree/main/web) for WASM |
|
| 33 |
| **Demo** | [localangle/cite-demo](https://huggingface.co/spaces/localangle/cite-demo) (in-browser) |
|
|
@@ -91,6 +91,8 @@ cite = CitePackage.load("path/to/package") # or from_pretrained("./path")
|
|
| 91 |
| `encoder/` | ModernBERT/Ettin config for the trimmed vocabulary |
|
| 92 |
| `tokenizer/` | Tokenizer with `[MASK]` as the block marker |
|
| 93 |
| `web/` | Browser package: `model.onnx` (~36 MB int8), `vocab_map.bin`, `cite-web.json`, `tokenizer/` |
|
|
|
|
|
|
|
| 94 |
|
| 95 |
PyTorch weights expand to fp32 at load; only storage is compressed. The `web/`
|
| 96 |
ONNX graph is what the static demo runs in WASM (soft cap 2048 tokens).
|
|
@@ -140,6 +142,33 @@ rank 4 (18%) to rank 2 (60%), still behind a funding paragraph that repeats the
|
|
| 140 |
"renovation". See `experiments/107-cite-abstract-probe`, `experiments/108-cite-broad-consistency`
|
| 141 |
and `experiments/109-cite-hard-negatives`.
|
| 142 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
## Latency
|
| 144 |
|
| 145 |
Batch-1, CPU, AMD EPYC (HF `cpu-upgrade`, 8 vCPU) for the related ONNX int8
|
|
|
|
| 27 |
| | |
|
| 28 |
| --- | --- |
|
| 29 |
| **Task** | Paragraph citation / needed-blocks |
|
| 30 |
+
| **Size** | ~42 MB weights (trimmed vocab, int8 embeddings, fp16 rest); a larger 68M build is under [`68m/`](https://huggingface.co/localangle/cite/tree/main/68m) |
|
| 31 |
| **Runtime** | CPU (PyTorch); browser (ONNX Runtime WASM) |
|
| 32 |
| **Weights** | [v0.3.0](https://huggingface.co/localangle/cite/tree/v0.3.0) · [`web/`](https://huggingface.co/localangle/cite/tree/main/web) for WASM |
|
| 33 |
| **Demo** | [localangle/cite-demo](https://huggingface.co/spaces/localangle/cite-demo) (in-browser) |
|
|
|
|
| 91 |
| `encoder/` | ModernBERT/Ettin config for the trimmed vocabulary |
|
| 92 |
| `tokenizer/` | Tokenizer with `[MASK]` as the block marker |
|
| 93 |
| `web/` | Browser package: `model.onnx` (~36 MB int8), `vocab_map.bin`, `cite-web.json`, `tokenizer/` |
|
| 94 |
+
| `68m/` | The 68M build as a full Python package (same layout as the repo root; ~107 MB weights) |
|
| 95 |
+
| `web-68m/` | Browser package for the 68M build (`model.onnx` ~88 MB int8) |
|
| 96 |
|
| 97 |
PyTorch weights expand to fp32 at load; only storage is compressed. The `web/`
|
| 98 |
ONNX graph is what the static demo runs in WASM (soft cap 2048 tokens).
|
|
|
|
| 142 |
"renovation". See `experiments/107-cite-abstract-probe`, `experiments/108-cite-broad-consistency`
|
| 143 |
and `experiments/109-cite-hard-negatives`.
|
| 144 |
|
| 145 |
+
## Cite 68M
|
| 146 |
+
|
| 147 |
+
A larger build, distilled from the same teacher with the same data and training steps, on
|
| 148 |
+
[Ettin-68M](https://huggingface.co/jhu-clsp/ettin-encoder-68m). It is more accurate, especially on broad
|
| 149 |
+
and abstract questions, and about 3x slower. The 32M build above stays the default.
|
| 150 |
+
|
| 151 |
+
```python
|
| 152 |
+
cite = CitePackage.from_pretrained("localangle/cite", subfolder="68m")
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
The [demo](https://huggingface.co/spaces/localangle/cite-demo) has a model switch; the 68M build downloads
|
| 156 |
+
about 88 MB on first use and is cached after that. Packaged thresholds: needed 0.79, contributing 0.17.
|
| 157 |
+
|
| 158 |
+
| | Cite 32M (v0.3.0) | Cite 68M |
|
| 159 |
+
| --- | --- | --- |
|
| 160 |
+
| Weights (package) | 41.8 MB | 107.0 MB |
|
| 161 |
+
| Browser `model.onnx` | 36.3 MB | 88.3 MB |
|
| 162 |
+
| v59 eval: coverage / exact / empty-set / any-extra rate | 0.692 / 0.725 / 0.897 / 0.066 | 0.729 / 0.763 / 0.923 / 0.042 |
|
| 163 |
+
| Abstract probe, 92 audited items: top-1 / needed recall | 0.859 / 0.859 | 0.902 / 0.891 |
|
| 164 |
+
| Abstract probe, all 117: top-1 | 0.752 | 0.786 |
|
| 165 |
+
| Browser (WASM, 1 thread): 7 paragraphs / 44 paragraphs | 185 ms / 1.4 s | 547 ms / 4.2 s |
|
| 166 |
+
|
| 167 |
+
On the short example article, for "What specific improvements will be made to the library as a result of the
|
| 168 |
+
renovation?", the improvements paragraph scores 93.5% on 68M (60% on 32M, behind a funding paragraph).
|
| 169 |
+
The same caveats apply: one run per size, a small hand-written probe, and browser timings from a single
|
| 170 |
+
machine. See `experiments/110-cite-68m`.
|
| 171 |
+
|
| 172 |
## Latency
|
| 173 |
|
| 174 |
Batch-1, CPU, AMD EPYC (HF `cpu-upgrade`, 8 vCPU) for the related ONNX int8
|
THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -4,6 +4,8 @@
|
|
| 4 |
|
| 5 |
This package's encoder is derived from
|
| 6 |
[`jhu-clsp/ettin-encoder-32m`](https://huggingface.co/jhu-clsp/ettin-encoder-32m)
|
|
|
|
|
|
|
| 7 |
(MIT License). Cite fine-tunes and compresses those weights; the original
|
| 8 |
license terms continue to apply to the Ettin contribution.
|
| 9 |
|
|
|
|
| 4 |
|
| 5 |
This package's encoder is derived from
|
| 6 |
[`jhu-clsp/ettin-encoder-32m`](https://huggingface.co/jhu-clsp/ettin-encoder-32m)
|
| 7 |
+
(MIT License); the `68m/` and `web-68m/` build is derived from
|
| 8 |
+
[`jhu-clsp/ettin-encoder-68m`](https://huggingface.co/jhu-clsp/ettin-encoder-68m)
|
| 9 |
(MIT License). Cite fine-tunes and compresses those weights; the original
|
| 10 |
license terms continue to apply to the Ettin contribution.
|
| 11 |
|
web-68m/cite-web.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"check": {
|
| 3 |
+
"max_abs_drift": 0.36005,
|
| 4 |
+
"ort_logits": [
|
| 5 |
+
-3.12337,
|
| 6 |
+
8.28555,
|
| 7 |
+
-10.1329
|
| 8 |
+
],
|
| 9 |
+
"torch_logits": [
|
| 10 |
+
-3.48341,
|
| 11 |
+
8.48436,
|
| 12 |
+
-10.23479
|
| 13 |
+
]
|
| 14 |
+
},
|
| 15 |
+
"cls_token_id": 50281,
|
| 16 |
+
"format": "cite-web-1",
|
| 17 |
+
"marker": "mask",
|
| 18 |
+
"mask_token_id": 50284,
|
| 19 |
+
"max_tokens_package": 7999,
|
| 20 |
+
"model": "model.onnx",
|
| 21 |
+
"platt": {
|
| 22 |
+
"bias": -0.12439946444662703,
|
| 23 |
+
"slope": 0.6250998652235936
|
| 24 |
+
},
|
| 25 |
+
"quantization": "int8-mixed-rr",
|
| 26 |
+
"sep_token_id": 50282,
|
| 27 |
+
"size_mb": 88.31,
|
| 28 |
+
"soft_cap": 2048,
|
| 29 |
+
"source_package": {
|
| 30 |
+
"base_model": {
|
| 31 |
+
"repo": "jhu-clsp/ettin-encoder-68m",
|
| 32 |
+
"revision": "ac19ae4bc51093b31c475665ac872a936d056cc2"
|
| 33 |
+
},
|
| 34 |
+
"experiment": "experiments/110-cite-68m",
|
| 35 |
+
"student": "cjdd3b/angle-core-private-evals/110-cite-68m-soup75"
|
| 36 |
+
},
|
| 37 |
+
"thresholds": {
|
| 38 |
+
"contributing": 0.17,
|
| 39 |
+
"needed": 0.79,
|
| 40 |
+
"needed_rule": "lowest threshold with graf_calibration empty-set accuracy >= 0.93 (104 amendment), per variant"
|
| 41 |
+
},
|
| 42 |
+
"tokenizer": "tokenizer",
|
| 43 |
+
"vocab_full": 50368,
|
| 44 |
+
"vocab_kept": 41655,
|
| 45 |
+
"vocab_map": "vocab_map.bin"
|
| 46 |
+
}
|
web-68m/model.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:61d488dc61ae924e76951108f765f5c6c1083d57cd4693fb83fb6963c834a1da
|
| 3 |
+
size 88311932
|
web-68m/tokenizer/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
web-68m/tokenizer/tokenizer_config.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"clean_up_tokenization_spaces": true,
|
| 4 |
+
"cls_token": "[CLS]",
|
| 5 |
+
"is_local": false,
|
| 6 |
+
"local_files_only": false,
|
| 7 |
+
"mask_token": "[MASK]",
|
| 8 |
+
"model_input_names": [
|
| 9 |
+
"input_ids",
|
| 10 |
+
"attention_mask"
|
| 11 |
+
],
|
| 12 |
+
"model_max_length": 8192,
|
| 13 |
+
"pad_token": "[PAD]",
|
| 14 |
+
"sep_token": "[SEP]",
|
| 15 |
+
"tokenizer_class": "TokenizersBackend",
|
| 16 |
+
"unk_token": "[UNK]"
|
| 17 |
+
}
|
web-68m/vocab_map.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4288bfd937dc3439678af6a9f2a48912087d1d0e9ee3411e7ebf1f86369b8664
|
| 3 |
+
size 201472
|