cjdd3b commited on
Commit
24819f9
·
verified ·
1 Parent(s): 5304001

Add Cite 68M (68m/ package, web-68m/ browser build), card

Browse files
68m/cite.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "cite-package-1",
3
+ "kept": 41655,
4
+ "marker": "mask",
5
+ "max_tokens": 7999,
6
+ "platt": {
7
+ "bias": -0.12439946444662703,
8
+ "slope": 0.6250998652235936
9
+ },
10
+ "readout": "mlm_head",
11
+ "source": {
12
+ "base_model": {
13
+ "repo": "jhu-clsp/ettin-encoder-68m",
14
+ "revision": "ac19ae4bc51093b31c475665ac872a936d056cc2"
15
+ },
16
+ "experiment": "experiments/110-cite-68m",
17
+ "student": "cjdd3b/angle-core-private-evals/110-cite-68m-soup75"
18
+ },
19
+ "storage": "token embeddings int8 per row; other weights fp16; computed in fp32",
20
+ "thresholds": {
21
+ "contributing": 0.17,
22
+ "needed": 0.79,
23
+ "needed_rule": "lowest threshold with graf_calibration empty-set accuracy >= 0.93 (104 amendment), per variant"
24
+ },
25
+ "vocabulary_full": 50368
26
+ }
68m/encoder/config.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "ModernBertModel"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 41651,
8
+ "causal_mask": false,
9
+ "classifier_activation": "gelu",
10
+ "classifier_bias": false,
11
+ "classifier_dropout": 0.0,
12
+ "classifier_pooling": "mean",
13
+ "cls_token_id": 41651,
14
+ "decoder_bias": true,
15
+ "deterministic_flash_attn": false,
16
+ "dtype": "float32",
17
+ "embedding_dropout": 0.0,
18
+ "eos_token_id": 41652,
19
+ "global_attn_every_n_layers": 3,
20
+ "gradient_checkpointing": false,
21
+ "hidden_activation": "gelu",
22
+ "hidden_size": 512,
23
+ "initializer_cutoff_factor": 2.0,
24
+ "initializer_range": 0.02,
25
+ "intermediate_size": 768,
26
+ "is_causal": false,
27
+ "layer_norm_eps": 1e-05,
28
+ "layer_types": [
29
+ "full_attention",
30
+ "sliding_attention",
31
+ "sliding_attention",
32
+ "full_attention",
33
+ "sliding_attention",
34
+ "sliding_attention",
35
+ "full_attention",
36
+ "sliding_attention",
37
+ "sliding_attention",
38
+ "full_attention",
39
+ "sliding_attention",
40
+ "sliding_attention",
41
+ "full_attention",
42
+ "sliding_attention",
43
+ "sliding_attention",
44
+ "full_attention",
45
+ "sliding_attention",
46
+ "sliding_attention",
47
+ "full_attention"
48
+ ],
49
+ "local_attention": 128,
50
+ "max_position_embeddings": 7999,
51
+ "mlp_bias": false,
52
+ "mlp_dropout": 0.0,
53
+ "model_type": "modernbert",
54
+ "norm_bias": false,
55
+ "norm_eps": 1e-05,
56
+ "num_attention_heads": 8,
57
+ "num_hidden_layers": 19,
58
+ "pad_token_id": 41653,
59
+ "position_embedding_type": "sans_pos",
60
+ "rope_parameters": {
61
+ "full_attention": {
62
+ "rope_theta": 160000.0,
63
+ "rope_type": "default"
64
+ },
65
+ "sliding_attention": {
66
+ "rope_theta": 160000.0,
67
+ "rope_type": "default"
68
+ }
69
+ },
70
+ "sep_token_id": 41652,
71
+ "sparse_pred_ignore_index": -100,
72
+ "sparse_prediction": false,
73
+ "tie_word_embeddings": true,
74
+ "transformers_version": "5.22.1",
75
+ "vocab_size": 41655
76
+ }
68m/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
68m/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "clean_up_tokenization_spaces": true,
4
+ "cls_token": "[CLS]",
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_input_names": [
9
+ "input_ids",
10
+ "attention_mask"
11
+ ],
12
+ "model_max_length": 8192,
13
+ "pad_token": "[PAD]",
14
+ "sep_token": "[SEP]",
15
+ "tokenizer_class": "TokenizersBackend",
16
+ "unk_token": "[UNK]"
17
+ }
68m/weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:441f386660492ddb915c9db88133809248b08307336a316a6f9ad0733fc6e0a5
3
+ size 106946686
README.md CHANGED
@@ -27,7 +27,7 @@ calibrated probability per paragraph and two thresholds (`needed`,
27
  | | |
28
  | --- | --- |
29
  | **Task** | Paragraph citation / needed-blocks |
30
- | **Size** | ~42 MB weights (trimmed vocab, int8 embeddings, fp16 rest) |
31
  | **Runtime** | CPU (PyTorch); browser (ONNX Runtime WASM) |
32
  | **Weights** | [v0.3.0](https://huggingface.co/localangle/cite/tree/v0.3.0) · [`web/`](https://huggingface.co/localangle/cite/tree/main/web) for WASM |
33
  | **Demo** | [localangle/cite-demo](https://huggingface.co/spaces/localangle/cite-demo) (in-browser) |
@@ -91,6 +91,8 @@ cite = CitePackage.load("path/to/package") # or from_pretrained("./path")
91
  | `encoder/` | ModernBERT/Ettin config for the trimmed vocabulary |
92
  | `tokenizer/` | Tokenizer with `[MASK]` as the block marker |
93
  | `web/` | Browser package: `model.onnx` (~36 MB int8), `vocab_map.bin`, `cite-web.json`, `tokenizer/` |
 
 
94
 
95
  PyTorch weights expand to fp32 at load; only storage is compressed. The `web/`
96
  ONNX graph is what the static demo runs in WASM (soft cap 2048 tokens).
@@ -140,6 +142,33 @@ rank 4 (18%) to rank 2 (60%), still behind a funding paragraph that repeats the
140
  "renovation". See `experiments/107-cite-abstract-probe`, `experiments/108-cite-broad-consistency`
141
  and `experiments/109-cite-hard-negatives`.
142
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  ## Latency
144
 
145
  Batch-1, CPU, AMD EPYC (HF `cpu-upgrade`, 8 vCPU) for the related ONNX int8
 
27
  | | |
28
  | --- | --- |
29
  | **Task** | Paragraph citation / needed-blocks |
30
+ | **Size** | ~42 MB weights (trimmed vocab, int8 embeddings, fp16 rest); a larger 68M build is under [`68m/`](https://huggingface.co/localangle/cite/tree/main/68m) |
31
  | **Runtime** | CPU (PyTorch); browser (ONNX Runtime WASM) |
32
  | **Weights** | [v0.3.0](https://huggingface.co/localangle/cite/tree/v0.3.0) · [`web/`](https://huggingface.co/localangle/cite/tree/main/web) for WASM |
33
  | **Demo** | [localangle/cite-demo](https://huggingface.co/spaces/localangle/cite-demo) (in-browser) |
 
91
  | `encoder/` | ModernBERT/Ettin config for the trimmed vocabulary |
92
  | `tokenizer/` | Tokenizer with `[MASK]` as the block marker |
93
  | `web/` | Browser package: `model.onnx` (~36 MB int8), `vocab_map.bin`, `cite-web.json`, `tokenizer/` |
94
+ | `68m/` | The 68M build as a full Python package (same layout as the repo root; ~107 MB weights) |
95
+ | `web-68m/` | Browser package for the 68M build (`model.onnx` ~88 MB int8) |
96
 
97
  PyTorch weights expand to fp32 at load; only storage is compressed. The `web/`
98
  ONNX graph is what the static demo runs in WASM (soft cap 2048 tokens).
 
142
  "renovation". See `experiments/107-cite-abstract-probe`, `experiments/108-cite-broad-consistency`
143
  and `experiments/109-cite-hard-negatives`.
144
 
145
+ ## Cite 68M
146
+
147
+ A larger build, distilled from the same teacher with the same data and training steps, on
148
+ [Ettin-68M](https://huggingface.co/jhu-clsp/ettin-encoder-68m). It is more accurate, especially on broad
149
+ and abstract questions, and about 3x slower. The 32M build above stays the default.
150
+
151
+ ```python
152
+ cite = CitePackage.from_pretrained("localangle/cite", subfolder="68m")
153
+ ```
154
+
155
+ The [demo](https://huggingface.co/spaces/localangle/cite-demo) has a model switch; the 68M build downloads
156
+ about 88 MB on first use and is cached after that. Packaged thresholds: needed 0.79, contributing 0.17.
157
+
158
+ | | Cite 32M (v0.3.0) | Cite 68M |
159
+ | --- | --- | --- |
160
+ | Weights (package) | 41.8 MB | 107.0 MB |
161
+ | Browser `model.onnx` | 36.3 MB | 88.3 MB |
162
+ | v59 eval: coverage / exact / empty-set / any-extra rate | 0.692 / 0.725 / 0.897 / 0.066 | 0.729 / 0.763 / 0.923 / 0.042 |
163
+ | Abstract probe, 92 audited items: top-1 / needed recall | 0.859 / 0.859 | 0.902 / 0.891 |
164
+ | Abstract probe, all 117: top-1 | 0.752 | 0.786 |
165
+ | Browser (WASM, 1 thread): 7 paragraphs / 44 paragraphs | 185 ms / 1.4 s | 547 ms / 4.2 s |
166
+
167
+ On the short example article, for "What specific improvements will be made to the library as a result of the
168
+ renovation?", the improvements paragraph scores 93.5% on 68M (60% on 32M, behind a funding paragraph).
169
+ The same caveats apply: one run per size, a small hand-written probe, and browser timings from a single
170
+ machine. See `experiments/110-cite-68m`.
171
+
172
  ## Latency
173
 
174
  Batch-1, CPU, AMD EPYC (HF `cpu-upgrade`, 8 vCPU) for the related ONNX int8
THIRD_PARTY_NOTICES.md CHANGED
@@ -4,6 +4,8 @@
4
 
5
  This package's encoder is derived from
6
  [`jhu-clsp/ettin-encoder-32m`](https://huggingface.co/jhu-clsp/ettin-encoder-32m)
 
 
7
  (MIT License). Cite fine-tunes and compresses those weights; the original
8
  license terms continue to apply to the Ettin contribution.
9
 
 
4
 
5
  This package's encoder is derived from
6
  [`jhu-clsp/ettin-encoder-32m`](https://huggingface.co/jhu-clsp/ettin-encoder-32m)
7
+ (MIT License); the `68m/` and `web-68m/` build is derived from
8
+ [`jhu-clsp/ettin-encoder-68m`](https://huggingface.co/jhu-clsp/ettin-encoder-68m)
9
  (MIT License). Cite fine-tunes and compresses those weights; the original
10
  license terms continue to apply to the Ettin contribution.
11
 
web-68m/cite-web.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "check": {
3
+ "max_abs_drift": 0.36005,
4
+ "ort_logits": [
5
+ -3.12337,
6
+ 8.28555,
7
+ -10.1329
8
+ ],
9
+ "torch_logits": [
10
+ -3.48341,
11
+ 8.48436,
12
+ -10.23479
13
+ ]
14
+ },
15
+ "cls_token_id": 50281,
16
+ "format": "cite-web-1",
17
+ "marker": "mask",
18
+ "mask_token_id": 50284,
19
+ "max_tokens_package": 7999,
20
+ "model": "model.onnx",
21
+ "platt": {
22
+ "bias": -0.12439946444662703,
23
+ "slope": 0.6250998652235936
24
+ },
25
+ "quantization": "int8-mixed-rr",
26
+ "sep_token_id": 50282,
27
+ "size_mb": 88.31,
28
+ "soft_cap": 2048,
29
+ "source_package": {
30
+ "base_model": {
31
+ "repo": "jhu-clsp/ettin-encoder-68m",
32
+ "revision": "ac19ae4bc51093b31c475665ac872a936d056cc2"
33
+ },
34
+ "experiment": "experiments/110-cite-68m",
35
+ "student": "cjdd3b/angle-core-private-evals/110-cite-68m-soup75"
36
+ },
37
+ "thresholds": {
38
+ "contributing": 0.17,
39
+ "needed": 0.79,
40
+ "needed_rule": "lowest threshold with graf_calibration empty-set accuracy >= 0.93 (104 amendment), per variant"
41
+ },
42
+ "tokenizer": "tokenizer",
43
+ "vocab_full": 50368,
44
+ "vocab_kept": 41655,
45
+ "vocab_map": "vocab_map.bin"
46
+ }
web-68m/model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61d488dc61ae924e76951108f765f5c6c1083d57cd4693fb83fb6963c834a1da
3
+ size 88311932
web-68m/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
web-68m/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "clean_up_tokenization_spaces": true,
4
+ "cls_token": "[CLS]",
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_input_names": [
9
+ "input_ids",
10
+ "attention_mask"
11
+ ],
12
+ "model_max_length": 8192,
13
+ "pad_token": "[PAD]",
14
+ "sep_token": "[SEP]",
15
+ "tokenizer_class": "TokenizersBackend",
16
+ "unk_token": "[UNK]"
17
+ }
web-68m/vocab_map.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4288bfd937dc3439678af6a9f2a48912087d1d0e9ee3411e7ebf1f86369b8664
3
+ size 201472