klpeng commited on
Commit
67d26d3
·
verified ·
1 Parent(s): 8b3159a

Upload folder using huggingface_hub

Browse files
Files changed (11) hide show
  1. .gitattributes +1 -0
  2. LICENSE-code +21 -0
  3. TERMS.md +12 -0
  4. config.json +46 -0
  5. gemini_ucd.py +186 -0
  6. labels.csv +3 -0
  7. model.safetensors +3 -0
  8. prevalences.csv +0 -0
  9. prevalences.json +53 -0
  10. prevalences.npz +3 -0
  11. requirements.txt +5 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ labels.csv filter=lfs diff=lfs merge=lfs -text
LICENSE-code ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Atomic Features contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
TERMS.md ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Artifact terms
2
+
3
+ The inference and validation Python code is covered by `LICENSE-code` (MIT).
4
+ That license does not assign a license to `model.safetensors`, `labels.csv`, or
5
+ the accompanying numerical artifacts. The source release did not specify a
6
+ redistribution license for weights or labels; this export preserves that status.
7
+ The owner should specify those terms when publishing the model.
8
+
9
+ The encoder uses embeddings produced by Google's Gemini embedding service;
10
+ Google's applicable terms govern use of that service. No Google model weights,
11
+ source texts, or real corpus embeddings are included here. Test vectors contain
12
+ synthetic inputs and their SAE outputs only.
config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format_version": 2,
3
+ "model_type": "gemini_ucd",
4
+ "model_name": "gemini-ucd-80k",
5
+ "input_dim": 3072,
6
+ "num_features": 80513,
7
+ "top_k": 128,
8
+ "dtype": "float32",
9
+ "input_unit_norm": false,
10
+ "activation": "topk(relu((x - b_dec) @ W_enc), k=128), among 80513 labeled concepts",
11
+ "weight_file": "model.safetensors",
12
+ "weight_layout": {
13
+ "W_enc": "embedding_dimension x num_features (physically pruned)",
14
+ "b_dec": "embedding_dimension"
15
+ },
16
+ "embedding_model": {
17
+ "model_id": "gemini-embedding-2-preview",
18
+ "dimensions": 3072,
19
+ "corpus_storage_dtype": "float16",
20
+ "recorded_task_type": "RETRIEVAL_DOCUMENT",
21
+ "compatibility": "Training embedding space only; other versions not validated"
22
+ },
23
+ "labels": {
24
+ "file": "labels.csv",
25
+ "count": 80513,
26
+ "id_column": "feature_id",
27
+ "release": "recall95_clarity_v1",
28
+ "inference_threshold": null,
29
+ "description_column": "discussion"
30
+ },
31
+ "source_checkpoint_sha256": "ca6c7cb9dcf318cb089eddd3a48b424d49cf25ac6219cd9bcb5bb355c3eee044",
32
+ "prevalences": {
33
+ "file": "prevalences.npz",
34
+ "csv_file": "prevalences.csv",
35
+ "metadata_file": "prevalences.json",
36
+ "thresholds": {
37
+ "0p05": 0.05,
38
+ "0p1": 0.1
39
+ },
40
+ "n_corpus_rows": 89827558,
41
+ "confidence_level": 0.99,
42
+ "upper_bound_method": "one-sided Clopper-Pearson",
43
+ "usage": "Heuristic corpus prevalences"
44
+ },
45
+ "feature_id_convention": "Contiguous zero-based UCD output coordinates."
46
+ }
gemini_ucd.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Universal Concept Dictionary with compact labeled outputs; LICENSE-code."""
2
+
3
+ import argparse
4
+ import csv
5
+ import json
6
+ from pathlib import Path
7
+
8
+ import numpy as np
9
+ import torch
10
+ from safetensors.torch import load_file
11
+ from scipy import sparse
12
+
13
+
14
+ class GeminiUCD(torch.nn.Module):
15
+ """Physically pruned encoder with TopK competition among labeled concepts."""
16
+
17
+ def __init__(self, config, tensors, labels):
18
+ super().__init__()
19
+ self.config = dict(config)
20
+ self.input_dim = config["input_dim"]
21
+ self.num_features = config["num_features"]
22
+ self.top_k = config["top_k"]
23
+ if config.get("format_version") != 2 or config.get("model_type") != "gemini_ucd":
24
+ raise ValueError("Unsupported UCD format")
25
+ if config.get("input_unit_norm") is not False:
26
+ raise ValueError("This encoder requires input_unit_norm=false")
27
+ if not isinstance(self.top_k, int) or not 0 < self.top_k <= self.num_features:
28
+ raise ValueError("Invalid top_k")
29
+ if set(tensors) != {"W_enc", "b_dec"}:
30
+ raise ValueError("Expected only W_enc and b_dec")
31
+ if tensors["W_enc"].shape != (self.input_dim, self.num_features):
32
+ raise ValueError("Invalid W_enc shape")
33
+ if tensors["b_dec"].shape != (self.input_dim,):
34
+ raise ValueError("Invalid b_dec shape")
35
+ for name, tensor in tensors.items():
36
+ if tensor.dtype != torch.float32:
37
+ raise ValueError("This release requires float32 weights")
38
+ self.register_buffer(name, tensor)
39
+ self.labels = dict(labels)
40
+ if set(self.labels) != set(range(self.num_features)) or any(
41
+ not str(label).strip() for label in self.labels.values()):
42
+ raise ValueError("Every UCD coordinate must have a nonempty label")
43
+ self.eval()
44
+
45
+ @classmethod
46
+ def from_pretrained(cls, model_id_or_path, *, revision=None, device="cpu",
47
+ cache_dir=None, local_files_only=False):
48
+ """Load a local folder or a Hub repo. Use a commit revision for reproducibility.
49
+
50
+ Only data files are downloaded; no remote Python code is imported.
51
+ """
52
+ directory = Path(model_id_or_path).expanduser()
53
+ if not directory.is_dir():
54
+ from huggingface_hub import snapshot_download
55
+
56
+ directory = Path(snapshot_download(
57
+ repo_id=str(model_id_or_path), revision=revision,
58
+ cache_dir=cache_dir, local_files_only=local_files_only,
59
+ allow_patterns=["config.json", "model.safetensors", "labels.csv"],
60
+ ))
61
+ config = json.loads((directory / "config.json").read_text())
62
+ labels = {}
63
+ with (directory / "labels.csv").open(newline="", encoding="utf-8") as handle:
64
+ for row in csv.DictReader(handle):
65
+ feature = int(row["feature_id"])
66
+ if feature in labels or not row["label"].strip():
67
+ raise ValueError("Duplicate feature ID or empty label")
68
+ labels[feature] = row["label"]
69
+ if len(labels) != config["labels"]["count"]:
70
+ raise ValueError("Label count disagrees with config")
71
+ tensors = load_file(str(directory / "model.safetensors"), device="cpu")
72
+ return cls(config, tensors, labels).to(device=device)
73
+
74
+ @torch.inference_mode()
75
+ def forward(self, embeddings):
76
+ """Return (concept_indices, values), both [batch, top_k], on model device.
77
+
78
+ Zero-valued slots are padding, not active features. This low-level method
79
+ processes one batch; encode()/encode_iter() handle batching automatically.
80
+ """
81
+ if self.W_enc.dtype != torch.float32 or self.b_dec.dtype != torch.float32:
82
+ raise ValueError("Keep this model in float32; reduced precision changes TopK")
83
+ if isinstance(embeddings, torch.Tensor):
84
+ if embeddings.is_complex() or embeddings.dtype == torch.bool:
85
+ raise ValueError("Expected real numeric embeddings")
86
+ x = embeddings.detach().to(device=self.W_enc.device, dtype=torch.float32)
87
+ else:
88
+ array = np.asarray(embeddings)
89
+ if array.dtype.kind not in "fiu":
90
+ raise ValueError("Expected real numeric embeddings")
91
+ # Copy only this batch, supporting read-only mmap inputs safely.
92
+ x = torch.from_numpy(np.array(array, dtype=np.float32, copy=True)).to(self.W_enc.device)
93
+ if x.ndim != 2 or x.shape[1] != self.input_dim:
94
+ raise ValueError(f"Expected embeddings with shape (N, {self.input_dim})")
95
+ if not torch.isfinite(x).all():
96
+ raise ValueError("Embeddings contain NaN or infinity")
97
+ with torch.autocast(device_type=self.W_enc.device.type, enabled=False):
98
+ scores = torch.relu((x - self.b_dec) @ self.W_enc)
99
+ if not torch.isfinite(scores).all():
100
+ raise ValueError("Nonfinite encoder scores; check embedding scale")
101
+ values, indices = torch.topk(scores, self.top_k, dim=-1)
102
+ return indices, values
103
+
104
+ def encode_iter(self, embeddings, *, batch_size=64):
105
+ """Yield CPU SciPy CSR batches, each with num_features columns.
106
+
107
+ Peak score storage is batch_size * num_features * 4 bytes.
108
+ Only labeled concepts are returned, in contiguous UCD coordinates.
109
+ """
110
+ if not isinstance(batch_size, int) or isinstance(batch_size, bool) or batch_size < 1:
111
+ raise ValueError("batch_size must be a positive integer")
112
+ if not isinstance(embeddings, (np.ndarray, torch.Tensor)):
113
+ raise TypeError("Expected a NumPy array or Torch tensor")
114
+ if embeddings.ndim != 2 or embeddings.shape[1] != self.input_dim:
115
+ raise ValueError(f"Expected embeddings with shape (N, {self.input_dim})")
116
+ if isinstance(embeddings, torch.Tensor):
117
+ if embeddings.is_complex() or embeddings.dtype == torch.bool:
118
+ raise ValueError("Expected real numeric embeddings")
119
+ elif embeddings.dtype.kind not in "fiu":
120
+ raise ValueError("Expected real numeric embeddings")
121
+ for start in range(0, len(embeddings), batch_size):
122
+ indices, values = self(embeddings[start:start + batch_size])
123
+ ids = indices.cpu().numpy().astype(np.int32, copy=False)
124
+ vals = values.cpu().numpy()
125
+ indptr = np.arange(0, vals.size + 1, self.top_k, dtype=np.int32)
126
+ result = sparse.csr_matrix(
127
+ (vals.ravel(), ids.ravel(), indptr),
128
+ shape=(len(vals), self.num_features),
129
+ )
130
+ result.eliminate_zeros()
131
+ result.sort_indices()
132
+ yield result
133
+
134
+ def encode(self, embeddings, *, batch_size=64):
135
+ """Return a float32 SciPy CSR matrix of shape (N, num_features).
136
+
137
+ The complete sparse output must fit in RAM; use encode_iter for streaming.
138
+ """
139
+ batches = list(self.encode_iter(embeddings, batch_size=batch_size))
140
+ return sparse.vstack(batches, format="csr") if batches else sparse.csr_matrix(
141
+ (0, self.num_features), dtype=np.float32
142
+ )
143
+
144
+ def explain(self, row, *, top_n=20):
145
+ """Describe one encoded CSR row, ordered by decreasing activation.
146
+
147
+ Every coordinate has a label. Values are activations, not probabilities.
148
+ """
149
+ if not sparse.issparse(row) or row.shape != (1, self.num_features):
150
+ raise ValueError("Pass one encoded sparse row, e.g. features[0]")
151
+ if not isinstance(top_n, int) or top_n < 0:
152
+ raise ValueError("top_n must be a nonnegative integer")
153
+ row = row.tocsr(copy=True)
154
+ row.sum_duplicates()
155
+ row.eliminate_zeros()
156
+ order = np.lexsort((row.indices, -row.data))[:top_n]
157
+ return [dict(feature_id=int(row.indices[i]), label=self.labels[int(row.indices[i])],
158
+ activation=float(row.data[i])) for i in order]
159
+
160
+
161
+ def main():
162
+ parser = argparse.ArgumentParser(description="Encode Gemini embeddings as sparse UCD concepts")
163
+ parser.add_argument("--model", required=True, help="Local folder or Hugging Face repo ID")
164
+ parser.add_argument("--revision", help="Hub commit SHA or tag")
165
+ parser.add_argument("--embeddings", required=True, type=Path, help="N x 3072 .npy array")
166
+ parser.add_argument("--output", required=True, type=Path, help="Output sparse .npz file")
167
+ parser.add_argument("--device", default="cpu", help="cpu, cuda, or cuda:0")
168
+ parser.add_argument("--batch-size", type=int, default=64)
169
+ args = parser.parse_args()
170
+ if args.output.exists():
171
+ parser.error(f"Output already exists: {args.output}")
172
+ if args.output.suffix != ".npz":
173
+ parser.error("--output must end in .npz")
174
+ if args.batch_size < 1:
175
+ parser.error("--batch-size must be positive")
176
+ model = GeminiUCD.from_pretrained(args.model, revision=args.revision, device=args.device)
177
+ embeddings = np.load(args.embeddings, mmap_mode="r", allow_pickle=False)
178
+ result = model.encode(embeddings, batch_size=args.batch_size)
179
+ args.output.parent.mkdir(parents=True, exist_ok=True)
180
+ with args.output.open("xb") as handle:
181
+ sparse.save_npz(handle, result)
182
+ print(json.dumps(dict(output=str(args.output), shape=list(result.shape), nnz=result.nnz)))
183
+
184
+
185
+ if __name__ == "__main__":
186
+ main()
labels.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f685a05f5eb16dc91a6b72e94d0130004acbfd835e1d37635f5b4b785b23efd7
3
+ size 47115831
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdc784c471876f89cb5a0e91837eea3e4e58f3de54ed96280f1e1928ed6609be
3
+ size 989356192
prevalences.csv ADDED
The diff for this file is too large to render. See raw diff
 
prevalences.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format_version": 1,
3
+ "files": [
4
+ "prevalences.csv",
5
+ "prevalences.npz"
6
+ ],
7
+ "num_features": 80513,
8
+ "n_corpus_rows": 89827558,
9
+ "thresholds": {
10
+ "0p05": 0.05,
11
+ "0p1": 0.1
12
+ },
13
+ "columns": {
14
+ "feature": "Zero-based UCD output coordinate; all coordinates have labels.",
15
+ "n_corpus_rows": "Denominator: all stored corpus rows, including duplicate texts and implicit zeros.",
16
+ "count_gt_0p05": "Number of corpus rows with post-TopK activation strictly > 0.05.",
17
+ "prevalence_gt_0p05": "count_gt_0p05 / n_corpus_rows; fraction, not percent.",
18
+ "prevalence_gt_0p05_upper_99": "One-sided 99% Clopper-Pearson upper bound for prevalence_gt_0p05.",
19
+ "count_gt_0p1": "Number of corpus rows with post-TopK activation strictly > 0.1.",
20
+ "prevalence_gt_0p1": "count_gt_0p1 / n_corpus_rows; fraction, not percent.",
21
+ "prevalence_gt_0p1_upper_99": "One-sided 99% Clopper-Pearson upper bound for prevalence_gt_0p1."
22
+ },
23
+ "units": "fraction in [0,1], not percent",
24
+ "source_checkpoint_sha256": "ca6c7cb9dcf318cb089eddd3a48b424d49cf25ac6219cd9bcb5bb355c3eee044",
25
+ "threshold_comparison": "Strict activation > threshold; stored float32 activations promoted to float64 before comparison with float64 decimal thresholds.",
26
+ "population": "All 89,827,558 stored Gemini corpus rows across 929 shards. Row-weighted, not text-deduplicated, and not held-out data.",
27
+ "confidence_bound": {
28
+ "method": "Clopper-Pearson exact binomial",
29
+ "confidence_level": 0.99,
30
+ "sidedness": "upper, one-sided",
31
+ "interval": "[0, upper_99]",
32
+ "alpha": 0.01,
33
+ "formula": "U = Beta.ppf(0.99, k + 1, n - k) for k < n; U = 1 for k = n.",
34
+ "zero_count_formula": "U = 1 - 0.01**(1/n), evaluated stably with -expm1(log(0.01)/n).",
35
+ "coverage": "Pointwise per neuron and threshold; no simultaneous or multiple-comparison adjustment.",
36
+ "interpretation": "Empirical prevalence is exact for the saved corpus. Population bounds assume independent Bernoulli observations from a fixed distribution; duplicate or correlated texts, dataset mixing, model selection and domain shift are not accounted for. These are model-based bounds, not distribution-free guarantees.",
37
+ "distinction": "99% one-sided upper confidence bound, not an activation percentile, Bayesian credible interval, or upper endpoint of a two-sided 99% interval."
38
+ },
39
+ "source": {
40
+ "statistics_file": "summary/feature_statistics.parquet",
41
+ "statistics_sha256": "593d31a7cc42b85f4dacc8c4c0082bec2d3ce9c2e5a48464a33c4587c5b654d6",
42
+ "pipeline_config_id": "53020913064a76fbb35f636a4a996512e2f350cf28a5d4e9c4b01dee3fb2e63c",
43
+ "activation_manifest_sha256": "644490bad91afd60d0ec080b29447a18ac3e0b92ef6e5271c98facbbd800c782",
44
+ "config_sha256": "a8f9a1de2af308c13de8de045f0a795c0e2256806e6ec47f3e39126520d65921",
45
+ "completion_sha256": "9f668601b244ca98d98b42db7111816c62707b92a36edb9158bfbfb68a7a427e",
46
+ "feature_partitions_cross_checked": 64
47
+ },
48
+ "generated_at_utc": "2026-09-29T14:35:07.641767+00:00",
49
+ "scipy_version": "1.15.3",
50
+ "method_reference": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats._result_classes.BinomTestResult.proportion_ci.html",
51
+ "usage": "Heuristic corpus prevalences",
52
+ "computation": "Computed from the source 131072-feature encoder with global top-128 selection, then restricted and remapped to the 80513 UCD coordinates."
53
+ }
prevalences.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d4ee99033cbe241e05df7a5442dc1612ea1926eb3d1e85958cc6cc4cc9f0a0e
3
+ size 2019239
requirements.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ numpy>=1.24
2
+ scipy>=1.11
3
+ torch>=2.6
4
+ safetensors>=0.4
5
+ huggingface-hub>=0.24