Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 redup.ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,132 @@
+---
+license: mit
+library_name: redup-topicmodel
+tags:
+- topic-modeling
+- bigartm
+- multilingual
+- feature-extraction
+---
+
+# REDUP multilingual topic model
+
+BigARTM topic-model artifacts for package `redup-topicmodel`
+(`redup.python.topicmodel`), created **2026-07-24**.
+
+This is a **custom** model dump (not Transformers / Diffusers). Load it with
+`huggingface_hub` + `redup-topicmodel` (and `bigartm==0.9.2`).
+`AutoModel` / `AutoTokenizer` are not supported.
+
+## Contents
+
+| Path | Role |
+|------|------|
+| `artm/parameters.bin` | BigARTM metadata + regularizers (pickle) |
+| `artm/p_wt.bin` | Topic–token matrix Φ for inference |
+| `tokenizers.json.gz` | Per-language BPE packs: `{lang: {vocab, merges}}` |
+| `config.json` | Hub metadata (languages, versions) |
+
+## Languages
+
+100 languages: af, am, ar, av, az, ba, be, bg, bn, bs, ca, ce, cs, cv, cy, da,
+de, el, en, eo, es, et, eu, fa, fi, fr, gd, gl, gu, he, hi, hr, hu, hy, ia, id,
+inh, is, it, ja, jv, ka, kaa, kbd, kk, kl, km, ko, krc, ky, la, lez, lo, lt, lv,
+mg, mhr, mi, mk, ml, mn, mo, ms, my, myv, ne, nl, no, oc, os, pl, pt, rm, rn, ro,
+ru, sah, sh, si, sk, sl, sm, so, sq, sr, sv, sw, ta, tg, th, tk, tr, tt, udm, uk,
+ur, uz, vi, yi, zh.
+
+Document modality `lang` selects the BPE pack; tokens are scored under BigARTM
+class id `@{lang}` (125 topics: `topic_0` … `topic_124`).
+
+## Install
+
+```bash
+pip install huggingface_hub redup-topicmodel bigartm==0.9.2
+```
+
+## Download
+
+```python
+from huggingface_hub import snapshot_download
+
+root = snapshot_download(repo_id="redup-ai/topicmodel-multilingual")
+# root/
+#   artm/parameters.bin
+#   artm/p_wt.bin
+#   tokenizers.json.gz
+#   config.json
+#   README.md
+```
+
+## Inference (recommended)
+
+High-level worker used by the gRPC service:
+
+```python
+import asyncio
+from types import SimpleNamespace
+
+from redup_topicmodel.topicmodel.inference import InferenceWorker
+
+
+def document(document_id: str, tokens: list[str], lang: str):
+    return SimpleNamespace(
+        document_id=document_id,
+        tokens=tokens,
+        modalities={"lang": lang},
+    )
+
+
+async def main(root: str):
+    worker = InferenceWorker({
+        "artifact_root": root,
+        "work_dir": "/tmp/topicmodel",
+    })
+    pack = SimpleNamespace(documents=[
+        document("doc-en", ["hello", "world"], "en"),
+        document("doc-ru", ["привет", "мир"], "ru"),
+    ])
+    result = await worker.get_documents_embedding("example", pack)
+    for embedding in result["embeddings"]:
+        print(len(embedding["values"]), sum(embedding["values"]))
+
+
+asyncio.run(main(root))
+```
+
+Each embedding is a length-125 topic distribution (≈ sums to 1).
+
+## Tokenizer only
+
+```python
+from redup_topicmodel.topicmodel.bpe import load_bpe_models
+
+bpe = load_bpe_models(f"{root}/tokenizers.json.gz")
+print(bpe["en"].encode("hello world").tokens)
+```
+
+## Load BigARTM weights only
+
+```python
+from redup_topicmodel.topicmodel.artm_loader import load_artm_model_inference
+
+model = load_artm_model_inference(f"{root}/artm")
+```
+
+## gRPC service config
+
+```yaml
+TopicModel:
+  artifact_root: /path/from/snapshot_download
+  work_dir: /tmp/topicmodel
+```
+
+## Notes
+
+- `parameters.bin` is a Python pickle produced by BigARTM; only load trusted artifacts.
+- `tokenizers.json.gz` is intentional: one file with 100 independent BPE packs.
+  This is **not** a Hugging Face `tokenizer.json` / `AutoTokenizer` export.
+
+## License
+
+MIT.

artm/p_wt.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:619f03c8d5ce5d400692ace45235a319d03e9667a6499d8854ec9014f9e6a645
+size 168434939

artm/parameters.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36b7ca0a60ef59970cfc1bc4b760e14999476f66c2f3196edced60e70d4ecbe1
+size 146709

config.json

@@ -0,0 +1,109 @@
+{
+  "model_name": "redup-multilingual-topicmodel",
+  "library": "redup-topicmodel",
+  "framework": "BigARTM",
+  "bigartm_version": "0.9.2",
+  "created": "2026-07-24",
+  "languages": [
+    "af",
+    "am",
+    "ar",
+    "av",
+    "az",
+    "ba",
+    "be",
+    "bg",
+    "bn",
+    "bs",
+    "ca",
+    "ce",
+    "cs",
+    "cv",
+    "cy",
+    "da",
+    "de",
+    "el",
+    "en",
+    "eo",
+    "es",
+    "et",
+    "eu",
+    "fa",
+    "fi",
+    "fr",
+    "gd",
+    "gl",
+    "gu",
+    "he",
+    "hi",
+    "hr",
+    "hu",
+    "hy",
+    "ia",
+    "id",
+    "inh",
+    "is",
+    "it",
+    "ja",
+    "jv",
+    "ka",
+    "kaa",
+    "kbd",
+    "kk",
+    "kl",
+    "km",
+    "ko",
+    "krc",
+    "ky",
+    "la",
+    "lez",
+    "lo",
+    "lt",
+    "lv",
+    "mg",
+    "mhr",
+    "mi",
+    "mk",
+    "ml",
+    "mn",
+    "mo",
+    "ms",
+    "my",
+    "myv",
+    "ne",
+    "nl",
+    "no",
+    "oc",
+    "os",
+    "pl",
+    "pt",
+    "rm",
+    "rn",
+    "ro",
+    "ru",
+    "sah",
+    "sh",
+    "si",
+    "sk",
+    "sl",
+    "sm",
+    "so",
+    "sq",
+    "sr",
+    "sv",
+    "sw",
+    "ta",
+    "tg",
+    "th",
+    "tk",
+    "tr",
+    "tt",
+    "udm",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "yi",
+    "zh"
+  ]
+}

tokenizers.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f831dd774fd65009e55716894b9a8c1f5c4aaf7e0f07d1e829dd34cfd6ff4d3
+size 94253290