Text Ranking
Transformers
ONNX
Safetensors
sentence-transformers
Transformers.js
modernbert
text-classification
reranker
cross-encoder
mmbert
text-embeddings-inference
Instructions to use atelasi/multilingual-reranker-small with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use atelasi/multilingual-reranker-small with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("atelasi/multilingual-reranker-small") model = AutoModelForSequenceClassification.from_pretrained("atelasi/multilingual-reranker-small", device_map="auto") - sentence-transformers
How to use atelasi/multilingual-reranker-small with sentence-transformers:
from sentence_transformers import CrossEncoder model = CrossEncoder("atelasi/multilingual-reranker-small") query = "Which planet is known as the Red Planet?" passages = [ "Venus is often called Earth's twin because of its similar size and proximity.", "Mars, known for its reddish appearance, is often referred to as the Red Planet.", "Jupiter, the largest planet in our solar system, has a prominent red spot.", "Saturn, famous for its rings, is sometimes mistaken for the Red Planet." ] scores = model.predict([(query, passage) for passage in passages]) print(scores) - Transformers.js
How to use atelasi/multilingual-reranker-small with Transformers.js:
// npm i @huggingface/transformers import { AutoTokenizer, AutoModelForSequenceClassification } from '@huggingface/transformers'; const tokenizer = await AutoTokenizer.from_pretrained('atelasi/multilingual-reranker-small'); const model = await AutoModelForSequenceClassification.from_pretrained('atelasi/multilingual-reranker-small'); const query = 'Which planet is known as the Red Planet?'; const documents = [ 'Mars, known for its reddish appearance, is often referred to as the Red Planet.', 'Venus is often called the twin of Earth because of its similar size and proximity.', ]; const inputs = tokenizer(new Array(documents.length).fill(query), { text_pair: documents, padding: true, truncation: true }); const { logits } = await model(inputs); console.log(logits.sigmoid().tolist()); // one relevance score per document - Notebooks
- Google Colab
- Kaggle
Download code/rerank/prep_webfaq.py from atelasi/multilingual-reranker-small: direct link, hf CLI and curl.
- Browser
- Download file 3.55 kB
-
https://huggingface.co/atelasi/multilingual-reranker-small/resolve/main/code/rerank/prep_webfaq.py
- Command line
-
hf download hf://atelasi/multilingual-reranker-small/code/rerank/prep_webfaq.py
-
curl -L -o prep_webfaq.py https://huggingface.co/atelasi/multilingual-reranker-small/resolve/main/code/rerank/prep_webfaq.py
3.55 kB
| """Real FAQ questions for reranker v1.2: PaDaS-Lab/webfaq (CC-BY-4.0; question/answer pairs from schema.org FAQ markup on websites), | |
| for the 45 of our 47 reranker languages it covers. Streams only the first lines of each language file (HfFileSystem range reads). | |
| Filters: question 10-300 and answer 40-2000 characters, no spam/gambling/adult domains, <= 50 pairs per site, unique questions and | |
| answers. Per language: a pool of POOL answers (the mining corpus) and QPL queries whose answer is in the pool (qtype 'faq'). | |
| CPU job. python rerank/prep_webfaq.py OUT_DIR [POOL=30000 QPL=2500 PID0=20000000] | |
| -> OUT_DIR/webfaq_passages.parquet (pid, text, lang) and OUT_DIR/q/q_webfaq.jsonl (pid, query, qtype, qlang)""" | |
| import json, os, random, re, sys, time | |
| from collections import Counter | |
| import pandas as pd | |
| from huggingface_hub import HfFileSystem | |
| OUT = sys.argv[1]; POOL, QPL, PID0 = (int(os.environ.get(k, d)) for k, d in (("POOL", 30000), ("QPL", 2500), ("PID0", 20_000_000))) | |
| ALIAS = {"arb": "ara", "cmn": "zho", "ekk": "est", "fil": "tgl", "lvs": "lav", "nob": "nor", "zsm": "msa"} | |
| OURS = ['arb_Arab', 'ben_Beng', 'bul_Cyrl', 'cat_Latn', 'ces_Latn', 'cmn_Hani', 'dan_Latn', 'deu_Latn', 'ekk_Latn', 'ell_Grek', 'eng_Latn', 'fas_Arab', | |
| 'fil_Latn', 'fin_Latn', 'fra_Latn', 'heb_Hebr', 'hin_Deva', 'hrv_Latn', 'hun_Latn', 'ind_Latn', 'ita_Latn', 'jpn_Jpan', 'kor_Hang', 'lit_Latn', | |
| 'lvs_Latn', 'mar_Deva', 'nld_Latn', 'nob_Latn', 'pol_Latn', 'por_Latn', 'ron_Latn', 'rus_Cyrl', 'slk_Latn', 'slv_Latn', 'spa_Latn', 'srp_Cyrl', | |
| 'swe_Latn', 'tam_Taml', 'tel_Telu', 'tha_Thai', 'tur_Latn', 'ukr_Cyrl', 'urd_Arab', 'vie_Latn'] | |
| SPAM = re.compile(r"casino|1win|1xbet|bet(ting)?|poker|slot|porn|sex|escort|xxx|viagra|cialis|loan|payday|crypto|forex|gambl|lottery|dating", re.I) | |
| fs = HfFileSystem(); rng = random.Random(0); P, Q, nxt = [], [], PID0 | |
| os.makedirs(f"{OUT}/q", exist_ok=True) | |
| for code in OURS: | |
| w = ALIAS.get(code[:3], code[:3]); pairs, seen_q, seen_a, per_site, n = [], set(), set(), Counter(), 0 | |
| for k in range(5): | |
| try: | |
| with fs.open(f"datasets/PaDaS-Lab/webfaq/data/{w}.jsonl", "r", block_size=8 << 20) as f: | |
| for line in f: | |
| n += 1 | |
| r = json.loads(line); q, a, o = r.get("question", "").strip(), r.get("answer", "").strip(), r.get("origin", "") | |
| if not (10 <= len(q) <= 300 and 40 <= len(a) <= 2000) or SPAM.search(o) or per_site[o] >= 50 or q in seen_q or a in seen_a: | |
| continue | |
| per_site[o] += 1; seen_q.add(q); seen_a.add(a); pairs.append((q, a)) | |
| if len(pairs) >= POOL or n > 40 * POOL: | |
| break | |
| break | |
| except Exception as e: | |
| print(code, "retry", k, repr(e)[:120], flush=True); time.sleep(30 * (k + 1)); pairs, seen_q, seen_a, per_site, n = [], set(), set(), Counter(), 0 | |
| ids = list(range(nxt, nxt + len(pairs))); nxt += len(pairs) | |
| P += [dict(pid=i, text=a, lang=code) for i, (_, a) in zip(ids, pairs)] | |
| qi = rng.sample(range(len(pairs)), min(QPL, len(pairs))) | |
| Q += [dict(pid=ids[i], query=pairs[i][0], qtype="faq", qlang=code) for i in qi] | |
| print(code, w, "lines", n, "pairs", len(pairs), "sites", len(per_site), "queries", len(qi), flush=True) | |
| pd.DataFrame(P).to_parquet(f"{OUT}/webfaq_passages.parquet") | |
| with open(f"{OUT}/q/q_webfaq.jsonl", "w") as f: | |
| for r in Q: | |
| f.write(json.dumps(r, ensure_ascii=False) + "\n") | |
| print("total passages", len(P), "queries", len(Q)) | |