aikacl commited on
Commit
e4329d1
·
verified ·
1 Parent(s): 7c311ea

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +189 -146
app.py CHANGED
@@ -1,161 +1,204 @@
1
- """
2
- KaLM-Reranker-V1-Nano on Hugging Face ZeroGPU Space.
3
-
4
- Key design choices for ZeroGPU:
5
- 1. `kalm_reranker.py` is downloaded from the model repo at startup (small file, no GPU needed).
6
- 2. The `KaLMReranker` instance itself is created lazily INSIDE the `@spaces.GPU`
7
- function, because at module-load time ZeroGPU has not yet allocated a CUDA device.
8
- 3. `@spaces.GPU(duration=120)` asks ZeroGPU for up to 120 s of GPU time per call —
9
- enough for one reranking batch on a 0.27B model.
10
- """
11
-
12
- from __future__ import annotations
13
-
14
  import os
15
- import sys
16
- from typing import List, Tuple
17
-
18
  import gradio as gr
19
- import spaces
20
- from huggingface_hub import hf_hub_download
21
-
22
- # --------------------------------------------------------------------------- #
23
- # 1. Bring in the `kalm_reranker.py` helper from the model repo.
24
- # This file is ~14 KB; downloading it once at startup is much cleaner than
25
- # copy-pasting the source into the Space repo.
26
- # --------------------------------------------------------------------------- #
27
  MODEL_ID = "KaLM-Embedding/KaLM-Reranker-V1-Nano"
28
-
29
- _kalm_helper_path = hf_hub_download(repo_id=MODEL_ID, filename="kalm_reranker.py")
30
- sys.path.insert(0, os.path.dirname(_kalm_helper_path))
31
-
32
- from kalm_reranker import KaLMReranker # noqa: E402 (path set at runtime)
33
-
34
- # --------------------------------------------------------------------------- #
35
- # 2. Lazy-loaded global reranker. Instantiated on first call inside @spaces.GPU.
36
- # --------------------------------------------------------------------------- #
37
- _reranker: KaLMReranker | None = None
38
-
39
-
40
- def get_reranker() -> KaLMReranker:
41
- global _reranker
42
- if _reranker is None:
43
- print(f"[KaLM] Loading model {MODEL_ID} on GPU ...", flush=True)
44
- _reranker = KaLMReranker(
45
- MODEL_ID,
46
- device="cuda", # ZeroGPU makes CUDA visible inside @spaces.GPU
47
- dtype="bfloat16", # BF16 matches the published checkpoint
48
- batch_size=32,
49
- query_max_length=512,
50
- max_length=1024, # encoder tokens for "<Document>: {passage}"
51
- chunk_size=4, # encoder chunk pooling (Matryoshka)
52
- )
53
- print("[KaLM] Model ready.", flush=True)
54
- return _reranker
55
-
56
-
57
- # --------------------------------------------------------------------------- #
58
- # 3. Inference function — wrapped with @spaces.GPU so ZeroGPU allocates a GPU.
59
- # --------------------------------------------------------------------------- #
60
- DEFAULT_QUERY = "What is the capital of China?"
61
- DEFAULT_DOCS = (
62
- "The capital of China is Beijing.\n"
63
- "Gravity attracts bodies toward one another.\n"
64
- "Beijing is located in northern China and has a population of over 20 million."
65
- )
66
  DEFAULT_INSTRUCTION = "Given a query, retrieve documents that answer the query."
67
-
68
-
69
- @spaces.GPU(duration=120)
70
- def rerank(
71
- query: str,
72
- documents_text: str,
73
- instruction: str,
74
- ) -> Tuple[str, list]:
75
- """Rerank documents against a query.
76
-
77
- Args:
78
- query: User query string.
79
- documents_text: Newline-separated candidate documents.
80
- instruction: Task instruction (e.g. "Given a query, retrieve ...").
81
-
82
- Returns:
83
- (formatted_text, raw_rankings_json)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  """
85
- query = (query or "").strip()
 
 
 
 
 
 
 
 
 
 
86
  if not query:
87
- return "ERROR: query is empty.", []
88
-
89
- docs: List[str] = [d.strip() for d in (documents_text or "").splitlines() if d.strip()]
90
  if not docs:
91
- return "ERROR: please provide at least one document (one per line).", []
92
-
93
- instruction = (instruction or "").strip() or None # fall back to model default
94
-
95
- r = get_reranker()
96
- rankings = r.rank(query, docs, instruction=instruction)
97
-
98
- # Pretty-print
99
- lines = [f"Top {len(rankings)} results for query: \"{query}\""]
100
- lines.append("-" * 60)
101
- for i, item in enumerate(rankings, start=1):
102
- cid = item["corpus_id"]
103
- score = item["score"]
104
- snippet = docs[cid][:100].replace("\n", " ")
105
- lines.append(f"#{i:>2} score={score:.6f} doc#{cid + 1} {snippet}{'...' if len(docs[cid]) > 100 else ''}")
106
- return "\n".join(lines), [
107
- {"rank": i, "corpus_id": item["corpus_id"], "score": float(item["score"]), "document": docs[item["corpus_id"]]}
108
- for i, item in enumerate(rankings, start=1)
109
- ]
110
-
111
-
112
- # --------------------------------------------------------------------------- #
113
- # 4. Gradio UI
114
- # --------------------------------------------------------------------------- #
115
- with gr.Blocks(title="KaLM-Reranker-V1-Nano", theme=gr.themes.Soft()) as demo:
116
- gr.Markdown(
117
- "# 🚀 KaLM-Reranker-V1-Nano · ZeroGPU Demo\n"
118
- "Encoder-decoder reranker (0.27B activated, BF16) served on Hugging Face ZeroGPU."
119
- )
120
-
121
- with gr.Row():
122
- query_in = gr.Textbox(
123
- label="Query",
124
- value=DEFAULT_QUERY,
125
- scale=2,
126
- placeholder="The user query ...",
127
  )
128
- instr_in = gr.Textbox(
129
- label="Instruction",
130
- value=DEFAULT_INSTRUCTION,
131
- scale=3,
132
- placeholder="e.g. Given a query, retrieve documents that answer the query.",
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  )
134
-
135
- docs_in = gr.Textbox(
136
- label="Candidate documents (one per line)",
137
- value=DEFAULT_DOCS,
138
- lines=8,
139
- placeholder="Paste one document per line ...",
 
 
 
 
 
 
 
 
 
 
 
140
  )
141
-
142
- btn = gr.Button("Rerank", variant="primary")
143
- out_text = gr.Textbox(label="Ranking", lines=10, show_copy_button=True)
144
- out_json = gr.JSON(label="Raw scores (rank / corpus_id / score / document)")
145
-
146
- btn.click(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
  fn=rerank,
148
- inputs=[query_in, docs_in, instr_in],
149
- outputs=[out_text, out_json],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
  )
151
-
152
  gr.Markdown(
153
- "---\n"
154
- "ℹ️ **First call after a cold start** takes ~30–60 s (the model is downloaded and moved "
155
- "onto a freshly-allocated ZeroGPU). Subsequent calls in the same session are fast.\n\n"
156
- "Model: [`KaLM-Embedding/KaLM-Reranker-V1-Nano`](https://huggingface.co/KaLM-Embedding/KaLM-Reranker-V1-Nano)"
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  )
158
-
159
-
160
  if __name__ == "__main__":
161
- demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ from typing import List
3
+
 
4
  import gradio as gr
5
+
6
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
7
+
8
+ from kalm_reranker import KaLMReranker
9
+
10
+
 
 
11
  MODEL_ID = "KaLM-Embedding/KaLM-Reranker-V1-Nano"
12
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  DEFAULT_INSTRUCTION = "Given a query, retrieve documents that answer the query."
14
+
15
+ MAX_DOCS = 20
16
+ MAX_DOC_CHARS = 4000
17
+
18
+
19
+ def load_model():
20
+ """
21
+ Load KaLM-Reranker once when the Space starts.
22
+
23
+ device=None and dtype=None allow the official wrapper to choose:
24
+ - BF16 on CUDA
25
+ - FP32 on CPU
26
+ """
27
+ reranker = KaLMReranker(
28
+ MODEL_ID,
29
+ device=None,
30
+ dtype=None,
31
+ batch_size=4,
32
+ query_max_length=512,
33
+ max_length=1024,
34
+ chunk_size=4,
35
+ )
36
+ return reranker
37
+
38
+
39
+ reranker = load_model()
40
+
41
+
42
+ def parse_documents(text: str) -> List[str]:
43
+ """
44
+ Split documents by blank lines.
45
+ Each candidate document should be separated by one empty line.
46
  """
47
+ docs = [doc.strip() for doc in text.split("\n\n") if doc.strip()]
48
+ docs = docs[:MAX_DOCS]
49
+ docs = [doc[:MAX_DOC_CHARS] for doc in docs]
50
+ return docs
51
+
52
+
53
+ def rerank(query: str, documents: str, instruction: str):
54
+ query = query.strip()
55
+ instruction = instruction.strip() or DEFAULT_INSTRUCTION
56
+ docs = parse_documents(documents)
57
+
58
  if not query:
59
+ return [], "Please input a query."
60
+
 
61
  if not docs:
62
+ return [], "Please input at least one candidate document."
63
+
64
+ try:
65
+ rankings = reranker.rank(
66
+ query=query,
67
+ documents=docs,
68
+ instruction=instruction,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
  )
70
+
71
+ table = []
72
+ for rank_idx, item in enumerate(rankings, start=1):
73
+ corpus_id = item["corpus_id"]
74
+ score = float(item["score"])
75
+ doc = docs[corpus_id]
76
+ table.append(
77
+ [
78
+ rank_idx,
79
+ corpus_id,
80
+ round(score, 6),
81
+ doc,
82
+ ]
83
+ )
84
+
85
+ summary = (
86
+ f"Reranked {len(docs)} documents with "
87
+ f"`{MODEL_ID}`. Higher score means more relevant."
88
  )
89
+ return table, summary
90
+
91
+ except Exception as error:
92
+ return [], f"Error during reranking: {repr(error)}"
93
+
94
+
95
+ with gr.Blocks(title="KaLM-Reranker-V1 Demo") as demo:
96
+ gr.Markdown(
97
+ """
98
+ # KaLM-Reranker-V1 Demo
99
+
100
+ **KaLM-Reranker-V1** is a fast but not late-interaction reranker for compressed document reranking.
101
+
102
+ Input a query and several candidate documents. The demo returns relevance scores and reranked results.
103
+
104
+ **Document format:** separate candidate documents with one blank line.
105
+ """
106
  )
107
+
108
+ with gr.Row():
109
+ with gr.Column(scale=1):
110
+ query = gr.Textbox(
111
+ label="Query",
112
+ value="What is the capital of China?",
113
+ lines=3,
114
+ )
115
+
116
+ instruction = gr.Textbox(
117
+ label="Instruction",
118
+ value=DEFAULT_INSTRUCTION,
119
+ lines=2,
120
+ )
121
+
122
+ documents = gr.Textbox(
123
+ label="Candidate Documents",
124
+ value=(
125
+ "The capital of China is Beijing.\n\n"
126
+ "Gravity attracts bodies toward one another.\n\n"
127
+ "Shanghai is a major city in China.\n\n"
128
+ "Paris is the capital of France."
129
+ ),
130
+ lines=14,
131
+ )
132
+
133
+ submit = gr.Button("Rerank", variant="primary")
134
+
135
+ with gr.Column(scale=1):
136
+ output_table = gr.Dataframe(
137
+ headers=["Rank", "Corpus ID", "Score", "Document"],
138
+ label="Reranking Results",
139
+ wrap=True,
140
+ )
141
+ output_summary = gr.Markdown()
142
+
143
+ submit.click(
144
  fn=rerank,
145
+ inputs=[query, documents, instruction],
146
+ outputs=[output_table, output_summary],
147
+ )
148
+
149
+ gr.Examples(
150
+ examples=[
151
+ [
152
+ "What is the capital of China?",
153
+ (
154
+ "The capital of China is Beijing.\n\n"
155
+ "Gravity attracts bodies toward one another.\n\n"
156
+ "Paris is the capital of France."
157
+ ),
158
+ DEFAULT_INSTRUCTION,
159
+ ],
160
+ [
161
+ "Which model is suitable for efficient reranking?",
162
+ (
163
+ "KaLM-Reranker-V1-Nano is designed for efficient reranking.\n\n"
164
+ "Large language models are often expensive for reranking.\n\n"
165
+ "Image classifiers are used for visual recognition."
166
+ ),
167
+ DEFAULT_INSTRUCTION,
168
+ ],
169
+ [
170
+ "What is KaLM-Reranker-V1 designed for?",
171
+ (
172
+ "KaLM-Reranker-V1 is a reranker for compressed document reranking.\n\n"
173
+ "KaLM-Embedding is a general-purpose embedding model.\n\n"
174
+ "Weather forecasting predicts future weather conditions."
175
+ ),
176
+ DEFAULT_INSTRUCTION,
177
+ ],
178
+ ],
179
+ inputs=[query, documents, instruction],
180
  )
181
+
182
  gr.Markdown(
183
+ """
184
+ ## Citation
185
+
186
+ If you find this demo useful, please cite:
187
+
188
+ ```bibtex
189
+ @misc{zhao2026kalmrerankerv1,
190
+ title={KaLM-Reranker-V1: Fast but Not Late Interaction for Compressed Document Reranking},
191
+ author={Xinping Zhao and Jiaxin Xu and Ziqi Dai and Xin Zhang and Shouzheng Huang and Danyu Tang and Xinshuo Hu and Meishan Zhang and Baotian Hu and Min Zhang},
192
+ year={2026},
193
+ eprint={2606.22807},
194
+ archivePrefix={arXiv},
195
+ primaryClass={cs.CL},
196
+ url={https://arxiv.org/abs/2606.22807},
197
+ }
198
+ ```
199
+ """
200
  )
201
+
202
+
203
  if __name__ == "__main__":
204
+ demo.launch()