ariG23498 HF Staff commited on
Commit
76904a5
·
verified ·
1 Parent(s): c127c63

Upload 4 files

Browse files
Files changed (4) hide show
  1. README.md +42 -8
  2. app.py +383 -0
  3. indexer.py +149 -0
  4. requirements.txt +2 -0
README.md CHANGED
@@ -1,15 +1,49 @@
1
  ---
2
- title: Transformers Code Snippets
3
- emoji: 🦀
4
- colorFrom: pink
5
- colorTo: purple
6
  sdk: gradio
7
  sdk_version: 6.30.0
8
- python_version: '3.13'
9
  app_file: app.py
10
  pinned: false
11
- license: apache-2.0
12
- short_description: Generates the correct transformers code snippet for models
 
 
 
 
13
  ---
14
 
15
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Transformers Playground
3
+ emoji: 🛝
4
+ colorFrom: yellow
5
+ colorTo: gray
6
  sdk: gradio
7
  sdk_version: 6.30.0
 
8
  app_file: app.py
9
  pinned: false
10
+ hf_oauth: true
11
+ hf_oauth_expiration_minutes: 480
12
+ hf_oauth_scopes:
13
+ - jobs
14
+ - read-repos
15
+ short_description: Copy official transformers snippets or run them on HF Jobs
16
  ---
17
 
18
+ # Transformers Playground
19
+
20
+ Pick any transformers model on the Hub, get the usage snippets from its
21
+ `model_doc` page (adapted to your checkpoint), copy them, or run them on
22
+ Hugging Face Jobs with your own account.
23
+
24
+ ## How it works
25
+
26
+ - `indexer.py` sparse-clones `docs/source/en/model_doc` from transformers,
27
+ extracts the `<hfoptions id="usage">` tabs and usage code blocks, drops
28
+ training and quantization examples (v1 is inference only), and writes
29
+ `snippets.json`.
30
+ - `app.py` matches a model's `config.json` `model_type` to its doc page,
31
+ swaps the doc's checkpoint for the selected model, adds a generic
32
+ `pipeline()` snippet from the model's task, and recommends the cheapest
33
+ Jobs flavor that fits the weights (live prices from `list_jobs_hardware`).
34
+ - Runs go through `run_job` with the user's OAuth token, so each user pays
35
+ for their own compute. Jobs need a PRO, Team or Enterprise account.
36
+
37
+ ## Setup
38
+
39
+ 1. Create the Space from these files. On first boot it builds the index
40
+ itself (about 30 seconds).
41
+ 2. Optional but recommended: publish the index as a dataset and refresh it
42
+ nightly, then set a Space variable `INDEX_REPO=<you>/transformers-snippets`.
43
+
44
+ ```bash
45
+ hf jobs scheduled run @daily python:3.12 --secrets HF_TOKEN -- bash -c \
46
+ "pip install -q huggingface_hub && \
47
+ curl -sL https://huggingface.co/spaces/<you>/transformers-playground/raw/main/indexer.py -o indexer.py && \
48
+ python indexer.py --push <you>/transformers-snippets"
49
+ ```
app.py ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Transformers Playground: find a model, grab its doc snippets, run them on HF Jobs."""
2
+ import json
3
+ import os
4
+ import re
5
+ import time
6
+
7
+ import gradio as gr
8
+ from huggingface_hub import (
9
+ HfApi, cancel_job, fetch_job_logs, hf_hub_download, inspect_job, run_job,
10
+ )
11
+ from huggingface_hub.utils import GatedRepoError, HfHubHTTPError, RepositoryNotFoundError
12
+
13
+ from indexer import build_index
14
+
15
+ INDEX_REPO = os.getenv("INDEX_REPO", "") # e.g. "your-org/transformers-snippets"
16
+ UV_IMAGE = "ghcr.io/astral-sh/uv:python3.12-bookworm"
17
+ api = HfApi()
18
+
19
+ # ---------------------------------------------------------------- index ----
20
+
21
+ def load_index():
22
+ if INDEX_REPO:
23
+ try:
24
+ path = hf_hub_download(INDEX_REPO, "snippets.json", repo_type="dataset")
25
+ with open(path) as f:
26
+ return json.load(f)
27
+ except Exception as e: # fall through to a local build
28
+ print(f"Could not load {INDEX_REPO}: {e}")
29
+ if os.path.exists("snippets.json"):
30
+ with open("snippets.json") as f:
31
+ return json.load(f)
32
+ print("Building snippet index from transformers@main ...")
33
+ index = build_index("main")
34
+ with open("snippets.json", "w") as f:
35
+ json.dump(index, f)
36
+ return index
37
+
38
+
39
+ INDEX = load_index()
40
+ _norm = lambda s: re.sub(r"[^a-z0-9]", "", (s or "").lower())
41
+ DOC_BY_NORM = {_norm(k): k for k in INDEX["docs"]}
42
+
43
+ # ------------------------------------------------------------- hardware ----
44
+
45
+ def _gb(s):
46
+ m = re.search(r"([\d.]+)", s or "")
47
+ return float(m.group(1)) if m else 0.0
48
+
49
+
50
+ def load_hardware():
51
+ try:
52
+ hw = api.list_jobs_hardware()
53
+ return [{
54
+ "name": h.name,
55
+ "ram": _gb(h.ram),
56
+ "vram": _gb(h.accelerator.vram) if h.accelerator else 0.0,
57
+ "gpus": int(h.accelerator.quantity) if h.accelerator else 0,
58
+ "gpu": h.accelerator.model if h.accelerator else None,
59
+ "usd_hr": h.unit_cost_usd * 60,
60
+ } for h in hw]
61
+ except Exception as e:
62
+ print(f"Hardware list unavailable ({e}); using a minimal fallback.")
63
+ return [
64
+ {"name": "cpu-basic", "ram": 16, "vram": 0, "gpus": 0, "gpu": None, "usd_hr": 0.01},
65
+ {"name": "t4-small", "ram": 15, "vram": 16, "gpus": 1, "gpu": "T4", "usd_hr": 0.40},
66
+ {"name": "l4x1", "ram": 30, "vram": 24, "gpus": 1, "gpu": "L4", "usd_hr": 0.80},
67
+ {"name": "a100-large", "ram": 142, "vram": 80, "gpus": 1, "gpu": "A100", "usd_hr": 2.50},
68
+ ]
69
+
70
+
71
+ HARDWARE = sorted(load_hardware(), key=lambda h: h["usd_hr"])
72
+ HW_BY_NAME = {h["name"]: h for h in HARDWARE}
73
+ BYTES = {"F64": 8, "F32": 4, "BF16": 2, "F16": 2, "F8_E4M3": 1, "F8_E5M2": 1, "I8": 1, "U8": 1, "I32": 4}
74
+
75
+
76
+ def estimate_gb(meta):
77
+ if not meta.get("params"):
78
+ return None
79
+ return meta["params"] * meta["bytes"] / 1e9 * 1.2 + 1.0 # weights + KV/activation headroom
80
+
81
+
82
+ def recommend(meta, needs_gpu, multi_gpu_ok):
83
+ need = estimate_gb(meta)
84
+ if need is None:
85
+ return "t4-small", "Model size unknown (no safetensors metadata), starting with a T4."
86
+ if not needs_gpu and meta["params"] <= 1.5e9:
87
+ for h in HARDWARE:
88
+ if h["gpus"] == 0 and h["ram"] >= need * 1.5:
89
+ return h["name"], f"~{need:.1f} GB needed; small enough for CPU."
90
+ for h in HARDWARE:
91
+ if h["gpus"] and h["vram"] >= need and h["ram"] >= need and (h["gpus"] == 1 or multi_gpu_ok):
92
+ return h["name"], f"~{need:.1f} GB needed; cheapest GPU that fits."
93
+ return HARDWARE[-1]["name"], f"~{need:.1f} GB needed; larger than any single flavor, expect OOM."
94
+
95
+
96
+ def hw_choices(meta):
97
+ need = estimate_gb(meta) or 0
98
+ out = []
99
+ for h in HARDWARE:
100
+ spec = f"{h['gpus']}x {h['gpu']}, {h['vram']:.0f} GB" if h["gpus"] else f"CPU, {h['ram']:.0f} GB RAM"
101
+ small = " (too small)" if (h["gpus"] and h["vram"] < need) else ""
102
+ out.append((f"{h['name']} [{spec}] ${h['usd_hr']:.2f}/h{small}", h["name"]))
103
+ return out
104
+
105
+ # ------------------------------------------------------------- snippets ----
106
+
107
+ SAMPLE_IMG = "http://images.cocodataset.org/val2017/000000039769.jpg"
108
+ SAMPLE_AUDIO = "https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac"
109
+ GENERIC_INPUTS = {
110
+ "text-generation": '"Plants create energy through a process known as", max_new_tokens=50',
111
+ "text-classification": '"I love how easy this library makes things!"',
112
+ "token-classification": '"My name is Sarah and I live in London."',
113
+ "fill-mask": 'f"Paris is the {pipe.tokenizer.mask_token} of France."',
114
+ "question-answering": 'question="Where do I live?", context="My name is Sarah and I live in London."',
115
+ "zero-shot-classification": '"I just bought a new laptop", candidate_labels=["tech", "cooking", "sports"]',
116
+ "summarization": '"The tower is 324 metres tall, about the same height as an 81-storey building, and the tallest structure in Paris."',
117
+ "translation": '"How old are you?"',
118
+ "feature-extraction": '"Transformers is great."',
119
+ "sentence-similarity": '"Transformers is great."',
120
+ "image-classification": f'"{SAMPLE_IMG}"',
121
+ "object-detection": f'"{SAMPLE_IMG}"',
122
+ "image-segmentation": f'"{SAMPLE_IMG}"',
123
+ "depth-estimation": f'"{SAMPLE_IMG}"',
124
+ "image-to-text": f'"{SAMPLE_IMG}"',
125
+ "zero-shot-image-classification": f'"{SAMPLE_IMG}", candidate_labels=["cats", "dogs", "birds"]',
126
+ "image-feature-extraction": f'"{SAMPLE_IMG}"',
127
+ "automatic-speech-recognition": f'"{SAMPLE_AUDIO}"',
128
+ "audio-classification": f'"{SAMPLE_AUDIO}"',
129
+ "image-text-to-text": (
130
+ f'text=[{{"role": "user", "content": [{{"type": "image", "url": "{SAMPLE_IMG}"}}, '
131
+ f'{{"type": "text", "text": "Describe this image."}}]}}], max_new_tokens=50'
132
+ ),
133
+ }
134
+ TASK_ALIAS = {"sentence-similarity": "feature-extraction"}
135
+
136
+
137
+ def generic_snippet(task):
138
+ if task not in GENERIC_INPUTS:
139
+ return None
140
+ code = (
141
+ "from transformers import pipeline\n\n"
142
+ f'pipe = pipeline(task="{TASK_ALIAS.get(task, task)}", model="MODEL_ID", device_map="auto")\n'
143
+ f"print(pipe({GENERIC_INPUTS[task]}))\n"
144
+ )
145
+ return {"label": "Generic pipeline (auto-generated)", "section": "", "language": "python",
146
+ "code": code, "checkpoint": "MODEL_ID", "needs_gpu": False}
147
+
148
+
149
+ def adapt(snippet, model_id):
150
+ code = snippet["code"]
151
+ if snippet.get("checkpoint"):
152
+ code = code.replace(snippet["checkpoint"], model_id)
153
+ return code
154
+
155
+
156
+ def snippet_task(code):
157
+ m = re.search(r'pipeline\(\s*(?:task\s*=\s*)?["\']([\w-]+)["\']', code)
158
+ return m.group(1) if m else None
159
+
160
+
161
+ def infer_deps(code, task):
162
+ deps = ["transformers", "torch", "accelerate"]
163
+ checks = {
164
+ "pillow": r"\bPIL\b|Image\.open|image",
165
+ "torchvision": r"torchvision|image|Image|vision|video",
166
+ "requests": r"\brequests\b",
167
+ "datasets": r"load_dataset|from datasets|import datasets",
168
+ "soundfile": r"audio|soundfile|\.flac|\.wav",
169
+ "librosa": r"librosa|audio|\.flac|\.wav",
170
+ "sentencepiece": r"sentencepiece|T5|Marian",
171
+ "num2words": r"SmolVLM|Idefics3",
172
+ }
173
+ blob = code + " " + (task or "")
174
+ deps += [d for d, pat in checks.items() if re.search(pat, blob)]
175
+ return ", ".join(deps)
176
+
177
+ # ------------------------------------------------------------ callbacks ----
178
+
179
+ def search_models(query, task):
180
+ models = api.list_models(search=query or None, filter="transformers",
181
+ pipeline_tag=task or None, sort="downloads", limit=30)
182
+ ids = [m.id for m in models]
183
+ return gr.update(choices=ids, value=ids[0] if ids else None)
184
+
185
+
186
+ def _token(oauth_token):
187
+ return oauth_token.token if oauth_token else None
188
+
189
+
190
+ def select_model(model_id, oauth_token: gr.OAuthToken | None):
191
+ empty = ("", gr.update(choices=[], value=None), [], {}, "", "", gr.update(), "", "")
192
+ if not model_id:
193
+ return empty
194
+ try:
195
+ info = api.model_info(model_id, token=_token(oauth_token))
196
+ except RepositoryNotFoundError:
197
+ return ("Model not found, or it is private and you are not signed in.",) + empty[1:]
198
+
199
+ cfg = info.config or {}
200
+ model_type = cfg.get("model_type")
201
+ arch = DOC_BY_NORM.get(_norm(model_type))
202
+ params, dtype_bytes = None, 2
203
+ if info.safetensors:
204
+ params = info.safetensors.total
205
+ main_dtype = max(info.safetensors.parameters, key=info.safetensors.parameters.get)
206
+ dtype_bytes = BYTES.get(main_dtype, 2)
207
+ meta = {"model_id": model_id, "params": params, "bytes": dtype_bytes,
208
+ "task": info.pipeline_tag, "gated": bool(info.gated)}
209
+
210
+ snippets = list(INDEX["docs"][arch]["snippets"]) if arch else []
211
+ generic = generic_snippet(info.pipeline_tag)
212
+ if generic:
213
+ snippets.append(generic)
214
+ snippets = [dict(s, adapted=adapt(s, model_id)) for s in snippets]
215
+
216
+ size = f"{params / 1e9:.2f}B params" if params else "size unknown"
217
+ lines = [f"### [{model_id}](https://huggingface.co/{model_id})",
218
+ f"`{model_type or 'unknown arch'}` | {info.pipeline_tag or 'no task tag'} | {size}"]
219
+ if arch:
220
+ d = INDEX["docs"][arch]
221
+ lines.append(f"Snippets from [{d['title']} docs]({d['doc_url']}) "
222
+ f"at transformers [`{INDEX['sha'][:8]}`]({d['source_url']})")
223
+ else:
224
+ lines.append("No model_doc page matched this architecture.")
225
+ if info.gated:
226
+ lines.append("This model is gated. Accept its license on the model page before running.")
227
+ if not snippets:
228
+ lines.append("No runnable snippets found for this model.")
229
+
230
+ choices = [(f"{s['label']} ({s['language']})", i) for i, s in enumerate(snippets)]
231
+ first = 0 if snippets else None
232
+ return ("\n\n".join(lines), gr.update(choices=choices, value=first), snippets, meta,
233
+ *snippet_outputs(first, snippets, meta))
234
+
235
+
236
+ def snippet_outputs(idx, snippets, meta):
237
+ if idx is None or not snippets:
238
+ return "", "", gr.update(choices=hw_choices(meta), value="cpu-basic"), "", ""
239
+ s = snippets[idx]
240
+ code = s["adapted"]
241
+ flavor, why = recommend(meta, s["needs_gpu"], "device_map" in code)
242
+ notes = [f"Recommended hardware: **{flavor}**. {why}"]
243
+ st = snippet_task(code)
244
+ if st and meta.get("task") and TASK_ALIAS.get(meta["task"], meta["task"]) != st:
245
+ notes.append(f"Heads up: this snippet uses the `{st}` task but the model is tagged "
246
+ f"`{meta['task']}`. It may need edits.")
247
+ if s["language"] != "python":
248
+ notes.append("Shell snippets are copy-only; run Python snippets on Jobs.")
249
+ return (code, s["code"], gr.update(choices=hw_choices(meta), value=flavor),
250
+ "\n\n".join(notes), infer_deps(code, meta.get("task")))
251
+
252
+
253
+ def cost_note(flavor, timeout):
254
+ h = HW_BY_NAME.get(flavor)
255
+ if not h:
256
+ return ""
257
+ return f"{flavor}: ${h['usd_hr']:.2f}/h. Worst case at a {int(timeout)} min timeout: ${h['usd_hr'] * timeout / 60:.2f}"
258
+
259
+
260
+ def build_script(code, deps, tf_source):
261
+ deps = [d.strip() for d in deps.split(",") if d.strip()]
262
+ if tf_source == "main":
263
+ deps = [d for d in deps if d != "transformers"]
264
+ deps.append("transformers @ https://github.com/huggingface/transformers/archive/refs/heads/main.zip")
265
+ header = "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n"
266
+ header += "".join(f'# "{d}",\n' for d in deps) + "# ]\n# ///\n\n"
267
+ return header + code
268
+
269
+
270
+ def run_snippet(code, idx, snippets, meta, flavor, deps, tf_source, timeout,
271
+ oauth_token: gr.OAuthToken | None):
272
+ token = _token(oauth_token)
273
+ if not token:
274
+ yield "Sign in with Hugging Face first. Jobs run on your account.", "", None
275
+ return
276
+ if idx is not None and snippets and snippets[idx]["language"] != "python":
277
+ yield "This is a shell snippet. Copy it to run locally.", "", None
278
+ return
279
+ try:
280
+ api.auth_check(meta["model_id"], token=token)
281
+ except GatedRepoError:
282
+ yield f"You don't have access to {meta['model_id']} yet. Accept its license on the model page.", "", None
283
+ return
284
+ except Exception:
285
+ pass # let the job surface any other access problem
286
+
287
+ script = build_script(code, deps, tf_source)
288
+ try:
289
+ job = run_job(
290
+ image=UV_IMAGE,
291
+ command=["bash", "-c", 'printf "%s" "$PLAYGROUND_SCRIPT" > /tmp/playground.py && uv run /tmp/playground.py'],
292
+ env={"PLAYGROUND_SCRIPT": script, "PYTHONUNBUFFERED": "1"},
293
+ secrets={"HF_TOKEN": token},
294
+ flavor=flavor,
295
+ timeout=f"{int(timeout)}m",
296
+ labels={"app": "transformers-playground"},
297
+ token=token,
298
+ )
299
+ except HfHubHTTPError as e:
300
+ hint = " Jobs need a PRO, Team or Enterprise account with credits." if e.response is not None and e.response.status_code in (402, 403) else ""
301
+ yield f"Could not start the job: {e}.{hint}", "", None
302
+ return
303
+
304
+ status = f"Job started on **{flavor}**. [Open job page]({job.url})"
305
+ logs, last = "", 0.0
306
+ yield status + "\n\nWaiting for logs...", logs, job.id
307
+ try:
308
+ for line in fetch_job_logs(job_id=job.id, follow=True, token=token):
309
+ logs += line.rstrip("\n") + "\n"
310
+ if time.time() - last > 0.5:
311
+ last = time.time()
312
+ yield status + "\n\nRunning...", logs, job.id
313
+ except Exception as e:
314
+ logs += f"\n[log stream interrupted: {e}]\n"
315
+ stage = inspect_job(job_id=job.id, token=token).status.stage
316
+ yield f"{status}\n\nFinished with status **{stage}**.", logs, job.id
317
+
318
+
319
+ def stop_job(job_id, oauth_token: gr.OAuthToken | None):
320
+ if not job_id or not oauth_token:
321
+ return "No running job to cancel."
322
+ cancel_job(job_id=job_id, token=oauth_token.token)
323
+ return f"Cancel requested for job `{job_id}`."
324
+
325
+ # ------------------------------------------------------------------- UI ----
326
+
327
+ TASKS = ["", "text-generation", "image-text-to-text", "text-classification", "token-classification",
328
+ "fill-mask", "question-answering", "summarization", "translation", "feature-extraction",
329
+ "automatic-speech-recognition", "audio-classification", "image-classification",
330
+ "object-detection", "image-segmentation", "depth-estimation", "zero-shot-classification"]
331
+
332
+ with gr.Blocks(title="Transformers Playground") as demo:
333
+ snippets_state, meta_state, job_state = gr.State([]), gr.State({}), gr.State(None)
334
+
335
+ with gr.Row(equal_height=True):
336
+ gr.Markdown("# Transformers Playground\nPick a model, copy the official usage snippet, "
337
+ "or run it on Hugging Face Jobs with your own account.")
338
+ gr.LoginButton(scale=0, min_width=240)
339
+
340
+ with gr.Row():
341
+ query = gr.Textbox(label="Search models", placeholder="llama, whisper, Qwen/Qwen2.5-0.5B-Instruct ...", scale=3)
342
+ task = gr.Dropdown(TASKS, value="", label="Task", scale=2)
343
+ search_btn = gr.Button("Search", scale=1)
344
+ model = gr.Dropdown(label="Model", choices=[], allow_custom_value=True,
345
+ info="Pick from results, or paste any model ID.")
346
+ model_md = gr.Markdown()
347
+
348
+ with gr.Row():
349
+ with gr.Column(scale=3):
350
+ snippet_pick = gr.Radio(label="Snippet", choices=[])
351
+ code = gr.Code(language="python", interactive=True, lines=18,
352
+ label="Snippet, adapted to your model (editable)", buttons=["copy", "download"])
353
+ with gr.Accordion("Original snippet from the docs", open=False):
354
+ original = gr.Code(language="python", interactive=False, buttons=["copy"])
355
+ with gr.Column(scale=2):
356
+ notes = gr.Markdown()
357
+ flavor = gr.Dropdown(label="Hardware", choices=hw_choices({}), value="cpu-basic")
358
+ timeout = gr.Slider(5, 120, value=30, step=5, label="Timeout (minutes)")
359
+ cost = gr.Markdown(cost_note("cpu-basic", 30))
360
+ deps = gr.Textbox(label="Dependencies", info="Comma-separated pip specs added to the job.")
361
+ tf_source = gr.Radio(["latest release", "main"], value="latest release",
362
+ label="transformers version",
363
+ info="Snippets come from the main docs; pick main if a newer API fails.")
364
+ with gr.Row():
365
+ run_btn = gr.Button("Run on HF Jobs", variant="primary")
366
+ stop_btn = gr.Button("Cancel job", variant="stop")
367
+ status = gr.Markdown()
368
+ logs = gr.Code(language="shell", label="Job logs", lines=14, interactive=False)
369
+
370
+ search_btn.click(search_models, [query, task], model)
371
+ query.submit(search_models, [query, task], model)
372
+ model.change(select_model, [model],
373
+ [model_md, snippet_pick, snippets_state, meta_state, code, original, flavor, notes, deps])
374
+ snippet_pick.input(snippet_outputs, [snippet_pick, snippets_state, meta_state],
375
+ [code, original, flavor, notes, deps])
376
+ flavor.change(cost_note, [flavor, timeout], cost)
377
+ timeout.change(cost_note, [flavor, timeout], cost)
378
+ run_btn.click(run_snippet, [code, snippet_pick, snippets_state, meta_state, flavor, deps, tf_source, timeout],
379
+ [status, logs, job_state])
380
+ stop_btn.click(stop_job, [job_state], status)
381
+
382
+ if __name__ == "__main__":
383
+ demo.launch(theme=gr.themes.Soft(primary_hue="amber", secondary_hue="slate"))
indexer.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Build the snippet index from transformers' docs/source/en/model_doc.
2
+
3
+ Run locally: python indexer.py --out snippets.json
4
+ Push to the Hub: python indexer.py --push your-org/transformers-snippets
5
+ (Schedule it as a daily HF Job to keep the index fresh.)
6
+ """
7
+ import argparse
8
+ import json
9
+ import os
10
+ import re
11
+ import shutil
12
+ import subprocess
13
+ import time
14
+
15
+ REPO = "huggingface/transformers"
16
+ DOC_DIR = "docs/source/en/model_doc"
17
+
18
+ FENCE = re.compile(r"```(py|python|bash|sh)\s*\n(.*?)```", re.S)
19
+ HFOPTIONS = re.compile(r'<hfoptions id="([^"]+)">(.*?)</hfoptions>', re.S)
20
+ HFOPTION = re.compile(r'<hfoption id="([^"]+)">(.*?)</hfoption>', re.S)
21
+ HEADING = re.compile(r"^(#{2,3})\s+(.+)$", re.M)
22
+ CHECKPOINT = re.compile(
23
+ r'(?:from_pretrained\(\s*|model\s*=\s*|--model\s+)["\']([\w.\-]+/[\w.\-]+)["\']?'
24
+ )
25
+
26
+ # v1 is inference-only: drop training and quantization snippets.
27
+ EXCLUDE = re.compile(
28
+ r"\bTrainer\b|\.train\(|TrainingArguments|SFTTrainer|trl\b|loss\.backward|optimizer\."
29
+ r"|BitsAndBytesConfig|TorchAoConfig|quantization_config|GPTQConfig|AwqConfig|HqqConfig"
30
+ )
31
+ INFERENCE_HINT = re.compile(r"pipeline\(|\.generate\(|from_pretrained|transformers (chat|run|serve)")
32
+
33
+
34
+ def fetch_docs(ref="main", workdir="/tmp/transformers-docs"):
35
+ """Shallow, sparse clone of just model_doc. No GitHub API calls, so no rate limits."""
36
+ shutil.rmtree(workdir, ignore_errors=True)
37
+ run = lambda *a, **kw: subprocess.run(a, check=True, capture_output=True, text=True, **kw)
38
+ run("git", "clone", "--quiet", "--depth", "1", "--filter=blob:none", "--sparse",
39
+ "--branch", ref, f"https://github.com/{REPO}", workdir)
40
+ run("git", "sparse-checkout", "set", DOC_DIR, cwd=workdir)
41
+ sha = run("git", "rev-parse", "HEAD", cwd=workdir).stdout.strip()
42
+ doc_root = os.path.join(workdir, DOC_DIR)
43
+ files = {}
44
+ for name in sorted(os.listdir(doc_root)):
45
+ if name.endswith(".md"):
46
+ with open(os.path.join(doc_root, name), encoding="utf-8") as f:
47
+ files[f"{DOC_DIR}/{name}"] = f.read()
48
+ return sha, files
49
+
50
+
51
+ def _section_for(text, pos):
52
+ """Nearest heading above a position, used as context for loose code blocks."""
53
+ title = ""
54
+ for m in HEADING.finditer(text):
55
+ if m.start() > pos:
56
+ break
57
+ title = m.group(2).strip()
58
+ return title
59
+
60
+
61
+ def _clean(code):
62
+ code = code.strip("\n")
63
+ return re.sub(r"^\s*<!--.*?-->\s*$", "", code, flags=re.M).strip("\n")
64
+
65
+
66
+ def _snippet(code, lang, label, section):
67
+ code = _clean(code)
68
+ if not code or EXCLUDE.search(code):
69
+ return None
70
+ if not INFERENCE_HINT.search(code):
71
+ return None
72
+ lang = "python" if lang in ("py", "python") else "bash"
73
+ ckpt = CHECKPOINT.search(code)
74
+ return {
75
+ "label": label,
76
+ "section": section,
77
+ "language": lang,
78
+ "code": code,
79
+ "checkpoint": ckpt.group(1) if ckpt else None,
80
+ "needs_gpu": bool(re.search(r'cuda|device_map|attn_implementation="flash', code)),
81
+ }
82
+
83
+
84
+ def parse_doc(text):
85
+ snippets, covered = [], []
86
+ # 1. Structured usage tabs: <hfoptions id="usage"><hfoption id="Pipeline">...
87
+ for group in HFOPTIONS.finditer(text):
88
+ covered.append(group.span())
89
+ section = _section_for(text, group.start())
90
+ for opt in HFOPTION.finditer(group.group(2)):
91
+ for fence in FENCE.finditer(opt.group(2)):
92
+ s = _snippet(fence.group(2), fence.group(1), opt.group(1), section)
93
+ if s:
94
+ snippets.append(s)
95
+ # 2. Loose code blocks under usage-like headings (older doc pages).
96
+ for fence in FENCE.finditer(text):
97
+ if any(a <= fence.start() < b for a, b in covered):
98
+ continue
99
+ section = _section_for(text, fence.start())
100
+ if not re.search(r"usage|example|inference|generat|quick|tips|notes", section, re.I):
101
+ continue
102
+ s = _snippet(fence.group(2), fence.group(1), section or "Example", section)
103
+ if s:
104
+ snippets.append(s)
105
+ # De-duplicate identical code.
106
+ seen, out = set(), []
107
+ for s in snippets:
108
+ if s["code"] not in seen:
109
+ seen.add(s["code"])
110
+ out.append(s)
111
+ return out
112
+
113
+
114
+ def build_index(ref="main"):
115
+ sha, files = fetch_docs(ref)
116
+ docs = {}
117
+ for path, text in files.items():
118
+ arch = os.path.basename(path)[:-3]
119
+ title = re.search(r"^#\s+(.+)$", text, re.M)
120
+ docs[arch] = {
121
+ "title": title.group(1).strip() if title else arch,
122
+ "doc_url": f"https://huggingface.co/docs/transformers/model_doc/{arch}",
123
+ "source_url": f"https://github.com/{REPO}/blob/{sha}/{path}",
124
+ "snippets": parse_doc(text),
125
+ }
126
+ return {"ref": ref, "sha": sha, "built_at": int(time.time()), "docs": docs}
127
+
128
+
129
+ if __name__ == "__main__":
130
+ ap = argparse.ArgumentParser()
131
+ ap.add_argument("--ref", default="main")
132
+ ap.add_argument("--out", default="snippets.json")
133
+ ap.add_argument("--push", help="dataset repo id to upload snippets.json to")
134
+ args = ap.parse_args()
135
+
136
+ index = build_index(args.ref)
137
+ with open(args.out, "w") as f:
138
+ json.dump(index, f)
139
+ n = sum(len(d["snippets"]) for d in index["docs"].values())
140
+ print(f"{len(index['docs'])} docs, {n} snippets @ {index['sha'][:8]}")
141
+
142
+ if args.push:
143
+ from huggingface_hub import HfApi
144
+ api = HfApi()
145
+ api.create_repo(args.push, repo_type="dataset", exist_ok=True)
146
+ api.upload_file(path_or_fileobj=args.out, path_in_repo="snippets.json",
147
+ repo_id=args.push, repo_type="dataset",
148
+ commit_message=f"Index transformers@{index['sha'][:8]}")
149
+ print(f"Pushed to https://huggingface.co/datasets/{args.push}")
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ huggingface_hub>=2.2,<3
2
+ requests