Johannes Janosovits commited on
Commit
0e3197c
Β·
1 Parent(s): 57e7244
Files changed (2) hide show
  1. app.py +286 -85
  2. requirements.txt +1 -4
app.py CHANGED
@@ -1,93 +1,294 @@
1
  import json
2
  import os
3
- from typing import Optional
4
-
5
- from fastapi import FastAPI, Header, HTTPException
6
- from huggingface_hub import HfApi, hf_hub_download, list_repo_files
7
- from pydantic import BaseModel
8
-
9
- app = FastAPI()
10
-
11
- WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET")
12
- HF_TOKEN = os.getenv("HF_TOKEN") # token with write access to the dataset repo
13
- #TARGET_DATASET = os.getenv("TARGET_DATASET") # e.g. "your-username/your-dataset"
14
- TARGET_DATASET = "MIMIR-AI-ROUTER/harness_evals"
15
- OUTPUT_FILENAME = "first_lines_summary.jsonl"
16
-
17
-
18
- # --- Pydantic models for the webhook payload ---
19
-
20
- class WebhookEvent(BaseModel):
21
- action: str
22
- scope: str
23
-
24
- class WebhookRepo(BaseModel):
25
- type: str
26
- name: str
27
- id: str
28
- private: bool
29
- headSha: str
30
-
31
- class WebhookPayload(BaseModel):
32
- event: WebhookEvent
33
- repo: WebhookRepo
34
-
35
-
36
- @app.post("/webhook")
37
- async def handle_webhook(
38
- payload: WebhookPayload,
39
- x_webhook_secret: Optional[str] = Header(default=None),
40
- ):
41
- # Validate secret
42
- if WEBHOOK_SECRET:
43
- if x_webhook_secret is None:
44
- raise HTTPException(status_code=401, detail="Missing secret")
45
- if x_webhook_secret != WEBHOOK_SECRET:
46
- raise HTTPException(status_code=403, detail="Invalid secret")
47
-
48
- # Only react to content updates on the target dataset
49
- if not (
50
- payload.event.action == "update"
51
- and payload.event.scope.startswith("repo.content")
52
- and payload.repo.name == TARGET_DATASET
53
- and payload.repo.type == "dataset"
54
- ):
55
- return {"processed": False, "reason": "Event not relevant"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
 
57
  api = HfApi(token=HF_TOKEN)
 
 
 
 
 
58
 
59
- # List all JSON files in the repo
60
- all_files = list(list_repo_files(TARGET_DATASET, repo_type="dataset", token=HF_TOKEN))
61
- json_files = [f for f in all_files if f.endswith(".json") or f.endswith(".jsonl")]
62
 
63
- results = []
64
- for filepath in json_files:
65
  try:
66
- local_path = hf_hub_download(
67
- repo_id=TARGET_DATASET,
68
- filename=filepath,
69
- repo_type="dataset",
70
- token=HF_TOKEN,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  )
72
- with open(local_path, "r", encoding="utf-8") as f:
73
- first_line = f.readline().strip()
74
- results.append({"file": filepath, "first_line": first_line})
75
- except Exception as e:
76
- results.append({"file": filepath, "error": str(e)})
77
-
78
- # Write summary to a JSONL file and upload it back to the dataset repo
79
- output_content = "\n".join(json.dumps(r) for r in results)
80
-
81
- api.upload_file(
82
- path_or_fileobj=output_content.encode("utf-8"),
83
- path_in_repo=OUTPUT_FILENAME,
84
- repo_id=TARGET_DATASET,
85
- repo_type="dataset",
86
- commit_message="Auto-update first_lines_summary.jsonl [webhook]",
87
- )
88
-
89
- return {"processed": True, "files_scanned": len(results)}
90
-
91
- @app.get("/")
92
- async def root():
93
- return {"status": "ok", "message": "Webhook server is running"}
 
1
  import json
2
  import os
3
+ from datetime import datetime
4
+
5
+ from huggingface_hub import HfApi, hf_hub_download
6
+ from shiny import App, reactive, render, ui
7
+
8
+ DATASET_REPO = "MIMIR-AI-ROUTER/harness_evals"
9
+ HF_BLOB_URL = f"https://huggingface.co/datasets/{DATASET_REPO}/blob/main"
10
+ HF_TOKEN = os.getenv("HF_TOKEN")
11
+
12
+ CSS = """
13
+ :root {
14
+ --bg: #1a1a2e;
15
+ --bg2: #16213e;
16
+ --bg3: #0f3460;
17
+ --accent: #e94560;
18
+ --link: #4a9eff;
19
+ --text: #e0e0e0;
20
+ --muted: #8888aa;
21
+ --border: #2a2a4a;
22
+ }
23
+ * { box-sizing: border-box; margin: 0; padding: 0; }
24
+ body { background: var(--bg); color: var(--text); font-family: system-ui, sans-serif; }
25
+ .wrap { max-width: 1300px; margin: 0 auto; padding: 24px 20px; }
26
+ h1 { font-size: 1.6em; border-bottom: 2px solid var(--accent); padding-bottom: 10px; margin-bottom: 6px; }
27
+ .sub { color: var(--muted); font-size: .85em; margin-bottom: 20px; }
28
+ .sub a { color: var(--link); text-decoration: none; }
29
+ .sub a:hover { text-decoration: underline; }
30
+ .controls { margin-bottom: 16px; }
31
+ .btn { background: var(--link); color: #fff; border: none; padding: 7px 14px;
32
+ border-radius: 4px; cursor: pointer; font-size: .85em; }
33
+ .btn:hover { opacity: .85; }
34
+ table { width: 100%; border-collapse: collapse; }
35
+ th { background: var(--bg2); color: var(--muted); font-size: .75em;
36
+ text-transform: uppercase; letter-spacing: .06em;
37
+ padding: 9px 12px; text-align: left; border-bottom: 2px solid var(--border); }
38
+ td { padding: 9px 12px; font-size: .88em; border-bottom: 1px solid var(--border); vertical-align: middle; }
39
+ tr.run-row { cursor: pointer; }
40
+ tr.run-row:hover td { background: var(--bg2); }
41
+ tr.run-row.active td { background: var(--bg3); }
42
+ .model { font-family: monospace; font-size: .82em; color: var(--link); font-weight: 600; }
43
+ .a { color: var(--link); text-decoration: none; font-size: .82em; }
44
+ .a:hover { text-decoration: underline; }
45
+ .detail { background: var(--bg2); border-radius: 6px; padding: 16px; margin-top: 20px; }
46
+ .detail h3 { font-size: .95em; color: var(--muted); font-weight: 400;
47
+ margin-bottom: 14px; padding-bottom: 8px; border-bottom: 1px solid var(--border); }
48
+ .detail th { background: var(--bg); }
49
+ .muted { color: var(--muted); }
50
+ .err { color: var(--accent); background: rgba(233,69,96,.1); padding: 10px 14px;
51
+ border-radius: 4px; border-left: 3px solid var(--accent); margin-bottom: 16px; }
52
+ .loading { color: var(--muted); padding: 40px; text-align: center; font-style: italic; }
53
+ """
54
+
55
+ # ── data loading ──────────────────────────────────────────────────────────────
56
+
57
+ _cache: dict = {"runs": None, "err": None}
58
+
59
+
60
+ def _summarize(groups: dict, tasks: list) -> str:
61
+ parts = []
62
+ for g_name, g_data in groups.items():
63
+ if not isinstance(g_data, dict):
64
+ continue
65
+ for k, v in g_data.items():
66
+ if "," not in k or not isinstance(v, float):
67
+ continue
68
+ mname = k.split(",")[0]
69
+ if not mname.endswith("_stderr"):
70
+ parts.append(f"{g_name}: {v:.3f}")
71
+ break
72
+ if not parts and tasks:
73
+ first_m = next(iter(tasks[0]["metrics"]), None)
74
+ if first_m:
75
+ vals = [t["metrics"][first_m]["value"] for t in tasks if first_m in t["metrics"]]
76
+ if vals:
77
+ parts.append(f"avg {first_m}: {sum(vals)/len(vals):.3f} (n={len(tasks)})")
78
+ return "; ".join(parts) or "β€”"
79
+
80
+
81
+ def load_runs(force: bool = False) -> tuple[list, str | None]:
82
+ if not force and _cache["runs"] is not None:
83
+ return _cache["runs"], _cache["err"]
84
 
85
  api = HfApi(token=HF_TOKEN)
86
+ try:
87
+ all_files = sorted(api.list_repo_files(DATASET_REPO, repo_type="dataset"))
88
+ except Exception as e:
89
+ _cache.update(runs=[], err=str(e))
90
+ return [], str(e)
91
 
92
+ result_files = [f for f in all_files if f.startswith("results_") and f.endswith(".json")]
93
+ sample_files = {f for f in all_files if f.startswith("samples_") and f.endswith(".jsonl")}
 
94
 
95
+ runs = []
96
+ for filename in sorted(result_files, reverse=True):
97
  try:
98
+ local = hf_hub_download(DATASET_REPO, filename, repo_type="dataset", token=HF_TOKEN)
99
+ with open(local) as f:
100
+ data = json.load(f)
101
+ except Exception:
102
+ continue
103
+
104
+ ts = filename.removeprefix("results_").removesuffix(".json")
105
+ model = data.get("model_name", "unknown")
106
+ date_ts = data.get("date")
107
+ date_str = datetime.fromtimestamp(date_ts).strftime("%Y-%m-%d %H:%M") if date_ts else "β€”"
108
+
109
+ tasks = []
110
+ for task_name, td in data.get("results", {}).items():
111
+ if not isinstance(td, dict):
112
+ continue
113
+ metrics: dict = {}
114
+ for k, v in td.items():
115
+ if k in ("name", "alias", "sample_len") or not isinstance(v, (int, float)):
116
+ continue
117
+ if "," in k:
118
+ mname, filt = k.rsplit(",", 1)
119
+ if not mname.endswith("_stderr"):
120
+ metrics[mname] = {
121
+ "value": v,
122
+ "stderr": td.get(f"{mname}_stderr,{filt}"),
123
+ }
124
+ samples_f = f"samples_{task_name}_{ts}.jsonl"
125
+ tasks.append({
126
+ "name": task_name,
127
+ "n": td.get("sample_len", 0),
128
+ "metrics": metrics,
129
+ "samples_file": samples_f if samples_f in sample_files else None,
130
+ })
131
+
132
+ runs.append({
133
+ "model": model,
134
+ "date": date_str,
135
+ "filename": filename,
136
+ "ts": ts,
137
+ "tasks": tasks,
138
+ "n_tasks": len(tasks),
139
+ "summary": _summarize(data.get("groups", {}), tasks),
140
+ })
141
+
142
+ _cache.update(runs=runs, err=None)
143
+ return runs, None
144
+
145
+
146
+ # ── UI ────────────────────────────────────────────────────────────────────────
147
+
148
+ app_ui = ui.page_fluid(
149
+ ui.tags.head(ui.tags.style(CSS)),
150
+ ui.div(
151
+ {"class": "wrap"},
152
+ ui.h1("LM Eval Results"),
153
+ ui.div(
154
+ {"class": "sub"},
155
+ "Dataset: ",
156
+ ui.tags.a(
157
+ DATASET_REPO,
158
+ href=f"https://huggingface.co/datasets/{DATASET_REPO}",
159
+ target="_blank",
160
+ ),
161
+ ),
162
+ ui.div(
163
+ {"class": "controls"},
164
+ ui.input_action_button("refresh", "β†Ί Refresh", class_="btn"),
165
+ ),
166
+ ui.output_ui("runs_table"),
167
+ ui.output_ui("task_detail"),
168
+ ui.tags.script(
169
+ "function selectRun(i){"
170
+ " Shiny.setInputValue('sel_idx', i, {priority:'event'});"
171
+ "}"
172
+ ),
173
+ ),
174
+ )
175
+
176
+
177
+ # ── server ────────────────────────────────────────────────────────────────────
178
+
179
+ def server(input, output, session):
180
+ sel = reactive.Value(None)
181
+
182
+ @reactive.effect
183
+ @reactive.event(input.refresh)
184
+ def _on_refresh():
185
+ _cache.update(runs=None, err=None)
186
+ sel.set(None)
187
+
188
+ @reactive.effect
189
+ @reactive.event(input.sel_idx)
190
+ def _on_select():
191
+ idx = input.sel_idx()
192
+ sel.set(None if sel() == idx else idx)
193
+
194
+ @output
195
+ @render.ui
196
+ def runs_table():
197
+ input.refresh() # reactive dependency
198
+ runs, err = load_runs()
199
+
200
+ if err:
201
+ return ui.div({"class": "err"}, f"Failed to load data: {err}")
202
+ if not runs:
203
+ return ui.div({"class": "loading"}, "No eval results found in the dataset.")
204
+
205
+ rows = []
206
+ for i, run in enumerate(runs):
207
+ active = "active" if sel() == i else ""
208
+ rows.append(
209
+ ui.tags.tr(
210
+ {"class": f"run-row {active}", "onclick": f"selectRun({i})"},
211
+ ui.tags.td(ui.span({"class": "model"}, run["model"])),
212
+ ui.tags.td(run["date"]),
213
+ ui.tags.td(str(run["n_tasks"])),
214
+ ui.tags.td(run["summary"]),
215
+ ui.tags.td(
216
+ ui.tags.a(
217
+ {"class": "a", "href": f"{HF_BLOB_URL}/{run['filename']}",
218
+ "target": "_blank", "onclick": "event.stopPropagation()"},
219
+ "πŸ“„ results",
220
+ )
221
+ ),
222
+ )
223
+ )
224
+
225
+ return ui.tags.table(
226
+ ui.tags.thead(
227
+ ui.tags.tr(
228
+ ui.tags.th("Model"),
229
+ ui.tags.th("Date"),
230
+ ui.tags.th("Tasks"),
231
+ ui.tags.th("Summary"),
232
+ ui.tags.th("File"),
233
+ )
234
+ ),
235
+ ui.tags.tbody(*rows),
236
+ )
237
+
238
+ @output
239
+ @render.ui
240
+ def task_detail():
241
+ idx = sel()
242
+ if idx is None:
243
+ return ui.div()
244
+ runs, _ = load_runs()
245
+ if not runs or idx >= len(runs):
246
+ return ui.div()
247
+
248
+ run = runs[idx]
249
+ task_rows = []
250
+ for task in sorted(run["tasks"], key=lambda t: t["name"]):
251
+ metric_parts = []
252
+ for mname, mdata in task["metrics"].items():
253
+ v = mdata["value"]
254
+ se = mdata.get("stderr")
255
+ metric_parts.append(
256
+ f"{mname}: {v:.3f}" + (f" Β±{se:.3f}" if se is not None else "")
257
+ )
258
+
259
+ samples_cell = ui.span({"class": "muted"}, "β€”")
260
+ if task["samples_file"]:
261
+ samples_cell = ui.tags.a(
262
+ {"class": "a",
263
+ "href": f"{HF_BLOB_URL}/{task['samples_file']}",
264
+ "target": "_blank"},
265
+ "πŸ“„ samples",
266
+ )
267
+
268
+ task_rows.append(
269
+ ui.tags.tr(
270
+ ui.tags.td(task["name"]),
271
+ ui.tags.td(str(task["n"])),
272
+ ui.tags.td("; ".join(metric_parts) or "β€”"),
273
+ ui.tags.td(samples_cell),
274
+ )
275
  )
276
+
277
+ return ui.div(
278
+ {"class": "detail"},
279
+ ui.tags.h3(f"{run['model']} Β· {run['date']} Β· {run['n_tasks']} tasks"),
280
+ ui.tags.table(
281
+ ui.tags.thead(
282
+ ui.tags.tr(
283
+ ui.tags.th("Task"),
284
+ ui.tags.th("N Samples"),
285
+ ui.tags.th("Metrics"),
286
+ ui.tags.th("Samples File"),
287
+ )
288
+ ),
289
+ ui.tags.tbody(*task_rows),
290
+ ),
291
+ )
292
+
293
+
294
+ app = App(app_ui, server)
 
 
 
requirements.txt CHANGED
@@ -1,5 +1,2 @@
1
- faicons
2
- fastapi
3
- uvicorn
4
  shiny
5
- huggingface_hub
 
 
 
 
1
  shiny
2
+ huggingface_hub