juliets Claude Opus 4.8 (1M context) commited on
Commit
e3e77dc
·
1 Parent(s): b945bbc

Run-all-models: run concurrently and stream rows as each finishes

Browse files

The first version ran the six models sequentially, so with the reasoning models
inheriting the Config default max_tokens (9192) a comparison could block for a
minute-plus with a blank table. Run them in a thread pool instead and make the
handler a generator that yields the table after each model returns: every model
shows "running" immediately and its verdict fills in as it lands, so total time is
about the slowest single model rather than the sum, and a stalled endpoint no
longer holds up the rest.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. app.py +56 -29
app.py CHANGED
@@ -167,15 +167,44 @@ def _comparison_note(model_info_dict, parsed, result) -> str:
167
  return ""
168
 
169
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
170
  def handle_run_all_models(test_input, current_policy, structured_rules, reasoning_effort,
171
  max_tokens, temperature, top_p, system_prompt_val, response_format_val,
172
  oauth_token: gr.OAuthToken | None = None):
173
- """Run the current content + policy against every model and return a comparison table.
174
 
175
  This is the workshop's core move — the same input judged by each model side by side.
176
- Each model is run independently; a model whose endpoint is down shows as unavailable
177
- rather than failing the whole comparison.
 
178
  """
 
 
179
  if not test_input or not test_input.strip():
180
  raise gr.Error("Please enter test content before running a comparison.")
181
  if not current_policy or current_policy == "*No policy loaded*":
@@ -185,32 +214,30 @@ def handle_run_all_models(test_input, current_policy, structured_rules, reasonin
185
  if hf_token is None:
186
  raise gr.Error("Please log in or set tokens to use Inference Providers. Check the sidebar.")
187
 
188
- rows = []
189
- for model in MODELS:
190
- model_id = model["id"]
191
- try:
192
- result = run_test(
193
- model_id=model_id, test_input=test_input, policy=current_policy, hf_token=hf_token,
194
- reasoning_effort=reasoning_effort, max_tokens=int(max_tokens),
195
- temperature=float(temperature), top_p=float(top_p),
196
- system_prompt=system_prompt_val, response_format=response_format_val,
197
- structured_rules=structured_rules,
198
- )
199
- parsed = parse_json_response(result.get("content", ""))
200
- label = parsed.get("label", -1)
201
- verdict = (
202
- "❌ Violation" if label == 1
203
- else "✅ No violation" if label == 0
204
- else "⚠️ Unclear"
205
- )
206
- note = _comparison_note(model, parsed, result)
207
- except gr.Error as e:
208
- verdict, note = "⚠️ Unavailable", str(e)[:200]
209
- except Exception as e:
210
- verdict, note = "⚠️ Error", str(e)[:200]
211
- rows.append([model["name"], verdict, note])
212
-
213
- return rows, gr.update(visible=True)
214
 
215
 
216
  # ============================================================================
 
167
  return ""
168
 
169
 
170
+ def _run_one_model(model, test_input, current_policy, structured_rules, reasoning_effort,
171
+ max_tokens, temperature, top_p, system_prompt_val, response_format_val, hf_token):
172
+ """Run a single model and return its [name, verdict, note] row. Never raises."""
173
+ try:
174
+ result = run_test(
175
+ model_id=model["id"], test_input=test_input, policy=current_policy, hf_token=hf_token,
176
+ reasoning_effort=reasoning_effort, max_tokens=int(max_tokens),
177
+ temperature=float(temperature), top_p=float(top_p),
178
+ system_prompt=system_prompt_val, response_format=response_format_val,
179
+ structured_rules=structured_rules,
180
+ )
181
+ parsed = parse_json_response(result.get("content", ""))
182
+ label = parsed.get("label", -1)
183
+ verdict = (
184
+ "❌ Violation" if label == 1
185
+ else "✅ No violation" if label == 0
186
+ else "⚠️ Unclear"
187
+ )
188
+ note = _comparison_note(model, parsed, result)
189
+ except gr.Error as e:
190
+ verdict, note = "⚠️ Unavailable", str(e)[:200]
191
+ except Exception as e:
192
+ verdict, note = "⚠️ Error", str(e)[:200]
193
+ return [model["name"], verdict, note]
194
+
195
+
196
  def handle_run_all_models(test_input, current_policy, structured_rules, reasoning_effort,
197
  max_tokens, temperature, top_p, system_prompt_val, response_format_val,
198
  oauth_token: gr.OAuthToken | None = None):
199
+ """Run the current content + policy against every model, streaming the comparison table.
200
 
201
  This is the workshop's core move — the same input judged by each model side by side.
202
+ Models run concurrently (reasoning models are slow, so sequential runs blocked for
203
+ minutes), and the table is yielded as each one finishes, so rows fill in live and a
204
+ model whose endpoint is down shows as unavailable rather than stalling the rest.
205
  """
206
+ from concurrent.futures import ThreadPoolExecutor
207
+
208
  if not test_input or not test_input.strip():
209
  raise gr.Error("Please enter test content before running a comparison.")
210
  if not current_policy or current_policy == "*No policy loaded*":
 
214
  if hf_token is None:
215
  raise gr.Error("Please log in or set tokens to use Inference Providers. Check the sidebar.")
216
 
217
+ # Start every model at once; show them all as "running" immediately so the table
218
+ # isn't blank while the slow reasoning models work.
219
+ status = {m["name"]: "⏳ running…" for m in MODELS}
220
+ notes = {m["name"]: "" for m in MODELS}
221
+ order = [m["name"] for m in MODELS]
222
+
223
+ def table():
224
+ return [[name, status[name], notes[name]] for name in order]
225
+
226
+ yield table(), gr.update(visible=True)
227
+
228
+ with ThreadPoolExecutor(max_workers=len(MODELS)) as executor:
229
+ futures = {
230
+ executor.submit(
231
+ _run_one_model, m, test_input, current_policy, structured_rules, reasoning_effort,
232
+ max_tokens, temperature, top_p, system_prompt_val, response_format_val, hf_token,
233
+ ): m for m in MODELS
234
+ }
235
+ from concurrent.futures import as_completed
236
+ for fut in as_completed(futures):
237
+ name, verdict, note = fut.result()
238
+ status[name] = verdict
239
+ notes[name] = note
240
+ yield table(), gr.update(visible=True)
 
 
241
 
242
 
243
  # ============================================================================