Mahir commited on
Commit
7ff4f6b
·
1 Parent(s): 6d6d6e3

Fix inference.py: sync heuristic fallback, clamp all scores

Browse files
Files changed (1) hide show
  1. inference.py +37 -12
inference.py CHANGED
@@ -285,23 +285,43 @@ async def main():
285
  args = parser.parse_args()
286
  if not HF_TOKEN:
287
  print("WARNING: No API key. Running heuristic baseline.", file=sys.stderr)
288
- import httpx as _httpx
289
- async def _run_heuristic():
290
- async with _httpx.AsyncClient(base_url=OPENENV_URL, timeout=30.0) as http:
291
- r = await http.post("/baseline")
292
- r.raise_for_status()
293
- data = r.json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
294
  print(f"\n{'='*60}\nHEURISTIC BASELINE\n{'='*60}")
295
- for tid, result in data.get("tasks", {}).items():
296
- mark = "✓ PASS" if result.get("passed") else "✗ FAIL"
297
- print(f" {tid:<38} {result['final_score']:.4f} {mark}")
298
- overall = data.get("overall_score", 0)
299
  print(f" {'OVERALL':<38} {overall:.4f}")
 
300
  with open("baseline_results.json", "w") as f:
301
  json.dump(data, f, indent=2)
302
  print("\n Results saved to baseline_results.json")
303
- asyncio.run(_run_heuristic())
304
- return
 
 
 
 
305
  client = OpenAI(api_key=HF_TOKEN, base_url=args.base_url)
306
  all_tasks = ["ticket_classification","response_drafting","queue_management","multi_turn_conversation","legal_clause_identification","legal_risk_flagging","legal_clause_redlining","clinical_triage_classification","clinical_esi_assignment","clinical_triage_note","pr_type_classification","pr_bug_identification","pr_review_comment"]
307
  tasks_to_run = all_tasks if args.task=="all" else [args.task]
@@ -323,6 +343,11 @@ async def main():
323
  print(f" ✗ CRASHED: {e}", file=sys.stderr); traceback.print_exc(file=sys.stderr)
324
  results[task_id] = {"final_score":0.0001,"passed":False,"reward_history":[],"metrics":{},"error":str(e)}
325
  print(f"[END] task={task_id} score=0.0000 steps=0", flush=True)
 
 
 
 
 
326
  scores = [r["final_score"] for r in results.values()]
327
  raw_overall = sum(scores)/len(scores) if scores else 0.0001
328
  overall = max(0.0001, min(0.9999, float(raw_overall)))
 
285
  args = parser.parse_args()
286
  if not HF_TOKEN:
287
  print("WARNING: No API key. Running heuristic baseline.", file=sys.stderr)
288
+ import urllib.request
289
+
290
+ def _sync_post(path):
291
+ req = urllib.request.Request(
292
+ OPENENV_URL + path, data=b"{}",
293
+ headers={"Content-Type": "application/json"}, method="POST"
294
+ )
295
+ with urllib.request.urlopen(req, timeout=60) as r:
296
+ return json.loads(r.read())
297
+
298
+ try:
299
+ data = _sync_post("/baseline")
300
+ tasks = data.get("tasks", {})
301
+ overall = float(data.get("overall_score", 0.5))
302
+
303
+ # Clamp all scores strictly within (0, 1)
304
+ for tid in tasks:
305
+ s = float(tasks[tid].get("final_score", 0.5))
306
+ tasks[tid]["final_score"] = max(0.0001, min(0.9999, s))
307
+ overall = max(0.0001, min(0.9999, overall))
308
+ data["overall_score"] = overall
309
+
310
  print(f"\n{'='*60}\nHEURISTIC BASELINE\n{'='*60}")
311
+ for tid, r in tasks.items():
312
+ mark = "✓ PASS" if r.get("passed") else "✗ FAIL"
313
+ print(f" {tid:<38} {r['final_score']:.4f} {mark}")
 
314
  print(f" {'OVERALL':<38} {overall:.4f}")
315
+
316
  with open("baseline_results.json", "w") as f:
317
  json.dump(data, f, indent=2)
318
  print("\n Results saved to baseline_results.json")
319
+
320
+ except Exception as e:
321
+ print(f"ERROR connecting to server: {e}", file=sys.stderr)
322
+ sys.exit(1)
323
+
324
+ sys.exit(0)
325
  client = OpenAI(api_key=HF_TOKEN, base_url=args.base_url)
326
  all_tasks = ["ticket_classification","response_drafting","queue_management","multi_turn_conversation","legal_clause_identification","legal_risk_flagging","legal_clause_redlining","clinical_triage_classification","clinical_esi_assignment","clinical_triage_note","pr_type_classification","pr_bug_identification","pr_review_comment"]
327
  tasks_to_run = all_tasks if args.task=="all" else [args.task]
 
343
  print(f" ✗ CRASHED: {e}", file=sys.stderr); traceback.print_exc(file=sys.stderr)
344
  results[task_id] = {"final_score":0.0001,"passed":False,"reward_history":[],"metrics":{},"error":str(e)}
345
  print(f"[END] task={task_id} score=0.0000 steps=0", flush=True)
346
+ # Clamp all individual task scores
347
+ for tid in results:
348
+ s = float(results[tid].get("final_score", 0.0001))
349
+ results[tid]["final_score"] = max(0.0001, min(0.9999, s))
350
+
351
  scores = [r["final_score"] for r in results.values()]
352
  raw_overall = sum(scores)/len(scores) if scores else 0.0001
353
  overall = max(0.0001, min(0.9999, float(raw_overall)))