Arrowman123 commited on
Commit
b4f3688
·
verified ·
1 Parent(s): 00855c5
Files changed (1) hide show
  1. inference.py +40 -42
inference.py CHANGED
@@ -15,8 +15,6 @@ TASK_NAME = "customer-support"
15
  BENCHMARK = "openenv"
16
 
17
  MAX_STEPS = 20
18
- MAX_TOTAL_REWARD = 20
19
- SUCCESS_SCORE_THRESHOLD = 0.6
20
 
21
 
22
  # LLM AGENT
@@ -38,7 +36,7 @@ refund / replace / reject / ask_proof
38
  )
39
  return response.choices[0].message.content.strip().lower()
40
  except Exception as e:
41
- print(f"[ERROR] LLM call failed: {e}")
42
  return "reject"
43
 
44
 
@@ -64,12 +62,8 @@ def log_end(**kwargs):
64
  print("[END]", kwargs, flush=True)
65
 
66
 
67
- # MAIN
68
  async def main():
69
- rewards = []
70
- steps_taken = 0
71
- success = False
72
-
73
  client = OpenAI(
74
  base_url=API_BASE_URL,
75
  api_key=API_KEY
@@ -77,59 +71,63 @@ async def main():
77
 
78
  log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
79
 
80
- try:
81
- res = safe_post(f"{SPACE_URL}/reset")
82
- result = res.json()
83
-
84
- state = result["observation"]["echoed_message"]
85
- done = result["done"]
86
-
87
- for step in range(1, MAX_STEPS + 1):
88
- if done:
89
- break
90
-
91
- message = get_model_message(client, state)
92
 
 
 
 
93
  res = safe_post(
94
- f"{SPACE_URL}/step",
95
- json={"message": message}
96
  )
97
  result = res.json()
98
 
99
- reward = result.get("reward", 0.0)
100
- done = result.get("done", False)
101
  state = result["observation"]["echoed_message"]
 
102
 
103
- rewards.append(reward)
104
- steps_taken = step
 
105
 
106
- log_step(
107
- step=step,
108
- action=message,
109
- reward=reward,
110
- done=done,
111
- error=None
112
- )
 
 
 
 
 
 
 
 
 
 
 
113
 
114
- score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
 
 
115
 
116
- # ensure score strictly between 0 and 1
117
- score = max(0.01, min(0.99, score))
118
 
119
- success = score >= SUCCESS_SCORE_THRESHOLD
120
 
121
  except Exception as e:
122
  print(f"[ERROR] {e}")
123
 
124
  finally:
125
  log_end(
126
- success=success,
127
- steps=steps_taken,
128
- score=score if 'score' in locals() else 0.0,
129
- rewards=rewards
130
  )
131
 
132
 
133
- # ENTRY
134
  if __name__ == "__main__":
135
  asyncio.run(main())
 
15
  BENCHMARK = "openenv"
16
 
17
  MAX_STEPS = 20
 
 
18
 
19
 
20
  # LLM AGENT
 
36
  )
37
  return response.choices[0].message.content.strip().lower()
38
  except Exception as e:
39
+ print(f"[ERROR] LLM failed: {e}")
40
  return "reject"
41
 
42
 
 
62
  print("[END]", kwargs, flush=True)
63
 
64
 
65
+ # MAIN
66
  async def main():
 
 
 
 
67
  client = OpenAI(
68
  base_url=API_BASE_URL,
69
  api_key=API_KEY
 
71
 
72
  log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
73
 
74
+ task_scores = []
 
 
 
 
 
 
 
 
 
 
 
75
 
76
+ try:
77
+ for task in ["easy", "medium", "hard"]:
78
+ # RESET TASK
79
  res = safe_post(
80
+ f"{SPACE_URL}/reset",
81
+ json={"task": task}
82
  )
83
  result = res.json()
84
 
 
 
85
  state = result["observation"]["echoed_message"]
86
+ done = result["done"]
87
 
88
+ for step in range(1, MAX_STEPS + 1):
89
+ if done:
90
+ break
91
 
92
+ message = get_model_message(client, state)
93
+
94
+ res = safe_post(
95
+ f"{SPACE_URL}/step",
96
+ json={"message": message}
97
+ )
98
+ result = res.json()
99
+
100
+ state = result["observation"]["echoed_message"]
101
+ done = result["done"]
102
+
103
+ log_step(
104
+ step=step,
105
+ action=message,
106
+ reward=result.get("reward", 0.0),
107
+ done=done,
108
+ error=None
109
+ )
110
 
111
+ # GET TASK SCORE FROM ENV
112
+ score = result.get("score", 0.5)
113
+ score = max(0.01, min(0.99, score))
114
 
115
+ task_scores.append(score)
 
116
 
117
+ final_score = sum(task_scores) / len(task_scores)
118
 
119
  except Exception as e:
120
  print(f"[ERROR] {e}")
121
 
122
  finally:
123
  log_end(
124
+ success=True,
125
+ steps=len(task_scores),
126
+ score=final_score if 'final_score' in locals() else 0.0,
127
+ rewards=task_scores
128
  )
129
 
130
 
131
+ # ENTRY
132
  if __name__ == "__main__":
133
  asyncio.run(main())