Arrowman123 commited on
Commit
25d5122
·
verified ·
1 Parent(s): 092f499
Files changed (1) hide show
  1. inference.py +52 -61
inference.py CHANGED
@@ -4,20 +4,25 @@ import requests
4
  import time
5
  from openai import OpenAI
6
 
7
- # REQUIRED VARIABLES
8
- API_BASE_URL = os.environ["API_BASE_URL"]
9
- API_KEY = os.environ["API_KEY"]
10
- MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")
11
 
12
- SPACE_URL = "https://arrowman123-customer-supp-env.hf.space"
 
13
 
14
- TASK_NAME = "customer-support"
15
- BENCHMARK = "openenv"
16
 
17
- MAX_STEPS = 20
 
 
18
 
 
 
19
 
20
- # LLM AGENT
 
21
  def get_model_message(client, state):
22
  prompt = f"""
23
  You are a customer support agent.
@@ -28,7 +33,6 @@ State:
28
  Choose ONLY ONE word:
29
  refund / replace / reject / ask_proof
30
  """
31
-
32
  try:
33
  response = client.chat.completions.create(
34
  model=MODEL_NAME,
@@ -36,7 +40,7 @@ refund / replace / reject / ask_proof
36
  )
37
  return response.choices[0].message.content.strip().lower()
38
  except Exception as e:
39
- print(f"[ERROR] LLM failed: {e}")
40
  return "reject"
41
 
42
 
@@ -46,83 +50,70 @@ def safe_post(url, json=None, retries=3):
46
  try:
47
  return requests.post(url, json=json, timeout=10)
48
  except Exception as e:
49
- print(f"[DEBUG] retry {i+1} due to {e}")
50
  time.sleep(2)
51
  raise Exception("Failed after retries")
52
 
 
 
53
 
54
- # LOGGING
55
- def log_start(**kwargs):
56
- print("[START]", kwargs, flush=True)
57
-
58
- def log_step(**kwargs):
59
- print("[STEP]", kwargs, flush=True)
 
 
60
 
61
- def log_end(**kwargs):
62
- print("[END]", kwargs, flush=True)
 
 
63
 
64
 
65
- # MAIN
66
  async def main():
67
- client = OpenAI(
68
- base_url=API_BASE_URL,
69
- api_key=API_KEY
70
- )
71
 
72
- log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
 
 
73
 
74
- task_scores = []
75
 
76
- try:
77
- # run 3 tasks
78
- for _ in range(3):
79
- res = safe_post(f"{SPACE_URL}/reset")
80
  result = res.json()
81
 
82
  state = result["observation"]["echoed_message"]
83
- done = result["done"]
84
 
85
  for step in range(1, MAX_STEPS + 1):
86
  if done:
87
  break
88
 
89
- message = get_model_message(client, state)
90
 
91
- res = safe_post(
92
- f"{SPACE_URL}/step",
93
- json={"message": message}
94
- )
95
  result = res.json()
96
 
97
- state = result["observation"]["echoed_message"]
98
- done = result["done"]
 
99
 
100
- log_step(
101
- step=step,
102
- action=message,
103
- reward=result.get("reward", 0.0),
104
- done=done,
105
- error=None
106
- )
107
 
108
- # GET SCORE FROM ENV
109
- score = result.get("score", 0.5)
110
- score = max(0.01, min(0.99, score))
111
 
112
- task_scores.append(score)
113
 
114
- final_score = sum(task_scores) / len(task_scores)
 
115
 
116
- except Exception as e:
117
- print(f"[ERROR] {e}")
118
-
119
- finally:
120
- log_end(
121
- success=True,
122
- steps=len(task_scores),
123
- score=final_score if 'final_score' in locals() else 0.0,
124
- rewards=task_scores
125
- )
126
 
127
 
128
  if __name__ == "__main__":
 
4
  import time
5
  from openai import OpenAI
6
 
7
+ # REQUIRED ENV VARIABLES
8
+ API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
9
+ MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")
10
+ HF_TOKEN = os.getenv("HF_TOKEN")
11
 
12
+ if HF_TOKEN is None:
13
+ raise ValueError("HF_TOKEN environment variable is required")
14
 
15
+ API_KEY = os.getenv("API_KEY", HF_TOKEN)
 
16
 
17
+ SPACE_URL = "https://arrowman123-customer-supp-env.hf.space"
18
+ BENCHMARK = "openenv"
19
+ MAX_STEPS = 20
20
 
21
+ # Task names must match openenv.yaml and cycle easy → medium → hard
22
+ TASK_NAMES = ["easy", "medium", "hard"]
23
 
24
+
25
+ # LLM AGENT
26
  def get_model_message(client, state):
27
  prompt = f"""
28
  You are a customer support agent.
 
33
  Choose ONLY ONE word:
34
  refund / replace / reject / ask_proof
35
  """
 
36
  try:
37
  response = client.chat.completions.create(
38
  model=MODEL_NAME,
 
40
  )
41
  return response.choices[0].message.content.strip().lower()
42
  except Exception as e:
43
+ print(f"[ERROR] LLM failed: {e}", flush=True)
44
  return "reject"
45
 
46
 
 
50
  try:
51
  return requests.post(url, json=json, timeout=10)
52
  except Exception as e:
53
+ print(f"[DEBUG] retry {i+1} due to {e}", flush=True)
54
  time.sleep(2)
55
  raise Exception("Failed after retries")
56
 
57
+ def log_start(task, env, model):
58
+ print(f"[START] task={task} env={env} model={model}", flush=True)
59
 
60
+ def log_step(step, action, reward, done, error=None):
61
+ error_str = "null" if error is None else str(error)
62
+ done_str = "true" if done else "false"
63
+ print(
64
+ f"[STEP] step={step} action={action} reward={reward:.2f} "
65
+ f"done={done_str} error={error_str}",
66
+ flush=True
67
+ )
68
 
69
+ def log_end(success, steps, rewards):
70
+ success_str = "true" if success else "false"
71
+ rewards_str = ",".join(f"{r:.2f}" for r in rewards)
72
+ print(f"[END] success={success_str} steps={steps} rewards={rewards_str}", flush=True)
73
 
74
 
75
+ # MAIN
76
  async def main():
77
+ # HF_TOKEN is used as the API key per guidelines
78
+ client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
 
 
79
 
80
+ for task_name in TASK_NAMES:
81
+ step_rewards = []
82
+ success = False
83
 
84
+ log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)
85
 
86
+ try:
87
+ res = safe_post(f"{SPACE_URL}/reset")
 
 
88
  result = res.json()
89
 
90
  state = result["observation"]["echoed_message"]
91
+ done = result["done"]
92
 
93
  for step in range(1, MAX_STEPS + 1):
94
  if done:
95
  break
96
 
97
+ action = get_model_message(client, state)
98
 
99
+ res = safe_post(f"{SPACE_URL}/step", json={"message": action})
 
 
 
100
  result = res.json()
101
 
102
+ state = result["observation"]["echoed_message"]
103
+ done = result["done"]
104
+ reward = float(result.get("reward", 0.0))
105
 
106
+ step_rewards.append(reward)
 
 
 
 
 
 
107
 
108
+ log_step(step=step, action=action, reward=reward, done=done, error=None)
 
 
109
 
110
+ success = True
111
 
112
+ except Exception as e:
113
+ print(f"[ERROR] task={task_name} {e}", flush=True)
114
 
115
+ finally:
116
+ log_end(success=success, steps=len(step_rewards), rewards=step_rewards)
 
 
 
 
 
 
 
 
117
 
118
 
119
  if __name__ == "__main__":