Omkar1806 commited on
Commit
d3b065a
·
verified ·
1 Parent(s): cdcc8e1

Update inference.py

Browse files
Files changed (1) hide show
  1. inference.py +12 -99
inference.py CHANGED
@@ -1,105 +1,18 @@
1
- import os
2
  import sys
3
- import numpy as np
4
- import random
5
- import re
6
- from openai import OpenAI
7
- from env import EmailTriageEnv
8
-
9
- # Mandatory Environment Variables
10
- API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
11
- MODEL_NAME = os.getenv("MODEL_NAME") or "meta-llama/Llama-3-70b-chat-hf"
12
- API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
13
- BENCHMARK = os.getenv("MY_ENV_V4_BENCHMARK", "email_gatekeeper_v1")
14
-
15
- client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
16
-
17
- def get_llm_action(email):
18
- """Smart classification logic with hybrid keywords"""
19
- desc = email.get('description', '').lower()
20
-
21
- # Keyword Priority Logic
22
- if any(k in desc for k in ["hack", "breach"]):
23
- return np.array([2, 1, 2], dtype=np.int64)
24
- elif any(k in desc for k in ["legal", "lawsuit", "threat"]):
25
- return np.array([2, 2, 2], dtype=np.int64)
26
- elif any(k in desc for k in ["refund", "dispute"]):
27
- return np.array([1, 2, 2], dtype=np.int64)
28
- elif any(k in desc for k in ["invoice", "billing", "overdue"]):
29
- return np.array([1, 0, 1], dtype=np.int64)
30
-
31
- # LLM Fallback
32
- try:
33
- prompt = f"Classify this email: {desc}. Output exactly 3 integers (0-2) separated by commas."
34
- response = client.chat.completions.create(
35
- model=MODEL_NAME,
36
- messages=[
37
- {"role": "system", "content": "You only output numbers like 1,0,2"},
38
- {"role": "user", "content": prompt}
39
- ],
40
- max_tokens=10,
41
- temperature=0
42
- )
43
- res = response.choices[0].message.content.strip()
44
- nums = re.findall(r'\d', res)
45
- actions = [int(n) for n in nums[:3]]
46
- while len(actions) < 3: actions.append(0)
47
- return np.array(actions, dtype=np.int64)
48
- except:
49
- return np.array([0, 0, 0], dtype=np.int64)
50
 
51
- def run_inference():
52
- tasks = ["easy", "medium", "hard"]
53
-
54
- for task_name in tasks:
55
- rewards = []
56
- steps_taken = 0
57
- success = False
58
- score = 0.0
59
-
60
- # 1. [START] Line
61
- print(f"[START] task={task_name} env={BENCHMARK} model={MODEL_NAME}", flush=True)
62
-
63
- try:
64
- env = EmailTriageEnv(task=task_name, shuffle=False)
65
- env.reset(seed=42)
66
- emails = list(env._queue)
67
-
68
- cumulative_reward = 0.0
69
- for i, email in enumerate(emails):
70
- step_num = i + 1
71
- action = get_llm_action(email)
72
-
73
- # Take Environment Step
74
- _, reward, done, _, info = env.step(action)
75
-
76
- cumulative_reward += reward
77
- rewards.append(float(reward))
78
- steps_taken = step_num
79
-
80
- # 2. [STEP] Line (Exactly as per example)
81
- action_str = f"classify({','.join(map(str, action))})"
82
- done_str = str(done).lower()
83
- print(f"[STEP] step={step_num} action={action_str} reward={reward:.2f} done={done_str} error=null", flush=True)
84
 
85
- # Calculate Final Score
86
- if len(emails) > 0:
87
- raw_score = cumulative_reward / len(emails)
88
- # Apply 0.99x safety clamp if perfect match
89
- if raw_score >= 0.99:
90
- score = 0.99 + random.uniform(0.001, 0.005)
91
- else:
92
- score = raw_score
93
-
94
- success = score >= 0.1
95
 
96
- except Exception as e:
97
- # Error handling to ensure [END] still prints
98
- pass
99
- finally:
100
- # 3. [END] Line (Mandatory)
101
- rewards_str = ",".join(f"{r:.2f}" for r in rewards) if rewards else "0.00"
102
- print(f"[END] success={str(success).lower()} steps={steps_taken} score={score:.3f} rewards={rewards_str}", flush=True)
103
 
104
  if __name__ == "__main__":
105
- run_inference()
 
 
1
  import sys
2
+ import os
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
 
4
+ # Path fix
5
+ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
+ from app import run_demo
 
 
 
 
 
 
 
 
 
8
 
9
+ def main():
10
+ print("--- 🚀 STARTING EVALUATION SCRIPT ---")
11
+ for level in ["easy", "medium", "hard"]:
12
+ print(f"\n--- Testing Difficulty: {level.upper()} ---")
13
+ output = run_demo(level)
14
+ print(output)
15
+ print("\n--- ✅ EVALUATION SCRIPT FINISHED ---")
16
 
17
  if __name__ == "__main__":
18
+ main()