File size: 3,564 Bytes
a451d32
 
 
 
e806f82
a451d32
25d5122
 
 
 
a451d32
25d5122
 
a451d32
25d5122
a451d32
25d5122
 
 
a451d32
25d5122
a451d32
25d5122
 
8b09cd1
 
 
a451d32
8b09cd1
 
a451d32
8b09cd1
 
 
 
 
 
 
 
 
 
25d5122
e806f82
8b09cd1
 
07792e4
a451d32
 
 
 
 
25d5122
a451d32
 
 
25d5122
 
a451d32
25d5122
 
 
 
 
 
 
 
a451d32
82107a5
25d5122
 
82107a5
a451d32
 
25d5122
a451d32
25d5122
8b09cd1
25d5122
 
82107a5
25d5122
a451d32
25d5122
a451d32
25d5122
82107a5
a451d32
 
 
25d5122
a451d32
b4f3688
 
 
a451d32
25d5122
b4f3688
25d5122
b4f3688
 
25d5122
 
 
b4f3688
25d5122
a451d32
25d5122
e806f82
82107a5
 
 
e806f82
25d5122
 
82107a5
a451d32
25d5122
82107a5
a451d32
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import asyncio
import os
import requests
import time
from openai import OpenAI

# REQUIRED ENV VARIABLES
API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
MODEL_NAME   = os.getenv("MODEL_NAME", "gpt-4o-mini")
HF_TOKEN     = os.getenv("HF_TOKEN")

if HF_TOKEN is None:
    raise ValueError("HF_TOKEN environment variable is required")

API_KEY = os.getenv("API_KEY", HF_TOKEN)

SPACE_URL  = "https://arrowman123-customer-supp-env.hf.space"
BENCHMARK  = "openenv"
MAX_STEPS  = 20

TASK_NAMES = ["easy", "medium", "hard"]


# LLM AGENT
def get_model_message(client, state):
    prompt = f"""
You are a customer support agent.

State:
{state}

Choose ONLY ONE word:
refund / replace / reject / ask_proof
"""
    try:
        response = client.chat.completions.create(
            model=MODEL_NAME,
            messages=[{"role": "user", "content": prompt}]
        )
        return response.choices[0].message.content.strip().lower()
    except Exception as e:
        print(f"[ERROR] LLM failed: {e}", flush=True)
        return "reject"


# SAFE REQUEST
def safe_post(url, json=None, retries=3):
    for i in range(retries):
        try:
            return requests.post(url, json=json, timeout=10)
        except Exception as e:
            print(f"[DEBUG] retry {i+1} due to {e}", flush=True)
            time.sleep(2)
    raise Exception("Failed after retries")

def log_start(task, env, model):
    print(f"[START] task={task} env={env} model={model}", flush=True)

def log_step(step, action, reward, done, error=None):
    error_str = "null" if error is None else str(error)
    done_str  = "true" if done else "false"
    print(
        f"[STEP] step={step} action={action} reward={reward:.2f} "
        f"done={done_str} error={error_str}",
        flush=True
    )

def log_end(success, steps, score, rewards):
    success_str = "true" if success else "false"
    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
    print(f"[END] success={success_str} steps={steps} score={score:.4f} rewards={rewards_str}", flush=True)


# MAIN
async def main():
    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)

    for task_name in TASK_NAMES:
        step_rewards = []
        score        = 0.5
        success      = False

        log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)

        try:
            res    = safe_post(f"{SPACE_URL}/reset", json={"task": task_name})
            result = res.json()

            state = result["observation"]["echoed_message"]
            done  = result["done"]

            for step in range(1, MAX_STEPS + 1):
                if done:
                    break

                action = get_model_message(client, state)

                res    = safe_post(f"{SPACE_URL}/step", json={"message": action})
                result = res.json()

                state  = result["observation"]["echoed_message"]
                done   = result["done"]
                reward = float(result.get("reward", 0.0))

                step_rewards.append(reward)

                log_step(step=step, action=action, reward=reward, done=done, error=None)

            raw_score = float(result.get("info", {}).get("score", 0.5))
            score     = max(0.05, min(0.95, raw_score))
            success   = True

        except Exception as e:
            print(f"[ERROR] task={task_name} {e}", flush=True)
            score = 0.1

        finally:
            log_end(success=success, steps=len(step_rewards), score=score, rewards=step_rewards)


if __name__ == "__main__":
    asyncio.run(main())