File size: 3,029 Bytes
a451d32
 
 
 
e806f82
a451d32
8b09cd1
e806f82
 
8b09cd1
a451d32
 
 
 
 
 
 
 
 
e806f82
8b09cd1
 
 
a451d32
8b09cd1
 
a451d32
8b09cd1
 
 
a451d32
8b09cd1
 
 
 
 
 
 
b4f3688
e806f82
8b09cd1
 
07792e4
a451d32
 
 
 
 
 
 
 
 
 
07792e4
a451d32
 
 
 
 
 
 
 
 
 
07792e4
a451d32
8b09cd1
 
 
 
 
a451d32
 
b4f3688
a451d32
b4f3688
07792e4
 
 
a451d32
 
 
b4f3688
a451d32
b4f3688
 
 
a451d32
b4f3688
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a451d32
07792e4
b4f3688
 
e806f82
b4f3688
e806f82
b4f3688
a451d32
 
 
 
 
 
b4f3688
 
 
 
a451d32
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import asyncio
import os
import requests
import time
from openai import OpenAI

# REQUIRED VARIABLES
API_BASE_URL = os.environ["API_BASE_URL"]
API_KEY = os.environ["API_KEY"]
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")

SPACE_URL = "https://arrowman123-customer-supp-env.hf.space"

TASK_NAME = "customer-support"
BENCHMARK = "openenv"

MAX_STEPS = 20


#  LLM AGENT 
def get_model_message(client, state):
    prompt = f"""
You are a customer support agent.

State:
{state}

Choose ONLY ONE word:
refund / replace / reject / ask_proof
"""

    try:
        response = client.chat.completions.create(
            model=MODEL_NAME,
            messages=[{"role": "user", "content": prompt}]
        )
        return response.choices[0].message.content.strip().lower()
    except Exception as e:
        print(f"[ERROR] LLM failed: {e}")
        return "reject"


# SAFE REQUEST
def safe_post(url, json=None, retries=3):
    for i in range(retries):
        try:
            return requests.post(url, json=json, timeout=10)
        except Exception as e:
            print(f"[DEBUG] retry {i+1} due to {e}")
            time.sleep(2)
    raise Exception("Failed after retries")


# LOGGING 
def log_start(**kwargs):
    print("[START]", kwargs, flush=True)

def log_step(**kwargs):
    print("[STEP]", kwargs, flush=True)

def log_end(**kwargs):
    print("[END]", kwargs, flush=True)


#  MAIN 
async def main():
    client = OpenAI(
        base_url=API_BASE_URL,
        api_key=API_KEY
    )

    log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)

    task_scores = []

    try:
        # run 3 tasks
        for _ in range(3):
            res = safe_post(f"{SPACE_URL}/reset")
            result = res.json()

            state = result["observation"]["echoed_message"]
            done = result["done"]

            for step in range(1, MAX_STEPS + 1):
                if done:
                    break

                message = get_model_message(client, state)

                res = safe_post(
                    f"{SPACE_URL}/step",
                    json={"message": message}
                )
                result = res.json()

                state = result["observation"]["echoed_message"]
                done = result["done"]

                log_step(
                    step=step,
                    action=message,
                    reward=result.get("reward", 0.0),
                    done=done,
                    error=None
                )

            #  GET SCORE FROM ENV
            score = result.get("score", 0.5)
            score = max(0.01, min(0.99, score))

            task_scores.append(score)

        final_score = sum(task_scores) / len(task_scores)

    except Exception as e:
        print(f"[ERROR] {e}")

    finally:
        log_end(
            success=True,
            steps=len(task_scores),
            score=final_score if 'final_score' in locals() else 0.0,
            rewards=task_scores
        )


if __name__ == "__main__":
    asyncio.run(main())