astroocra commited on
Commit
d9da212
·
1 Parent(s): a569f45

remove gradio

Browse files
Files changed (2) hide show
  1. server/app.py +24 -212
  2. server/models.py +30 -51
server/app.py CHANGED
@@ -1,249 +1,61 @@
1
- """Full FastAPI app for the Eco-LLM Inference Routing environment."""
2
- from __future__ import annotations
3
-
4
  import uvicorn
5
- import gradio as gr
6
- from fastapi import FastAPI, HTTPException
7
- from gradio.routes import mount_gradio_app
8
  from openenv.core.env_server.http_server import create_app
9
  from pydantic import BaseModel
10
  from typing import List
11
 
12
- # Your local logic imports
13
  from server.env import EcoLLMInferenceRoutingEnvironment
14
  from server.models import RLAction, RLObservation, Strategy, ModelChoice
15
  from server.tasks import TASKS
16
 
17
- # ─────────────────────────────────────────
18
- # 🎯 GRADER CONSTANTS
19
- # Each task defines [WORST, BEST] episode-reward bounds.
20
- # WORST = lowest achievable cumulative reward (naïve NONE+SMALL agent).
21
- # BEST = highest achievable cumulative reward (optimal strategy).
22
- # score = soft_clamp( (episode_reward - WORST) / (BEST - WORST), 0.01, 0.99 )
23
- # This guarantees scores are strictly between 0 and 1.
24
- # ─────────────────────────────────────────
 
 
25
  _GRADER_BOUNDS = {
26
- # task_1: 1 query, min_model=MEDIUM
27
- # WORST: NONE+SMALL → wrong → -(0.8×0.1 + 0.1×1.0) = -0.18
28
- # BEST: EARLY_EXIT+MEDIUM → 1 - 0.8×0.3 - 0.1×2.0 + 0.1 = 0.66
29
  "task_1": {"worst": -0.18, "best": 0.66},
30
- # task_2: 3 queries (min: SMALL, MEDIUM, LARGE); large_model_penalty=-0.2
31
- # WORST: NONE+SMALL → q1 correct (SMALL ok), q2/q3 wrong
32
- # = (1-0.08-0.1) + (0-0.06-0.1) + (0-0.04-0.1) = 0.52
33
- # BEST: EARLY_EXIT per query (SMALL/MEDIUM/LARGE) = 0.92+0.72+0.16 = 1.80
34
  "task_2": {"worst": 0.52, "best": 1.80},
35
- # task_3: 5 queries (min: SMALL/kb, MEDIUM, SMALL/kb, LARGE, MEDIUM)
36
- # WORST: NONE+SMALL → q1/q3 correct, q2/q4/q5 wrong = 1.24
37
- # BEST: CALL_KB(q1)+EE+MEDIUM(q2)+USE_CACHE(q3)+EE+LARGE(q4)+EE+MEDIUM(q5)
38
- # = 0.81+0.72+1.50+0.42+0.75 = 4.20
39
  "task_3": {"worst": 1.24, "best": 4.20},
40
  }
41
- _GRADER_EPS = 0.01
42
-
43
-
44
- def _run_episode(task_id: str, actions: list[dict]) -> float:
45
- """Replay a fixed list of actions and return the normalised score."""
46
- env = EcoLLMInferenceRoutingEnvironment()
47
- obs = env.reset(task_id=task_id)
48
- episode_reward = 0.0
49
-
50
- for raw_action in actions:
51
- if obs.done:
52
- break
53
- action = RLAction(
54
- strategy=Strategy(raw_action["strategy"]),
55
- model_choice=ModelChoice(raw_action["model_choice"]),
56
- exit_flag=bool(raw_action.get("exit_flag", False)),
57
- )
58
- obs = env.step(action)
59
- episode_reward += obs.reward_details.total_reward
60
-
61
- bounds = _GRADER_BOUNDS[task_id]
62
- worst, best = bounds["worst"], bounds["best"]
63
- normalised = (episode_reward - worst) / (best - worst)
64
- score = max(_GRADER_EPS, min(1.0 - _GRADER_EPS, normalised))
65
- return round(score, 6)
66
-
67
-
68
- # ─────────────────────────────────────────
69
- # 🏗️ REQUEST / RESPONSE MODELS
70
- # ─────────────────────────────────────────
71
 
72
  class GraderAction(BaseModel):
73
  strategy: str
74
  model_choice: str
75
  exit_flag: bool = False
76
 
77
-
78
  class GraderRequest(BaseModel):
79
  task_id: str
80
  actions: List[GraderAction]
81
 
82
-
83
- class GraderResponse(BaseModel):
84
- task_id: str
85
- score: float # strictly in (0, 1)
86
- episode_reward: float
87
- worst_bound: float
88
- best_bound: float
89
-
90
-
91
- # 1. Create the Master App FIRST
92
- app = FastAPI()
93
-
94
- # ✅ REQUIRED FOR EVALUATOR
95
  @app.get("/health")
96
  def health():
97
  return {"status": "ok"}
98
 
99
-
100
- # ─────────────────────────────────────────
101
- # 📋 GET /tasks – task catalogue
102
- # ─────────────────────────────────────────
103
-
104
  @app.get("/tasks")
105
  def list_tasks():
106
- """Return the catalogue of available tasks with grader bounds."""
107
- return {
108
- task_id: {
109
- "task_id": spec.task_id,
110
- "difficulty": spec.difficulty,
111
- "description": spec.description,
112
- "num_queries": len(spec.queries),
113
- "grader_bounds": _GRADER_BOUNDS[task_id],
114
- }
115
- for task_id, spec in TASKS.items()
116
- }
117
-
118
 
119
- # ─────────────────────────────────────────
120
- # 🏆 POST /grader – deterministic grader
121
- # ─────────────────────────────────────────
122
-
123
- @app.post("/grader", response_model=GraderResponse)
124
  def grade_episode(request: GraderRequest):
125
- """
126
- Deterministically grade a completed episode.
127
-
128
- Supply the task_id and the ordered list of actions taken during the
129
- episode. The endpoint replays those actions in a fresh environment
130
- instance and returns a score strictly between 0 and 1.
131
-
132
- Score formula
133
- -------------
134
- score = soft_clamp(
135
- (episode_reward - WORST) / (BEST - WORST),
136
- lo=0.01, hi=0.99
137
- )
138
-
139
- where WORST and BEST are the empirically determined lower and upper
140
- bounds for each task (see GET /tasks for exact values).
141
- """
142
- task_id = request.task_id
143
- if task_id not in _GRADER_BOUNDS:
144
- raise HTTPException(
145
- status_code=400,
146
- detail=f"Unknown task_id {task_id!r}. Valid: {list(_GRADER_BOUNDS)}"
147
- )
148
-
149
- raw_actions = [a.model_dump() for a in request.actions]
150
-
151
- # Replay to measure episode_reward independently for the response
152
  env = EcoLLMInferenceRoutingEnvironment()
153
- obs = env.reset(task_id=task_id)
154
- episode_reward = 0.0
155
- for raw_action in raw_actions:
156
- if obs.done:
157
- break
158
- action = RLAction(
159
- strategy=Strategy(raw_action["strategy"]),
160
- model_choice=ModelChoice(raw_action["model_choice"]),
161
- exit_flag=bool(raw_action.get("exit_flag", False)),
162
- )
163
  obs = env.step(action)
164
- episode_reward += obs.reward_details.total_reward
165
-
166
- bounds = _GRADER_BOUNDS[task_id]
167
- worst, best = bounds["worst"], bounds["best"]
168
- normalised = (episode_reward - worst) / (best - worst)
169
- score = max(_GRADER_EPS, min(1.0 - _GRADER_EPS, normalised))
170
-
171
- return GraderResponse(
172
- task_id=task_id,
173
- score=round(score, 6),
174
- episode_reward=round(episode_reward, 6),
175
- worst_bound=worst,
176
- best_bound=best,
177
- )
178
-
179
- # ─────────────────────────────────────────
180
- # 🎯 GRADIO UI DEFINITION
181
- # ─────────────────────────────────────────
182
-
183
- def reset_demo(task_id: str, state: dict) -> tuple:
184
- demo_env = EcoLLMInferenceRoutingEnvironment()
185
- obs = demo_env.reset(task_id=task_id)
186
- updated_state = {"env": demo_env, "obs": obs, "history": [], "total_reward": 0.0}
187
-
188
- log = f"🔄 Reset — Task: {task_id}\n📋 Query: {obs.query}\n🌍 Carbon: {obs.carbon_intensity:.2f}"
189
- return log, "", 0.0, updated_state
190
-
191
- def step_demo(strategy: str, model: str, exit_flag: bool, state: dict) -> tuple:
192
- demo_env = state.get("env")
193
- if not demo_env: return "⚠️ Reset first!", "", 0.0, state
194
-
195
- action = RLAction(strategy=Strategy(strategy), model_choice=ModelChoice(model), exit_flag=exit_flag)
196
- new_obs = demo_env.step(action)
197
- rd = new_obs.reward_details
198
- total_reward = state.get("total_reward", 0.0) + rd.total_reward
199
-
200
- log = f"Step Result: {'✅' if rd.correct else '❌'}\nReward: {rd.total_reward:+.3f}"
201
- return log, str(rd), round(total_reward, 3), {**state, "obs": new_obs, "total_reward": total_reward}
202
-
203
- with gr.Blocks(title="Eco-LLM Routing") as demo:
204
- demo_state = gr.State({"env": None, "obs": None, "history": [], "total_reward": 0.0})
205
- gr.Markdown("# 🌱 Eco-LLM Inference Routing")
206
-
207
- with gr.Row():
208
- task_selector = gr.Dropdown(choices=["task_1", "task_2", "task_3"], value="task_1", label="Task")
209
- reset_btn = gr.Button("🔄 Reset", variant="primary")
210
-
211
- obs_box = gr.Textbox(label="State", lines=4, interactive=False)
212
 
213
- with gr.Row():
214
- strategy_dd = gr.Dropdown(choices=["NONE", "USE_CACHE", "DO_CASCADE", "EARLY_EXIT", "WAIT", "CALL_KB"], value="NONE", label="Strategy")
215
- model_dd = gr.Dropdown(choices=["SMALL", "MEDIUM", "LARGE"], value="SMALL", label="Model")
216
- exit_cb = gr.Checkbox(label="Exit Flag", value=False)
217
-
218
- step_btn = gr.Button("▶️ Step")
219
- reward_box = gr.Textbox(label="Step Detail", interactive=False)
220
- total_num = gr.Number(label="Total Reward", interactive=False)
221
-
222
- reset_btn.click(reset_demo, [task_selector, demo_state], [obs_box, reward_box, total_num, demo_state])
223
- step_btn.click(step_demo, [strategy_dd, model_dd, exit_cb, demo_state], [obs_box, reward_box, total_num, demo_state])
224
-
225
- # ─────────────────────────────────────────
226
- # 🚀 ROUTING SETUP
227
- # ─────────────────────────────────────────
228
-
229
- # 2. MOUNT GRADIO TO THE ROOT ("/")
230
- # This ensures Gradio's internal JS/CSS assets load without path issues.
231
- app = mount_gradio_app(app, demo, path="/")
232
-
233
- # 3. Create and mount OpenEnv to "/api"
234
- # This keeps the environment logic accessible but gets its UI out of the way.
235
- openenv_logic = create_app(
236
- EcoLLMInferenceRoutingEnvironment,
237
- RLAction,
238
- RLObservation,
239
- env_name="eco_llm_inference_routing",
240
- max_concurrent_envs=4,
241
- )
242
- app.mount("/api", openenv_logic)
243
-
244
-
245
- def main(host: str = "0.0.0.0", port: int = 7860) -> None:
246
- uvicorn.run(app, host=host, port=port)
247
 
248
  if __name__ == "__main__":
249
- main()
 
 
 
 
1
  import uvicorn
2
+ from fastapi import HTTPException
 
 
3
  from openenv.core.env_server.http_server import create_app
4
  from pydantic import BaseModel
5
  from typing import List
6
 
 
7
  from server.env import EcoLLMInferenceRoutingEnvironment
8
  from server.models import RLAction, RLObservation, Strategy, ModelChoice
9
  from server.tasks import TASKS
10
 
11
+ # 1. Create native OpenEnv app
12
+ app = create_app(
13
+ EcoLLMInferenceRoutingEnvironment,
14
+ RLAction,
15
+ RLObservation,
16
+ env_name="eco_llm_inference_routing",
17
+ max_concurrent_envs=4,
18
+ )
19
+
20
+ # 2. Grading Metadata
21
  _GRADER_BOUNDS = {
 
 
 
22
  "task_1": {"worst": -0.18, "best": 0.66},
 
 
 
 
23
  "task_2": {"worst": 0.52, "best": 1.80},
 
 
 
 
24
  "task_3": {"worst": 1.24, "best": 4.20},
25
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
 
27
  class GraderAction(BaseModel):
28
  strategy: str
29
  model_choice: str
30
  exit_flag: bool = False
31
 
 
32
  class GraderRequest(BaseModel):
33
  task_id: str
34
  actions: List[GraderAction]
35
 
36
+ # 3. Add required extra endpoints to the OpenEnv app
 
 
 
 
 
 
 
 
 
 
 
 
37
  @app.get("/health")
38
  def health():
39
  return {"status": "ok"}
40
 
 
 
 
 
 
41
  @app.get("/tasks")
42
  def list_tasks():
43
+ return {tid: {"task_id": tid, "bounds": _GRADER_BOUNDS.get(tid)} for tid in TASKS.keys()}
 
 
 
 
 
 
 
 
 
 
 
44
 
45
+ @app.post("/grader")
 
 
 
 
46
  def grade_episode(request: GraderRequest):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  env = EcoLLMInferenceRoutingEnvironment()
48
+ obs = env.reset(task_id=request.task_id)
49
+ total_reward = 0.0
50
+ for a in request.actions:
51
+ if obs.done: break
52
+ action = RLAction(strategy=Strategy(a.strategy), model_choice=ModelChoice(a.model_choice), exit_flag=a.exit_flag)
 
 
 
 
 
53
  obs = env.step(action)
54
+ total_reward += obs.reward_details.total_reward
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
+ bounds = _GRADER_BOUNDS[request.task_id]
57
+ score = (total_reward - bounds["worst"]) / (bounds["best"] - bounds["worst"])
58
+ return {"task_id": request.task_id, "score": max(0.01, min(0.99, score)), "reward": total_reward}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
 
60
  if __name__ == "__main__":
61
+ uvicorn.run(app, host="0.0.0.0", port=7860)
server/models.py CHANGED
@@ -1,14 +1,10 @@
1
  """Pydantic models for the Eco-LLM Inference Routing environment."""
2
-
3
  from __future__ import annotations
4
-
5
  from enum import Enum
6
  from typing import Any
7
-
8
  from openenv.core.env_server.types import Action, Observation, State
9
  from pydantic import BaseModel, Field
10
 
11
-
12
  class Strategy(str, Enum):
13
  NONE = "NONE"
14
  USE_CACHE = "USE_CACHE"
@@ -17,66 +13,49 @@ class Strategy(str, Enum):
17
  WAIT = "WAIT"
18
  CALL_KB = "CALL_KB"
19
 
20
-
21
  class ModelChoice(str, Enum):
22
  SMALL = "SMALL"
23
  MEDIUM = "MEDIUM"
24
  LARGE = "LARGE"
25
 
26
-
27
  class RLAction(Action):
28
  """Action taken by the routing agent at each step."""
29
-
30
- strategy: Strategy = Field(..., description="Routing strategy to apply")
31
- model_choice: ModelChoice = Field(..., description="Primary model for inference")
32
- exit_flag: bool = Field(..., description="Whether to terminate a cascade early")
33
-
34
 
35
  class RLReward(BaseModel):
36
- """Structured reward breakdown for deterministic grading."""
37
-
38
- score: float = Field(..., description="1.0 if correct else 0.0")
39
- energy_cost: float = Field(..., description="Energy consumed this step")
40
- latency_cost: float = Field(..., description="Latency incurred this step")
41
- carbon_penalty: float = Field(..., description="Carbon-weighted energy penalty")
42
- latency_penalty: float = Field(..., description="Latency penalty with mu=0.01")
43
- bonuses: float = Field(..., description="Sum of bonuses or task-specific adjustments")
44
- total_reward: float = Field(..., description="Final reward for the step")
45
- correct: bool = Field(..., description="Whether the selected answer was correct")
46
- strategy_trace: list[str] = Field(
47
- default_factory=list,
48
- description="Ordered record of sub-actions executed during the step",
49
- )
50
-
51
 
52
  class RLObservation(Observation):
53
- """Observation returned after reset or step."""
54
-
55
- query: str = Field(..., description="Current query to answer")
56
- cache_contents: list[str] = Field(
57
- default_factory=list,
58
- description="Queries answered correctly and available in cache",
59
- )
60
- carbon_intensity: float = Field(..., ge=0.0, le=1.0, description="Current carbon intensity")
61
- reward_details: RLReward = Field(..., description="Structured reward payload")
62
- kb_available: bool = Field(default=False, description="Whether KB is available for current query")
63
- reward: float = Field(default=0.0, description="Step reward for this transition")
64
- done: bool = Field(default=False, description="Whether episode has terminated")
65
- metadata: dict[str, Any] = Field(default_factory=dict, description="Additional state metadata")
66
-
67
 
68
  class RLState(State):
69
- """Internal environment state."""
70
-
71
- episode_id: str = Field(..., description="Unique episode identifier")
72
- step_count: int = Field(default=0, ge=0, description="Total steps taken in episode")
73
- task_id: str = Field(..., description="Active task identifier")
74
- step_index: int = Field(..., ge=0, description="Index within the current task trajectory")
75
- query_index: int = Field(..., ge=0, description="Index of the active query")
76
- total_energy: float = Field(default=0.0, ge=0.0, description="Cumulative energy cost")
77
- total_latency: float = Field(default=0.0, ge=0.0, description="Cumulative latency cost")
78
- cache_contents: list[str] = Field(default_factory=list, description="Current cache contents")
79
- carbon_history_index: int = Field(default=0, ge=0, description="Current carbon history pointer")
80
 
81
  def public_payload(self) -> dict[str, Any]:
82
  return self.model_dump()
 
1
  """Pydantic models for the Eco-LLM Inference Routing environment."""
 
2
  from __future__ import annotations
 
3
  from enum import Enum
4
  from typing import Any
 
5
  from openenv.core.env_server.types import Action, Observation, State
6
  from pydantic import BaseModel, Field
7
 
 
8
  class Strategy(str, Enum):
9
  NONE = "NONE"
10
  USE_CACHE = "USE_CACHE"
 
13
  WAIT = "WAIT"
14
  CALL_KB = "CALL_KB"
15
 
 
16
  class ModelChoice(str, Enum):
17
  SMALL = "SMALL"
18
  MEDIUM = "MEDIUM"
19
  LARGE = "LARGE"
20
 
 
21
  class RLAction(Action):
22
  """Action taken by the routing agent at each step."""
23
+ # We use explicit defaults to ensure the UI renders dropdowns
24
+ strategy: Strategy = Field(default=Strategy.NONE, description="Routing strategy to apply")
25
+ model_choice: ModelChoice = Field(default=ModelChoice.SMALL, description="Primary model size")
26
+ exit_flag: bool = Field(default=False, description="Terminate cascade early")
 
27
 
28
  class RLReward(BaseModel):
29
+ score: float = 0.0
30
+ energy_cost: float = 0.0
31
+ latency_cost: float = 0.0
32
+ carbon_penalty: float = 0.0
33
+ latency_penalty: float = 0.0
34
+ bonuses: float = 0.0
35
+ total_reward: float = 0.0
36
+ correct: bool = False
37
+ strategy_trace: list[str] = Field(default_factory=list)
 
 
 
 
 
 
38
 
39
  class RLObservation(Observation):
40
+ query: str = ""
41
+ cache_contents: list[str] = Field(default_factory=list)
42
+ carbon_intensity: float = 0.0
43
+ reward_details: RLReward = Field(default_factory=RLReward)
44
+ kb_available: bool = False
45
+ reward: float = 0.0
46
+ done: bool = False
47
+ metadata: dict[str, Any] = Field(default_factory=dict)
 
 
 
 
 
 
48
 
49
  class RLState(State):
50
+ episode_id: str = ""
51
+ step_count: int = 0
52
+ task_id: str = ""
53
+ step_index: int = 0
54
+ query_index: int = 0
55
+ total_energy: float = 0.0
56
+ total_latency: float = 0.0
57
+ cache_contents: list[str] = Field(default_factory=list)
58
+ carbon_history_index: int = 0
 
 
59
 
60
  def public_payload(self) -> dict[str, Any]:
61
  return self.model_dump()