Berlm commited on
Commit
fa2e96f
·
verified ·
1 Parent(s): 2e46bc9

Upload folder using huggingface_hub

Browse files
inference.py CHANGED
@@ -8,22 +8,6 @@ from typing import Any
8
 
9
  from openai import OpenAI
10
 
11
- try:
12
- if os.environ.get("DEBUG", "false").lower() == "true":
13
- from server.debug_logging import debug_enabled, write_debug_event
14
- else:
15
- raise ImportError("Debug logging is not enabled")
16
- except ImportError: # pragma: no cover
17
-
18
- def debug_enabled() -> bool:
19
- return False
20
-
21
- def write_debug_event(
22
- event_type: str, payload: dict[str, Any] | None = None
23
- ) -> None:
24
- return None
25
-
26
-
27
  from agent_action_tools import (
28
  AgentToolInvocation,
29
  build_openai_tools,
@@ -33,6 +17,7 @@ from agent_action_tools import (
33
  from client import PptAgentEnv
34
  from models import PptAgentAction
35
 
 
36
  HF_TOKEN = os.getenv("HF_TOKEN")
37
 
38
  API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
@@ -348,26 +333,6 @@ def _history_entry_from_tool_call(
348
  }
349
 
350
 
351
- def _message_debug_payload(message: Any) -> dict[str, Any]:
352
- tool_calls = getattr(message, "tool_calls", None) or []
353
- return {
354
- "content": getattr(message, "content", None),
355
- "tool_calls": [
356
- {
357
- "id": getattr(tool_call, "id", None),
358
- "type": getattr(tool_call, "type", None),
359
- "function": {
360
- "name": getattr(getattr(tool_call, "function", None), "name", None),
361
- "arguments": getattr(
362
- getattr(tool_call, "function", None), "arguments", None
363
- ),
364
- },
365
- }
366
- for tool_call in tool_calls
367
- ],
368
- }
369
-
370
-
371
  def choose_action(
372
  client: OpenAI, observation: Any, history: list[dict[str, Any]], max_steps: int
373
  ) -> tuple[PptAgentAction, str, AgentToolInvocation]:
@@ -383,20 +348,6 @@ def choose_action(
383
  },
384
  ]
385
  raw_response: dict[str, Any] | None = None
386
- request_payload = {
387
- "stage": "chat.tools",
388
- "model": MODEL_NAME,
389
- "base_url": API_BASE_URL,
390
- "temperature": TEMPERATURE,
391
- "max_tokens": 8192,
392
- "tool_choice": "required",
393
- "messages": messages,
394
- "tools": OPENAI_TOOLS,
395
- }
396
-
397
- if debug_enabled():
398
- write_debug_event("llm.request", request_payload)
399
-
400
  try:
401
  completion = client.chat.completions.create(
402
  model=MODEL_NAME,
@@ -407,15 +358,6 @@ def choose_action(
407
  tool_choice="required",
408
  )
409
  message = completion.choices[0].message
410
- if debug_enabled():
411
- write_debug_event(
412
- "llm.response",
413
- {
414
- "stage": "chat.tools",
415
- "model": MODEL_NAME,
416
- "message": _message_debug_payload(message),
417
- },
418
- )
419
  invocation, raw_response = _extract_tool_invocation(message)
420
  _validate_tool_choice(invocation, observation)
421
  action = tool_invocation_to_action(
@@ -439,26 +381,7 @@ def choose_action(
439
  invocation,
440
  )
441
  except Exception as error:
442
- if debug_enabled():
443
- write_debug_event(
444
- "llm.error",
445
- {
446
- "stage": "chat.tools",
447
- "model": MODEL_NAME,
448
- "base_url": API_BASE_URL,
449
- "error_type": type(error).__name__,
450
- "error": str(error),
451
- "response": raw_response,
452
- },
453
- )
454
- print(f"[DEBUG] choose_action failed: {error}", flush=True)
455
- if raw_response is not None:
456
- print(
457
- "[DEBUG] choose_action response="
458
- + json.dumps(raw_response, separators=(",", ":")),
459
- flush=True,
460
- )
461
- raise
462
 
463
 
464
  async def run_task(difficulty: str, max_steps: int) -> None:
@@ -475,7 +398,7 @@ async def run_task(difficulty: str, max_steps: int) -> None:
475
  result = await env.reset(difficulty=difficulty)
476
  observation = result.observation
477
  log_start(
478
- task=observation.task_name or "unknown",
479
  env=BENCHMARK,
480
  model=MODEL_NAME,
481
  )
@@ -508,11 +431,6 @@ async def run_task(difficulty: str, max_steps: int) -> None:
508
  )
509
  if result.done:
510
  break
511
- else:
512
- print(
513
- f"[DEBUG] reached max steps {max_steps} without completion",
514
- flush=True,
515
- )
516
 
517
  success = bool(
518
  started
@@ -521,8 +439,8 @@ async def run_task(difficulty: str, max_steps: int) -> None:
521
  and observation.score > 0.0
522
  )
523
  score = float(observation.score or 0.0)
 
524
  except Exception as error:
525
- print(f"[DEBUG] inference failed: {error}", flush=True)
526
  success = False
527
  finally:
528
  log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
 
8
 
9
  from openai import OpenAI
10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  from agent_action_tools import (
12
  AgentToolInvocation,
13
  build_openai_tools,
 
17
  from client import PptAgentEnv
18
  from models import PptAgentAction
19
 
20
+
21
  HF_TOKEN = os.getenv("HF_TOKEN")
22
 
23
  API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
 
333
  }
334
 
335
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
336
  def choose_action(
337
  client: OpenAI, observation: Any, history: list[dict[str, Any]], max_steps: int
338
  ) -> tuple[PptAgentAction, str, AgentToolInvocation]:
 
348
  },
349
  ]
350
  raw_response: dict[str, Any] | None = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
351
  try:
352
  completion = client.chat.completions.create(
353
  model=MODEL_NAME,
 
358
  tool_choice="required",
359
  )
360
  message = completion.choices[0].message
 
 
 
 
 
 
 
 
 
361
  invocation, raw_response = _extract_tool_invocation(message)
362
  _validate_tool_choice(invocation, observation)
363
  action = tool_invocation_to_action(
 
381
  invocation,
382
  )
383
  except Exception as error:
384
+ pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
385
 
386
 
387
  async def run_task(difficulty: str, max_steps: int) -> None:
 
398
  result = await env.reset(difficulty=difficulty)
399
  observation = result.observation
400
  log_start(
401
+ task=difficulty,
402
  env=BENCHMARK,
403
  model=MODEL_NAME,
404
  )
 
431
  )
432
  if result.done:
433
  break
 
 
 
 
 
434
 
435
  success = bool(
436
  started
 
439
  and observation.score > 0.0
440
  )
441
  score = float(observation.score or 0.0)
442
+
443
  except Exception as error:
 
444
  success = False
445
  finally:
446
  log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
inference_debug.py ADDED
@@ -0,0 +1,537 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import json
5
+ import os
6
+ import textwrap
7
+ from typing import Any
8
+
9
+ from openai import OpenAI
10
+
11
+ try:
12
+ if os.environ.get("DEBUG", "false").lower() == "true":
13
+ from server.debug_logging import debug_enabled, write_debug_event
14
+ else:
15
+ raise ImportError("Debug logging is not enabled")
16
+ except ImportError: # pragma: no cover
17
+
18
+ def debug_enabled() -> bool:
19
+ return False
20
+
21
+ def write_debug_event(
22
+ event_type: str, payload: dict[str, Any] | None = None
23
+ ) -> None:
24
+ return None
25
+
26
+
27
+ from agent_action_tools import (
28
+ AgentToolInvocation,
29
+ build_openai_tools,
30
+ parse_tool_invocation,
31
+ tool_invocation_to_action,
32
+ )
33
+ from client import PptAgentEnv
34
+ from models import PptAgentAction
35
+
36
+ HF_TOKEN = os.getenv("HF_TOKEN")
37
+
38
+ API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
39
+ MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen3.5-27B")
40
+ ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:7860")
41
+ TEMPERATURE = 0.0
42
+ TASKS = [("easy", 10), ("medium", 15), ("hard", 20)]
43
+
44
+ BENCHMARK = "ppt_agent"
45
+
46
+ OPENAI_TOOLS = build_openai_tools()
47
+
48
+ _FALLBACK_SAVE_PATH = "outputs/presentation.pptx"
49
+
50
+ SYSTEM_PROMPT = textwrap.dedent(
51
+ """
52
+ You are an elite presentation designer and PowerPoint author. Your job is to choose exactly one next environment tool call that moves the deck toward a polished, professional final presentation. Make sure to follow the output schema and tool-use rules precisely.
53
+
54
+ You are given:
55
+ - the full task prompt
56
+ - the source context
57
+ - the current slide count
58
+ - the remaining step budget
59
+ - the last action error/result
60
+ - recent action history
61
+ - the current theme
62
+ - slide constraints
63
+
64
+ Available tools:
65
+ - create_slide
66
+ - update_slide
67
+ - delete_slide
68
+ - set_theme
69
+ - save_presentation
70
+
71
+ Return no prose. Produce exactly one tool call.
72
+
73
+ Presentation quality bar:
74
+ - Make slides look like strong professional consulting, strategy, product, or investor decks.
75
+ - Build a clear narrative, not a random collection of slides.
76
+ - Prefer strong hierarchy, clean alignment, generous whitespace, and restrained color usage.
77
+ - Avoid clutter, overcrowded text, and weak visual balance.
78
+ - Use charts or tables when they communicate evidence better than paragraphs.
79
+ - Use concise text with meaningful headlines and clear supporting points.
80
+ - Keep the design consistent across the whole deck.
81
+
82
+ Design guidance:
83
+ - Prefer a cohesive deck-level theme. Use set_theme early if the current theme is weak or mismatched to the task.
84
+ - Keep typography consistent across slides.
85
+ - Prefer readable layouts with clear margins and spacing.
86
+ - Use word_wrap, space_before_pt, space_after_pt, and line_spacing when helpful for clean text layout.
87
+
88
+ Content guidance:
89
+ - Ground claims in the provided source context.
90
+ - Do not invent facts, numbers, or citations not supported by the inputs.
91
+ - If the task implies a specific slide structure, satisfy it.
92
+
93
+ Tool-use rules:
94
+ - Use only the tool schema fields exactly as defined.
95
+ - Never invent shorthand slide DSL fields such as:
96
+ - background -> use the schema field, not legacy DSL keys
97
+ - type: title/body -> use type: text
98
+ - font -> use style.font_name
99
+ - size -> use numeric style.font_size_pt values
100
+ - color -> use style.color_hex or other schema color fields
101
+ - Use the provided tools directly. Do not emit plans, explanations, or pseudo-actions.
102
+
103
+ Shape rules:
104
+ - Text shapes require explicit geometry in slide inches: x, y, w, h.
105
+ - For new text shapes use type: text.
106
+ - For accent bars use type: accent_bar.
107
+ - For charts use type: chart with ct, cd, x, y, w, h.
108
+ - For tables use type: table with td, x, y, w, h.
109
+ - For images use type: image with img, x, y, and optional w, h.
110
+
111
+ Update rules:
112
+ - Use create_slide to add a new slide.
113
+ - Use update_slide to revise an existing slide.
114
+ - update_slide requires si.
115
+ - Existing shapes must be updated by id.
116
+ - Use known_named_shapes_by_slide and prior tool results to find the correct id.
117
+ - Only use delete_slide when a slide is clearly wrong, redundant, or should be removed.
118
+ - Do not iterate too much by updating and deleting slides. Aim to get each new slide right on the first try.
119
+ - Use save_presentation when the deck is complete enough and ready to submit.
120
+
121
+ Theme token rules:
122
+ - You may use these theme tokens inside payload values:
123
+ - <bg>, <surface>, <accent>, <primary>, <secondary>, <font>
124
+ - Prefer theme tokens over hardcoded repeated color and font values for consistent design.
125
+ - Use set_theme to update or add more theme tokens in the theme
126
+ - Omitted theme keys remain unchanged.
127
+
128
+ Recommended workflow:
129
+ - If needed, set or refine the theme first.
130
+ - Think about the slide structure and then create it in one go.
131
+ - Call update_slide only if you think it is needed
132
+ - You have a finite max step budget. Do not let remaining_steps reach 0 before calling save_presentation.
133
+ - Save only when the deck is coherent, complete, and professionally presented.
134
+
135
+ Example create_slide payload:
136
+ {
137
+ "bg": "<bg>",
138
+ "shapes": [
139
+ {
140
+ "type": "text",
141
+ "name": "title",
142
+ "text": "Harbor Retail Expansion 2026",
143
+ "x": 0.7,
144
+ "y": 0.6,
145
+ "w": 8.6,
146
+ "h": 0.8,
147
+ "style": {
148
+ "font_name": "<font>",
149
+ "font_size_pt": 28,
150
+ "color_hex": "<primary>",
151
+ "bold": true,
152
+ "word_wrap": true,
153
+ "space_after_pt": 4
154
+ }
155
+ },
156
+ {
157
+ "type": "text",
158
+ "name": "subtitle",
159
+ "text": "Growth priorities, risks, and operating implications",
160
+ "x": 0.7,
161
+ "y": 1.45,
162
+ "w": 7.8,
163
+ "h": 0.9,
164
+ "style": {
165
+ "font_name": "<font>",
166
+ "font_size_pt": 16,
167
+ "color_hex": "<secondary>",
168
+ "word_wrap": true,
169
+ "line_spacing": 1.15
170
+ }
171
+ }
172
+ ]
173
+ }
174
+
175
+ Example update_slide payload:
176
+ {
177
+ "si": 1,
178
+ "upd": [
179
+ {
180
+ "type": "text",
181
+ "id": 11,
182
+ "text": "Harbor Retail Expansion 2026",
183
+ "style": {
184
+ "font_name": "<font>",
185
+ "font_size_pt": 28,
186
+ "color_hex": "<primary>",
187
+ "bold": true
188
+ }
189
+ }
190
+ ],
191
+ "add": [
192
+ {
193
+ "type": "accent_bar",
194
+ "name": "accent",
195
+ "hex": "<accent>"
196
+ }
197
+ ]
198
+ }
199
+
200
+ Example set_theme payload:
201
+ {
202
+ "bg": "#F8FAFC",
203
+ "surface": "#FFFFFF",
204
+ "accent": "#2563EB",
205
+ "primary": "#0F172A",
206
+ "secondary": "#475569",
207
+ "font": "Aptos",
208
+ "ts": 28,
209
+ "bs": 16,
210
+ "cs": 10
211
+ }
212
+ """
213
+ ).strip()
214
+
215
+
216
+ def log_start(task: str, env: str, model: str) -> None:
217
+ print(f"[START] task={task} env={env} model={model}", flush=True)
218
+
219
+
220
+ def log_step(
221
+ step: int, action: str, reward: float, done: bool, error: str | None
222
+ ) -> None:
223
+ error_value = (error or "null").replace("\n", " ")
224
+ print(
225
+ f"[STEP] step={step} action={action} reward={reward:.2f} done={str(done).lower()} error={error_value}",
226
+ flush=True,
227
+ )
228
+
229
+
230
+ def log_end(success: bool, steps: int, score: float, rewards: list[float]) -> None:
231
+ rewards_str = ",".join(f"{reward:.2f}" for reward in rewards)
232
+ print(
233
+ f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",
234
+ flush=True,
235
+ )
236
+
237
+
238
+ def _observation_metadata(observation: Any) -> dict[str, Any]:
239
+ metadata = getattr(observation, "metadata", None)
240
+ return metadata if isinstance(metadata, dict) else {}
241
+
242
+
243
+ def _planning_payload(
244
+ observation: Any, history: list[dict[str, Any]], max_steps: int
245
+ ) -> dict[str, Any]:
246
+ metadata = _observation_metadata(observation)
247
+ current_theme = metadata.get("current_theme")
248
+ known_named_shapes = metadata.get("known_named_shapes_by_slide")
249
+ slide_constraints = metadata.get("slide_constraints")
250
+ default_save_path = metadata.get("default_save_path")
251
+ metadata_max_steps = metadata.get("max_steps")
252
+ effective_max_steps = (
253
+ min(metadata_max_steps, max_steps)
254
+ if isinstance(metadata_max_steps, int)
255
+ else max_steps
256
+ )
257
+
258
+ return {
259
+ "task_prompt": observation.task_prompt,
260
+ "source_context": observation.source_context,
261
+ "slide_count": observation.slide_count,
262
+ "remaining_steps": max(
263
+ 0,
264
+ effective_max_steps - int(metadata.get("step_count", 0)),
265
+ ),
266
+ "last_action_error": observation.last_action_error,
267
+ "last_action_result": observation.last_action_result,
268
+ "recent_actions": history[-5:],
269
+ "known_named_shapes_by_slide": (
270
+ known_named_shapes if isinstance(known_named_shapes, dict) else {}
271
+ ),
272
+ "current_theme": current_theme if isinstance(current_theme, dict) else {},
273
+ "slide_constraints": (
274
+ slide_constraints if isinstance(slide_constraints, dict) else {}
275
+ ),
276
+ "default_save_path": (
277
+ default_save_path
278
+ if isinstance(default_save_path, str) and default_save_path
279
+ else _FALLBACK_SAVE_PATH
280
+ ),
281
+ }
282
+
283
+
284
+ def _validate_tool_choice(invocation: AgentToolInvocation, observation: Any) -> None:
285
+ if invocation.tool_name == "save_presentation":
286
+ constraints = _observation_metadata(observation).get("slide_constraints")
287
+ min_slides = (
288
+ constraints.get("min_slides") if isinstance(constraints, dict) else None
289
+ )
290
+ if isinstance(min_slides, int) and observation.slide_count < min_slides:
291
+ raise ValueError(
292
+ "cannot save before reaching the required minimum slide count"
293
+ )
294
+ return
295
+
296
+ if invocation.tool_name in {"update_slide", "delete_slide"}:
297
+ slide_index = invocation.arguments.get("slide_index")
298
+ if not isinstance(slide_index, int):
299
+ raise ValueError(f"{invocation.tool_name} requires slide_index")
300
+ if slide_index > observation.slide_count:
301
+ raise ValueError(
302
+ f"slide_index {slide_index} is out of range for {observation.slide_count} slides"
303
+ )
304
+
305
+
306
+ def _extract_tool_invocation(
307
+ message: Any,
308
+ ) -> tuple[AgentToolInvocation, dict[str, Any]]:
309
+ raw_tool_calls = getattr(message, "tool_calls", None) or []
310
+ tool_calls = list(raw_tool_calls)
311
+ if not tool_calls:
312
+ raise ValueError("LLM must emit at least one tool call")
313
+
314
+ tool_call = tool_calls[0]
315
+ function = getattr(tool_call, "function", None)
316
+ tool_name = getattr(function, "name", None)
317
+ arguments = getattr(function, "arguments", None) or "{}"
318
+ if not isinstance(tool_name, str) or not tool_name:
319
+ raise ValueError("Tool call missing function name")
320
+ return (
321
+ parse_tool_invocation(tool_name, arguments),
322
+ {
323
+ "tool_name": tool_name,
324
+ "arguments": arguments,
325
+ "ignored_tool_calls": max(0, len(tool_calls) - 1),
326
+ },
327
+ )
328
+
329
+
330
+ def _history_entry_from_tool_call(
331
+ invocation: AgentToolInvocation, observation: Any
332
+ ) -> dict[str, Any]:
333
+ last_action_result = (
334
+ observation.last_action_result
335
+ if isinstance(observation.last_action_result, dict)
336
+ else {}
337
+ )
338
+ slide_index = last_action_result.get("slide_index")
339
+ raw_tool_result = last_action_result.get("tool_result")
340
+ tool_result = raw_tool_result if isinstance(raw_tool_result, dict) else None
341
+
342
+ return {
343
+ "tool_name": invocation.tool_name,
344
+ "arguments": invocation.arguments,
345
+ "slide_index": slide_index if isinstance(slide_index, int) else None,
346
+ "tool_result": tool_result,
347
+ "error": observation.last_action_error,
348
+ }
349
+
350
+
351
+ def _message_debug_payload(message: Any) -> dict[str, Any]:
352
+ tool_calls = getattr(message, "tool_calls", None) or []
353
+ return {
354
+ "content": getattr(message, "content", None),
355
+ "tool_calls": [
356
+ {
357
+ "id": getattr(tool_call, "id", None),
358
+ "type": getattr(tool_call, "type", None),
359
+ "function": {
360
+ "name": getattr(getattr(tool_call, "function", None), "name", None),
361
+ "arguments": getattr(
362
+ getattr(tool_call, "function", None), "arguments", None
363
+ ),
364
+ },
365
+ }
366
+ for tool_call in tool_calls
367
+ ],
368
+ }
369
+
370
+
371
+ def choose_action(
372
+ client: OpenAI, observation: Any, history: list[dict[str, Any]], max_steps: int
373
+ ) -> tuple[PptAgentAction, str, AgentToolInvocation]:
374
+ metadata = _observation_metadata(observation)
375
+ messages = [
376
+ {"role": "system", "content": SYSTEM_PROMPT},
377
+ {
378
+ "role": "user",
379
+ "content": json.dumps(
380
+ _planning_payload(observation, history, max_steps),
381
+ separators=(",", ":"),
382
+ ),
383
+ },
384
+ ]
385
+ raw_response: dict[str, Any] | None = None
386
+ request_payload = {
387
+ "stage": "chat.tools",
388
+ "model": MODEL_NAME,
389
+ "base_url": API_BASE_URL,
390
+ "temperature": TEMPERATURE,
391
+ "max_tokens": 8192,
392
+ "tool_choice": "required",
393
+ "messages": messages,
394
+ "tools": OPENAI_TOOLS,
395
+ }
396
+
397
+ if debug_enabled():
398
+ write_debug_event("llm.request", request_payload)
399
+
400
+ try:
401
+ completion = client.chat.completions.create(
402
+ model=MODEL_NAME,
403
+ messages=messages,
404
+ temperature=TEMPERATURE,
405
+ max_tokens=8192,
406
+ tools=OPENAI_TOOLS,
407
+ tool_choice="required",
408
+ )
409
+ message = completion.choices[0].message
410
+ if debug_enabled():
411
+ write_debug_event(
412
+ "llm.response",
413
+ {
414
+ "stage": "chat.tools",
415
+ "model": MODEL_NAME,
416
+ "message": _message_debug_payload(message),
417
+ },
418
+ )
419
+ invocation, raw_response = _extract_tool_invocation(message)
420
+ _validate_tool_choice(invocation, observation)
421
+ action = tool_invocation_to_action(
422
+ invocation,
423
+ default_save_path=(
424
+ metadata.get("default_save_path")
425
+ if isinstance(metadata.get("default_save_path"), str)
426
+ else _FALLBACK_SAVE_PATH
427
+ ),
428
+ )
429
+ return (
430
+ action,
431
+ json.dumps(
432
+ {
433
+ "action_type": action.action_type,
434
+ "slide_index": action.slide_index,
435
+ "payload": action.payload,
436
+ },
437
+ separators=(",", ":"),
438
+ ),
439
+ invocation,
440
+ )
441
+ except Exception as error:
442
+ if debug_enabled():
443
+ write_debug_event(
444
+ "llm.error",
445
+ {
446
+ "stage": "chat.tools",
447
+ "model": MODEL_NAME,
448
+ "base_url": API_BASE_URL,
449
+ "error_type": type(error).__name__,
450
+ "error": str(error),
451
+ "response": raw_response,
452
+ },
453
+ )
454
+ print(f"[DEBUG] choose_action failed: {error}", flush=True)
455
+ if raw_response is not None:
456
+ print(
457
+ "[DEBUG] choose_action response="
458
+ + json.dumps(raw_response, separators=(",", ":")),
459
+ flush=True,
460
+ )
461
+ raise
462
+
463
+
464
+ async def run_task(difficulty: str, max_steps: int) -> None:
465
+ client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
466
+ rewards: list[float] = []
467
+ history: list[dict[str, Any]] = []
468
+ steps_taken = 0
469
+ score = 0.0
470
+ success = False
471
+ started = False
472
+
473
+ try:
474
+ async with PptAgentEnv(base_url=ENV_BASE_URL, message_timeout_s=180.0) as env:
475
+ result = await env.reset(difficulty=difficulty)
476
+ observation = result.observation
477
+ log_start(
478
+ task=observation.task_name or "unknown",
479
+ env=BENCHMARK,
480
+ model=MODEL_NAME,
481
+ )
482
+ started = True
483
+
484
+ for step in range(1, max_steps + 1):
485
+ if result.done:
486
+ break
487
+ # choose_action performs a synchronous model request. Keep it off
488
+ # the event loop so websocket heartbeat traffic can still flow.
489
+ action, action_str, invocation = await asyncio.to_thread(
490
+ choose_action,
491
+ client,
492
+ observation,
493
+ history,
494
+ max_steps,
495
+ )
496
+ result = await env.step(action)
497
+ observation = result.observation
498
+ reward = float(result.reward or 0.0)
499
+ rewards.append(reward)
500
+ history.append(_history_entry_from_tool_call(invocation, observation))
501
+ steps_taken = step
502
+ log_step(
503
+ step=step,
504
+ action=action_str,
505
+ reward=reward,
506
+ done=result.done,
507
+ error=observation.last_action_error,
508
+ )
509
+ if result.done:
510
+ break
511
+ else:
512
+ print(
513
+ f"[DEBUG] reached max steps {max_steps} without completion",
514
+ flush=True,
515
+ )
516
+
517
+ success = bool(
518
+ started
519
+ and result.done
520
+ and not observation.last_action_error
521
+ and observation.score > 0.0
522
+ )
523
+ score = float(observation.score or 0.0)
524
+ except Exception as error:
525
+ print(f"[DEBUG] inference failed: {error}", flush=True)
526
+ success = False
527
+ finally:
528
+ log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
529
+
530
+
531
+ async def main() -> None:
532
+ for difficulty, max_steps in TASKS:
533
+ await run_task(difficulty, max_steps)
534
+
535
+
536
+ if __name__ == "__main__":
537
+ asyncio.run(main())
openenv.yaml CHANGED
@@ -2,5 +2,5 @@ spec_version: 1
2
  name: ppt_agent
3
  type: space
4
  runtime: fastapi
5
- app: server.app:app
6
  port: 7860
 
2
  name: ppt_agent
3
  type: space
4
  runtime: fastapi
5
+ app: server:app
6
  port: 7860
server/app.py CHANGED
@@ -28,12 +28,7 @@ Usage:
28
  python -m server.app
29
  """
30
 
31
- try:
32
- from openenv.core.env_server.http_server import create_app
33
- except Exception as e: # pragma: no cover
34
- raise ImportError(
35
- "openenv is required for the web interface. Install dependencies with '\n uv sync\n'"
36
- ) from e
37
 
38
  try:
39
  from ..models import PptAgentAction, PptAgentObservation
 
28
  python -m server.app
29
  """
30
 
31
+ from openenv.core.env_server.http_server import create_app
 
 
 
 
 
32
 
33
  try:
34
  from ..models import PptAgentAction, PptAgentObservation
server/llm_client.py CHANGED
@@ -51,8 +51,6 @@ class LLMClient:
51
  ),
52
  )
53
  api_key = os.environ.get("JUDGE_API_KEY", os.environ.get("HF_TOKEN"))
54
- if not api_key:
55
- raise ValueError("JUDGE_API_KEY or HF_TOKEN must be set")
56
 
57
  self.client = OpenAI(base_url=self.base_url, api_key=api_key)
58
  logger.info(
 
51
  ),
52
  )
53
  api_key = os.environ.get("JUDGE_API_KEY", os.environ.get("HF_TOKEN"))
 
 
54
 
55
  self.client = OpenAI(base_url=self.base_url, api_key=api_key)
56
  logger.info(
server/utils/reward_metrics.py CHANGED
@@ -14,7 +14,7 @@ _NUMBER_PATTERN = re.compile(r"\b\d+(?:\.\d+)?%?\b")
14
  _TEXT_UNIT_SPLIT_PATTERN = re.compile(r"(?<=[.!?])\s+|\n+")
15
  _DEFAULT_SENTENCE_MODEL = os.environ.get(
16
  "SENTENCE_TRANSFORMER_MODEL",
17
- "sentence-transformers/all-MiniLM-L6-v2",
18
  )
19
 
20
 
 
14
  _TEXT_UNIT_SPLIT_PATTERN = re.compile(r"(?<=[.!?])\s+|\n+")
15
  _DEFAULT_SENTENCE_MODEL = os.environ.get(
16
  "SENTENCE_TRANSFORMER_MODEL",
17
+ "all-mpnet-base-v2",
18
  )
19
 
20
 
validate-submission.sh ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+
3
+ set -uo pipefail
4
+
5
+ DOCKER_BUILD_TIMEOUT=600
6
+ if [ -t 1 ]; then
7
+ RED='\033[0;31m'
8
+ GREEN='\033[0;32m'
9
+ YELLOW='\033[1;33m'
10
+ BOLD='\033[1m'
11
+ NC='\033[0m'
12
+ else
13
+ RED='' GREEN='' YELLOW='' BOLD='' NC=''
14
+ fi
15
+
16
+ run_with_timeout() {
17
+ local secs="$1"; shift
18
+ if command -v timeout &>/dev/null; then
19
+ timeout "$secs" "$@"
20
+ elif command -v gtimeout &>/dev/null; then
21
+ gtimeout "$secs" "$@"
22
+ else
23
+ "$@" &
24
+ local pid=$!
25
+ ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
26
+ local watcher=$!
27
+ wait "$pid" 2>/dev/null
28
+ local rc=$?
29
+ kill "$watcher" 2>/dev/null
30
+ wait "$watcher" 2>/dev/null
31
+ return $rc
32
+ fi
33
+ }
34
+
35
+ portable_mktemp() {
36
+ local prefix="${1:-validate}"
37
+ mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
38
+ }
39
+
40
+ CLEANUP_FILES=()
41
+ cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
42
+ trap cleanup EXIT
43
+
44
+ PING_URL="${1:-}"
45
+ REPO_DIR="${2:-.}"
46
+
47
+ if [ -z "$PING_URL" ]; then
48
+ printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
49
+ exit 1
50
+ fi
51
+
52
+ if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
53
+ printf "Error: directory '%s' not found\n" "${2:-.}"
54
+ exit 1
55
+ fi
56
+
57
+ PING_URL="${PING_URL%/}"
58
+ PASS=0
59
+
60
+ log() { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
61
+ pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
62
+ fail() { log "${RED}FAILED${NC} -- $1"; }
63
+ hint() { printf " ${YELLOW}Hint:${NC} %b\n" "$1"; }
64
+ stop_at() {
65
+ printf "\n"
66
+ printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
67
+ exit 1
68
+ }
69
+
70
+ printf "\n${BOLD}========================================${NC}\n"
71
+ printf "${BOLD} OpenEnv Submission Validator${NC}\n"
72
+ printf "${BOLD}========================================${NC}\n"
73
+ log "Repo: $REPO_DIR"
74
+ log "Ping URL: $PING_URL"
75
+ printf "\n"
76
+
77
+ log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
78
+
79
+ CURL_OUTPUT=$(portable_mktemp "validate-curl")
80
+ CLEANUP_FILES+=("$CURL_OUTPUT")
81
+ HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
82
+ -H "Content-Type: application/json" -d '{}' \
83
+ "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
84
+
85
+ if [ "$HTTP_CODE" = "200" ]; then
86
+ pass "HF Space is live and responds to /reset"
87
+ elif [ "$HTTP_CODE" = "000" ]; then
88
+ fail "HF Space not reachable (connection failed or timed out)"
89
+ hint "Check your network connection and that the Space is running."
90
+ stop_at "Step 1"
91
+ else
92
+ fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
93
+ hint "Make sure your Space is running and the URL is correct."
94
+ stop_at "Step 1"
95
+ fi
96
+
97
+ log "${BOLD}Step 2/3: Running docker build${NC} ..."
98
+ log "Build command: docker build --progress=plain $REPO_DIR"
99
+
100
+ if ! command -v docker &>/dev/null; then
101
+ fail "docker command not found"
102
+ hint "Install Docker: https://docs.docker.com/get-docker/"
103
+ stop_at "Step 2"
104
+ fi
105
+
106
+ BUILD_OK=false
107
+ BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build --progress=plain "$REPO_DIR" 2>&1) && BUILD_OK=true
108
+
109
+ if [ "$BUILD_OK" = true ]; then
110
+ pass "Docker build succeeded"
111
+ else
112
+ fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
113
+ printf "%s\n" "$BUILD_OUTPUT" | tail -20
114
+ stop_at "Step 2"
115
+ fi
116
+
117
+ log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
118
+
119
+ if ! command -v openenv &>/dev/null; then
120
+ fail "openenv command not found"
121
+ hint "Install it: pip install openenv-core"
122
+ stop_at "Step 3"
123
+ fi
124
+
125
+ VALIDATE_OK=false
126
+ VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
127
+
128
+ if [ "$VALIDATE_OK" = true ]; then
129
+ pass "openenv validate passed"
130
+ [ -n "$VALIDATE_OUTPUT" ] && log " $VALIDATE_OUTPUT"
131
+ else
132
+ fail "openenv validate failed"
133
+ printf "%s\n" "$VALIDATE_OUTPUT"
134
+ stop_at "Step 3"
135
+ fi
136
+
137
+ printf "\n${BOLD}========================================${NC}\n"
138
+ printf "${GREEN}${BOLD} All 3/3 checks passed!${NC}\n"
139
+ printf "${GREEN}${BOLD} Your submission is ready to submit.${NC}\n"
140
+ printf "${BOLD}========================================${NC}\n\n"