Upload folder using huggingface_hub
Browse files- inference.py +4 -86
- inference_debug.py +537 -0
- openenv.yaml +1 -1
- server/app.py +1 -6
- server/llm_client.py +0 -2
- server/utils/reward_metrics.py +1 -1
- validate-submission.sh +140 -0
inference.py
CHANGED
|
@@ -8,22 +8,6 @@ from typing import Any
|
|
| 8 |
|
| 9 |
from openai import OpenAI
|
| 10 |
|
| 11 |
-
try:
|
| 12 |
-
if os.environ.get("DEBUG", "false").lower() == "true":
|
| 13 |
-
from server.debug_logging import debug_enabled, write_debug_event
|
| 14 |
-
else:
|
| 15 |
-
raise ImportError("Debug logging is not enabled")
|
| 16 |
-
except ImportError: # pragma: no cover
|
| 17 |
-
|
| 18 |
-
def debug_enabled() -> bool:
|
| 19 |
-
return False
|
| 20 |
-
|
| 21 |
-
def write_debug_event(
|
| 22 |
-
event_type: str, payload: dict[str, Any] | None = None
|
| 23 |
-
) -> None:
|
| 24 |
-
return None
|
| 25 |
-
|
| 26 |
-
|
| 27 |
from agent_action_tools import (
|
| 28 |
AgentToolInvocation,
|
| 29 |
build_openai_tools,
|
|
@@ -33,6 +17,7 @@ from agent_action_tools import (
|
|
| 33 |
from client import PptAgentEnv
|
| 34 |
from models import PptAgentAction
|
| 35 |
|
|
|
|
| 36 |
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 37 |
|
| 38 |
API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
|
|
@@ -348,26 +333,6 @@ def _history_entry_from_tool_call(
|
|
| 348 |
}
|
| 349 |
|
| 350 |
|
| 351 |
-
def _message_debug_payload(message: Any) -> dict[str, Any]:
|
| 352 |
-
tool_calls = getattr(message, "tool_calls", None) or []
|
| 353 |
-
return {
|
| 354 |
-
"content": getattr(message, "content", None),
|
| 355 |
-
"tool_calls": [
|
| 356 |
-
{
|
| 357 |
-
"id": getattr(tool_call, "id", None),
|
| 358 |
-
"type": getattr(tool_call, "type", None),
|
| 359 |
-
"function": {
|
| 360 |
-
"name": getattr(getattr(tool_call, "function", None), "name", None),
|
| 361 |
-
"arguments": getattr(
|
| 362 |
-
getattr(tool_call, "function", None), "arguments", None
|
| 363 |
-
),
|
| 364 |
-
},
|
| 365 |
-
}
|
| 366 |
-
for tool_call in tool_calls
|
| 367 |
-
],
|
| 368 |
-
}
|
| 369 |
-
|
| 370 |
-
|
| 371 |
def choose_action(
|
| 372 |
client: OpenAI, observation: Any, history: list[dict[str, Any]], max_steps: int
|
| 373 |
) -> tuple[PptAgentAction, str, AgentToolInvocation]:
|
|
@@ -383,20 +348,6 @@ def choose_action(
|
|
| 383 |
},
|
| 384 |
]
|
| 385 |
raw_response: dict[str, Any] | None = None
|
| 386 |
-
request_payload = {
|
| 387 |
-
"stage": "chat.tools",
|
| 388 |
-
"model": MODEL_NAME,
|
| 389 |
-
"base_url": API_BASE_URL,
|
| 390 |
-
"temperature": TEMPERATURE,
|
| 391 |
-
"max_tokens": 8192,
|
| 392 |
-
"tool_choice": "required",
|
| 393 |
-
"messages": messages,
|
| 394 |
-
"tools": OPENAI_TOOLS,
|
| 395 |
-
}
|
| 396 |
-
|
| 397 |
-
if debug_enabled():
|
| 398 |
-
write_debug_event("llm.request", request_payload)
|
| 399 |
-
|
| 400 |
try:
|
| 401 |
completion = client.chat.completions.create(
|
| 402 |
model=MODEL_NAME,
|
|
@@ -407,15 +358,6 @@ def choose_action(
|
|
| 407 |
tool_choice="required",
|
| 408 |
)
|
| 409 |
message = completion.choices[0].message
|
| 410 |
-
if debug_enabled():
|
| 411 |
-
write_debug_event(
|
| 412 |
-
"llm.response",
|
| 413 |
-
{
|
| 414 |
-
"stage": "chat.tools",
|
| 415 |
-
"model": MODEL_NAME,
|
| 416 |
-
"message": _message_debug_payload(message),
|
| 417 |
-
},
|
| 418 |
-
)
|
| 419 |
invocation, raw_response = _extract_tool_invocation(message)
|
| 420 |
_validate_tool_choice(invocation, observation)
|
| 421 |
action = tool_invocation_to_action(
|
|
@@ -439,26 +381,7 @@ def choose_action(
|
|
| 439 |
invocation,
|
| 440 |
)
|
| 441 |
except Exception as error:
|
| 442 |
-
|
| 443 |
-
write_debug_event(
|
| 444 |
-
"llm.error",
|
| 445 |
-
{
|
| 446 |
-
"stage": "chat.tools",
|
| 447 |
-
"model": MODEL_NAME,
|
| 448 |
-
"base_url": API_BASE_URL,
|
| 449 |
-
"error_type": type(error).__name__,
|
| 450 |
-
"error": str(error),
|
| 451 |
-
"response": raw_response,
|
| 452 |
-
},
|
| 453 |
-
)
|
| 454 |
-
print(f"[DEBUG] choose_action failed: {error}", flush=True)
|
| 455 |
-
if raw_response is not None:
|
| 456 |
-
print(
|
| 457 |
-
"[DEBUG] choose_action response="
|
| 458 |
-
+ json.dumps(raw_response, separators=(",", ":")),
|
| 459 |
-
flush=True,
|
| 460 |
-
)
|
| 461 |
-
raise
|
| 462 |
|
| 463 |
|
| 464 |
async def run_task(difficulty: str, max_steps: int) -> None:
|
|
@@ -475,7 +398,7 @@ async def run_task(difficulty: str, max_steps: int) -> None:
|
|
| 475 |
result = await env.reset(difficulty=difficulty)
|
| 476 |
observation = result.observation
|
| 477 |
log_start(
|
| 478 |
-
task=
|
| 479 |
env=BENCHMARK,
|
| 480 |
model=MODEL_NAME,
|
| 481 |
)
|
|
@@ -508,11 +431,6 @@ async def run_task(difficulty: str, max_steps: int) -> None:
|
|
| 508 |
)
|
| 509 |
if result.done:
|
| 510 |
break
|
| 511 |
-
else:
|
| 512 |
-
print(
|
| 513 |
-
f"[DEBUG] reached max steps {max_steps} without completion",
|
| 514 |
-
flush=True,
|
| 515 |
-
)
|
| 516 |
|
| 517 |
success = bool(
|
| 518 |
started
|
|
@@ -521,8 +439,8 @@ async def run_task(difficulty: str, max_steps: int) -> None:
|
|
| 521 |
and observation.score > 0.0
|
| 522 |
)
|
| 523 |
score = float(observation.score or 0.0)
|
|
|
|
| 524 |
except Exception as error:
|
| 525 |
-
print(f"[DEBUG] inference failed: {error}", flush=True)
|
| 526 |
success = False
|
| 527 |
finally:
|
| 528 |
log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
|
|
|
|
| 8 |
|
| 9 |
from openai import OpenAI
|
| 10 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
from agent_action_tools import (
|
| 12 |
AgentToolInvocation,
|
| 13 |
build_openai_tools,
|
|
|
|
| 17 |
from client import PptAgentEnv
|
| 18 |
from models import PptAgentAction
|
| 19 |
|
| 20 |
+
|
| 21 |
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 22 |
|
| 23 |
API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
|
|
|
|
| 333 |
}
|
| 334 |
|
| 335 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 336 |
def choose_action(
|
| 337 |
client: OpenAI, observation: Any, history: list[dict[str, Any]], max_steps: int
|
| 338 |
) -> tuple[PptAgentAction, str, AgentToolInvocation]:
|
|
|
|
| 348 |
},
|
| 349 |
]
|
| 350 |
raw_response: dict[str, Any] | None = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
try:
|
| 352 |
completion = client.chat.completions.create(
|
| 353 |
model=MODEL_NAME,
|
|
|
|
| 358 |
tool_choice="required",
|
| 359 |
)
|
| 360 |
message = completion.choices[0].message
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
invocation, raw_response = _extract_tool_invocation(message)
|
| 362 |
_validate_tool_choice(invocation, observation)
|
| 363 |
action = tool_invocation_to_action(
|
|
|
|
| 381 |
invocation,
|
| 382 |
)
|
| 383 |
except Exception as error:
|
| 384 |
+
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
|
| 386 |
|
| 387 |
async def run_task(difficulty: str, max_steps: int) -> None:
|
|
|
|
| 398 |
result = await env.reset(difficulty=difficulty)
|
| 399 |
observation = result.observation
|
| 400 |
log_start(
|
| 401 |
+
task=difficulty,
|
| 402 |
env=BENCHMARK,
|
| 403 |
model=MODEL_NAME,
|
| 404 |
)
|
|
|
|
| 431 |
)
|
| 432 |
if result.done:
|
| 433 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 434 |
|
| 435 |
success = bool(
|
| 436 |
started
|
|
|
|
| 439 |
and observation.score > 0.0
|
| 440 |
)
|
| 441 |
score = float(observation.score or 0.0)
|
| 442 |
+
|
| 443 |
except Exception as error:
|
|
|
|
| 444 |
success = False
|
| 445 |
finally:
|
| 446 |
log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
|
inference_debug.py
ADDED
|
@@ -0,0 +1,537 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import asyncio
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import textwrap
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from openai import OpenAI
|
| 10 |
+
|
| 11 |
+
try:
|
| 12 |
+
if os.environ.get("DEBUG", "false").lower() == "true":
|
| 13 |
+
from server.debug_logging import debug_enabled, write_debug_event
|
| 14 |
+
else:
|
| 15 |
+
raise ImportError("Debug logging is not enabled")
|
| 16 |
+
except ImportError: # pragma: no cover
|
| 17 |
+
|
| 18 |
+
def debug_enabled() -> bool:
|
| 19 |
+
return False
|
| 20 |
+
|
| 21 |
+
def write_debug_event(
|
| 22 |
+
event_type: str, payload: dict[str, Any] | None = None
|
| 23 |
+
) -> None:
|
| 24 |
+
return None
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
from agent_action_tools import (
|
| 28 |
+
AgentToolInvocation,
|
| 29 |
+
build_openai_tools,
|
| 30 |
+
parse_tool_invocation,
|
| 31 |
+
tool_invocation_to_action,
|
| 32 |
+
)
|
| 33 |
+
from client import PptAgentEnv
|
| 34 |
+
from models import PptAgentAction
|
| 35 |
+
|
| 36 |
+
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 37 |
+
|
| 38 |
+
API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
|
| 39 |
+
MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen3.5-27B")
|
| 40 |
+
ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:7860")
|
| 41 |
+
TEMPERATURE = 0.0
|
| 42 |
+
TASKS = [("easy", 10), ("medium", 15), ("hard", 20)]
|
| 43 |
+
|
| 44 |
+
BENCHMARK = "ppt_agent"
|
| 45 |
+
|
| 46 |
+
OPENAI_TOOLS = build_openai_tools()
|
| 47 |
+
|
| 48 |
+
_FALLBACK_SAVE_PATH = "outputs/presentation.pptx"
|
| 49 |
+
|
| 50 |
+
SYSTEM_PROMPT = textwrap.dedent(
|
| 51 |
+
"""
|
| 52 |
+
You are an elite presentation designer and PowerPoint author. Your job is to choose exactly one next environment tool call that moves the deck toward a polished, professional final presentation. Make sure to follow the output schema and tool-use rules precisely.
|
| 53 |
+
|
| 54 |
+
You are given:
|
| 55 |
+
- the full task prompt
|
| 56 |
+
- the source context
|
| 57 |
+
- the current slide count
|
| 58 |
+
- the remaining step budget
|
| 59 |
+
- the last action error/result
|
| 60 |
+
- recent action history
|
| 61 |
+
- the current theme
|
| 62 |
+
- slide constraints
|
| 63 |
+
|
| 64 |
+
Available tools:
|
| 65 |
+
- create_slide
|
| 66 |
+
- update_slide
|
| 67 |
+
- delete_slide
|
| 68 |
+
- set_theme
|
| 69 |
+
- save_presentation
|
| 70 |
+
|
| 71 |
+
Return no prose. Produce exactly one tool call.
|
| 72 |
+
|
| 73 |
+
Presentation quality bar:
|
| 74 |
+
- Make slides look like strong professional consulting, strategy, product, or investor decks.
|
| 75 |
+
- Build a clear narrative, not a random collection of slides.
|
| 76 |
+
- Prefer strong hierarchy, clean alignment, generous whitespace, and restrained color usage.
|
| 77 |
+
- Avoid clutter, overcrowded text, and weak visual balance.
|
| 78 |
+
- Use charts or tables when they communicate evidence better than paragraphs.
|
| 79 |
+
- Use concise text with meaningful headlines and clear supporting points.
|
| 80 |
+
- Keep the design consistent across the whole deck.
|
| 81 |
+
|
| 82 |
+
Design guidance:
|
| 83 |
+
- Prefer a cohesive deck-level theme. Use set_theme early if the current theme is weak or mismatched to the task.
|
| 84 |
+
- Keep typography consistent across slides.
|
| 85 |
+
- Prefer readable layouts with clear margins and spacing.
|
| 86 |
+
- Use word_wrap, space_before_pt, space_after_pt, and line_spacing when helpful for clean text layout.
|
| 87 |
+
|
| 88 |
+
Content guidance:
|
| 89 |
+
- Ground claims in the provided source context.
|
| 90 |
+
- Do not invent facts, numbers, or citations not supported by the inputs.
|
| 91 |
+
- If the task implies a specific slide structure, satisfy it.
|
| 92 |
+
|
| 93 |
+
Tool-use rules:
|
| 94 |
+
- Use only the tool schema fields exactly as defined.
|
| 95 |
+
- Never invent shorthand slide DSL fields such as:
|
| 96 |
+
- background -> use the schema field, not legacy DSL keys
|
| 97 |
+
- type: title/body -> use type: text
|
| 98 |
+
- font -> use style.font_name
|
| 99 |
+
- size -> use numeric style.font_size_pt values
|
| 100 |
+
- color -> use style.color_hex or other schema color fields
|
| 101 |
+
- Use the provided tools directly. Do not emit plans, explanations, or pseudo-actions.
|
| 102 |
+
|
| 103 |
+
Shape rules:
|
| 104 |
+
- Text shapes require explicit geometry in slide inches: x, y, w, h.
|
| 105 |
+
- For new text shapes use type: text.
|
| 106 |
+
- For accent bars use type: accent_bar.
|
| 107 |
+
- For charts use type: chart with ct, cd, x, y, w, h.
|
| 108 |
+
- For tables use type: table with td, x, y, w, h.
|
| 109 |
+
- For images use type: image with img, x, y, and optional w, h.
|
| 110 |
+
|
| 111 |
+
Update rules:
|
| 112 |
+
- Use create_slide to add a new slide.
|
| 113 |
+
- Use update_slide to revise an existing slide.
|
| 114 |
+
- update_slide requires si.
|
| 115 |
+
- Existing shapes must be updated by id.
|
| 116 |
+
- Use known_named_shapes_by_slide and prior tool results to find the correct id.
|
| 117 |
+
- Only use delete_slide when a slide is clearly wrong, redundant, or should be removed.
|
| 118 |
+
- Do not iterate too much by updating and deleting slides. Aim to get each new slide right on the first try.
|
| 119 |
+
- Use save_presentation when the deck is complete enough and ready to submit.
|
| 120 |
+
|
| 121 |
+
Theme token rules:
|
| 122 |
+
- You may use these theme tokens inside payload values:
|
| 123 |
+
- <bg>, <surface>, <accent>, <primary>, <secondary>, <font>
|
| 124 |
+
- Prefer theme tokens over hardcoded repeated color and font values for consistent design.
|
| 125 |
+
- Use set_theme to update or add more theme tokens in the theme
|
| 126 |
+
- Omitted theme keys remain unchanged.
|
| 127 |
+
|
| 128 |
+
Recommended workflow:
|
| 129 |
+
- If needed, set or refine the theme first.
|
| 130 |
+
- Think about the slide structure and then create it in one go.
|
| 131 |
+
- Call update_slide only if you think it is needed
|
| 132 |
+
- You have a finite max step budget. Do not let remaining_steps reach 0 before calling save_presentation.
|
| 133 |
+
- Save only when the deck is coherent, complete, and professionally presented.
|
| 134 |
+
|
| 135 |
+
Example create_slide payload:
|
| 136 |
+
{
|
| 137 |
+
"bg": "<bg>",
|
| 138 |
+
"shapes": [
|
| 139 |
+
{
|
| 140 |
+
"type": "text",
|
| 141 |
+
"name": "title",
|
| 142 |
+
"text": "Harbor Retail Expansion 2026",
|
| 143 |
+
"x": 0.7,
|
| 144 |
+
"y": 0.6,
|
| 145 |
+
"w": 8.6,
|
| 146 |
+
"h": 0.8,
|
| 147 |
+
"style": {
|
| 148 |
+
"font_name": "<font>",
|
| 149 |
+
"font_size_pt": 28,
|
| 150 |
+
"color_hex": "<primary>",
|
| 151 |
+
"bold": true,
|
| 152 |
+
"word_wrap": true,
|
| 153 |
+
"space_after_pt": 4
|
| 154 |
+
}
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
"type": "text",
|
| 158 |
+
"name": "subtitle",
|
| 159 |
+
"text": "Growth priorities, risks, and operating implications",
|
| 160 |
+
"x": 0.7,
|
| 161 |
+
"y": 1.45,
|
| 162 |
+
"w": 7.8,
|
| 163 |
+
"h": 0.9,
|
| 164 |
+
"style": {
|
| 165 |
+
"font_name": "<font>",
|
| 166 |
+
"font_size_pt": 16,
|
| 167 |
+
"color_hex": "<secondary>",
|
| 168 |
+
"word_wrap": true,
|
| 169 |
+
"line_spacing": 1.15
|
| 170 |
+
}
|
| 171 |
+
}
|
| 172 |
+
]
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
Example update_slide payload:
|
| 176 |
+
{
|
| 177 |
+
"si": 1,
|
| 178 |
+
"upd": [
|
| 179 |
+
{
|
| 180 |
+
"type": "text",
|
| 181 |
+
"id": 11,
|
| 182 |
+
"text": "Harbor Retail Expansion 2026",
|
| 183 |
+
"style": {
|
| 184 |
+
"font_name": "<font>",
|
| 185 |
+
"font_size_pt": 28,
|
| 186 |
+
"color_hex": "<primary>",
|
| 187 |
+
"bold": true
|
| 188 |
+
}
|
| 189 |
+
}
|
| 190 |
+
],
|
| 191 |
+
"add": [
|
| 192 |
+
{
|
| 193 |
+
"type": "accent_bar",
|
| 194 |
+
"name": "accent",
|
| 195 |
+
"hex": "<accent>"
|
| 196 |
+
}
|
| 197 |
+
]
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
Example set_theme payload:
|
| 201 |
+
{
|
| 202 |
+
"bg": "#F8FAFC",
|
| 203 |
+
"surface": "#FFFFFF",
|
| 204 |
+
"accent": "#2563EB",
|
| 205 |
+
"primary": "#0F172A",
|
| 206 |
+
"secondary": "#475569",
|
| 207 |
+
"font": "Aptos",
|
| 208 |
+
"ts": 28,
|
| 209 |
+
"bs": 16,
|
| 210 |
+
"cs": 10
|
| 211 |
+
}
|
| 212 |
+
"""
|
| 213 |
+
).strip()
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def log_start(task: str, env: str, model: str) -> None:
|
| 217 |
+
print(f"[START] task={task} env={env} model={model}", flush=True)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def log_step(
|
| 221 |
+
step: int, action: str, reward: float, done: bool, error: str | None
|
| 222 |
+
) -> None:
|
| 223 |
+
error_value = (error or "null").replace("\n", " ")
|
| 224 |
+
print(
|
| 225 |
+
f"[STEP] step={step} action={action} reward={reward:.2f} done={str(done).lower()} error={error_value}",
|
| 226 |
+
flush=True,
|
| 227 |
+
)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def log_end(success: bool, steps: int, score: float, rewards: list[float]) -> None:
|
| 231 |
+
rewards_str = ",".join(f"{reward:.2f}" for reward in rewards)
|
| 232 |
+
print(
|
| 233 |
+
f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",
|
| 234 |
+
flush=True,
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _observation_metadata(observation: Any) -> dict[str, Any]:
|
| 239 |
+
metadata = getattr(observation, "metadata", None)
|
| 240 |
+
return metadata if isinstance(metadata, dict) else {}
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def _planning_payload(
|
| 244 |
+
observation: Any, history: list[dict[str, Any]], max_steps: int
|
| 245 |
+
) -> dict[str, Any]:
|
| 246 |
+
metadata = _observation_metadata(observation)
|
| 247 |
+
current_theme = metadata.get("current_theme")
|
| 248 |
+
known_named_shapes = metadata.get("known_named_shapes_by_slide")
|
| 249 |
+
slide_constraints = metadata.get("slide_constraints")
|
| 250 |
+
default_save_path = metadata.get("default_save_path")
|
| 251 |
+
metadata_max_steps = metadata.get("max_steps")
|
| 252 |
+
effective_max_steps = (
|
| 253 |
+
min(metadata_max_steps, max_steps)
|
| 254 |
+
if isinstance(metadata_max_steps, int)
|
| 255 |
+
else max_steps
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
return {
|
| 259 |
+
"task_prompt": observation.task_prompt,
|
| 260 |
+
"source_context": observation.source_context,
|
| 261 |
+
"slide_count": observation.slide_count,
|
| 262 |
+
"remaining_steps": max(
|
| 263 |
+
0,
|
| 264 |
+
effective_max_steps - int(metadata.get("step_count", 0)),
|
| 265 |
+
),
|
| 266 |
+
"last_action_error": observation.last_action_error,
|
| 267 |
+
"last_action_result": observation.last_action_result,
|
| 268 |
+
"recent_actions": history[-5:],
|
| 269 |
+
"known_named_shapes_by_slide": (
|
| 270 |
+
known_named_shapes if isinstance(known_named_shapes, dict) else {}
|
| 271 |
+
),
|
| 272 |
+
"current_theme": current_theme if isinstance(current_theme, dict) else {},
|
| 273 |
+
"slide_constraints": (
|
| 274 |
+
slide_constraints if isinstance(slide_constraints, dict) else {}
|
| 275 |
+
),
|
| 276 |
+
"default_save_path": (
|
| 277 |
+
default_save_path
|
| 278 |
+
if isinstance(default_save_path, str) and default_save_path
|
| 279 |
+
else _FALLBACK_SAVE_PATH
|
| 280 |
+
),
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def _validate_tool_choice(invocation: AgentToolInvocation, observation: Any) -> None:
|
| 285 |
+
if invocation.tool_name == "save_presentation":
|
| 286 |
+
constraints = _observation_metadata(observation).get("slide_constraints")
|
| 287 |
+
min_slides = (
|
| 288 |
+
constraints.get("min_slides") if isinstance(constraints, dict) else None
|
| 289 |
+
)
|
| 290 |
+
if isinstance(min_slides, int) and observation.slide_count < min_slides:
|
| 291 |
+
raise ValueError(
|
| 292 |
+
"cannot save before reaching the required minimum slide count"
|
| 293 |
+
)
|
| 294 |
+
return
|
| 295 |
+
|
| 296 |
+
if invocation.tool_name in {"update_slide", "delete_slide"}:
|
| 297 |
+
slide_index = invocation.arguments.get("slide_index")
|
| 298 |
+
if not isinstance(slide_index, int):
|
| 299 |
+
raise ValueError(f"{invocation.tool_name} requires slide_index")
|
| 300 |
+
if slide_index > observation.slide_count:
|
| 301 |
+
raise ValueError(
|
| 302 |
+
f"slide_index {slide_index} is out of range for {observation.slide_count} slides"
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def _extract_tool_invocation(
|
| 307 |
+
message: Any,
|
| 308 |
+
) -> tuple[AgentToolInvocation, dict[str, Any]]:
|
| 309 |
+
raw_tool_calls = getattr(message, "tool_calls", None) or []
|
| 310 |
+
tool_calls = list(raw_tool_calls)
|
| 311 |
+
if not tool_calls:
|
| 312 |
+
raise ValueError("LLM must emit at least one tool call")
|
| 313 |
+
|
| 314 |
+
tool_call = tool_calls[0]
|
| 315 |
+
function = getattr(tool_call, "function", None)
|
| 316 |
+
tool_name = getattr(function, "name", None)
|
| 317 |
+
arguments = getattr(function, "arguments", None) or "{}"
|
| 318 |
+
if not isinstance(tool_name, str) or not tool_name:
|
| 319 |
+
raise ValueError("Tool call missing function name")
|
| 320 |
+
return (
|
| 321 |
+
parse_tool_invocation(tool_name, arguments),
|
| 322 |
+
{
|
| 323 |
+
"tool_name": tool_name,
|
| 324 |
+
"arguments": arguments,
|
| 325 |
+
"ignored_tool_calls": max(0, len(tool_calls) - 1),
|
| 326 |
+
},
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def _history_entry_from_tool_call(
|
| 331 |
+
invocation: AgentToolInvocation, observation: Any
|
| 332 |
+
) -> dict[str, Any]:
|
| 333 |
+
last_action_result = (
|
| 334 |
+
observation.last_action_result
|
| 335 |
+
if isinstance(observation.last_action_result, dict)
|
| 336 |
+
else {}
|
| 337 |
+
)
|
| 338 |
+
slide_index = last_action_result.get("slide_index")
|
| 339 |
+
raw_tool_result = last_action_result.get("tool_result")
|
| 340 |
+
tool_result = raw_tool_result if isinstance(raw_tool_result, dict) else None
|
| 341 |
+
|
| 342 |
+
return {
|
| 343 |
+
"tool_name": invocation.tool_name,
|
| 344 |
+
"arguments": invocation.arguments,
|
| 345 |
+
"slide_index": slide_index if isinstance(slide_index, int) else None,
|
| 346 |
+
"tool_result": tool_result,
|
| 347 |
+
"error": observation.last_action_error,
|
| 348 |
+
}
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _message_debug_payload(message: Any) -> dict[str, Any]:
|
| 352 |
+
tool_calls = getattr(message, "tool_calls", None) or []
|
| 353 |
+
return {
|
| 354 |
+
"content": getattr(message, "content", None),
|
| 355 |
+
"tool_calls": [
|
| 356 |
+
{
|
| 357 |
+
"id": getattr(tool_call, "id", None),
|
| 358 |
+
"type": getattr(tool_call, "type", None),
|
| 359 |
+
"function": {
|
| 360 |
+
"name": getattr(getattr(tool_call, "function", None), "name", None),
|
| 361 |
+
"arguments": getattr(
|
| 362 |
+
getattr(tool_call, "function", None), "arguments", None
|
| 363 |
+
),
|
| 364 |
+
},
|
| 365 |
+
}
|
| 366 |
+
for tool_call in tool_calls
|
| 367 |
+
],
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def choose_action(
|
| 372 |
+
client: OpenAI, observation: Any, history: list[dict[str, Any]], max_steps: int
|
| 373 |
+
) -> tuple[PptAgentAction, str, AgentToolInvocation]:
|
| 374 |
+
metadata = _observation_metadata(observation)
|
| 375 |
+
messages = [
|
| 376 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 377 |
+
{
|
| 378 |
+
"role": "user",
|
| 379 |
+
"content": json.dumps(
|
| 380 |
+
_planning_payload(observation, history, max_steps),
|
| 381 |
+
separators=(",", ":"),
|
| 382 |
+
),
|
| 383 |
+
},
|
| 384 |
+
]
|
| 385 |
+
raw_response: dict[str, Any] | None = None
|
| 386 |
+
request_payload = {
|
| 387 |
+
"stage": "chat.tools",
|
| 388 |
+
"model": MODEL_NAME,
|
| 389 |
+
"base_url": API_BASE_URL,
|
| 390 |
+
"temperature": TEMPERATURE,
|
| 391 |
+
"max_tokens": 8192,
|
| 392 |
+
"tool_choice": "required",
|
| 393 |
+
"messages": messages,
|
| 394 |
+
"tools": OPENAI_TOOLS,
|
| 395 |
+
}
|
| 396 |
+
|
| 397 |
+
if debug_enabled():
|
| 398 |
+
write_debug_event("llm.request", request_payload)
|
| 399 |
+
|
| 400 |
+
try:
|
| 401 |
+
completion = client.chat.completions.create(
|
| 402 |
+
model=MODEL_NAME,
|
| 403 |
+
messages=messages,
|
| 404 |
+
temperature=TEMPERATURE,
|
| 405 |
+
max_tokens=8192,
|
| 406 |
+
tools=OPENAI_TOOLS,
|
| 407 |
+
tool_choice="required",
|
| 408 |
+
)
|
| 409 |
+
message = completion.choices[0].message
|
| 410 |
+
if debug_enabled():
|
| 411 |
+
write_debug_event(
|
| 412 |
+
"llm.response",
|
| 413 |
+
{
|
| 414 |
+
"stage": "chat.tools",
|
| 415 |
+
"model": MODEL_NAME,
|
| 416 |
+
"message": _message_debug_payload(message),
|
| 417 |
+
},
|
| 418 |
+
)
|
| 419 |
+
invocation, raw_response = _extract_tool_invocation(message)
|
| 420 |
+
_validate_tool_choice(invocation, observation)
|
| 421 |
+
action = tool_invocation_to_action(
|
| 422 |
+
invocation,
|
| 423 |
+
default_save_path=(
|
| 424 |
+
metadata.get("default_save_path")
|
| 425 |
+
if isinstance(metadata.get("default_save_path"), str)
|
| 426 |
+
else _FALLBACK_SAVE_PATH
|
| 427 |
+
),
|
| 428 |
+
)
|
| 429 |
+
return (
|
| 430 |
+
action,
|
| 431 |
+
json.dumps(
|
| 432 |
+
{
|
| 433 |
+
"action_type": action.action_type,
|
| 434 |
+
"slide_index": action.slide_index,
|
| 435 |
+
"payload": action.payload,
|
| 436 |
+
},
|
| 437 |
+
separators=(",", ":"),
|
| 438 |
+
),
|
| 439 |
+
invocation,
|
| 440 |
+
)
|
| 441 |
+
except Exception as error:
|
| 442 |
+
if debug_enabled():
|
| 443 |
+
write_debug_event(
|
| 444 |
+
"llm.error",
|
| 445 |
+
{
|
| 446 |
+
"stage": "chat.tools",
|
| 447 |
+
"model": MODEL_NAME,
|
| 448 |
+
"base_url": API_BASE_URL,
|
| 449 |
+
"error_type": type(error).__name__,
|
| 450 |
+
"error": str(error),
|
| 451 |
+
"response": raw_response,
|
| 452 |
+
},
|
| 453 |
+
)
|
| 454 |
+
print(f"[DEBUG] choose_action failed: {error}", flush=True)
|
| 455 |
+
if raw_response is not None:
|
| 456 |
+
print(
|
| 457 |
+
"[DEBUG] choose_action response="
|
| 458 |
+
+ json.dumps(raw_response, separators=(",", ":")),
|
| 459 |
+
flush=True,
|
| 460 |
+
)
|
| 461 |
+
raise
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
async def run_task(difficulty: str, max_steps: int) -> None:
|
| 465 |
+
client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
|
| 466 |
+
rewards: list[float] = []
|
| 467 |
+
history: list[dict[str, Any]] = []
|
| 468 |
+
steps_taken = 0
|
| 469 |
+
score = 0.0
|
| 470 |
+
success = False
|
| 471 |
+
started = False
|
| 472 |
+
|
| 473 |
+
try:
|
| 474 |
+
async with PptAgentEnv(base_url=ENV_BASE_URL, message_timeout_s=180.0) as env:
|
| 475 |
+
result = await env.reset(difficulty=difficulty)
|
| 476 |
+
observation = result.observation
|
| 477 |
+
log_start(
|
| 478 |
+
task=observation.task_name or "unknown",
|
| 479 |
+
env=BENCHMARK,
|
| 480 |
+
model=MODEL_NAME,
|
| 481 |
+
)
|
| 482 |
+
started = True
|
| 483 |
+
|
| 484 |
+
for step in range(1, max_steps + 1):
|
| 485 |
+
if result.done:
|
| 486 |
+
break
|
| 487 |
+
# choose_action performs a synchronous model request. Keep it off
|
| 488 |
+
# the event loop so websocket heartbeat traffic can still flow.
|
| 489 |
+
action, action_str, invocation = await asyncio.to_thread(
|
| 490 |
+
choose_action,
|
| 491 |
+
client,
|
| 492 |
+
observation,
|
| 493 |
+
history,
|
| 494 |
+
max_steps,
|
| 495 |
+
)
|
| 496 |
+
result = await env.step(action)
|
| 497 |
+
observation = result.observation
|
| 498 |
+
reward = float(result.reward or 0.0)
|
| 499 |
+
rewards.append(reward)
|
| 500 |
+
history.append(_history_entry_from_tool_call(invocation, observation))
|
| 501 |
+
steps_taken = step
|
| 502 |
+
log_step(
|
| 503 |
+
step=step,
|
| 504 |
+
action=action_str,
|
| 505 |
+
reward=reward,
|
| 506 |
+
done=result.done,
|
| 507 |
+
error=observation.last_action_error,
|
| 508 |
+
)
|
| 509 |
+
if result.done:
|
| 510 |
+
break
|
| 511 |
+
else:
|
| 512 |
+
print(
|
| 513 |
+
f"[DEBUG] reached max steps {max_steps} without completion",
|
| 514 |
+
flush=True,
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
success = bool(
|
| 518 |
+
started
|
| 519 |
+
and result.done
|
| 520 |
+
and not observation.last_action_error
|
| 521 |
+
and observation.score > 0.0
|
| 522 |
+
)
|
| 523 |
+
score = float(observation.score or 0.0)
|
| 524 |
+
except Exception as error:
|
| 525 |
+
print(f"[DEBUG] inference failed: {error}", flush=True)
|
| 526 |
+
success = False
|
| 527 |
+
finally:
|
| 528 |
+
log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
async def main() -> None:
|
| 532 |
+
for difficulty, max_steps in TASKS:
|
| 533 |
+
await run_task(difficulty, max_steps)
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
if __name__ == "__main__":
|
| 537 |
+
asyncio.run(main())
|
openenv.yaml
CHANGED
|
@@ -2,5 +2,5 @@ spec_version: 1
|
|
| 2 |
name: ppt_agent
|
| 3 |
type: space
|
| 4 |
runtime: fastapi
|
| 5 |
-
app: server
|
| 6 |
port: 7860
|
|
|
|
| 2 |
name: ppt_agent
|
| 3 |
type: space
|
| 4 |
runtime: fastapi
|
| 5 |
+
app: server:app
|
| 6 |
port: 7860
|
server/app.py
CHANGED
|
@@ -28,12 +28,7 @@ Usage:
|
|
| 28 |
python -m server.app
|
| 29 |
"""
|
| 30 |
|
| 31 |
-
|
| 32 |
-
from openenv.core.env_server.http_server import create_app
|
| 33 |
-
except Exception as e: # pragma: no cover
|
| 34 |
-
raise ImportError(
|
| 35 |
-
"openenv is required for the web interface. Install dependencies with '\n uv sync\n'"
|
| 36 |
-
) from e
|
| 37 |
|
| 38 |
try:
|
| 39 |
from ..models import PptAgentAction, PptAgentObservation
|
|
|
|
| 28 |
python -m server.app
|
| 29 |
"""
|
| 30 |
|
| 31 |
+
from openenv.core.env_server.http_server import create_app
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
try:
|
| 34 |
from ..models import PptAgentAction, PptAgentObservation
|
server/llm_client.py
CHANGED
|
@@ -51,8 +51,6 @@ class LLMClient:
|
|
| 51 |
),
|
| 52 |
)
|
| 53 |
api_key = os.environ.get("JUDGE_API_KEY", os.environ.get("HF_TOKEN"))
|
| 54 |
-
if not api_key:
|
| 55 |
-
raise ValueError("JUDGE_API_KEY or HF_TOKEN must be set")
|
| 56 |
|
| 57 |
self.client = OpenAI(base_url=self.base_url, api_key=api_key)
|
| 58 |
logger.info(
|
|
|
|
| 51 |
),
|
| 52 |
)
|
| 53 |
api_key = os.environ.get("JUDGE_API_KEY", os.environ.get("HF_TOKEN"))
|
|
|
|
|
|
|
| 54 |
|
| 55 |
self.client = OpenAI(base_url=self.base_url, api_key=api_key)
|
| 56 |
logger.info(
|
server/utils/reward_metrics.py
CHANGED
|
@@ -14,7 +14,7 @@ _NUMBER_PATTERN = re.compile(r"\b\d+(?:\.\d+)?%?\b")
|
|
| 14 |
_TEXT_UNIT_SPLIT_PATTERN = re.compile(r"(?<=[.!?])\s+|\n+")
|
| 15 |
_DEFAULT_SENTENCE_MODEL = os.environ.get(
|
| 16 |
"SENTENCE_TRANSFORMER_MODEL",
|
| 17 |
-
"
|
| 18 |
)
|
| 19 |
|
| 20 |
|
|
|
|
| 14 |
_TEXT_UNIT_SPLIT_PATTERN = re.compile(r"(?<=[.!?])\s+|\n+")
|
| 15 |
_DEFAULT_SENTENCE_MODEL = os.environ.get(
|
| 16 |
"SENTENCE_TRANSFORMER_MODEL",
|
| 17 |
+
"all-mpnet-base-v2",
|
| 18 |
)
|
| 19 |
|
| 20 |
|
validate-submission.sh
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
|
| 3 |
+
set -uo pipefail
|
| 4 |
+
|
| 5 |
+
DOCKER_BUILD_TIMEOUT=600
|
| 6 |
+
if [ -t 1 ]; then
|
| 7 |
+
RED='\033[0;31m'
|
| 8 |
+
GREEN='\033[0;32m'
|
| 9 |
+
YELLOW='\033[1;33m'
|
| 10 |
+
BOLD='\033[1m'
|
| 11 |
+
NC='\033[0m'
|
| 12 |
+
else
|
| 13 |
+
RED='' GREEN='' YELLOW='' BOLD='' NC=''
|
| 14 |
+
fi
|
| 15 |
+
|
| 16 |
+
run_with_timeout() {
|
| 17 |
+
local secs="$1"; shift
|
| 18 |
+
if command -v timeout &>/dev/null; then
|
| 19 |
+
timeout "$secs" "$@"
|
| 20 |
+
elif command -v gtimeout &>/dev/null; then
|
| 21 |
+
gtimeout "$secs" "$@"
|
| 22 |
+
else
|
| 23 |
+
"$@" &
|
| 24 |
+
local pid=$!
|
| 25 |
+
( sleep "$secs" && kill "$pid" 2>/dev/null ) &
|
| 26 |
+
local watcher=$!
|
| 27 |
+
wait "$pid" 2>/dev/null
|
| 28 |
+
local rc=$?
|
| 29 |
+
kill "$watcher" 2>/dev/null
|
| 30 |
+
wait "$watcher" 2>/dev/null
|
| 31 |
+
return $rc
|
| 32 |
+
fi
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
portable_mktemp() {
|
| 36 |
+
local prefix="${1:-validate}"
|
| 37 |
+
mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
CLEANUP_FILES=()
|
| 41 |
+
cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
|
| 42 |
+
trap cleanup EXIT
|
| 43 |
+
|
| 44 |
+
PING_URL="${1:-}"
|
| 45 |
+
REPO_DIR="${2:-.}"
|
| 46 |
+
|
| 47 |
+
if [ -z "$PING_URL" ]; then
|
| 48 |
+
printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
|
| 49 |
+
exit 1
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
|
| 53 |
+
printf "Error: directory '%s' not found\n" "${2:-.}"
|
| 54 |
+
exit 1
|
| 55 |
+
fi
|
| 56 |
+
|
| 57 |
+
PING_URL="${PING_URL%/}"
|
| 58 |
+
PASS=0
|
| 59 |
+
|
| 60 |
+
log() { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
|
| 61 |
+
pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
|
| 62 |
+
fail() { log "${RED}FAILED${NC} -- $1"; }
|
| 63 |
+
hint() { printf " ${YELLOW}Hint:${NC} %b\n" "$1"; }
|
| 64 |
+
stop_at() {
|
| 65 |
+
printf "\n"
|
| 66 |
+
printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
|
| 67 |
+
exit 1
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
printf "\n${BOLD}========================================${NC}\n"
|
| 71 |
+
printf "${BOLD} OpenEnv Submission Validator${NC}\n"
|
| 72 |
+
printf "${BOLD}========================================${NC}\n"
|
| 73 |
+
log "Repo: $REPO_DIR"
|
| 74 |
+
log "Ping URL: $PING_URL"
|
| 75 |
+
printf "\n"
|
| 76 |
+
|
| 77 |
+
log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
|
| 78 |
+
|
| 79 |
+
CURL_OUTPUT=$(portable_mktemp "validate-curl")
|
| 80 |
+
CLEANUP_FILES+=("$CURL_OUTPUT")
|
| 81 |
+
HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
|
| 82 |
+
-H "Content-Type: application/json" -d '{}' \
|
| 83 |
+
"$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
|
| 84 |
+
|
| 85 |
+
if [ "$HTTP_CODE" = "200" ]; then
|
| 86 |
+
pass "HF Space is live and responds to /reset"
|
| 87 |
+
elif [ "$HTTP_CODE" = "000" ]; then
|
| 88 |
+
fail "HF Space not reachable (connection failed or timed out)"
|
| 89 |
+
hint "Check your network connection and that the Space is running."
|
| 90 |
+
stop_at "Step 1"
|
| 91 |
+
else
|
| 92 |
+
fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
|
| 93 |
+
hint "Make sure your Space is running and the URL is correct."
|
| 94 |
+
stop_at "Step 1"
|
| 95 |
+
fi
|
| 96 |
+
|
| 97 |
+
log "${BOLD}Step 2/3: Running docker build${NC} ..."
|
| 98 |
+
log "Build command: docker build --progress=plain $REPO_DIR"
|
| 99 |
+
|
| 100 |
+
if ! command -v docker &>/dev/null; then
|
| 101 |
+
fail "docker command not found"
|
| 102 |
+
hint "Install Docker: https://docs.docker.com/get-docker/"
|
| 103 |
+
stop_at "Step 2"
|
| 104 |
+
fi
|
| 105 |
+
|
| 106 |
+
BUILD_OK=false
|
| 107 |
+
BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build --progress=plain "$REPO_DIR" 2>&1) && BUILD_OK=true
|
| 108 |
+
|
| 109 |
+
if [ "$BUILD_OK" = true ]; then
|
| 110 |
+
pass "Docker build succeeded"
|
| 111 |
+
else
|
| 112 |
+
fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
|
| 113 |
+
printf "%s\n" "$BUILD_OUTPUT" | tail -20
|
| 114 |
+
stop_at "Step 2"
|
| 115 |
+
fi
|
| 116 |
+
|
| 117 |
+
log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
|
| 118 |
+
|
| 119 |
+
if ! command -v openenv &>/dev/null; then
|
| 120 |
+
fail "openenv command not found"
|
| 121 |
+
hint "Install it: pip install openenv-core"
|
| 122 |
+
stop_at "Step 3"
|
| 123 |
+
fi
|
| 124 |
+
|
| 125 |
+
VALIDATE_OK=false
|
| 126 |
+
VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
|
| 127 |
+
|
| 128 |
+
if [ "$VALIDATE_OK" = true ]; then
|
| 129 |
+
pass "openenv validate passed"
|
| 130 |
+
[ -n "$VALIDATE_OUTPUT" ] && log " $VALIDATE_OUTPUT"
|
| 131 |
+
else
|
| 132 |
+
fail "openenv validate failed"
|
| 133 |
+
printf "%s\n" "$VALIDATE_OUTPUT"
|
| 134 |
+
stop_at "Step 3"
|
| 135 |
+
fi
|
| 136 |
+
|
| 137 |
+
printf "\n${BOLD}========================================${NC}\n"
|
| 138 |
+
printf "${GREEN}${BOLD} All 3/3 checks passed!${NC}\n"
|
| 139 |
+
printf "${GREEN}${BOLD} Your submission is ready to submit.${NC}\n"
|
| 140 |
+
printf "${BOLD}========================================${NC}\n\n"
|