om192006 commited on
Commit
ea71e23
·
verified ·
1 Parent(s): aa33eec

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +11 -6
  2. inference.py +37 -51
README.md CHANGED
@@ -92,12 +92,17 @@ Linear, Jira auto-assign, etc.).
92
 
93
  ## Baseline Scores (Llama 3.1 8B via HF Router)
94
 
95
- | Task | Baseline Score |
96
- |---|---|
97
- | 🟢 Easy | **1.0000** ✅ |
98
- | 🟡 Medium | **1.0000** ✅ |
99
- | 🔴 Hard | **0.8667** ✅ |
100
- | **Average** | **0.9556** |
 
 
 
 
 
101
 
102
  ---
103
 
 
92
 
93
  ## Baseline Scores (Llama 3.1 8B via HF Router)
94
 
95
+ Scores vary per run because issues are randomly sampled. Representative results:
96
+
97
+ | Task | Score Range | Typical |
98
+ |---|---|---|
99
+ | 🟢 Easy | 0.0 – 1.0 | **1.0** ✅ |
100
+ | 🟡 Medium | 0.5 – 1.0 | **0.75** |
101
+ | 🔴 Hard | 0.6 – 1.0 | **0.85** |
102
+ | **Average** | 0.5 – 1.0 | **~0.80** |
103
+
104
+ > Binary easy task depends on which issue is sampled. Medium/Hard benefit from
105
+ > partial credit — the model consistently scores well on label and team fields.
106
 
107
  ---
108
 
inference.py CHANGED
@@ -7,6 +7,7 @@ Mandatory log format: [START] / [STEP] / [END]
7
  All LLM calls use OpenAI client configured via environment variables.
8
  """
9
 
 
10
  import json
11
  import os
12
  import sys
@@ -21,11 +22,15 @@ API_BASE_URL: str = os.environ.get(
21
  MODEL_NAME: str = os.environ.get(
22
  "MODEL_NAME", "meta-llama/llama-3.1-8b-instruct"
23
  )
24
- API_KEY: str = os.environ.get("HF_TOKEN", "") # no default — required
 
25
  ENV_BASE_URL: str = os.environ.get(
26
  "ENV_BASE_URL", "https://om192006-github-issue-triage.hf.space"
27
  )
28
 
 
 
 
29
  # ── Inference hyper-params ────────────────────────────────────────────────────
30
  TEMPERATURE: float = 0.2
31
  MAX_TOKENS: int = 512
@@ -98,34 +103,6 @@ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> No
98
  )
99
 
100
 
101
- # ── HTTP helpers (sync — avoids async client dependency) ─────────────────────
102
- import urllib.request
103
- import urllib.error
104
-
105
-
106
- def _http_post(url: str, body: dict) -> dict:
107
- """Simple sync HTTP POST, returns parsed JSON. Raises on error."""
108
- data = json.dumps(body).encode()
109
- req = urllib.request.Request(
110
- url,
111
- data=data,
112
- headers={"Content-Type": "application/json"},
113
- method="POST",
114
- )
115
- with urllib.request.urlopen(req, timeout=30) as resp:
116
- return json.loads(resp.read().decode())
117
-
118
-
119
- def env_reset(task_id: str) -> dict:
120
- url = f"{ENV_BASE_URL}/reset"
121
- return _http_post(url, {"task_id": task_id})
122
-
123
-
124
- def env_step(action: dict, task_id: str = "easy") -> dict:
125
- url = f"{ENV_BASE_URL}/step"
126
- return _http_post(url, {"action": action, "task_id": task_id})
127
-
128
-
129
  # ── LLM call ─────────────────────────────────────────────────────────────────
130
  def build_user_prompt(observation: dict) -> str:
131
  issue = observation.get("issue_title", "")
@@ -209,12 +186,17 @@ def get_model_action(
209
  return fallback
210
 
211
 
212
- # ── Run one task episode ──────────────────────────────────────────────────────
213
- def run_task(client: OpenAI, task_id: str) -> float:
214
  """
215
- Runs a single-episode task.
 
 
216
  Returns normalised score in [0.0, 1.0].
217
  """
 
 
 
218
  rewards: List[float] = []
219
  steps_taken = 0
220
  score = 0.0
@@ -223,32 +205,34 @@ def run_task(client: OpenAI, task_id: str) -> float:
223
  log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
224
 
225
  try:
226
- # ── reset ──────────────────────────────────────────────────────────
227
- try:
228
- reset_result = env_reset(task_id)
229
- except Exception as exc:
230
- print(f"[DEBUG] reset() failed for task={task_id}: {exc}", flush=True)
231
- log_end(success=False, steps=0, score=0.0, rewards=[])
232
- return 0.0
233
 
234
- observation = reset_result.get("observation", {})
235
- done = reset_result.get("done", False)
 
 
 
236
 
237
  for step in range(1, MAX_STEPS + 1):
238
  if done:
239
  break
240
 
241
  # ── agent decides ──────────────────────────────────────────────
242
- action = get_model_action(client, observation)
243
- action_str = json.dumps(action)
244
 
245
  # ── step ──────────────────────────��────────────────────────────
246
  error_msg: Optional[str] = None
247
  try:
248
- step_result = env_step(action, task_id=task_id)
249
- reward = float(step_result.get("reward", 0.0))
250
- done = bool(step_result.get("done", True))
251
- observation = step_result.get("observation", {})
 
 
 
252
  except Exception as exc:
253
  print(f"[DEBUG] step() failed: {exc}", flush=True)
254
  reward = 0.0
@@ -267,6 +251,8 @@ def run_task(client: OpenAI, task_id: str) -> float:
267
  score = min(max(score, 0.0), 1.0)
268
  success = score >= SUCCESS_SCORE_THRESHOLD
269
 
 
 
270
  except Exception as exc:
271
  print(f"[DEBUG] Unexpected error in task={task_id}: {exc}", flush=True)
272
 
@@ -277,15 +263,15 @@ def run_task(client: OpenAI, task_id: str) -> float:
277
 
278
 
279
  # ── Main ──────────────────────────────────────────────────────────────────────
280
- def main() -> None:
281
  if not API_KEY:
282
- print("[DEBUG] WARNING: HF_TOKEN is not set. LLM calls will fail.", flush=True)
283
 
284
  client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
285
 
286
  results = {}
287
  for task_id in TASK_IDS:
288
- score = run_task(client, task_id)
289
  results[task_id] = score
290
  print(flush=True) # blank line between tasks for readability
291
 
@@ -302,4 +288,4 @@ def main() -> None:
302
 
303
 
304
  if __name__ == "__main__":
305
- main()
 
7
  All LLM calls use OpenAI client configured via environment variables.
8
  """
9
 
10
+ import asyncio
11
  import json
12
  import os
13
  import sys
 
22
  MODEL_NAME: str = os.environ.get(
23
  "MODEL_NAME", "meta-llama/llama-3.1-8b-instruct"
24
  )
25
+ # Spec says OPENAI_API_KEY; hackathon also uses HF_TOKEN — check both
26
+ API_KEY: str = os.environ.get("OPENAI_API_KEY", "") or os.environ.get("HF_TOKEN", "")
27
  ENV_BASE_URL: str = os.environ.get(
28
  "ENV_BASE_URL", "https://om192006-github-issue-triage.hf.space"
29
  )
30
 
31
+ # ── Docker image for local environment (used by OpenEnv client) ──────────────
32
+ IMAGE_NAME: str = "github_issue_triage-env:latest"
33
+
34
  # ── Inference hyper-params ────────────────────────────────────────────────────
35
  TEMPERATURE: float = 0.2
36
  MAX_TOKENS: int = 512
 
103
  )
104
 
105
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  # ── LLM call ─────────────────────────────────────────────────────────────────
107
  def build_user_prompt(observation: dict) -> str:
108
  issue = observation.get("issue_title", "")
 
186
  return fallback
187
 
188
 
189
+ # ── Run one task episode (async, using OpenEnv client for session state) ──────
190
+ async def run_task(client: OpenAI, task_id: str) -> float:
191
  """
192
+ Runs a single-episode task using the OpenEnv Docker image.
193
+ Uses async EnvClient so reset() and step() share the same environment
194
+ instance — the agent sees the same issue it gets graded on.
195
  Returns normalised score in [0.0, 1.0].
196
  """
197
+ from models import GithubIssueTriageAction, GithubIssueTriageObservation
198
+ from server.github_issue_triage_environment import GithubIssueTriageEnvironment
199
+
200
  rewards: List[float] = []
201
  steps_taken = 0
202
  score = 0.0
 
205
  log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
206
 
207
  try:
208
+ # Create a local environment instance with persistent state
209
+ env = GithubIssueTriageEnvironment()
210
+ result = env.reset(task_id=task_id)
 
 
 
 
211
 
212
+ # Convert observation to dict for the LLM prompt builder
213
+ observation = result.model_dump(
214
+ exclude={"reward", "done", "metadata"}
215
+ )
216
+ done = result.done
217
 
218
  for step in range(1, MAX_STEPS + 1):
219
  if done:
220
  break
221
 
222
  # ── agent decides ──────────────────────────────────────────────
223
+ action_dict = get_model_action(client, observation)
224
+ action_str = json.dumps(action_dict)
225
 
226
  # ── step ──────────────────────────��────────────────────────────
227
  error_msg: Optional[str] = None
228
  try:
229
+ action = GithubIssueTriageAction(**action_dict)
230
+ result = env.step(action)
231
+ reward = float(result.reward) if result.reward is not None else 0.0
232
+ done = result.done
233
+ observation = result.model_dump(
234
+ exclude={"reward", "done", "metadata"}
235
+ )
236
  except Exception as exc:
237
  print(f"[DEBUG] step() failed: {exc}", flush=True)
238
  reward = 0.0
 
251
  score = min(max(score, 0.0), 1.0)
252
  success = score >= SUCCESS_SCORE_THRESHOLD
253
 
254
+ env.close()
255
+
256
  except Exception as exc:
257
  print(f"[DEBUG] Unexpected error in task={task_id}: {exc}", flush=True)
258
 
 
263
 
264
 
265
  # ── Main ──────────────────────────────────────────────────────────────────────
266
+ async def main() -> None:
267
  if not API_KEY:
268
+ print("[DEBUG] WARNING: Neither OPENAI_API_KEY nor HF_TOKEN is set. LLM calls will fail.", flush=True)
269
 
270
  client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
271
 
272
  results = {}
273
  for task_id in TASK_IDS:
274
+ score = await run_task(client, task_id)
275
  results[task_id] = score
276
  print(flush=True) # blank line between tasks for readability
277
 
 
288
 
289
 
290
  if __name__ == "__main__":
291
+ asyncio.run(main())