Download inference.py from TanujInsane/document-classification-env: direct link, hf CLI and curl.
- Browser
- Download file 4.71 kB
-
https://huggingface.co/spaces/TanujInsane/document-classification-env/resolve/main/inference.py
- Command line
-
hf download hf://spaces/TanujInsane/document-classification-env/inference.py
-
curl -L -o inference.py https://huggingface.co/spaces/TanujInsane/document-classification-env/resolve/main/inference.py
4.71 kB
| import os | |
| import re | |
| import traceback | |
| from openai import OpenAI | |
| from grading import AgentGrader | |
| from constants import CATEGORIES | |
| # LLM API configuration from environment variables (Grader injected) | |
| API_BASE_URL = os.environ.get("API_BASE_URL") | |
| API_KEY = os.environ.get("API_KEY") | |
| # Initialize LLM client | |
| client = None | |
| if API_BASE_URL and API_KEY: | |
| client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY) | |
| # Per-difficulty LLM call budget (to avoid 30-minute timeout) | |
| # After budget is used, we fall back to a fast keyword classifier | |
| MAX_LLM_CALLS_PER_DIFF = 15 | |
| llm_budget = { | |
| "easy": MAX_LLM_CALLS_PER_DIFF, | |
| "medium": MAX_LLM_CALLS_PER_DIFF, | |
| "hard": MAX_LLM_CALLS_PER_DIFF, | |
| } | |
| KEYWORD_HINTS = { | |
| "billing": 1, | |
| "invoice": 1, | |
| "payment": 1, | |
| "charge": 1, | |
| "refund": 3, | |
| "dispute": 2, | |
| "technical": 4, | |
| "bug": 4, | |
| "error": 4, | |
| "support": 3, | |
| "urgent": 3, | |
| "hr": 5, | |
| "payroll": 5, | |
| "legal": 6, | |
| "contract": 6, | |
| "executive": 7, | |
| "finance": 8, | |
| "marketing": 9, | |
| "operations": 10, | |
| } | |
| def keyword_fallback(content, difficulty): | |
| """Fast keyword-based classification when LLM budget is exhausted.""" | |
| difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"]) | |
| max_valid = max(difficulty_cats.keys()) | |
| content_lower = content.lower() | |
| for keyword, category_idx in KEYWORD_HINTS.items(): | |
| if keyword in content_lower and category_idx <= max_valid: | |
| return category_idx | |
| return 0 # Default to first category | |
| def get_llm_prediction(content, difficulty): | |
| """Uses LLM to classify document content. Falls back to keyword classifier after budget.""" | |
| global llm_budget | |
| # Use fast fallback if no client or budget exhausted | |
| if client is None or llm_budget.get(difficulty, 0) <= 0: | |
| return keyword_fallback(content, difficulty) | |
| # Consume one LLM call from budget | |
| llm_budget[difficulty] -= 1 | |
| difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"]) | |
| category_list = "\n".join([f"- {idx}: {name}" for idx, name in difficulty_cats.items()]) | |
| prompt = f"""You are a document classification expert. | |
| Classify the following customer support document into exactly one of these categories: | |
| {category_list} | |
| Document Content: | |
| {content} | |
| Respond ONLY with the category index (integer). Do not include any other text.""" | |
| try: | |
| response = client.chat.completions.create( | |
| model="gpt-4o", | |
| messages=[{"role": "user", "content": prompt}], | |
| temperature=0, | |
| max_tokens=10, | |
| timeout=10, | |
| ) | |
| prediction = response.choices[0].message.content.strip() | |
| match = re.search(r'\d+', prediction) | |
| if match: | |
| return int(match.group()) | |
| except Exception as e: | |
| print(f"LLM Reasoning Error: {e}") | |
| return keyword_fallback(content, difficulty) | |
| def main(): | |
| difficulties = ["easy", "medium", "hard"] | |
| scores = {} | |
| for diff in difficulties: | |
| try: | |
| # START line per difficulty | |
| print( | |
| f"[START] task={diff} env=DocumentClassification-v1 model=llm_agent", | |
| flush=True, | |
| ) | |
| # Create grader for this difficulty | |
| grader = AgentGrader(task_difficulty=diff, num_episodes=1) | |
| # Agent function using the LLM | |
| def agent_fn(obs, _diff=diff): | |
| return get_llm_prediction(obs["content"], _diff) | |
| # Run grading | |
| result = grader.grade(agent_fn) | |
| score = float(result["score"]) | |
| # Clamp to strictly between 0 and 1 | |
| if score <= 0.0: | |
| score = 0.05 | |
| elif score >= 1.0: | |
| score = 0.95 | |
| scores[diff] = score | |
| # Approximate steps using num_documents from task_config | |
| steps = grader.env.task_config.get("num_documents", 0) | |
| # END line per difficulty | |
| print( | |
| f"[END] success=true steps={steps} score={score:.4f}", | |
| flush=True, | |
| ) | |
| except Exception: | |
| # Fallback: still produce a valid END line | |
| scores[diff] = 0.05 | |
| print( | |
| f"[END] success=false steps=0 score=0.0500", | |
| flush=True, | |
| ) | |
| traceback.print_exc() | |
| # Final summary similar to the example | |
| print("===== FINAL SCORES =====", flush=True) | |
| for diff in difficulties: | |
| if diff in scores: | |
| print(f"{diff}: {scores[diff]:.4f}", flush=True) | |
| if scores: | |
| avg = sum(scores.values()) / len(scores) | |
| print(f"Average: {avg:.4f}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |