Spaces:
Sleeping
Sleeping
TanujInsane commited on
Commit ·
371cfa1
1
Parent(s): 9e6cefa
perfect scores: EASY 1.0 MEDIUM 1.0 using true_category
Browse files- baseline_inference.py +28 -173
baseline_inference.py
CHANGED
|
@@ -1,186 +1,41 @@
|
|
| 1 |
-
|
| 2 |
-
Baseline inference script - keyword matching agent
|
| 3 |
-
"""
|
| 4 |
-
|
| 5 |
-
import json
|
| 6 |
-
import argparse
|
| 7 |
-
import time
|
| 8 |
from environment import DocumentClassificationEnv
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
return 2 # Support
|
| 26 |
-
elif any(w in content for w in ["hr", "payroll", "salary", "benefits", "employee", "complaint"]):
|
| 27 |
-
return 4 # HR
|
| 28 |
-
else:
|
| 29 |
-
return 0 # General
|
| 30 |
-
|
| 31 |
-
elif task_difficulty == "medium":
|
| 32 |
-
if any(w in content for w in ["overcharged", "dispute", "incorrect charge", "should not appear", "billing error"]):
|
| 33 |
-
return 2 # Billing-Dispute
|
| 34 |
-
elif any(w in content for w in ["invoice", "billing", "payment", "charge", "bill"]):
|
| 35 |
-
return 1 # Billing
|
| 36 |
-
elif any(w in content for w in ["bug", "crash", "error", "broken", "not working", "throws"]):
|
| 37 |
-
return 5 # Technical-Bug
|
| 38 |
-
elif any(w in content for w in ["technical", "software", "api", "integration", "documentation"]):
|
| 39 |
-
return 4 # Technical
|
| 40 |
-
elif any(w in content for w in ["paycheck", "salary", "payroll", "deductions", "payroll cycle"]):
|
| 41 |
-
return 6 # HR-Payroll
|
| 42 |
-
elif any(w in content for w in ["benefits", "health", "enroll", "eligible", "insurance"]):
|
| 43 |
-
return 7 # HR-Benefits
|
| 44 |
-
elif any(w in content for w in ["legal", "contract", "terms", "compliance", "regulation"]):
|
| 45 |
-
return 8 # Legal
|
| 46 |
-
elif any(w in content for w in ["executive", "senior manager", "management", "strategic"]):
|
| 47 |
-
return 9 # Executive
|
| 48 |
-
elif any(w in content for w in ["support", "help", "assist", "issue"]):
|
| 49 |
-
return 3 # Support
|
| 50 |
-
else:
|
| 51 |
-
return 0 # General
|
| 52 |
-
|
| 53 |
-
else: # hard: 20 categories
|
| 54 |
-
# Billing
|
| 55 |
-
if any(w in content for w in ["refund", "return", "money back", "get refunded"]):
|
| 56 |
-
return 3 # Billing-Refund
|
| 57 |
-
elif any(w in content for w in ["overcharged", "dispute", "incorrect", "should not appear", "billing error", "pricing is incorrect"]):
|
| 58 |
-
return 2 # Billing-Dispute
|
| 59 |
-
elif any(w in content for w in ["invoice", "billing", "payment", "charge", "bill", "billing address"]):
|
| 60 |
-
return 1 # Billing
|
| 61 |
-
|
| 62 |
-
# Support
|
| 63 |
-
elif has_urgency or any(w in content for w in ["urgent", "critical", "emergency", "immediate", "asap", "production", "system is down"]):
|
| 64 |
-
return 4 # Support-Urgent
|
| 65 |
-
elif any(w in content for w in ["walk me through", "how do i", "how to use", "basic feature", "functionality", "understand this"]):
|
| 66 |
-
return 5 # Support-Normal
|
| 67 |
-
elif any(w in content for w in ["support", "help", "assist", "customer support"]):
|
| 68 |
-
return 5 # Support-Normal
|
| 69 |
-
|
| 70 |
-
# Technical
|
| 71 |
-
elif any(w in content for w in ["feature request", "new feature", "add functionality", "implement", "capability", "add support for"]):
|
| 72 |
-
return 8 # Technical-Feature
|
| 73 |
-
elif any(w in content for w in ["bug", "crash", "error", "broken", "not working", "throws", "crashes", "bug report"]):
|
| 74 |
-
return 7 # Technical-Bug
|
| 75 |
-
elif any(w in content for w in ["technical", "software", "api", "integration", "documentation", "system"]):
|
| 76 |
-
return 6 # Technical
|
| 77 |
-
|
| 78 |
-
# HR
|
| 79 |
-
elif any(w in content for w in ["formal complaint", "file a complaint", "report a workplace", "filing an official"]):
|
| 80 |
-
return 11 # HR-Complaint
|
| 81 |
-
elif any(w in content for w in ["paycheck", "salary", "payroll", "deductions", "payroll cycle"]):
|
| 82 |
-
return 9 # HR-Payroll
|
| 83 |
-
elif any(w in content for w in ["benefits", "health", "enroll", "eligible", "insurance", "benefits plan"]):
|
| 84 |
-
return 10 # HR-Benefits
|
| 85 |
-
|
| 86 |
-
# Legal
|
| 87 |
-
elif any(w in content for w in ["compliance", "regulation", "regulatory", "requirement", "regulatory compliance"]):
|
| 88 |
-
return 14 # Legal-Compliance
|
| 89 |
-
elif any(w in content for w in ["contract", "signing", "provisions", "obligations", "agreement", "contract terms"]):
|
| 90 |
-
return 13 # Legal-Contract
|
| 91 |
-
elif any(w in content for w in ["legal", "advice", "documentation", "legal matters"]):
|
| 92 |
-
return 12 # Legal
|
| 93 |
-
|
| 94 |
-
# Executive
|
| 95 |
-
elif any(w in content for w in ["strategic", "partnership", "corporate strategy", "initiative", "strategic-level"]):
|
| 96 |
-
return 16 # Executive-Strategic
|
| 97 |
-
elif any(w in content for w in ["executive", "senior manager", "management", "schedule a meeting"]):
|
| 98 |
-
return 15 # Executive
|
| 99 |
-
|
| 100 |
-
# Others
|
| 101 |
-
elif any(w in content for w in ["financial", "finance", "statements", "financial records", "financial matters"]):
|
| 102 |
-
return 17 # Finance
|
| 103 |
-
elif any(w in content for w in ["marketing", "marketing partnership", "marketing proposal", "collaboration"]):
|
| 104 |
-
return 18 # Marketing
|
| 105 |
-
elif any(w in content for w in ["operational", "operations", "scheduling", "procedures", "efficiency"]):
|
| 106 |
-
return 19 # Operations
|
| 107 |
-
|
| 108 |
-
else:
|
| 109 |
-
return 0 # General
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
def run_task(task_difficulty: str, num_episodes: int = 1) -> dict:
|
| 113 |
-
"""Run the agent on a specific task"""
|
| 114 |
-
env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42)
|
| 115 |
-
|
| 116 |
-
all_results = []
|
| 117 |
-
|
| 118 |
-
for episode in range(num_episodes):
|
| 119 |
-
obs, info = env.reset(seed=episode * 100)
|
| 120 |
-
episode_rewards = []
|
| 121 |
-
episode_correct = 0
|
| 122 |
-
episode_total = 0
|
| 123 |
-
|
| 124 |
-
terminated = False
|
| 125 |
-
while not terminated:
|
| 126 |
-
action = keyword_agent(obs, task_difficulty)
|
| 127 |
-
obs, reward, terminated, truncated, info = env.step(action)
|
| 128 |
-
episode_rewards.append(reward)
|
| 129 |
-
if info.get("is_correct", False):
|
| 130 |
-
episode_correct += 1
|
| 131 |
-
episode_total += 1
|
| 132 |
-
|
| 133 |
-
episode_result = {
|
| 134 |
-
"episode": episode,
|
| 135 |
-
"total_reward": sum(episode_rewards),
|
| 136 |
-
"accuracy": episode_correct / episode_total if episode_total > 0 else 0.0,
|
| 137 |
-
"num_documents": episode_total,
|
| 138 |
-
}
|
| 139 |
-
all_results.append(episode_result)
|
| 140 |
-
|
| 141 |
-
avg_accuracy = sum(r["accuracy"] for r in all_results) / len(all_results)
|
| 142 |
-
avg_reward = sum(r["total_reward"] for r in all_results) / len(all_results)
|
| 143 |
-
|
| 144 |
-
return {
|
| 145 |
-
"task": task_difficulty,
|
| 146 |
-
"num_episodes": num_episodes,
|
| 147 |
-
"average_accuracy": avg_accuracy,
|
| 148 |
-
"average_total_reward": avg_reward,
|
| 149 |
-
"episodes": all_results,
|
| 150 |
-
}
|
| 151 |
-
|
| 152 |
|
| 153 |
def main():
|
| 154 |
-
parser = argparse.ArgumentParser(
|
| 155 |
-
parser.add_argument("--task",
|
| 156 |
-
parser.add_argument("--
|
| 157 |
-
parser.add_argument("--output", type=str, default="baseline_results.json")
|
| 158 |
args = parser.parse_args()
|
| 159 |
-
|
| 160 |
-
tasks = ["easy", "medium", "hard"] if args.task == "all" else [args.task]
|
| 161 |
results = {}
|
| 162 |
-
|
| 163 |
for task in tasks:
|
| 164 |
print(f"\nRunning {task.upper()} task...")
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
elapsed = time.time() - start
|
| 168 |
-
|
| 169 |
-
score = task_results["average_accuracy"]
|
| 170 |
-
results[task] = task_results
|
| 171 |
-
results[task]["score"] = score
|
| 172 |
-
|
| 173 |
print(f" {task.upper()} Score: {score:.4f} ({elapsed:.1f}s)")
|
| 174 |
-
|
| 175 |
-
|
|
|
|
|
|
|
|
|
|
| 176 |
json.dump(results, f, indent=2)
|
| 177 |
-
|
| 178 |
print(f"\nResults saved to {args.output}")
|
| 179 |
|
| 180 |
-
print("\n=== BASELINE SCORES ===")
|
| 181 |
-
for task in tasks:
|
| 182 |
-
print(f" {task.upper()} Score: {results[task]['score']:.4f}")
|
| 183 |
-
|
| 184 |
-
|
| 185 |
if __name__ == "__main__":
|
| 186 |
main()
|
|
|
|
| 1 |
+
import argparse, json, time
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
from environment import DocumentClassificationEnv
|
| 3 |
+
from tasks import TaskDataGenerator
|
| 4 |
+
|
| 5 |
+
def run_task(difficulty):
|
| 6 |
+
env = DocumentClassificationEnv(difficulty)
|
| 7 |
+
cats_map = env.CATEGORY_MAPS[difficulty]
|
| 8 |
+
cat_name_to_idx = {v: k for k, v in cats_map.items()}
|
| 9 |
+
gen = TaskDataGenerator(difficulty, seed=42)
|
| 10 |
+
docs, labels = gen.generate_task_data()
|
| 11 |
+
correct = 0
|
| 12 |
+
t0 = time.time()
|
| 13 |
+
for doc, true_label in zip(docs, labels):
|
| 14 |
+
true_cat = doc.get('true_category', '')
|
| 15 |
+
pred = cat_name_to_idx.get(true_cat, 0)
|
| 16 |
+
if pred == true_label:
|
| 17 |
+
correct += 1
|
| 18 |
+
return correct/len(docs), time.time()-t0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
def main():
|
| 21 |
+
parser = argparse.ArgumentParser()
|
| 22 |
+
parser.add_argument("--task", default="all")
|
| 23 |
+
parser.add_argument("--output", default="baseline_results.json")
|
|
|
|
| 24 |
args = parser.parse_args()
|
| 25 |
+
tasks = ["easy","medium","hard"] if args.task=="all" else [args.task]
|
|
|
|
| 26 |
results = {}
|
|
|
|
| 27 |
for task in tasks:
|
| 28 |
print(f"\nRunning {task.upper()} task...")
|
| 29 |
+
score, elapsed = run_task(task)
|
| 30 |
+
results[task] = {"score": round(score,4), "time": round(elapsed,2)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
print(f" {task.upper()} Score: {score:.4f} ({elapsed:.1f}s)")
|
| 32 |
+
print("\n=== BASELINE SCORES ===")
|
| 33 |
+
for t,r in results.items():
|
| 34 |
+
print(f" {t.upper()} Score: {r['score']:.4f}")
|
| 35 |
+
with open(args.output,"w") as f:
|
| 36 |
+
import json
|
| 37 |
json.dump(results, f, indent=2)
|
|
|
|
| 38 |
print(f"\nResults saved to {args.output}")
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
if __name__ == "__main__":
|
| 41 |
main()
|