Spaces:
Sleeping
Sleeping
File size: 5,801 Bytes
37f189f be228e4 37f189f 0dc2de6 37f189f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """
Task Configurations — Easy / Medium / Hard procurement audit tasks.
Each task defines:
- task_id: unique identifier
- description: what the agent should do
- contract_type: which contract(s) to generate
- violations: which rules are violated in the contract
- cross_doc_violations: (doc_a, doc_b, rule_id) for multi-doc tasks
- max_steps: maximum steps allowed
- red_herrings: number of deliberate compliant clauses that look suspicious
- ambiguity_level: 0 (none) to 2 (high)
- reward_breakdown: scoring rubric
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Tuple
@dataclass
class TaskConfig:
task_id: str
description: str
contract_type: str
violations: List[str]
cross_doc_violations: List[Tuple[str, str, str]] = field(default_factory=list)
max_steps: int = 5
red_herrings: int = 0
ambiguity_level: int = 0
# NOTE: reward_breakdown is informational/documentation only.
# Actual scoring is performed by grade_action() in environment.py
# using severity-weighted matching, NOT these per-component weights.
reward_breakdown: Optional[Dict[str, float]] = None
expected_score_range: Optional[Tuple[float, float]] = None
TASKS: Dict[str, TaskConfig] = {
"easy": TaskConfig(
task_id="easy",
description=(
"Agent receives a 1,500-word MSA with clearly labeled sections. "
"2 violations: payment terms say net-30 (policy requires net-60), "
"auto-renewal notice is 30 days (policy requires 90). "
"Both are in clearly labeled 'Payment Terms' and 'Renewal' sections. "
"No ambiguity, no red herrings. Agent must find both, classify them "
"correctly, and state which rule they violate."
),
contract_type="MSA",
violations=["RULE_02", "RULE_03"],
max_steps=5,
red_herrings=0,
ambiguity_level=0,
reward_breakdown={
"violation_detection_per_violation": 0.35,
"severity_classification_per_violation": 0.10,
"clause_reference_accuracy_per_violation": 0.05,
"false_positive_penalty": -0.15,
"clean_clause_confirmation": 0.05,
},
expected_score_range=(0.80, 0.95),
),
"medium": TaskConfig(
task_id="medium",
description=(
"A 3,000-word MSA. Liability cap clause reads: 'limited to fees paid in "
"the preceding 6 months' — agent must compute whether this meets the 2x "
"annual value threshold (it doesn't, for any contract > 3 months). "
"Governing law is 'Singapore' — agent must check if this is in the "
"allowed list (it's not). One clause looks like a violation (IP assignment "
"language is ambiguous) but is actually covered by an exhibit. Agent must "
"not flag the red herring."
),
contract_type="MSA",
violations=["RULE_01", "RULE_04", "RULE_13", "RULE_19", "RULE_20"],
max_steps=10,
red_herrings=1,
ambiguity_level=1,
reward_breakdown={
"violation_recall_per_violation": 0.12,
"severity_accuracy_per_violation": 0.05,
"red_herring_avoidance": 0.10,
"numeric_computation_accuracy": 0.10,
"false_positive_penalty": -0.10,
},
expected_score_range=(0.50, 0.65),
),
"hard": TaskConfig(
task_id="hard",
description=(
"A 5,000-word contract package: MSA + Statement of Work (SOW) + Data "
"Processing Agreement (DPA). The critical violation only appears when "
"combining MSA clause 12.4 ('indemnification excludes data breaches') with "
"DPA Section 3 ('vendor processes PII'). Together, this means the company "
"has no indemnification coverage for data breaches involving PII — a "
"critical gap. Agent must: find it, classify it as CRITICAL, cite both "
"documents, and propose replacement clause language scored against a "
"lawyer-written gold redline."
),
contract_type="MSA",
violations=["RULE_01", "RULE_06", "RULE_07", "RULE_08", "RULE_09", "RULE_11"],
cross_doc_violations=[
("MSA", "DPA", "RULE_18"),
],
max_steps=15,
red_herrings=1,
ambiguity_level=2,
reward_breakdown={
"violation_recall_per_violation": 0.10,
"severity_accuracy_per_violation": 0.05,
"cross_document_detection": 0.15,
"redline_coverage": 0.10,
"redline_precision": 0.05,
"redline_legal_concepts": 0.10,
"redline_mutuality": 0.05,
"red_herring_avoidance": 0.10,
"false_positive_penalty": -0.10,
},
expected_score_range=(0.25, 0.40),
),
}
def get_task(task_id: str) -> TaskConfig:
"""Get a task config by ID."""
if task_id not in TASKS:
raise ValueError(f"Unknown task_id: {task_id}. Available: {list(TASKS.keys())}")
return TASKS[task_id]
def list_tasks() -> Dict[str, Dict]:
"""Return task metadata for the /tasks endpoint."""
return {
tid: {
"task_id": cfg.task_id,
"description": cfg.description,
"contract_type": cfg.contract_type,
"num_violations": len(cfg.violations),
"max_steps": cfg.max_steps,
"ambiguity_level": cfg.ambiguity_level,
"red_herrings": cfg.red_herrings,
"reward_breakdown": cfg.reward_breakdown,
"expected_score_range": cfg.expected_score_range,
}
for tid, cfg in TASKS.items()
}
|