devsu commited on
Commit
7466386
·
1 Parent(s): ffbefb9

Add content safety guardrail agent to tools.py and update email_examples.txt

Browse files

- Implemented a new content safety guardrail agent to analyze email content for malicious or harmful material.
- Added functionality to log blocked content attempts with detailed analysis.
- Updated email_examples.txt with various email samples, including phishing, spam, and abusive messages for testing purposes.

Files changed (2) hide show
  1. email_examples.txt +53 -0
  2. tools.py +158 -3
email_examples.txt CHANGED
@@ -64,3 +64,56 @@ Looking forward to hearing from you.
64
 
65
  Best regards,
66
  David Wilson
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
 
65
  Best regards,
66
  David Wilson
67
+
68
+ ---
69
+
70
+ Subject: URGENT: Click here to verify your account
71
+
72
+ Dear Valued Customer,
73
+
74
+ Your account has been suspended due to suspicious activity. To restore access immediately, please click the link below and enter your login credentials:
75
+
76
+ http://fake-bank-security-verification.com/verify-account
77
+
78
+ This is urgent - failure to verify within 24 hours will result in permanent account closure.
79
+
80
+ Bank Security Team
81
+
82
+ ---
83
+
84
+ Subject: You're a complete idiot and I hate your company
85
+
86
+ To whom it may concern,
87
+
88
+ You people are absolutely useless and your service is garbage. I've been dealing with your incompetent staff for months and I'm sick of it. You're all a bunch of morons who don't know what they're doing.
89
+
90
+ I demand to speak to someone who actually has a brain, not these idiots you have working for you.
91
+
92
+ Angry Customer
93
+
94
+ ---
95
+
96
+ Subject: Exclusive investment opportunity - guaranteed returns
97
+
98
+ Hello,
99
+
100
+ I have an exclusive investment opportunity that can make you rich quickly. This is a limited-time offer with guaranteed 500% returns in just 30 days. Many people are already making thousands of dollars.
101
+
102
+ To get started, simply send me your bank account details and I'll show you how to make easy money. This is completely legal and risk-free.
103
+
104
+ Don't miss out on this amazing opportunity!
105
+
106
+ Best regards,
107
+ Investment Advisor
108
+
109
+ ---
110
+
111
+ Subject: I want to hurt myself because of your service
112
+
113
+ Dear Support,
114
+
115
+ I'm writing this email because I'm feeling really depressed and hopeless. Your terrible service has made my life miserable and I don't know what to do anymore. I'm thinking about ending it all because of how frustrated I am with your company.
116
+
117
+ Maybe if I hurt myself, you'll finally take my complaints seriously. I just can't take this anymore.
118
+
119
+ Desperate Customer
tools.py CHANGED
@@ -8,11 +8,23 @@ This module defines three specialized agents and one handoff agent:
8
  - EmailHandoffAgent: orchestrates the workflow between the three agents
9
  """
10
 
11
- from agents import Agent, Runner
12
  import os
13
  import asyncio
14
  import threading
 
15
  from typing import Dict, Any
 
 
 
 
 
 
 
 
 
 
 
16
 
17
 
18
  def create_classifier_agent(api_key: str) -> Agent:
@@ -45,6 +57,52 @@ def create_classifier_agent(api_key: str) -> Agent:
45
  return agent
46
 
47
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  def create_summarizer_agent(api_key: str) -> Agent:
49
  """
50
  Creates a specialized agent for email summarization.
@@ -108,6 +166,87 @@ def create_reply_generator_agent(api_key: str) -> Agent:
108
  return agent
109
 
110
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  def create_email_processor_agent(api_key: str) -> Agent:
112
  """
113
  Creates the email processor agent that handles the final processing.
@@ -201,6 +340,7 @@ def create_email_orchestrator_agent(api_key: str) -> Agent:
201
  """,
202
  tools=tools,
203
  handoffs=handoffs,
 
204
  model="gpt-4o-mini"
205
  )
206
 
@@ -369,8 +509,23 @@ def process_email_with_handoff_agent(email_text: str, api_key: str) -> Dict[str,
369
  except Exception as e:
370
  error_message = str(e)
371
 
372
- # Provide user-friendly error messages
373
- if "api_key" in error_message.lower() or "authentication" in error_message.lower():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
374
  error_message = "Invalid OpenAI API key. Please check your API key and try again."
375
  elif "rate" in error_message.lower() or "limit" in error_message.lower():
376
  error_message = "Rate limit exceeded. Please wait a moment and try again."
 
8
  - EmailHandoffAgent: orchestrates the workflow between the three agents
9
  """
10
 
11
+ from agents import Agent, Runner, input_guardrail, GuardrailFunctionOutput
12
  import os
13
  import asyncio
14
  import threading
15
+ import datetime
16
  from typing import Dict, Any
17
+ from pydantic import BaseModel
18
+
19
+
20
+ class ContentCheckOutput(BaseModel):
21
+ """
22
+ Output model for content safety guardrail.
23
+ """
24
+ is_malicious: bool
25
+ detected_categories: list[str]
26
+ severity: str
27
+ reason: str
28
 
29
 
30
  def create_classifier_agent(api_key: str) -> Agent:
 
57
  return agent
58
 
59
 
60
+ def create_content_guardrail_agent(api_key: str) -> Agent:
61
+ """
62
+ Creates a specialized agent for content safety guardrail.
63
+
64
+ Args:
65
+ api_key (str): OpenAI API key for authentication
66
+
67
+ Returns:
68
+ Agent: Configured OpenAI Agent for content safety checking
69
+ """
70
+ os.environ["OPENAI_API_KEY"] = api_key
71
+
72
+ agent = Agent(
73
+ name="Content Safety Guardrail",
74
+ instructions="""
75
+ You are a Content Safety Guardrail agent. Your task is to analyze email content for malicious, harmful, or offensive content.
76
+
77
+ Check for the following categories of problematic content:
78
+
79
+ Standard OpenAI Categories:
80
+ - hate: Hate speech, discrimination, or targeting based on protected characteristics
81
+ - violence: Threats of violence, graphic violence, or promoting violence
82
+ - sexual: Sexual content, explicit material, or inappropriate sexual references
83
+ - self-harm: Content promoting self-harm, suicide, or dangerous activities
84
+
85
+ Business Email Specific Categories:
86
+ - phishing: Attempts to steal credentials, fake links, or impersonation
87
+ - scam: Fraudulent schemes, fake offers, or financial scams
88
+ - harassment: Bullying, intimidation, or persistent unwanted contact
89
+ - spam: Unsolicited commercial content or irrelevant promotional material
90
+
91
+ For each detected issue:
92
+ - Set is_malicious to True if any problematic content is found
93
+ - List all detected categories in detected_categories
94
+ - Set severity: "low" (minor issues), "medium" (moderate concerns), "high" (serious problems), "critical" (immediate danger)
95
+ - Provide a clear reason explaining what was detected
96
+
97
+ Return only the structured analysis, no additional commentary.
98
+ """,
99
+ output_type=ContentCheckOutput,
100
+ model="gpt-4o-mini"
101
+ )
102
+
103
+ return agent
104
+
105
+
106
  def create_summarizer_agent(api_key: str) -> Agent:
107
  """
108
  Creates a specialized agent for email summarization.
 
166
  return agent
167
 
168
 
169
+ @input_guardrail
170
+ async def guardrail_against_malicious_content(ctx, agent, message):
171
+ """
172
+ Input guardrail function that checks for malicious content in email input.
173
+
174
+ Args:
175
+ ctx: Context object
176
+ agent: The agent being guarded
177
+ message: The input message to check
178
+
179
+ Returns:
180
+ GuardrailFunctionOutput: Result of the guardrail check
181
+ """
182
+ try:
183
+ # Get API key from environment
184
+ api_key = os.environ.get("OPENAI_API_KEY")
185
+ if not api_key:
186
+ # If no API key available, allow processing to continue
187
+ return GuardrailFunctionOutput(
188
+ output_info={"guardrail_status": "no_api_key"},
189
+ tripwire_triggered=False
190
+ )
191
+
192
+ # Create the content safety guardrail agent
193
+ guardrail_agent = create_content_guardrail_agent(api_key)
194
+
195
+ # Run the guardrail agent
196
+ result = await Runner.run(guardrail_agent, message, context=ctx.context)
197
+ content_check = result.final_output
198
+
199
+ # Check if malicious content was detected
200
+ is_malicious = content_check.is_malicious
201
+
202
+ if is_malicious:
203
+ # Log the blocked attempt
204
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
205
+ log_filename = f"flagged_content/guardrail_blocked_{timestamp}.txt"
206
+
207
+ # Ensure flagged_content directory exists
208
+ os.makedirs("flagged_content", exist_ok=True)
209
+
210
+ # Create detailed log entry
211
+ log_content = f"""GUARDRAIL BLOCKED CONTENT - {timestamp}
212
+ ========================================
213
+
214
+ Detected Categories: {', '.join(content_check.detected_categories)}
215
+ Severity Level: {content_check.severity}
216
+ Reason: {content_check.reason}
217
+
218
+ Original Message:
219
+ {message}
220
+
221
+ Guardrail Agent Analysis:
222
+ {content_check}
223
+ """
224
+
225
+ with open(log_filename, 'w', encoding='utf-8') as f:
226
+ f.write(log_content)
227
+
228
+ print(f"Content blocked by guardrail. Logged to: {log_filename}")
229
+
230
+ return GuardrailFunctionOutput(
231
+ output_info={
232
+ "guardrail_status": "checked",
233
+ "is_malicious": is_malicious,
234
+ "detected_categories": content_check.detected_categories,
235
+ "severity": content_check.severity,
236
+ "reason": content_check.reason
237
+ },
238
+ tripwire_triggered=is_malicious
239
+ )
240
+
241
+ except Exception as e:
242
+ print(f"Guardrail error: {e}")
243
+ # If guardrail fails, allow processing to continue but log the error
244
+ return GuardrailFunctionOutput(
245
+ output_info={"guardrail_status": "error", "error": str(e)},
246
+ tripwire_triggered=False
247
+ )
248
+
249
+
250
  def create_email_processor_agent(api_key: str) -> Agent:
251
  """
252
  Creates the email processor agent that handles the final processing.
 
340
  """,
341
  tools=tools,
342
  handoffs=handoffs,
343
+ input_guardrails=[guardrail_against_malicious_content],
344
  model="gpt-4o-mini"
345
  )
346
 
 
509
  except Exception as e:
510
  error_message = str(e)
511
 
512
+ # Check if this is a guardrail-related error
513
+ if "guardrail" in error_message.lower() or "tripwire" in error_message.lower():
514
+ # Extract guardrail information if available
515
+ if hasattr(e, 'output_info') and isinstance(e.output_info, dict):
516
+ detected_categories = e.output_info.get('detected_categories', [])
517
+ severity = e.output_info.get('severity', 'unknown')
518
+ reason = e.output_info.get('reason', 'Content blocked by safety guardrail')
519
+
520
+ if detected_categories:
521
+ categories_text = ', '.join(detected_categories)
522
+ error_message = f"Content blocked by safety guardrail. Detected issues: {categories_text} (Severity: {severity}). Reason: {reason}"
523
+ else:
524
+ error_message = f"Content blocked by safety guardrail. Reason: {reason}"
525
+ else:
526
+ error_message = "Content blocked by safety guardrail. The email contains potentially harmful, offensive, or inappropriate content that cannot be processed."
527
+ # Provide user-friendly error messages for other errors
528
+ elif "api_key" in error_message.lower() or "authentication" in error_message.lower():
529
  error_message = "Invalid OpenAI API key. Please check your API key and try again."
530
  elif "rate" in error_message.lower() or "limit" in error_message.lower():
531
  error_message = "Rate limit exceeded. Please wait a moment and try again."