Spaces:
Sleeping
Sleeping
Add enrichment agent config with prioritized sources, rate limits, and retry policies
3005446 verified | { | |
| "agent_name": "recovery_enrichment_agent", | |
| "version": "1.0.0", | |
| "description": "Configuration for recovery and enrichment agent with prioritized data sources", | |
| "enrichment_sources": { | |
| "priority_order": [ | |
| "crunchbase_api", | |
| "clearbit_api", | |
| "linkedin_browser", | |
| "company_website_scraper", | |
| "apollo_api" | |
| ], | |
| "source_configs": { | |
| "crunchbase_api": { | |
| "enabled": true, | |
| "rate_limit": { | |
| "requests_per_minute": 60, | |
| "requests_per_hour": 3000, | |
| "burst_size": 10 | |
| }, | |
| "retry_policy": { | |
| "max_retries": 3, | |
| "initial_backoff_seconds": 2, | |
| "backoff_multiplier": 2, | |
| "max_backoff_seconds": 60 | |
| }, | |
| "timeout_seconds": 30, | |
| "required_fields": ["company_name", "domain"] | |
| }, | |
| "clearbit_api": { | |
| "enabled": true, | |
| "rate_limit": { | |
| "requests_per_minute": 100, | |
| "requests_per_hour": 5000, | |
| "burst_size": 20 | |
| }, | |
| "retry_policy": { | |
| "max_retries": 3, | |
| "initial_backoff_seconds": 1, | |
| "backoff_multiplier": 2, | |
| "max_backoff_seconds": 30 | |
| }, | |
| "timeout_seconds": 20, | |
| "required_fields": ["domain"] | |
| }, | |
| "linkedin_browser": { | |
| "enabled": true, | |
| "rate_limit": { | |
| "requests_per_minute": 10, | |
| "requests_per_hour": 200, | |
| "burst_size": 3 | |
| }, | |
| "retry_policy": { | |
| "max_retries": 2, | |
| "initial_backoff_seconds": 5, | |
| "backoff_multiplier": 3, | |
| "max_backoff_seconds": 120 | |
| }, | |
| "timeout_seconds": 60, | |
| "requires_human_validation": true, | |
| "fallback_to_browser_agent": true | |
| }, | |
| "company_website_scraper": { | |
| "enabled": true, | |
| "rate_limit": { | |
| "requests_per_minute": 30, | |
| "requests_per_hour": 1000, | |
| "burst_size": 5 | |
| }, | |
| "retry_policy": { | |
| "max_retries": 2, | |
| "initial_backoff_seconds": 3, | |
| "backoff_multiplier": 2, | |
| "max_backoff_seconds": 45 | |
| }, | |
| "timeout_seconds": 45, | |
| "user_agent_rotation": true | |
| }, | |
| "apollo_api": { | |
| "enabled": true, | |
| "rate_limit": { | |
| "requests_per_minute": 50, | |
| "requests_per_hour": 2000, | |
| "burst_size": 15 | |
| }, | |
| "retry_policy": { | |
| "max_retries": 3, | |
| "initial_backoff_seconds": 2, | |
| "backoff_multiplier": 2, | |
| "max_backoff_seconds": 60 | |
| }, | |
| "timeout_seconds": 25 | |
| } | |
| } | |
| }, | |
| "queue_processing": { | |
| "batch_size": 10, | |
| "concurrent_workers": 3, | |
| "queue_check_interval_seconds": 5, | |
| "max_queue_age_hours": 24, | |
| "priority_weights": { | |
| "high": 3, | |
| "medium": 2, | |
| "low": 1 | |
| } | |
| }, | |
| "agent_handoff": { | |
| "browser_agent_fallback": { | |
| "enabled": true, | |
| "trigger_conditions": [ | |
| "api_rate_limit_exceeded", | |
| "authentication_required", | |
| "captcha_detected", | |
| "data_quality_insufficient" | |
| ], | |
| "max_browser_attempts": 2, | |
| "timeout_seconds": 120 | |
| }, | |
| "human_validation": { | |
| "enabled": true, | |
| "trigger_conditions": [ | |
| "conflicting_data_sources", | |
| "confidence_score_below_threshold", | |
| "sensitive_data_detected", | |
| "legal_compliance_check" | |
| ], | |
| "confidence_threshold": 0.75, | |
| "escalation_queue": "human_review" | |
| } | |
| }, | |
| "logging": { | |
| "activity_log_path": "activity_log.csv", | |
| "log_level": "INFO", | |
| "log_rotation": { | |
| "enabled": true, | |
| "max_size_mb": 50, | |
| "backup_count": 5 | |
| }, | |
| "tracked_events": [ | |
| "enrichment_started", | |
| "enrichment_completed", | |
| "enrichment_failed", | |
| "source_attempted", | |
| "source_succeeded", | |
| "source_failed", | |
| "rate_limit_hit", | |
| "retry_attempted", | |
| "browser_agent_triggered", | |
| "human_validation_required", | |
| "escalation_created" | |
| ] | |
| }, | |
| "data_quality": { | |
| "min_confidence_score": 0.6, | |
| "required_enrichment_fields": [ | |
| "company_name", | |
| "industry", | |
| "employee_count", | |
| "founded_year" | |
| ], | |
| "validation_rules": { | |
| "email_format": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", | |
| "phone_format": "^\\+?[1-9]\\d{1,14}$", | |
| "url_format": "^https?://.*" | |
| } | |
| }, | |
| "error_handling": { | |
| "circuit_breaker": { | |
| "enabled": true, | |
| "failure_threshold": 5, | |
| "timeout_seconds": 60, | |
| "half_open_max_calls": 3 | |
| }, | |
| "dead_letter_queue": { | |
| "enabled": true, | |
| "max_retry_count": 5, | |
| "retention_hours": 72 | |
| } | |
| }, | |
| "integration": { | |
| "upstream_agents": ["query_search_agent", "activity_ingestion_agent"], | |
| "downstream_agents": ["validation_agent", "export_agent"], | |
| "message_format": "json", | |
| "queue_type": "priority_queue" | |
| } | |
| } |