Add enrichment agent config with prioritized sources, rate limits, and retry policies

docs/configs/enrichment_config.json

@@ -0,0 +1,199 @@
+{
+  "agent_name": "recovery_enrichment_agent",
+  "version": "1.0.0",
+  "description": "Configuration for recovery and enrichment agent with prioritized data sources",
+  
+  "enrichment_sources": {
+    "priority_order": [
+      "crunchbase_api",
+      "clearbit_api",
+      "linkedin_browser",
+      "company_website_scraper",
+      "apollo_api"
+    ],
+    
+    "source_configs": {
+      "crunchbase_api": {
+        "enabled": true,
+        "rate_limit": {
+          "requests_per_minute": 60,
+          "requests_per_hour": 3000,
+          "burst_size": 10
+        },
+        "retry_policy": {
+          "max_retries": 3,
+          "initial_backoff_seconds": 2,
+          "backoff_multiplier": 2,
+          "max_backoff_seconds": 60
+        },
+        "timeout_seconds": 30,
+        "required_fields": ["company_name", "domain"]
+      },
+      
+      "clearbit_api": {
+        "enabled": true,
+        "rate_limit": {
+          "requests_per_minute": 100,
+          "requests_per_hour": 5000,
+          "burst_size": 20
+        },
+        "retry_policy": {
+          "max_retries": 3,
+          "initial_backoff_seconds": 1,
+          "backoff_multiplier": 2,
+          "max_backoff_seconds": 30
+        },
+        "timeout_seconds": 20,
+        "required_fields": ["domain"]
+      },
+      
+      "linkedin_browser": {
+        "enabled": true,
+        "rate_limit": {
+          "requests_per_minute": 10,
+          "requests_per_hour": 200,
+          "burst_size": 3
+        },
+        "retry_policy": {
+          "max_retries": 2,
+          "initial_backoff_seconds": 5,
+          "backoff_multiplier": 3,
+          "max_backoff_seconds": 120
+        },
+        "timeout_seconds": 60,
+        "requires_human_validation": true,
+        "fallback_to_browser_agent": true
+      },
+      
+      "company_website_scraper": {
+        "enabled": true,
+        "rate_limit": {
+          "requests_per_minute": 30,
+          "requests_per_hour": 1000,
+          "burst_size": 5
+        },
+        "retry_policy": {
+          "max_retries": 2,
+          "initial_backoff_seconds": 3,
+          "backoff_multiplier": 2,
+          "max_backoff_seconds": 45
+        },
+        "timeout_seconds": 45,
+        "user_agent_rotation": true
+      },
+      
+      "apollo_api": {
+        "enabled": true,
+        "rate_limit": {
+          "requests_per_minute": 50,
+          "requests_per_hour": 2000,
+          "burst_size": 15
+        },
+        "retry_policy": {
+          "max_retries": 3,
+          "initial_backoff_seconds": 2,
+          "backoff_multiplier": 2,
+          "max_backoff_seconds": 60
+        },
+        "timeout_seconds": 25
+      }
+    }
+  },
+  
+  "queue_processing": {
+    "batch_size": 10,
+    "concurrent_workers": 3,
+    "queue_check_interval_seconds": 5,
+    "max_queue_age_hours": 24,
+    "priority_weights": {
+      "high": 3,
+      "medium": 2,
+      "low": 1
+    }
+  },
+  
+  "agent_handoff": {
+    "browser_agent_fallback": {
+      "enabled": true,
+      "trigger_conditions": [
+        "api_rate_limit_exceeded",
+        "authentication_required",
+        "captcha_detected",
+        "data_quality_insufficient"
+      ],
+      "max_browser_attempts": 2,
+      "timeout_seconds": 120
+    },
+    
+    "human_validation": {
+      "enabled": true,
+      "trigger_conditions": [
+        "conflicting_data_sources",
+        "confidence_score_below_threshold",
+        "sensitive_data_detected",
+        "legal_compliance_check"
+      ],
+      "confidence_threshold": 0.75,
+      "escalation_queue": "human_review"
+    }
+  },
+  
+  "logging": {
+    "activity_log_path": "activity_log.csv",
+    "log_level": "INFO",
+    "log_rotation": {
+      "enabled": true,
+      "max_size_mb": 50,
+      "backup_count": 5
+    },
+    "tracked_events": [
+      "enrichment_started",
+      "enrichment_completed",
+      "enrichment_failed",
+      "source_attempted",
+      "source_succeeded",
+      "source_failed",
+      "rate_limit_hit",
+      "retry_attempted",
+      "browser_agent_triggered",
+      "human_validation_required",
+      "escalation_created"
+    ]
+  },
+  
+  "data_quality": {
+    "min_confidence_score": 0.6,
+    "required_enrichment_fields": [
+      "company_name",
+      "industry",
+      "employee_count",
+      "founded_year"
+    ],
+    "validation_rules": {
+      "email_format": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
+      "phone_format": "^\\+?[1-9]\\d{1,14}$",
+      "url_format": "^https?://.*"
+    }
+  },
+  
+  "error_handling": {
+    "circuit_breaker": {
+      "enabled": true,
+      "failure_threshold": 5,
+      "timeout_seconds": 60,
+      "half_open_max_calls": 3
+    },
+    "dead_letter_queue": {
+      "enabled": true,
+      "max_retry_count": 5,
+      "retention_hours": 72
+    }
+  },
+  
+  "integration": {
+    "upstream_agents": ["query_search_agent", "activity_ingestion_agent"],
+    "downstream_agents": ["validation_agent", "export_agent"],
+    "message_format": "json",
+    "queue_type": "priority_queue"
+  }
+}