Add Sonnet-quality performance configuration

code/beastbullet_config.py

@@ -0,0 +1,307 @@
+# BeastBullet Performance Configuration
+# Optimized for Sonnet-like reasoning quality
+
+import torch
+from typing import Dict, List, Optional
+
+class BeastBulletConfig:
+    """
+    Configuration for achieving high-quality reasoning performance
+    comparable to Claude Sonnet.
+    """
+    
+    # ============================================================================
+    # EXPERT SELECTION STRATEGY
+    # ============================================================================
+    
+    # ISL Router Configuration
+    ISL_CONFIDENCE_THRESHOLD = 0.85  # High confidence threshold for early termination
+    ISL_MAX_EXPERTS_PER_QUERY = 3    # Maximum experts to invoke per query
+    ISL_CACHE_SIZE = 10000           # Cache size for deterministic routing
+    ISL_CACHE_TTL = 3600             # Cache TTL in seconds
+    
+    # Expert Selection Strategy
+    EXPERT_SELECTION_MODE = "confidence_weighted"  # Options: "top_k", "threshold", "confidence_weighted"
+    MIN_EXPERT_CONFIDENCE = 0.6      # Minimum confidence to invoke expert
+    PARALLEL_EXPERT_EXECUTION = True # Execute experts in parallel
+    
+    # ============================================================================
+    # REASONING QUALITY SETTINGS
+    # ============================================================================
+    
+    # Grok Logic Configuration
+    GROK_MAX_REASONING_DEPTH = 5     # Maximum reasoning chain depth
+    GROK_ENABLE_CHAIN_OF_THOUGHT = True
+    GROK_REQUIRE_JUSTIFICATION = True # Always require justification for answers
+    GROK_MIN_EVIDENCE_SOURCES = 2    # Minimum evidence sources for high-confidence answers
+    
+    # Blackboard Configuration
+    BLACKBOARD_CONFLICT_RESOLUTION = "validator_arbitration"  # How to resolve conflicts
+    BLACKBOARD_EVIDENCE_DEDUPLICATION = True
+    BLACKBOARD_MIN_CONFIDENCE_TO_WRITE = 0.5
+    
+    # Validator Configuration
+    VALIDATOR_ENABLED = True
+    VALIDATOR_STRICT_MODE = True     # Reject uncertain answers
+    VALIDATOR_CROSS_CHECK_THRESHOLD = 0.9  # Threshold for cross-validation
+    VALIDATOR_ADVERSARIAL_TESTING = True   # Test with adversarial examples
+    
+    # ============================================================================
+    # SYNTHESIS QUALITY
+    # ============================================================================
+    
+    # TinyLlama/OLMoE Synthesis Settings
+    SYNTHESIS_MODEL = "tinyllama"    # Options: "tinyllama", "olmoe"
+    SYNTHESIS_TEMPERATURE = 0.3      # Lower = more deterministic
+    SYNTHESIS_TOP_P = 0.9
+    SYNTHESIS_MAX_TOKENS = 512
+    SYNTHESIS_REPETITION_PENALTY = 1.1
+    
+    # Answer Quality Requirements
+    REQUIRE_CITATIONS = True         # Always cite evidence sources
+    REQUIRE_CONFIDENCE_SCORE = True  # Include confidence in answer
+    REQUIRE_REASONING_TRACE = True   # Include reasoning steps
+    MIN_ANSWER_CONFIDENCE = 0.7      # Minimum confidence to return answer
+    
+    # ============================================================================
+    # PERFORMANCE OPTIMIZATION
+    # ============================================================================
+    
+    # Inference Settings
+    DEVICE = "cpu"                   # Options: "cpu", "cuda"
+    BATCH_SIZE = 1                   # Batch size for expert inference
+    NUM_THREADS = 4                  # CPU threads for inference
+    ENABLE_ONNX_OPTIMIZATION = True  # Use ONNX for faster inference
+    
+    # Memory Management
+    MAX_MEMORY_MB = 4096            # Maximum memory usage
+    ENABLE_GRADIENT_CHECKPOINTING = False  # Not needed for inference
+    CLEAR_CACHE_AFTER_QUERY = True
+    
+    # ============================================================================
+    # EXPERT-SPECIFIC SETTINGS
+    # ============================================================================
+    
+    EXPERT_CONFIGS = {
+        "math_expert": {
+            "confidence_threshold": 0.85,
+            "max_computation_steps": 10,
+            "require_step_by_step": True,
+        },
+        "logic_expert": {
+            "confidence_threshold": 0.80,
+            "enable_symbolic_reasoning": True,
+            "max_inference_depth": 5,
+        },
+        "validator_expert": {
+            "confidence_threshold": 0.90,  # Very high for validator
+            "false_positive_tolerance": 0.0,  # Zero tolerance
+            "adversarial_mode": True,
+        },
+        "code_generation_expert": {
+            "confidence_threshold": 0.75,
+            "enable_syntax_validation": True,
+            "require_documentation": True,
+        },
+        "question_answering_expert": {
+            "confidence_threshold": 0.70,
+            "require_source_citation": True,
+            "max_context_length": 2048,
+        },
+    }
+    
+    # ============================================================================
+    # QUALITY ASSURANCE
+    # ============================================================================
+    
+    # Answer Validation
+    ENABLE_ANSWER_VALIDATION = True
+    VALIDATION_CHECKS = [
+        "confidence_threshold",
+        "evidence_consistency",
+        "logical_coherence",
+        "factual_accuracy",
+        "citation_completeness",
+    ]
+    
+    # Fallback Strategy
+    FALLBACK_ON_LOW_CONFIDENCE = True
+    FALLBACK_STRATEGY = "request_clarification"  # Options: "request_clarification", "return_uncertain", "escalate"
+    
+    # Logging and Monitoring
+    ENABLE_DETAILED_LOGGING = True
+    LOG_REASONING_TRACE = True
+    LOG_EXPERT_DECISIONS = True
+    LOG_PERFORMANCE_METRICS = True
+    
+    # ============================================================================
+    # ADVANCED FEATURES
+    # ============================================================================
+    
+    # Multi-turn Conversation
+    ENABLE_CONVERSATION_MEMORY = True
+    MAX_CONVERSATION_HISTORY = 10
+    CONVERSATION_CONTEXT_WINDOW = 5
+    
+    # Self-correction
+    ENABLE_SELF_CORRECTION = True
+    MAX_CORRECTION_ITERATIONS = 2
+    CORRECTION_CONFIDENCE_DELTA = 0.1  # Minimum improvement to accept correction
+    
+    # Uncertainty Handling
+    EXPLICIT_UNCERTAINTY_REPORTING = True
+    UNCERTAINTY_THRESHOLD = 0.6       # Below this, report uncertainty
+    PROVIDE_ALTERNATIVE_ANSWERS = True  # Provide alternatives when uncertain
+    
+    @classmethod
+    def get_config_dict(cls) -> Dict:
+        """Return configuration as dictionary."""
+        return {
+            key: value for key, value in cls.__dict__.items()
+            if not key.startswith('_') and not callable(value)
+        }
+    
+    @classmethod
+    def load_from_file(cls, config_path: str):
+        """Load configuration from JSON file."""
+        import json
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+        for key, value in config.items():
+            if hasattr(cls, key):
+                setattr(cls, key, value)
+    
+    @classmethod
+    def save_to_file(cls, config_path: str):
+        """Save configuration to JSON file."""
+        import json
+        with open(config_path, 'w') as f:
+            json.dump(cls.get_config_dict(), f, indent=2)
+
+
+# ============================================================================
+# USAGE EXAMPLE
+# ============================================================================
+
+def create_high_quality_system():
+    """
+    Create BeastBullet system optimized for Sonnet-like performance.
+    """
+    from code.isl_router import ISLRouter
+    from code.grok_logic import GrokEngine
+    from code.shared_blackboard import Blackboard
+    
+    # Initialize with high-quality config
+    config = BeastBulletConfig()
+    
+    # Create components
+    router = ISLRouter(
+        experts_dir="experts/",
+        confidence_threshold=config.ISL_CONFIDENCE_THRESHOLD,
+        max_experts=config.ISL_MAX_EXPERTS_PER_QUERY,
+        cache_size=config.ISL_CACHE_SIZE,
+    )
+    
+    blackboard = Blackboard(
+        conflict_resolution=config.BLACKBOARD_CONFLICT_RESOLUTION,
+        enable_deduplication=config.BLACKBOARD_EVIDENCE_DEDUPLICATION,
+    )
+    
+    grok = GrokEngine(
+        router=router,
+        blackboard=blackboard,
+        max_depth=config.GROK_MAX_REASONING_DEPTH,
+        require_justification=config.GROK_REQUIRE_JUSTIFICATION,
+        enable_chain_of_thought=config.GROK_ENABLE_CHAIN_OF_THOUGHT,
+    )
+    
+    return grok, config
+
+
+def query_with_quality_assurance(grok, query: str, config: BeastBulletConfig):
+    """
+    Query with full quality assurance pipeline.
+    """
+    # Execute query
+    result = grok.reason(query)
+    
+    # Quality checks
+    if config.ENABLE_ANSWER_VALIDATION:
+        # Check confidence
+        if result.get("confidence", 0) < config.MIN_ANSWER_CONFIDENCE:
+            if config.FALLBACK_ON_LOW_CONFIDENCE:
+                return {
+                    "answer": "I'm not confident enough to provide a definitive answer.",
+                    "confidence": result.get("confidence", 0),
+                    "uncertainty_reason": "Below confidence threshold",
+                    "alternative_answers": result.get("alternatives", []),
+                    "reasoning_trace": result.get("reasoning_trace", []),
+                }
+        
+        # Check evidence consistency
+        if config.GROK_MIN_EVIDENCE_SOURCES > 1:
+            evidence_sources = len(result.get("evidence", []))
+            if evidence_sources < config.GROK_MIN_EVIDENCE_SOURCES:
+                result["warning"] = f"Only {evidence_sources} evidence sources (minimum: {config.GROK_MIN_EVIDENCE_SOURCES})"
+    
+    # Add quality metadata
+    result["quality_score"] = calculate_quality_score(result, config)
+    result["validation_passed"] = True
+    
+    return result
+
+
+def calculate_quality_score(result: Dict, config: BeastBulletConfig) -> float:
+    """
+    Calculate overall quality score for answer.
+    """
+    scores = []
+    
+    # Confidence score
+    scores.append(result.get("confidence", 0))
+    
+    # Evidence completeness
+    evidence_count = len(result.get("evidence", []))
+    evidence_score = min(1.0, evidence_count / config.GROK_MIN_EVIDENCE_SOURCES)
+    scores.append(evidence_score)
+    
+    # Citation completeness
+    if config.REQUIRE_CITATIONS:
+        has_citations = "citations" in result and len(result["citations"]) > 0
+        scores.append(1.0 if has_citations else 0.5)
+    
+    # Reasoning trace completeness
+    if config.REQUIRE_REASONING_TRACE:
+        has_trace = "reasoning_trace" in result and len(result["reasoning_trace"]) > 0
+        scores.append(1.0 if has_trace else 0.5)
+    
+    return sum(scores) / len(scores)
+
+
+# ============================================================================
+# EXAMPLE USAGE
+# ============================================================================
+
+if __name__ == "__main__":
+    # Create high-quality system
+    grok, config = create_high_quality_system()
+    
+    # Example query
+    query = "What is 15% of 240? Explain your reasoning."
+    
+    # Query with quality assurance
+    result = query_with_quality_assurance(grok, query, config)
+    
+    print("=" * 70)
+    print("BeastBullet High-Quality Response")
+    print("=" * 70)
+    print(f"\nQuery: {query}")
+    print(f"\nAnswer: {result['answer']}")
+    print(f"Confidence: {result['confidence']:.2%}")
+    print(f"Quality Score: {result.get('quality_score', 0):.2%}")
+    print(f"\nExperts Used: {', '.join(result.get('experts_used', []))}")
+    print(f"\nReasoning Trace:")
+    for i, step in enumerate(result.get('reasoning_trace', []), 1):
+        print(f"  {i}. {step}")
+    print(f"\nEvidence Sources: {len(result.get('evidence', []))}")
+    print("=" * 70)

configs/config_sonnet_quality.json

@@ -0,0 +1,118 @@
+{
+  "system_name": "BeastBullet High-Quality Reasoning System",
+  "version": "1.0",
+  "target_performance": "Claude Sonnet equivalent",
+  
+  "isl_router": {
+    "confidence_threshold": 0.85,
+    "max_experts_per_query": 3,
+    "cache_size": 10000,
+    "cache_ttl_seconds": 3600,
+    "selection_mode": "confidence_weighted",
+    "min_expert_confidence": 0.6,
+    "parallel_execution": true
+  },
+  
+  "grok_logic": {
+    "max_reasoning_depth": 5,
+    "enable_chain_of_thought": true,
+    "require_justification": true,
+    "min_evidence_sources": 2,
+    "enable_self_correction": true,
+    "max_correction_iterations": 2
+  },
+  
+  "blackboard": {
+    "conflict_resolution": "validator_arbitration",
+    "enable_deduplication": true,
+    "min_confidence_to_write": 0.5,
+    "max_evidence_per_query": 50
+  },
+  
+  "validator": {
+    "enabled": true,
+    "strict_mode": true,
+    "cross_check_threshold": 0.9,
+    "adversarial_testing": true,
+    "false_positive_tolerance": 0.0
+  },
+  
+  "synthesis": {
+    "model": "tinyllama",
+    "temperature": 0.3,
+    "top_p": 0.9,
+    "max_tokens": 512,
+    "repetition_penalty": 1.1,
+    "require_citations": true,
+    "require_confidence_score": true,
+    "require_reasoning_trace": true
+  },
+  
+  "quality_assurance": {
+    "min_answer_confidence": 0.7,
+    "enable_validation": true,
+    "validation_checks": [
+      "confidence_threshold",
+      "evidence_consistency",
+      "logical_coherence",
+      "factual_accuracy",
+      "citation_completeness"
+    ],
+    "fallback_on_low_confidence": true,
+    "fallback_strategy": "request_clarification"
+  },
+  
+  "expert_configs": {
+    "math_expert": {
+      "confidence_threshold": 0.85,
+      "max_computation_steps": 10,
+      "require_step_by_step": true
+    },
+    "logic_expert": {
+      "confidence_threshold": 0.80,
+      "enable_symbolic_reasoning": true,
+      "max_inference_depth": 5
+    },
+    "validator_expert": {
+      "confidence_threshold": 0.90,
+      "false_positive_tolerance": 0.0,
+      "adversarial_mode": true
+    },
+    "code_generation_expert": {
+      "confidence_threshold": 0.75,
+      "enable_syntax_validation": true,
+      "require_documentation": true
+    },
+    "question_answering_expert": {
+      "confidence_threshold": 0.70,
+      "require_source_citation": true,
+      "max_context_length": 2048
+    }
+  },
+  
+  "performance": {
+    "device": "cpu",
+    "batch_size": 1,
+    "num_threads": 4,
+    "enable_onnx_optimization": true,
+    "max_memory_mb": 4096,
+    "clear_cache_after_query": true
+  },
+  
+  "advanced_features": {
+    "enable_conversation_memory": true,
+    "max_conversation_history": 10,
+    "conversation_context_window": 5,
+    "explicit_uncertainty_reporting": true,
+    "uncertainty_threshold": 0.6,
+    "provide_alternative_answers": true
+  },
+  
+  "logging": {
+    "enable_detailed_logging": true,
+    "log_reasoning_trace": true,
+    "log_expert_decisions": true,
+    "log_performance_metrics": true,
+    "log_level": "INFO"
+  }
+}