Hugging Face's logo

SetMD
/
beastbullet-experts

ONNX
Model card Files
xet
 Community
beastbullet-experts / code /beastbullet_config.py
SetMD's picture
SetMD
Add Sonnet-quality performance configuration
d323efe
Raw History Blame Contribute Delete
11.6 kB
# BeastBullet Performance Configuration
# Optimized for Sonnet-like reasoning quality
import torch
from typing import Dict, List, Optional
class BeastBulletConfig:
"""
Configuration for achieving high-quality reasoning performance
comparable to Claude Sonnet.
"""
# ============================================================================
# EXPERT SELECTION STRATEGY
# ============================================================================
# ISL Router Configuration
ISL_CONFIDENCE_THRESHOLD = 0.85 # High confidence threshold for early termination
ISL_MAX_EXPERTS_PER_QUERY = 3 # Maximum experts to invoke per query
ISL_CACHE_SIZE = 10000 # Cache size for deterministic routing
ISL_CACHE_TTL = 3600 # Cache TTL in seconds
# Expert Selection Strategy
EXPERT_SELECTION_MODE = "confidence_weighted" # Options: "top_k", "threshold", "confidence_weighted"
MIN_EXPERT_CONFIDENCE = 0.6 # Minimum confidence to invoke expert
PARALLEL_EXPERT_EXECUTION = True # Execute experts in parallel
# ============================================================================
# REASONING QUALITY SETTINGS
# ============================================================================
# Grok Logic Configuration
GROK_MAX_REASONING_DEPTH = 5 # Maximum reasoning chain depth
GROK_ENABLE_CHAIN_OF_THOUGHT = True
GROK_REQUIRE_JUSTIFICATION = True # Always require justification for answers
GROK_MIN_EVIDENCE_SOURCES = 2 # Minimum evidence sources for high-confidence answers
# Blackboard Configuration
BLACKBOARD_CONFLICT_RESOLUTION = "validator_arbitration" # How to resolve conflicts
BLACKBOARD_EVIDENCE_DEDUPLICATION = True
BLACKBOARD_MIN_CONFIDENCE_TO_WRITE = 0.5
# Validator Configuration
VALIDATOR_ENABLED = True
VALIDATOR_STRICT_MODE = True # Reject uncertain answers
VALIDATOR_CROSS_CHECK_THRESHOLD = 0.9 # Threshold for cross-validation
VALIDATOR_ADVERSARIAL_TESTING = True # Test with adversarial examples
# ============================================================================
# SYNTHESIS QUALITY
# ============================================================================
# TinyLlama/OLMoE Synthesis Settings
SYNTHESIS_MODEL = "tinyllama" # Options: "tinyllama", "olmoe"
SYNTHESIS_TEMPERATURE = 0.3 # Lower = more deterministic
SYNTHESIS_TOP_P = 0.9
SYNTHESIS_MAX_TOKENS = 512
SYNTHESIS_REPETITION_PENALTY = 1.1
# Answer Quality Requirements
REQUIRE_CITATIONS = True # Always cite evidence sources
REQUIRE_CONFIDENCE_SCORE = True # Include confidence in answer
REQUIRE_REASONING_TRACE = True # Include reasoning steps
MIN_ANSWER_CONFIDENCE = 0.7 # Minimum confidence to return answer
# ============================================================================
# PERFORMANCE OPTIMIZATION
# ============================================================================
# Inference Settings
DEVICE = "cpu" # Options: "cpu", "cuda"
BATCH_SIZE = 1 # Batch size for expert inference
NUM_THREADS = 4 # CPU threads for inference
ENABLE_ONNX_OPTIMIZATION = True # Use ONNX for faster inference
# Memory Management
MAX_MEMORY_MB = 4096 # Maximum memory usage
ENABLE_GRADIENT_CHECKPOINTING = False # Not needed for inference
CLEAR_CACHE_AFTER_QUERY = True
# ============================================================================
# EXPERT-SPECIFIC SETTINGS
# ============================================================================
EXPERT_CONFIGS = {
"math_expert": {
"confidence_threshold": 0.85,
"max_computation_steps": 10,
"require_step_by_step": True,
},
"logic_expert": {
"confidence_threshold": 0.80,
"enable_symbolic_reasoning": True,
"max_inference_depth": 5,
},
"validator_expert": {
"confidence_threshold": 0.90, # Very high for validator
"false_positive_tolerance": 0.0, # Zero tolerance
"adversarial_mode": True,
},
"code_generation_expert": {
"confidence_threshold": 0.75,
"enable_syntax_validation": True,
"require_documentation": True,
},
"question_answering_expert": {
"confidence_threshold": 0.70,
"require_source_citation": True,
"max_context_length": 2048,
},
}
# ============================================================================
# QUALITY ASSURANCE
# ============================================================================
# Answer Validation
ENABLE_ANSWER_VALIDATION = True
VALIDATION_CHECKS = [
"confidence_threshold",
"evidence_consistency",
"logical_coherence",
"factual_accuracy",
"citation_completeness",
]
# Fallback Strategy
FALLBACK_ON_LOW_CONFIDENCE = True
FALLBACK_STRATEGY = "request_clarification" # Options: "request_clarification", "return_uncertain", "escalate"
# Logging and Monitoring
ENABLE_DETAILED_LOGGING = True
LOG_REASONING_TRACE = True
LOG_EXPERT_DECISIONS = True
LOG_PERFORMANCE_METRICS = True
# ============================================================================
# ADVANCED FEATURES
# ============================================================================
# Multi-turn Conversation
ENABLE_CONVERSATION_MEMORY = True
MAX_CONVERSATION_HISTORY = 10
CONVERSATION_CONTEXT_WINDOW = 5
# Self-correction
ENABLE_SELF_CORRECTION = True
MAX_CORRECTION_ITERATIONS = 2
CORRECTION_CONFIDENCE_DELTA = 0.1 # Minimum improvement to accept correction
# Uncertainty Handling
EXPLICIT_UNCERTAINTY_REPORTING = True
UNCERTAINTY_THRESHOLD = 0.6 # Below this, report uncertainty
PROVIDE_ALTERNATIVE_ANSWERS = True # Provide alternatives when uncertain
@classmethod
def get_config_dict(cls) -> Dict:
"""Return configuration as dictionary."""
return {
key: value for key, value in cls.__dict__.items()
if not key.startswith('_') and not callable(value)
}
@classmethod
def load_from_file(cls, config_path: str):
"""Load configuration from JSON file."""
import json
with open(config_path, 'r') as f:
config = json.load(f)
for key, value in config.items():
if hasattr(cls, key):
setattr(cls, key, value)
@classmethod
def save_to_file(cls, config_path: str):
"""Save configuration to JSON file."""
import json
with open(config_path, 'w') as f:
json.dump(cls.get_config_dict(), f, indent=2)
# ============================================================================
# USAGE EXAMPLE
# ============================================================================
def create_high_quality_system():
"""
Create BeastBullet system optimized for Sonnet-like performance.
"""
from code.isl_router import ISLRouter
from code.grok_logic import GrokEngine
from code.shared_blackboard import Blackboard
# Initialize with high-quality config
config = BeastBulletConfig()
# Create components
router = ISLRouter(
experts_dir="experts/",
confidence_threshold=config.ISL_CONFIDENCE_THRESHOLD,
max_experts=config.ISL_MAX_EXPERTS_PER_QUERY,
cache_size=config.ISL_CACHE_SIZE,
)
blackboard = Blackboard(
conflict_resolution=config.BLACKBOARD_CONFLICT_RESOLUTION,
enable_deduplication=config.BLACKBOARD_EVIDENCE_DEDUPLICATION,
)
grok = GrokEngine(
router=router,
blackboard=blackboard,
max_depth=config.GROK_MAX_REASONING_DEPTH,
require_justification=config.GROK_REQUIRE_JUSTIFICATION,
enable_chain_of_thought=config.GROK_ENABLE_CHAIN_OF_THOUGHT,
)
return grok, config
def query_with_quality_assurance(grok, query: str, config: BeastBulletConfig):
"""
Query with full quality assurance pipeline.
"""
# Execute query
result = grok.reason(query)
# Quality checks
if config.ENABLE_ANSWER_VALIDATION:
# Check confidence
if result.get("confidence", 0) < config.MIN_ANSWER_CONFIDENCE:
if config.FALLBACK_ON_LOW_CONFIDENCE:
return {
"answer": "I'm not confident enough to provide a definitive answer.",
"confidence": result.get("confidence", 0),
"uncertainty_reason": "Below confidence threshold",
"alternative_answers": result.get("alternatives", []),
"reasoning_trace": result.get("reasoning_trace", []),
}
# Check evidence consistency
if config.GROK_MIN_EVIDENCE_SOURCES > 1:
evidence_sources = len(result.get("evidence", []))
if evidence_sources < config.GROK_MIN_EVIDENCE_SOURCES:
result["warning"] = f"Only {evidence_sources} evidence sources (minimum: {config.GROK_MIN_EVIDENCE_SOURCES})"
# Add quality metadata
result["quality_score"] = calculate_quality_score(result, config)
result["validation_passed"] = True
return result
def calculate_quality_score(result: Dict, config: BeastBulletConfig) -> float:
"""
Calculate overall quality score for answer.
"""
scores = []
# Confidence score
scores.append(result.get("confidence", 0))
# Evidence completeness
evidence_count = len(result.get("evidence", []))
evidence_score = min(1.0, evidence_count / config.GROK_MIN_EVIDENCE_SOURCES)
scores.append(evidence_score)
# Citation completeness
if config.REQUIRE_CITATIONS:
has_citations = "citations" in result and len(result["citations"]) > 0
scores.append(1.0 if has_citations else 0.5)
# Reasoning trace completeness
if config.REQUIRE_REASONING_TRACE:
has_trace = "reasoning_trace" in result and len(result["reasoning_trace"]) > 0
scores.append(1.0 if has_trace else 0.5)
return sum(scores) / len(scores)
# ============================================================================
# EXAMPLE USAGE
# ============================================================================
if __name__ == "__main__":
# Create high-quality system
grok, config = create_high_quality_system()
# Example query
query = "What is 15% of 240? Explain your reasoning."
# Query with quality assurance
result = query_with_quality_assurance(grok, query, config)
print("=" * 70)
print("BeastBullet High-Quality Response")
print("=" * 70)
print(f"\nQuery: {query}")
print(f"\nAnswer: {result['answer']}")
print(f"Confidence: {result['confidence']:.2%}")
print(f"Quality Score: {result.get('quality_score', 0):.2%}")
print(f"\nExperts Used: {', '.join(result.get('experts_used', []))}")
print(f"\nReasoning Trace:")
for i, step in enumerate(result.get('reasoning_trace', []), 1):
print(f" {i}. {step}")
print(f"\nEvidence Sources: {len(result.get('evidence', []))}")
print("=" * 70)