improve-eval-2.0 / debug_backward_pass.py
mdkrasnow
database and logging changes
f886976
Raw History Blame Contribute Delete
12.8 kB
#!/usr/bin/env python
"""
Debug script specifically for the backward pass.
This script directly tests the backward pass functionality with
detailed inspection of each step to help identify why it's failing.
"""
import os
import sys
import logging
import inspect
from pathlib import Path
import json
import traceback
# Configure detailed logging
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler('backward_pass_debug.log')
]
)
logger = logging.getLogger(__name__)
# Add parent directory to path for imports
sys.path.append(str(Path(__file__).parent))
# Import components
from data_labelling.archer.archer import Archer
from data_labelling.archer.helpers.prompt import Prompt
from data_labelling.archer.backwardPass.danielson_model import DanielsonModel
from data_labelling.archer.backwardPass.promptOptimizer import PromptOptimizer, ADALFLOW_AVAILABLE
def inspect_object(obj, name="object", max_depth=1, current_depth=0):
"""
Recursively inspect an object and log its attributes.
Args:
obj: The object to inspect
name: Name to use in logging
max_depth: Maximum recursion depth
current_depth: Current recursion depth
"""
indent = " " * current_depth
if current_depth > max_depth:
logger.debug(f"{indent}{name}: [max depth reached]")
return
# Log the object type
obj_type = type(obj).__name__
logger.debug(f"{indent}{name} (type: {obj_type}):")
# For basic types, just log the value
if isinstance(obj, (str, int, float, bool, type(None))):
logger.debug(f"{indent} value: {obj}")
return
# For lists, log length and first few items
if isinstance(obj, list):
logger.debug(f"{indent} length: {len(obj)}")
if len(obj) > 0:
for i, item in enumerate(obj[:3]):
if current_depth < max_depth:
inspect_object(item, f"{name}[{i}]", max_depth, current_depth + 1)
if len(obj) > 3:
logger.debug(f"{indent} ... ({len(obj) - 3} more items)")
return
# For dictionaries, log keys and first few values
if isinstance(obj, dict):
logger.debug(f"{indent} keys: {list(obj.keys())}")
if len(obj) > 0:
for k, v in list(obj.items())[:3]:
if current_depth < max_depth:
inspect_object(v, f"{name}['{k}']", max_depth, current_depth + 1)
if len(obj) > 3:
logger.debug(f"{indent} ... ({len(obj) - 3} more items)")
return
# For other objects, log attributes
try:
attrs = dir(obj)
attrs = [a for a in attrs if not a.startswith('__')]
logger.debug(f"{indent} attributes: {attrs[:10]}" +
(f" ... ({len(attrs) - 10} more)" if len(attrs) > 10 else ""))
# Log a few important attributes for known types
if obj_type == 'Prompt':
logger.debug(f"{indent} content: {obj.content[:50]}...")
logger.debug(f"{indent} score: {getattr(obj, 'score', 'N/A')}")
logger.debug(f"{indent} generation: {getattr(obj, 'generation', 'N/A')}")
elif obj_type == 'Parameter' and hasattr(obj, 'data'):
logger.debug(f"{indent} data: {obj.data[:50]}...")
logger.debug(f"{indent} role_desc: {getattr(obj, 'role_desc', 'N/A')}")
elif obj_type == 'Archer':
logger.debug(f"{indent} adalflow_enabled: {getattr(obj, 'adalflow_enabled', 'N/A')}")
if hasattr(obj, 'active_prompts'):
logger.debug(f"{indent} active_prompts: {len(obj.active_prompts)}")
except Exception as e:
logger.debug(f"{indent} error inspecting: {str(e)}")
def debug_adalflow_status():
"""Debug the AdaLFlow availability and status."""
logger.info("=== CHECKING ADALFLOW STATUS ===")
logger.info(f"ADALFLOW_AVAILABLE: {ADALFLOW_AVAILABLE}")
# Try to import AdaLFlow components directly
try:
logger.info("Attempting to import AdaLFlow components directly")
try:
from adalflow.optim.parameter import Parameter, ParameterType
logger.info(" βœ“ Successfully imported Parameter and ParameterType")
except ImportError as e:
logger.error(f" βœ— Failed to import Parameter: {str(e)}")
try:
from adalflow.optim.text_grad.tgd_optimizer import TGDOptimizer
logger.info(" βœ“ Successfully imported TGDOptimizer")
except ImportError as e:
logger.error(f" βœ— Failed to import TGDOptimizer: {str(e)}")
try:
from adalflow.core import Generator
logger.info(" βœ“ Successfully imported Generator")
except ImportError as e:
logger.error(f" βœ— Failed to import Generator: {str(e)}")
except Exception as e:
logger.error(f"Error during AdaLFlow imports: {str(e)}")
# Check if we're using the mock classes
if not ADALFLOW_AVAILABLE:
logger.info("Using mock AdaLFlow classes")
optimizer = PromptOptimizer(
model_name="test-model",
adalflow_enabled=True
)
logger.info(f"Mock optimizer.optimizer type: {type(optimizer.optimizer).__name__}")
logger.info(f"Mock optimizer.generator type: {type(optimizer.generator).__name__}")
else:
logger.info("Using real AdaLFlow classes")
def debug_backward_pass_steps():
"""Debug each step of the backward pass in isolation."""
logger.info("=== DEBUGGING BACKWARD PASS STEPS ===")
# Create test objects
initial_prompts = [
Prompt("Test prompt 1: Generate an analysis for component {component_id} from {input}"),
Prompt("Test prompt 2: Create an evaluation for component {component_id} using {input}")
]
# Create test evaluations
test_evaluations = [
(
initial_prompts[0],
"Generated content for prompt 1",
{"score": 3.5, "feedback": "Test feedback for prompt 1"}
),
(
initial_prompts[1],
"Generated content for prompt 2",
{"score": 4.0, "feedback": "Test feedback for prompt 2"}
)
]
# Create optimizer with logging for each step
optimizer = PromptOptimizer(
model_name="test-model",
temperature=0.7,
adalflow_enabled=True, # Test with AdaLFlow enabled
openrouter_api_key="test_api_key"
)
# Log the optimizer status
logger.info("Optimizer status:")
logger.info(f" adalflow_enabled: {optimizer.adalflow_enabled}")
logger.info(f" ADALFLOW_AVAILABLE: {ADALFLOW_AVAILABLE}")
# Debug the _wrap_prompts_as_params method
logger.info("Testing _wrap_prompts_as_params method")
try:
parameters = optimizer._wrap_prompts_as_params(initial_prompts)
logger.info(f" βœ“ Created {len(parameters)} parameters")
for i, param in enumerate(parameters):
logger.info(f" Parameter {i}: {param.data[:30]}...")
except Exception as e:
logger.error(f" βœ— Error in _wrap_prompts_as_params: {str(e)}")
traceback.print_exc()
# Debug parameters and optimization
logger.info("Creating parameters and testing backward")
try:
# Create feedback and score maps
feedback_map = {"0": "Test feedback 1", "1": "Test feedback 2"}
score_map = {"0": 3.5, "1": 4.0}
if optimizer.adalflow_enabled and ADALFLOW_AVAILABLE:
# Define a mock backward method for debugging
def mock_backward(self):
logger.info(f"Mock backward called for {self.role_desc}")
# Here you can add additional checks
# Replace the actual backward method with our mock
from adalflow.optim.parameter import Parameter
original_backward = Parameter.backward
Parameter.backward = mock_backward
# Run the optimization
try:
logger.info("Running optimization with mock backward")
new_prompts = optimizer.optimize(initial_prompts, feedback_map, score_map)
logger.info(f" βœ“ Optimization completed, got {len(new_prompts)} new prompts")
# Inspect the results
for i, prompt in enumerate(new_prompts):
logger.info(f" Prompt {i}: {prompt.content[:50]}...")
finally:
# Restore original method
Parameter.backward = original_backward
else:
logger.info("Running fallback optimization (AdaLFlow not available)")
new_prompts = optimizer.optimize(initial_prompts, feedback_map, score_map)
logger.info(f" βœ“ Fallback optimization completed, got {len(new_prompts)} new prompts")
except Exception as e:
logger.error(f" βœ— Error in optimization: {str(e)}")
traceback.print_exc()
# Now debug with a real Archer object
logger.info("Testing backward pass with Archer")
try:
# Create an Archer instance with AdaLFlow enabled
archer = Archer(
generator_model_name="test-model",
evaluator_model_name="test-model",
optimizer_model_name="test-model",
knowledge_base=["./data_labelling/eval"],
rubric="Test rubric",
initial_prompts=initial_prompts,
openrouter_api_key="test_api_key",
adalflow_enabled=True
)
# Replace optimizer with our debug version
archer.optimizer = optimizer
# Set the active prompts
archer.active_prompts = initial_prompts
# Log Archer status
logger.info(f"Archer adalflow_enabled: {archer.adalflow_enabled}")
logger.info(f"Archer optimizer.adalflow_enabled: {archer.optimizer.adalflow_enabled}")
# Run the backward pass
logger.info("Running Archer backward pass")
archer.run_backward_pass(test_evaluations)
# Check if the active prompts were updated
logger.info(f"Active prompts after backward pass: {len(archer.active_prompts)}")
for i, prompt in enumerate(archer.active_prompts):
logger.info(f" Prompt {i}: {prompt.content[:50]}...")
except Exception as e:
logger.error(f"Error in Archer backward pass: {str(e)}")
traceback.print_exc()
def debug_danielson_backward_pass():
"""Debug the backward pass with the Danielson model."""
logger.info("=== DEBUGGING DANIELSON MODEL BACKWARD PASS ===")
# Create the Danielson model
model = DanielsonModel(adalflow_enabled=True)
# Log model status
logger.info(f"Model name: {model.name}")
logger.info(f"Model adalflow_enabled: {model.adalflow_enabled}")
logger.info(f"Model has {len(model.prompts)} prompts")
logger.info(f"Model has {len(model.adalflow_params) if hasattr(model, 'adalflow_params') else 0} adalflow_params")
# Create feedback and score maps
feedback_map = {
"context_analysis": "Could be more specific",
"component_evaluation_base": "Needs better structure"
}
score_map = {
"context_analysis": 3.5,
"component_evaluation_base": 4.0
}
# Create optimizer
optimizer = PromptOptimizer(
model_name="test-model",
temperature=0.7,
adalflow_enabled=True,
openrouter_api_key="test_api_key"
)
# Debug optimize_model
logger.info("Testing optimize_model with Danielson model")
try:
result = optimizer.optimize_model(model, feedback_map, score_map)
logger.info(f"optimize_model result: {result}")
except Exception as e:
logger.error(f"Error in optimize_model: {str(e)}")
traceback.print_exc()
def main():
"""Run all debug functions."""
logger.info("Starting backward pass debugging")
try:
# Check AdaLFlow status
debug_adalflow_status()
# Debug backward pass steps
debug_backward_pass_steps()
# Debug Danielson model backward pass
debug_danielson_backward_pass()
logger.info("Debugging completed")
except Exception as e:
logger.error(f"Unhandled exception in debugging: {str(e)}")
traceback.print_exc()
if __name__ == "__main__":
main()